解码h264videostream获取图像缓冲区

我跟着这个post来解码我的h264videostream帧。

我的数据框如下：

我的代码：

NSString * const naluTypesStrings[] = { @"0: Unspecified (non-VCL)", @"1: Coded slice of a non-IDR picture (VCL)", // P frame @"2: Coded slice data partition A (VCL)", @"3: Coded slice data partition B (VCL)", @"4: Coded slice data partition C (VCL)", @"5: Coded slice of an IDR picture (VCL)", // I frame @"6: Supplemental enhancement information (SEI) (non-VCL)", @"7: Sequence parameter set (non-VCL)", // SPS parameter @"8: Picture parameter set (non-VCL)", // PPS parameter @"9: Access unit delimiter (non-VCL)", @"10: End of sequence (non-VCL)", @"11: End of stream (non-VCL)", @"12: Filler data (non-VCL)", @"13: Sequence parameter set extension (non-VCL)", @"14: Prefix NAL unit (non-VCL)", @"15: Subset sequence parameter set (non-VCL)", @"16: Reserved (non-VCL)", @"17: Reserved (non-VCL)", @"18: Reserved (non-VCL)", @"19: Coded slice of an auxiliary coded picture without partitioning (non-VCL)", @"20: Coded slice extension (non-VCL)", @"21: Coded slice extension for depth view components (non-VCL)", @"22: Reserved (non-VCL)", @"23: Reserved (non-VCL)", @"24: STAP-A Single-time aggregation packet (non-VCL)", @"25: STAP-B Single-time aggregation packet (non-VCL)", @"26: MTAP16 Multi-time aggregation packet (non-VCL)", @"27: MTAP24 Multi-time aggregation packet (non-VCL)", @"28: FU-A Fragmentation unit (non-VCL)", @"29: FU-B Fragmentation unit (non-VCL)", @"30: Unspecified (non-VCL)", @"31: Unspecified (non-VCL)", }; -(void) receivedRawVideoFrame:(uint8_t *)frame withSize:(uint32_t)frameSize isIFrame:(int)isIFrame { // First NAUL is AUD so i skiped 6 frame frame = &frame[6]; frameSize -= 6; OSStatus status; uint8_t *data = NULL; uint8_t *pps = NULL; uint8_t *sps = NULL; // I know what my H.264 data source's NALUs look like so I know start code index is always 0. // if you don't know where it starts, you can use a for loop similar to how i find the 2nd and 3rd start codes int startCodeIndex = 0; int secondStartCodeIndex = 0; int thirdStartCodeIndex = 0; int fourthStartCodeIndex = 0; long blockLength = 0; CMSampleBufferRef sampleBuffer = NULL; CMBlockBufferRef blockBuffer = NULL; int nalu_type = (frame[startCodeIndex + 4] & 0x1F); NSLog(@"~~~~~~~ Received NALU Type \"%@\" ~~~~~~~~", naluTypesStrings[nalu_type]); // if we havent already set up our format description with our SPS PPS parameters, we // can't process any frames except type 7 that has our parameters if (nalu_type != 7 && _formatDesc == NULL) { NSLog(@"Video error: Frame is not an I Frame and format description is null"); return; } // NALU type 7 is the SPS parameter NALU if (nalu_type == 7) { // find where the second PPS start code begins, (the 0x00 00 00 01 code) // from which we also get the length of the first SPS code for (int i = startCodeIndex + 4; i < frameSize; i++) { if (frame[i] == 0x00 && frame[i+1] == 0x00 && frame[i+2] == 0x00 && frame[i+3] == 0x01) { secondStartCodeIndex = i; _spsSize = secondStartCodeIndex; // includes the header in the size break; } } // find what the second NALU type is nalu_type = (frame[secondStartCodeIndex + 4] & 0x1F); NSLog(@"~~~~~~~ Received NALU Type \"%@\" ~~~~~~~~", naluTypesStrings[nalu_type]); } // type 8 is the PPS parameter NALU if(nalu_type == 8) { // find where the NALU after this one starts so we know how long the PPS parameter is for (int i = secondStartCodeIndex + 4; i < frameSize; i++) { if (frame[i] == 0x00 && frame[i+1] == 0x00 && frame[i+2] == 0x01) { thirdStartCodeIndex = i; _ppsSize = thirdStartCodeIndex - _spsSize; break; } } // _ppsSize = frameSize - -_spsSize; // allocate enough data to fit the SPS and PPS parameters into our data objects. // VTD doesn't want you to include the start code header (4 bytes long) so we add the - 4 here sps = malloc(_spsSize - 4); pps = malloc(_ppsSize - 4); // copy in the actual sps and pps values, again ignoring the 4 byte header memcpy (sps, &frame[4], _spsSize-4); memcpy (pps, &frame[_spsSize+4], _ppsSize-4); // now we set our H264 parameters uint8_t* parameterSetPointers[2] = {sps, pps}; size_t parameterSetSizes[2] = {(_spsSize-4), (_ppsSize-4)}; status = CMVideoFormatDescriptionCreateFromH264ParameterSets(kCFAllocatorDefault, 2, (const uint8_t *const*)parameterSetPointers, parameterSetSizes, 4, &_formatDesc); NSLog(@"\t\t Creation of CMVideoFormatDescription: %@", (status == noErr) ? @"successful!" : @"failed..."); if(status != noErr) NSLog(@"\t\t Format Description ERROR type: %d", (int)status); // See if decomp session can convert from previous format description // to the new one, if not we need to remake the decomp session. // This snippet was not necessary for my applications but it could be for yours // BOOL needNewDecompSession = (VTDecompressionSessionCanAcceptFormatDescription(_decompressionSession, _formatDesc) == NO); // if(needNewDecompSession) // { // [self createDecompSession]; // } // now lets handle the IDR frame that (should) come after the parameter sets // I say "should" because that's how I expect my H264 stream to work, YMMV nalu_type = (frame[thirdStartCodeIndex + 3] & 0x1F); NSLog(@"~~~~~~~ Received NALU Type \"%@\" ~~~~~~~~", naluTypesStrings[nalu_type]); } // create our VTDecompressionSession. This isnt neccessary if you choose to use AVSampleBufferDisplayLayer if((status == noErr) && (_decompressionSession == NULL)) { [self createDecompSession]; } if(nalu_type == 6) { for (int i = thirdStartCodeIndex + 3; i < frameSize; i++) { if (frame[i] == 0x00 && frame[i+1] == 0x00 && frame[i+2] == 0x01) { fourthStartCodeIndex = i; _seiSize = fourthStartCodeIndex - (_spsSize + _ppsSize); break; } } // do stuff here // [...] nalu_type = (frame[fourthStartCodeIndex + 3] & 0x1F); } // type 5 is an IDR frame NALU. The SPS and PPS NALUs should always be followed by an IDR (or IFrame) NALU, as far as I know if(nalu_type == 5) { // find the offset, or where the SPS and PPS NALUs end and the IDR frame NALU begins int offset = fourthStartCodeIndex; int fifthStartCodeIndex = 0; for (int i = fourthStartCodeIndex + 3; i < frameSize; i++) { if (frame[i] == 0x00 && frame[i+1] == 0x00 && frame[i+2] == 0x01) { fifthStartCodeIndex = i; break; } } blockLength = fifthStartCodeIndex - offset; data = malloc(blockLength); data = memcpy(data, &frame[offset], blockLength); // replace the start code header on this NALU with its size. // AVCC format requires that you do this. // htonl converts the unsigned int from host to network byte order uint32_t dataLength32 = htonl (blockLength - 3); memcpy (data, &dataLength32, sizeof (uint32_t)); // create a block buffer from the IDR NALU status = CMBlockBufferCreateWithMemoryBlock(NULL, data, // memoryBlock to hold buffered data blockLength, // block length of the mem block in bytes. kCFAllocatorNull, NULL, 0, // offsetToData blockLength, // dataLength of relevant bytes, starting at offsetToData 0, &blockBuffer); NSLog(@"\t\t BlockBufferCreation: \t %@", (status == kCMBlockBufferNoErr) ? @"successful!" : @"failed..."); blockLength = frameSize - fifthStartCodeIndex; data = malloc(blockLength); data = memcpy(data, &frame[offset], blockLength); // replace the start code header on this NALU with its size. // AVCC format requires that you do this. // htonl converts the unsigned int from host to network byte order uint32_t dataLength32_1 = htonl (blockLength - 3); memcpy (data, &dataLength32_1, sizeof (uint32_t)); status = CMBlockBufferAppendMemoryBlock(blockBuffer, data, blockLength, kCFAllocatorNull, NULL, 0, blockLength, 0); NSLog(@"\t\t BlockBufferAppend: \t %@", (status == kCMBlockBufferNoErr) ? @"successful!" : @"failed..."); } // NALU type 1 is non-IDR (or PFrame) picture if (nalu_type == 1) { // non-IDR frames do not have an offset due to SPS and PSS, so the approach // is similar to the IDR frames just without the offset blockLength = frameSize; data = malloc(blockLength); data = memcpy(data, &frame[0], blockLength); // again, replace the start header with the size of the NALU uint32_t dataLength32 = htonl (blockLength - 4); memcpy (data, &dataLength32, sizeof (uint32_t)); status = CMBlockBufferCreateWithMemoryBlock(NULL, data, // memoryBlock to hold data. If NULL, block will be alloc when needed blockLength, // overall length of the mem block in bytes kCFAllocatorNull, NULL, 0, // offsetToData blockLength, // dataLength of relevant data bytes, starting at offsetToData 0, &blockBuffer); NSLog(@"\t\t BlockBufferCreation: \t %@", (status == kCMBlockBufferNoErr) ? @"successful!" : @"failed..."); } // now create our sample buffer from the block buffer, if(status == noErr) { // here I'm not bothering with any timing specifics since in my case we displayed all frames immediately const size_t sampleSize = blockLength; sampleBuffer = NULL; status = CMSampleBufferCreate(kCFAllocatorDefault, blockBuffer, true, NULL, NULL, _formatDesc, 1, 0, NULL, 1, &sampleSize, &sampleBuffer); NSLog(@"\t\t SampleBufferCreate: \t %@", (status == noErr) ? @"successful!" : @"failed..."); } if(status == noErr) { // set some values of the sample buffer's attachments CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, YES); CFMutableDictionaryRef dict = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachments, 0); CFDictionarySetValue(dict, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue); // either send the samplebuffer to a VTDecompressionSession or to an AVSampleBufferDisplayLayer [self render:sampleBuffer]; } // free memory to avoid a memory leak, do the same for sps, pps and blockbuffer if (NULL != data) { free (data); data = NULL; } } -(void) createDecompSession { // make sure to destroy the old VTD session _decompressionSession = NULL; VTDecompressionOutputCallbackRecord callBackRecord; callBackRecord.decompressionOutputCallback = decompressionSessionDecodeFrameCallback; // this is necessary if you need to make calls to Objective C "self" from within in the callback method. callBackRecord.decompressionOutputRefCon = (__bridge void *)self; // you can set some desired attributes for the destination pixel buffer. I didn't use this but you may // if you need to set some attributes, be sure to uncomment the dictionary in VTDecompressionSessionCreate NSDictionary *destinationImageBufferAttributes = [NSDictionary dictionaryWithObjectsAndKeys: [NSNumber numberWithBool:YES], (id)kCVPixelBufferOpenGLESCompatibilityKey, nil]; OSStatus status = VTDecompressionSessionCreate(NULL, _formatDesc, NULL, (__bridge CFDictionaryRef)(destinationImageBufferAttributes), &callBackRecord, &_decompressionSession); NSLog(@"Video Decompression Session Create: \t %@", (status == noErr) ? @"successful!" : @"failed..."); if(status != noErr) NSLog(@"\t\t VTD ERROR type: %d", (int)status); } void decompressionSessionDecodeFrameCallback(void *decompressionOutputRefCon, void *sourceFrameRefCon, OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef imageBuffer, CMTime presentationTimeStamp, CMTime presentationDuration) { // THISCLASSNAME *streamManager = (__bridge THISCLASSNAME *)decompressionOutputRefCon; if (status != noErr) { NSError *error = [NSError errorWithDomain:NSOSStatusErrorDomain code:status userInfo:nil]; NSLog(@"Decompressed error: %@", error); } else { NSLog(@"Decompressed sucessfully"); // do something with your resulting CVImageBufferRef that is your decompressed frame // [streamManager displayDecodedFrame:imageBuffer]; } } - (void) render:(CMSampleBufferRef)sampleBuffer { VTDecodeFrameFlags flags = kVTDecodeFrame_1xRealTimePlayback; VTDecodeInfoFlags flagOut; NSDate* currentTime = [NSDate date]; VTDecompressionSessionDecodeFrame(_decompressionSession, sampleBuffer, flags, (void*)CFBridgingRetain(currentTime), &flagOut); CFRelease(sampleBuffer); // if you're using AVSampleBufferDisplayLayer, you only need to use this line of code // [videoLayer enqueueSampleBuffer:sampleBuffer]; } @end 

输出：

 [2110:825268] ~~~~~~~ Received NALU Type "7: Sequence parameter set (non-VCL)" ~~~~~~~~ [2110:825268] ~~~~~~~ Received NALU Type "8: Picture parameter set (non-VCL)" ~~~~~~~~ [2110:825268] Creation of CMVideoFormatDescription: successful! [2110:825268] ~~~~~~~ Received NALU Type "6: Supplemental enhancement information (SEI) (non-VCL)" ~~~~~~~~ [2110:825268] BlockBufferCreation: successful! [2110:825268] BlockBufferAppend: successful! [2110:825268] SampleBufferCreate: successful! [2110:825277] Decompressed error: Error Domain=NSOSStatusErrorDomain Code=-12909 "(null)" 

请指出我错过了什么。