大华设备rtsp的sdp中获取的sps错误相关处理

李国帅取自日志 2012/04/12 09:03:17

东西比较乱，作为参考

参数集：序列参数集和图像参数集

序列参数集包括一个图像序列的所有信息，即两个IDR图像间的所有图像信息

图像参数集包括一个图像的所有分片的所有相关信息，包括图像类型、序列号等，解码时某些序列号的丢失可用来校验信息包的丢失与否。

多个不同的序列和图像参数集存储在解码器中，编码器依据每个编码分片的头部存储位置来选择适当的参数集，图像及本身也包括使用的序列参数集等参考信息。

正文

大华的设置中，从sdp中获取的sps可能是错误的，必须等待第一个关键帧到来，才能从中分析出正确的参数。

所以不能急于播放，必须等待关键帧的到来。

void CRTPClientSession::AddVideoData(char*pData, UInt32 nLen, UInt32 inTimestamp, UInt32 inSeqNum, Bool16 inMark )
{//For video, Combine packet if necessaryif(m_nFirstFrame == 1)//make sure the first frame is start with key frame
{int nFrameType =WaitIFrame(pData,inTimestamp);if (nFrameType != 1)return;m_nFirstFrame= 0;TraceMsg("ProcessRTPPacket first received, Key frame, timestamp=%u, id=%d\n", inTimestamp, m_nSessionID);}//unsigned char start_sequence[]= {0, 0, 1};unsigned char start_sequence[]= {0, 0, 0, 1};unsignedchar end_sequence[]= {0x00, 0x00, 0x01, 0x09, 0x10};//帧末添加帧间分隔符//TraceMsg("ProcessRTPPacket received, frame inMark=%d,inSeqNum=%d,timestamp=%u, nLen=%d\n",inMark,inSeqNum, inTimestamp, nLen);if(m_nLengthBuffer == 0 || (m_nLastPacketTimestamp != inTimestamp))//begin new frame
{if(m_nLastPacketTimestamp !=inTimestamp){//TraceMsg("ProcessRTPPacket Invalid combining Frame: hasn't been end. timestamp = %u\n", inTimestamp);//_ASSERT(m_nLastPacketTimestamp!=0);m_nLengthBuffer = 0; //may be a new frame arrived? drop exist packets which is bad.重新开始一帧。
}int nValid =CheckVideoData(pData);if (nValid < 0)return;m_nLastPacketTimestamp=inTimestamp;m_nLastPacketSeqNum=inSeqNum;if(inMark)//deliver it if this frame is end
{m_cReadWrite.Lock();
#ifdef _DEBUG//memcpy(m_cbFrameBuffer, pData, nLen);//start with a new frame//memcpy(m_cbFrameBuffer+nLen,end_sequence,sizeof(end_sequence));//int lenNew = sizeof(end_sequence);////lenNew += nLen;//m_nLengthBuffer = lenNew;
memcpy(m_cbFrameBuffer,&start_sequence,sizeof(start_sequence));//加入起始码int lenNew = sizeof(start_sequence);//            memcpy(m_cbFrameBuffer+lenNew,pData,nLen);lenNew+=nLen;m_nLengthBuffer=lenNew;BYTE pDstData[200000];int lDataLenOut = 200000;//单一帧包m_VideoHandler->DecodeRecvData((BYTE*)m_cbFrameBuffer, m_nLengthBuffer,pDstData,lDataLenOut);#endif //_DEBUG//m_pRTPRecv->AddData(TRUE, (BYTE*)m_cbFrameBuffer, m_nLengthBuffer, inTimestamp);
m_nLengthBuffer= 0;m_cReadWrite.Lock();}else//分帧模式第一个包
{//如何获取sps,现在直接从媒体流读取，以后要把sdp中的数据使用base64解码。//判断帧类型如果是idr帧，需要在帧头加上sps和pps，格式为：起始码（0x00000001）+sps+起始码+pps+起始码+完整帧数据int lenNew = 0;//            USHORT nTag = ((USHORT*)pData)[0];if(nTag == 0x857C || nTag == 0x817C)//如何判断idr帧0x857C
{memcpy(m_cbFrameBuffer+lenNew,&start_sequence,sizeof(start_sequence));lenNew+= sizeof(start_sequence);////unsigned char sps[]= {0x67,0x42,0xe0,0x1e,0x67,0x42,0xe0,0x1e,0xe2,0x58};//海康4cif spsunsigned char sps[]= {0x36,0x37,0x34,0x32,0x65,0x30,0x31,0x34,0x64,0x61,0x30,0x35,0x38,0x32,0x35,0x31};//dah 4cifmemcpy(m_cbFrameBuffer+lenNew,&sps,sizeof(sps));lenNew+= sizeof(sps);memcpy(m_cbFrameBuffer,&start_sequence,sizeof(start_sequence));lenNew+= sizeof(start_sequence);////unsigned char pps[]= {0x68,0xce,0x3c,0x80};//海康4cifunsigned char pps[]= {0x36,0x38,0x63,0x65,0x33,0x30,0x61,0x34,0x38,0x30};//dah 4cifmemcpy(m_cbFrameBuffer+lenNew,&pps,sizeof(pps));lenNew+= sizeof(pps);}memcpy(m_cbFrameBuffer+lenNew,&start_sequence,sizeof(start_sequence));lenNew+= sizeof(start_sequence);////将FU-A包根据rtp的包序号和FU-A的header字节，组成完整264帧；unsigned char head1 = (BYTE)(*pData);//获取第一个字节unsigned char head2 = (BYTE)(*(pData+1));//获取第二个字节//unsigned char type = head1 & 0x1f;//获取FU indicator的类型域，//unsigned char flag = head2 & 0xe0;//获取FU header的前三位，判断当前是分包的开始、中间或结束.80开始，40结束BYTE tagNal = (head1 & 0xe0) | (head2 & 0x1f);memcpy(m_cbFrameBuffer+lenNew,&tagNal,1);memcpy(m_cbFrameBuffer+lenNew+1,pData+2,nLen-2);lenNew+= nLen-2+1;m_nLengthBuffer=lenNew;}return;}else//combine packets
{if(m_nLastPacketSeqNum + 1 !=inSeqNum){TraceMsg("ProcessRTPPacket Invalid Frame: missing any packets. inSeqNum = %u\n", inSeqNum);m_nLengthBuffer= 0; //// check seqNum, WARNNING 序号错乱,丢弃随后的所有帧
}else{m_nLastPacketSeqNum=inSeqNum;//goes here, the packet is validif(inMark)//分帧模式最后一包
{m_cReadWrite.Lock();
#ifdef _DEBUGint lenNew =m_nLengthBuffer;memcpy(m_cbFrameBuffer+lenNew, pData+2, nLen-2);lenNew+= nLen -2;//memcpy(m_cbFrameBuffer+lenNew,end_sequence,sizeof(end_sequence));//m_nLengthBuffer += sizeof(end_sequence);
BYTE pDstData[200000];int lDataLenOut = 200000;m_VideoHandler->DecodeRecvData((BYTE*)m_cbFrameBuffer, m_nLengthBuffer,pDstData,lDataLenOut);#endif //_DEBUG//m_pRTPRecv->AddData(TRUE, (BYTE*)m_cbFrameBuffer, m_nLengthBuffer, m_nLastPacketTimestamp);m_nLengthBuffer = 0;m_cReadWrite.Lock();}else////分帧模式中间包
{int lenNew =m_nLengthBuffer;memcpy(m_cbFrameBuffer+lenNew, pData+2, nLen-2);lenNew+= nLen -2;m_nLengthBuffer=lenNew;}}}
}

更改为

void CRTPClientSession::AddVideoDataH264(char*pData, UInt32 nLen, UInt32 inTimestamp, UInt32 inSeqNum, Bool16 inMark )
{unsignedchar head1 = (BYTE)(*pData);//获取第一个字节unsigned char type = head1 & 0x1f;//获取FU indicator的类型域，if(type==0x7)//判断NAL的类型为sps
{memcpy(m_cbSPS,pData,nLen);m_nSPS=nLen;return;}else if (type == 0x8)//判断NAL的类型为pps
{memcpy(m_cbPPS,pData,nLen);m_nPPS=nLen;return;}if(m_nSPS == 0 || m_nPPS == 0){return;//没有获取到配置参数
}unsignedchar start_sequence[]= {0x00, 0x00, 0x00, 0x01};//{0, 0, 1};//    unsigned char end_sequence[]= {0x00, 0x00, 0x01, 0x09, 0x10};//帧末添加帧间分隔符if(type==0x1c)//判断NAL的类型为0x1c=28，说明是FU-A分片
{//TraceMsg("ProcessRTPPacket received, frame inMark=%d,inSeqNum=%d,timestamp=%u, nLen=%d\n",inMark,inSeqNum, inTimestamp, nLen);
unsignedchar head2 = (BYTE)(*(pData+1));//获取第二个字节unsigned char flag = head2 & 0xe0;//获取FU header的前三位，判断当前是分包的开始、中间或结束.80开始，40结束unsigned char typeSingle = head2 & 0x1f;//获取single nal的类型域，int nFrameType = -1;//无效帧if(m_nGetFirstFrame == 0)//make sure the first frame is start with key frame
{if(typeSingle == 0x1){nFrameType= 0;//普通帧
}else if(typeSingle == 0x5){nFrameType= 1;//关键帧m_nGetFirstFrame = 1;TraceMsg("ProcessRTPPacket first received, Key frame, timestamp=%u, id=%d\n", inTimestamp, m_nSessionID);}else{_ASSERT(0);}}if (m_nGetFirstFrame == 0)return;//make sure the first frame is start with key frameif (flag==0x80)//开始//分帧模式第一个包
{ASSERT(m_nLengthBuffer== 0 || (m_nLastPacketTimestamp !=inTimestamp));//TraceMsg("ProcessRTPPacket Invalid combining Frame: hasn't been end. timestamp = %u\n", inTimestamp);
m_nLengthBuffer= 0; //may be a new frame arrived? drop exist packets which is bad.重新开始一帧。
m_nLastPacketTimestamp=inTimestamp;m_nLastPacketSeqNum=inSeqNum;int lenNew = 0;//            USHORT nTag = ((USHORT*)pData)[0];if(nFrameType == 1){memcpy(m_cbFrameBuffer+lenNew,start_sequence,sizeof(start_sequence));lenNew+= sizeof(start_sequence);//memcpy(m_cbFrameBuffer+lenNew,m_cbSPS,m_nSPS);lenNew+=m_nSPS;memcpy(m_cbFrameBuffer+lenNew,start_sequence,sizeof(start_sequence));lenNew+= sizeof(start_sequence);//memcpy(m_cbFrameBuffer+lenNew,m_cbPPS,m_nPPS);lenNew+=m_nPPS;}memcpy(m_cbFrameBuffer+lenNew,start_sequence,sizeof(start_sequence));lenNew+= sizeof(start_sequence);////将FU-A包根据rtp的包序号和FU-A的header字节，组成完整264帧；BYTE tagNal = (head1 & 0xe0) | (head2 & 0x1f);memcpy(m_cbFrameBuffer+lenNew,&tagNal,1);memcpy(m_cbFrameBuffer+lenNew+1,pData+2,nLen-2);lenNew+= nLen-2+1;m_nLengthBuffer=lenNew;}else if(flag==0x40)//结束
{ASSERT(inMark);//分帧模式最后一包if(m_nLastPacketSeqNum + 1 !=inSeqNum){TraceMsg("ProcessRTPPacket Invalid Frame: missing any packets. inSeqNum = %u\n", inSeqNum);m_nLengthBuffer= 0; //// check seqNum, WARNNING 序号错乱,丢弃随后的所有帧return;}m_nLastPacketSeqNum=inSeqNum;m_cReadWrite.Lock();int lenNew =m_nLengthBuffer;memcpy(m_cbFrameBuffer+lenNew, pData+2, nLen-2);lenNew+= nLen -2;m_nLengthBuffer+=lenNew;memcpy(m_cbFrameBuffer+lenNew,end_sequence,sizeof(end_sequence));m_nLengthBuffer+= sizeof(end_sequence);
#ifdef _DEBUG//BYTE pDstData[200000];//int lDataLenOut = 200000;//m_VideoHandler->DecodeRecvData((BYTE*)m_cbFrameBuffer, m_nLengthBuffer,pDstData,lDataLenOut);
#endif //_DEBUGm_pRTPRecv->AddData(TRUE, (BYTE*)m_cbFrameBuffer, m_nLengthBuffer, m_nLastPacketTimestamp);m_nLengthBuffer= 0;m_cReadWrite.Lock();}else//中间
{if(m_nLastPacketSeqNum + 1 !=inSeqNum){TraceMsg("ProcessRTPPacket Invalid Frame: missing any packets. inSeqNum = %u\n", inSeqNum);m_nLengthBuffer= 0; //// check seqNum, WARNNING 序号错乱,丢弃随后的所有帧return;}m_nLastPacketSeqNum=inSeqNum;////分帧模式中间包int lenNew =m_nLengthBuffer;memcpy(m_cbFrameBuffer+lenNew, pData+2, nLen-2);lenNew+= nLen -2;m_nLengthBuffer=lenNew;}}else if (type>=0x1 && type<=0x17)//单包帧数据
{ASSERT(inMark);m_nLastPacketTimestamp=inTimestamp;m_nLastPacketSeqNum=inSeqNum;int lenNew = 0;m_cReadWrite.Lock();memcpy(m_cbFrameBuffer+lenNew,&start_sequence,sizeof(start_sequence));//加入起始码lenNew += sizeof(start_sequence);//memcpy(m_cbFrameBuffer+lenNew,pData,nLen);lenNew+=nLen;memcpy(m_cbFrameBuffer+lenNew,end_sequence,sizeof(end_sequence));lenNew+= sizeof(end_sequence);//m_nLengthBuffer=lenNew;#ifdef _DEBUG//BYTE pDstData[200000];//int lDataLenOut = 200000;//单一帧包//m_VideoHandler->DecodeRecvData((BYTE*)m_cbFrameBuffer, m_nLengthBuffer,pDstData,lDataLenOut);
#endif //_DEBUGm_pRTPRecv->AddData(TRUE, (BYTE*)m_cbFrameBuffer, m_nLengthBuffer, inTimestamp);m_nLengthBuffer= 0;m_cReadWrite.Lock();}else{_ASSERT(0);return;/*不处理*/}
}

------------------

如何获取sps,

1、现在直接从抓包媒体流读取

2、要把sdp中的数据使用base64解码。

3、直接从rtp的Nal中读取，Nal类型7和5

unsigned char head1 = (BYTE)(*pData);//获取第一个字节unsigned char type = head1 & 0x1f;//获取FU indicator的类型域，if(type==0x7)//判断NAL的类型为sps
{memcpy(m_cbSPS,pData,nLen);m_nSPS=nLen;return;}else if (type == 0x8)//判断NAL的类型为pps
{memcpy(m_cbPPS,pData,nLen);m_nPPS=nLen;return;}//判断帧类型如果是idr帧，需要在帧头加上sps和pps，格式为：起始码（0x00000001）+sps+起始码+pps+起始码+完整帧数据unsigned char head2 = (BYTE)(*(pData+1));//获取第二个字节unsigned char flag = head2 & 0xe0;//获取FU header的前三位，判断当前是分包的开始、中间或结束.80开始，40结束unsigned char typeSingle = head2 & 0x1f;//获取single nal的类型域，int nFrameType = -1;//无效帧if(typeSingle == 0x1){nFrameType= 0;//普通帧
}else if(typeSingle == 0x5){nFrameType= 1;//关键帧//如何判断idr帧0x857Cm_nGetFirstFrame = 1;TraceMsg("ProcessRTPPacket first received, Key frame, timestamp=%u, id=%d\n", inTimestamp, m_nSessionID);}

----------------------

FU-A

The FU indicator octet has the following format:

+---------------+

|0|1|2|3|4|5|6|7|

+-+-+-+-+-+-+-+-+

|F|NRI| Type |

+---------------+

The FU header has the following format:

+---------------+

|0|1|2|3|4|5|6|7|

+-+-+-+-+-+-+-+-+

|S|E|R| Type |

+---------------+

例子中

所有的音频帧都是关键帧，也是mark边界帧。

所有rtp负载,大华和海康都是这样的（主机序）

普通帧包以开始817c，中间017c，结束417c。网络序（7c81）

关键帧包以开始857c，中间057c，结束457c。

FU indicator == 0x7C

FU header == 0x85等

FU header的一个Type指的是FU-A转变singleNAL之后的type。

2012/04/13 10:05:11

-----------------

在获取sps和pps的时候，应该直接写入程序中，而不是一定要在关键帧的头部。使用大华的流媒体，就有这种情况。

unsigned char head1 = (BYTE)(*pData);//获取第一个字节unsigned char type = head1 & 0x1f;//获取FU indicator的类型域，if (bPadding != 0x0){int nPadding = pData[nLen-1];nLen-= nPadding;//最后一个字节是长度，==包含长度字节在内的大小。
}if(type==0x7)//判断NAL的类型为sps
{memcpy(m_cbSPS,pData,nLen);m_nSPS=nLen;return;}else if (type == 0x8)//判断NAL的类型为pps
{memcpy(m_cbPPS,pData,nLen);m_nPPS=nLen;return;}if(m_nSPS == 0 || m_nPPS == 0){return;//没有获取到配置参数
}int lenNew = 0;//需要分散到获取pps，sps之后。不仅仅遇到关键帧需要填写pps，sps，收到这些东西也需要写一遍if(nFrameType == 1){memcpy(m_cbFrameBuffer+lenNew,start_sequence,sizeof(start_sequence));lenNew+= sizeof(start_sequence);//memcpy(m_cbFrameBuffer+lenNew,m_cbSPS,m_nSPS);lenNew+=m_nSPS;//m_nLengthBuffer = lenNew;//m_pRTPRecv->AddData(TRUE, (BYTE*)m_cbFrameBuffer, m_nLengthBuffer, inTimestamp);//lenNew = 0;//        memcpy(m_cbFrameBuffer+lenNew,start_sequence,sizeof(start_sequence));lenNew+= sizeof(start_sequence);//memcpy(m_cbFrameBuffer+lenNew,m_cbPPS,m_nPPS);lenNew+=m_nPPS;//m_nLengthBuffer = lenNew;//m_pRTPRecv->AddData(TRUE, (BYTE*)m_cbFrameBuffer, m_nLengthBuffer, inTimestamp);
}//lenNew = 0;//    memcpy(m_cbFrameBuffer+lenNew,start_sequence,sizeof(start_sequence));lenNew+= sizeof(start_sequence);////将FU-A包根据rtp的包序号和FU-A的header字节，组成完整264帧；BYTE tagNal = (head1 & 0xe0) | (head2 & 0x1f);memcpy(m_cbFrameBuffer+lenNew,&tagNal,1);memcpy(m_cbFrameBuffer+lenNew+1,pData+2,nLen-2);lenNew+= nLen-2+1;m_nLengthBuffer= lenNew;

更改为

unsigned char start_sequence[]= {0x00, 0x00, 0x00, 0x01};//{0, 0, 1};//

unsigned char end_sequence[]= {0x00, 0x00, 0x01, 0x09, 0x10};//帧末添加帧间分隔符

int lenNew = 0;//    if(type==0x7)//判断NAL的类型为sps
{if (memcmp(m_cbSPS,pData,nLen) == 0){return;}memcpy(m_cbSPS,pData,nLen);m_nSPS=nLen;m_cReadWrite.Lock();memcpy(m_cbFrameBuffer+lenNew,start_sequence,sizeof(start_sequence));//加入起始码lenNew += sizeof(start_sequence);//memcpy(m_cbFrameBuffer+lenNew,m_cbSPS,m_nSPS);lenNew+=m_nSPS;m_nLengthBuffer=lenNew;m_pRTPRecv->AddData(TRUE, (BYTE*)m_cbFrameBuffer, m_nLengthBuffer, m_nLastPacketTimestamp);m_nLengthBuffer= 0;m_cReadWrite.Unlock();return;}

使用MPC - Video decoder并不会闪烁，而是用自己写的解码filter会出现闪烁。原因是每次设置pps和sps图像都会闪动。

nal_unit_type 看这个字段的

avcodec_decode_video函数里有处理

nal_unit_type NAL 单元和 RBSP 语法结构的内容 C

0 未指定

1 一个非IDR图像的编码条带slice_layer_without_partitioning_rbsp( )

2 编码条带数据分割块A slice_data_partition_a_layer_rbsp( )

3 编码条带数据分割块B slice_data_partition_b_layer_rbsp( )

4 编码条带数据分割块C slice_data_partition_c_layer_rbsp( )

5 IDR图像的编码条带 slice_layer_without_partitioning_rbsp( )

6 辅助增强信息 (SEI) sei_rbsp( )

7 序列参数集 seq_parameter_set_rbsp( )

8 图像参数集 pic_parameter_set_rbsp( )

9 访问单元分隔符access_unit_delimiter_rbsp( )

10 序列结尾end_of_seq_rbsp( )

11 流结尾end_of_stream_rbsp( )

12 填充数据filler_data_rbsp( )

13 序列参数集扩展seq_parameter_set_extension_rbsp( )

14..18 保留

19 未分割的辅助编码图像的编码条带slice_layer_without_partitioning_rbsp( )

20..23 保留

24..31 未指定

-----

sps pps和以前的不一样，但是vlc可以播放，我们播放器无法通过。vlc可能通过其他的字段计算长宽

v=0o=StreamingServer 3580774467 1109299840000 IN IP4 192.168.10.12s=\sample_h264_1mbit.mp4
u=http:///
e=admin@
c=IN IP4 0.0.0.0b=AS:2097279t=0 0a=control:*
a=isma-compliance:2,2.0,2a=range:npt=0-  70.00000m=video 0 RTP/AVP 96b=AS:2097151a=rtpmap:96 H264/90000a=control:trackID=3a=cliprect:0,0,480,380a=framesize:96 380-480a=fmtp:96 packetization-mode=1;profile-level-id=4D401E;sprop-parameter-sets=J01AHqkYMB73oA==,KM4C+IA=
a=mpeg4-esid:201m=audio 0 RTP/AVP 97b=AS:127a=rtpmap:97 mpeg4-generic/48000/2a=control:trackID=4a=fmtp:97 profile-level-id=15;mode=AAC-hbr;sizelength=13;indexlength=3;indexdeltalength=3;config=1190
a=mpeg4-esid:101

base64形式 J01AHqkYMB73oA==

二进制形式 0x 27 4d 40 1e a9 18 30 1e f7 a0 00

KM4C+IA=

0x28 ce 02 f8 80

可以解析出来384*480

以前有个海康设备151

sprop-parameter-sets=Z0LgHmdC4B7iWBYCSQgAADhAAA==,aM48gA==

重复的字节

Z0LgHmdC4B7iWBYCSQgAADhAAA==

0x67 42 e0 1e 67 42 e0 1e e2 58 16 02 49 08 00 00 38 40 00 00

aM48gA==

0x68 ce 3c 80 00

大华244

sprop-parameter-sets=Njc0MmUwMTRkYTA1ODI1MQ==,NjhjZTMwYTQ4MA==;

Njc0MmUwMTRkYTA1ODI1MQ==

0x36 37 34 32 65 30 31 34 64 61 30 35 38 32 35 31 00

实际上它把二进制数作为字符串，然后又变成二进制进行传输 6742e014da058251

NjhjZTMwYTQ4MA

0x36 38 63 65 33 30 61 34 38 30 00

对应的字符串为 68ce30a480

-------------------------------