/* * Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved. * * This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit). * * Use of this source code is governed by MIT-like license that can be found in the * LICENSE file in the root of the source tree. All contributing project authors * may be found in the AUTHORS file in the root of the source tree. */ #include "H264.h" #include "H264Rtmp.h" #include "H264Rtp.h" #include "Util/logger.h" #include "Util/base64.h" #include "Common/Parser.h" #include "Common/config.h" #include "Extension/Factory.h" #ifdef ENABLE_MP4 #include "mpeg4-avc.h" #endif #include #include using namespace std; using namespace toolkit; namespace mediakit { // ---- 内部比特流工具 ---- namespace { // 去除 RBSP 防竞争字节 (0x00 0x00 0x03 -> 0x00 0x00) static std::vector rbsp_from_nalu(const uint8_t *data, size_t size) { std::vector out; out.reserve(size); for (size_t i = 0; i < size; ) { if (i + 2 < size && data[i] == 0x00 && data[i+1] == 0x00 && data[i+2] == 0x03) { out.push_back(0x00); out.push_back(0x00); i += 3; } else { out.push_back(data[i++]); } } return out; } struct BitStream { const uint8_t *buf; size_t size; // bytes size_t pos; // bit position BitStream(const uint8_t *b, size_t s) : buf(b), size(s), pos(0) {} bool eof() const { return pos >= size * 8; } size_t bits_left() const { return size * 8 - pos; } uint32_t read_bits(int n) { if (n < 0 || n > 32 || (size_t)n > bits_left()) { throw std::runtime_error("eof"); } uint32_t val = 0; for (int i = 0; i < n; i++) { val = (val << 1) | ((buf[pos / 8] >> (7 - pos % 8)) & 1); pos++; } return val; } void skip_bits(int n) { if (n < 0 || (size_t)n > bits_left()) { throw std::runtime_error("eof"); } pos += n; } uint32_t read_ue() { // Exp-Golomb unsigned int zeros = 0; while (!eof() && read_bits(1) == 0) zeros++; if (zeros == 0) return 0; if (zeros >= 32) throw std::runtime_error("exp-golomb overflow"); return (1u << zeros) - 1 + read_bits(zeros); } int32_t read_se() { // Exp-Golomb signed uint32_t v = read_ue(); return (v & 1) ? (int32_t)((v + 1) >> 1) : -(int32_t)(v >> 1); } }; } // anonymous namespace // ---- H264 SPS 解析 ---- static bool getAVCInfo(const char *sps_raw, size_t sps_len, int &iVideoWidth, int &iVideoHeight, float &iVideoFps) { if (sps_len < 4) return false; // sps_raw[0] 是 NAL header,从第 1 字节开始是 RBSP auto rbsp = rbsp_from_nalu((const uint8_t *)sps_raw + 1, sps_len - 1); if (rbsp.size() < 3) return false; try { BitStream bs(rbsp.data(), rbsp.size()); uint8_t profile_idc = (uint8_t)bs.read_bits(8); // profile_idc bs.skip_bits(8); // constraint flags + reserved bs.skip_bits(8); // level_idc bs.read_ue(); // seq_parameter_set_id uint32_t chroma_format_idc = 1; if (profile_idc == 100 || profile_idc == 110 || profile_idc == 122 || profile_idc == 244 || profile_idc == 44 || profile_idc == 83 || profile_idc == 86 || profile_idc == 118 || profile_idc == 128) { chroma_format_idc = bs.read_ue(); if (chroma_format_idc > 3) { return false; } if (chroma_format_idc == 3) bs.skip_bits(1); // separate_colour_plane_flag bs.read_ue(); // bit_depth_luma_minus8 bs.read_ue(); // bit_depth_chroma_minus8 bs.skip_bits(1); // qpprime_y_zero_transform_bypass_flag if (bs.read_bits(1)) { // seq_scaling_matrix_present_flag int cnt = (chroma_format_idc != 3) ? 8 : 12; for (int i = 0; i < cnt; i++) { if (bs.read_bits(1)) { // seq_scaling_list_present_flag int sz = (i < 6) ? 16 : 64; int last = 8, next = 8; for (int j = 0; j < sz; j++) { if (next != 0) next = (last + bs.read_se() + 256) % 256; last = (next == 0) ? last : next; } } } } } bs.read_ue(); // log2_max_frame_num_minus4 uint32_t pic_order_cnt_type = bs.read_ue(); if (pic_order_cnt_type == 0) { bs.read_ue(); // log2_max_pic_order_cnt_lsb_minus4 } else if (pic_order_cnt_type == 1) { bs.skip_bits(1); // delta_pic_order_always_zero_flag bs.read_se(); // offset_for_non_ref_pic bs.read_se(); // offset_for_top_to_bottom_field uint32_t n = bs.read_ue(); for (uint32_t i = 0; i < n; i++) bs.read_se(); } bs.read_ue(); // max_num_ref_frames bs.skip_bits(1); // gaps_in_frame_num_value_allowed_flag uint32_t pic_width_in_mbs_minus1 = bs.read_ue(); uint32_t pic_height_in_map_units_minus1 = bs.read_ue(); uint32_t frame_mbs_only_flag = bs.read_bits(1); if (!frame_mbs_only_flag) bs.skip_bits(1); // mb_adaptive_frame_field_flag bs.skip_bits(1); // direct_8x8_inference_flag uint32_t crop_left = 0, crop_right = 0, crop_top = 0, crop_bottom = 0; if (bs.read_bits(1)) { // frame_cropping_flag crop_left = bs.read_ue(); crop_right = bs.read_ue(); crop_top = bs.read_ue(); crop_bottom = bs.read_ue(); } uint32_t crop_unit_x = 1; uint32_t crop_unit_y = 2 - frame_mbs_only_flag; if (chroma_format_idc == 1) { crop_unit_x = 2; crop_unit_y = 2 * (2 - frame_mbs_only_flag); } else if (chroma_format_idc == 2) { crop_unit_x = 2; crop_unit_y = 2 - frame_mbs_only_flag; } uint64_t raw_width = (uint64_t)(pic_width_in_mbs_minus1 + 1) * 16; uint64_t raw_height = (uint64_t)(pic_height_in_map_units_minus1 + 1) * 16 * (2 - frame_mbs_only_flag); uint64_t crop_w = ((uint64_t)crop_left + crop_right) * crop_unit_x; uint64_t crop_h = ((uint64_t)crop_top + crop_bottom) * crop_unit_y; if (crop_w >= raw_width || crop_h >= raw_height) return false; iVideoWidth = (int)(raw_width - crop_w); iVideoHeight = (int)(raw_height - crop_h); iVideoFps = 0.0f; if (bs.read_bits(1)) { // vui_parameters_present_flag if (bs.read_bits(1)) { // aspect_ratio_info_present_flag uint32_t ar = bs.read_bits(8); if (ar == 255) bs.skip_bits(32); // sar width+height } if (bs.read_bits(1)) bs.skip_bits(1); // overscan if (bs.read_bits(1)) { // video_signal_type_present_flag bs.skip_bits(3 + 1); // video_format + video_full_range_flag if (bs.read_bits(1)) bs.skip_bits(24); // colour_description_present_flag } if (bs.read_bits(1)) { // chroma_loc_info_present_flag bs.read_ue(); bs.read_ue(); } if (bs.read_bits(1)) { // timing_info_present_flag uint32_t num_units_in_tick = bs.read_bits(32); uint32_t time_scale = bs.read_bits(32); bs.skip_bits(1); // fixed_frame_rate_flag if (num_units_in_tick > 0) { iVideoFps = (float)time_scale / (2.0f * (float)num_units_in_tick); } } } return iVideoWidth > 0 && iVideoHeight > 0; } catch (...) { return iVideoWidth > 0 && iVideoHeight > 0; } } bool getAVCInfo(const string &strSps, int &iVideoWidth, int &iVideoHeight, float &iVideoFps) { return getAVCInfo(strSps.data(), strSps.size(), iVideoWidth, iVideoHeight, iVideoFps); } static const char *memfind(const char *buf, ssize_t len, const char *subbuf, ssize_t sublen) { for (auto i = 0; i < len - sublen; ++i) { if (memcmp(buf + i, subbuf, sublen) == 0) { return buf + i; } } return NULL; } void splitH264( const char *ptr, size_t len, size_t prefix, const std::function &cb) { auto start = ptr + prefix; auto end = ptr + len; size_t next_prefix; while (true) { auto next_start = memfind(start, end - start, "\x00\x00\x01", 3); if (next_start) { // 找到下一帧 [AUTO-TRANSLATED:7161f54a] // Find the next frame if (*(next_start - 1) == 0x00) { // 这个是00 00 00 01开头 [AUTO-TRANSLATED:b0d79e9e] // This starts with 00 00 00 01 next_start -= 1; next_prefix = 4; } else { // 这个是00 00 01开头 [AUTO-TRANSLATED:18ae81d8] // This starts with 00 00 01 next_prefix = 3; } // 记得加上本帧prefix长度 [AUTO-TRANSLATED:8bde5d52] // Remember to add the prefix length of this frame cb(start - prefix, next_start - start + prefix, prefix); // 搜索下一帧末尾的起始位置 [AUTO-TRANSLATED:8976b719] // Search for the starting position of the end of the next frame start = next_start + next_prefix; // 记录下一帧的prefix长度 [AUTO-TRANSLATED:756aee4e] // Record the prefix length of the next frame prefix = next_prefix; continue; } // 未找到下一帧,这是最后一帧 [AUTO-TRANSLATED:58365453] // The next frame was not found, this is the last frame cb(start - prefix, end - start + prefix, prefix); break; } } size_t prefixSize(const char *ptr, size_t len) { if (len < 4) { return 0; } if (ptr[0] != 0x00 || ptr[1] != 0x00) { // 不是0x00 00开头 [AUTO-TRANSLATED:c406f0da] // Not 0x00 00 at the beginning return 0; } if (ptr[2] == 0x00 && ptr[3] == 0x01) { // 是0x00 00 00 01 [AUTO-TRANSLATED:70caae72] // It is 0x00 00 00 01 return 4; } if (ptr[2] == 0x01) { // 是0x00 00 01 [AUTO-TRANSLATED:78b4a3c9] // It is 0x00 00 01 return 3; } return 0; } //////////////////////////////////////////////////////////////////////////////////////////////////// H264Track::H264Track(const string &sps, const string &pps, int sps_prefix_len, int pps_prefix_len) { _sps = sps.substr(sps_prefix_len); _pps = pps.substr(pps_prefix_len); H264Track::update(); } CodecId H264Track::getCodecId() const { return CodecH264; } int H264Track::getVideoHeight() const { return _height; } int H264Track::getVideoWidth() const { return _width; } float H264Track::getVideoFps() const { return _fps; } bool H264Track::ready() const { return !_sps.empty() && !_pps.empty(); } bool H264Track::inputFrame(const Frame::Ptr &frame) { using H264FrameInternal = FrameInternal; int type = H264_TYPE(frame->data()[frame->prefixSize()]); if ((type == H264Frame::NAL_B_P || type == H264Frame::NAL_IDR) && ready()) { return inputFrame_l(frame); } // 非I/B/P帧情况下,split一下,防止多个帧粘合在一起 [AUTO-TRANSLATED:b69c6e75] // In the case of non-I/B/P frames, split it to prevent multiple frames from sticking together bool ret = false; splitH264(frame->data(), frame->size(), frame->prefixSize(), [&](const char *ptr, size_t len, size_t prefix) { H264FrameInternal::Ptr sub_frame = std::make_shared(frame, (char *)ptr, len, prefix); if (inputFrame_l(sub_frame)) { ret = true; } }); return ret; } toolkit::Buffer::Ptr H264Track::getExtraData() const { CHECK(ready()); #ifdef ENABLE_MP4 struct mpeg4_avc_t avc; memset(&avc, 0, sizeof(avc)); string sps_pps = string("\x00\x00\x00\x01", 4) + _sps + string("\x00\x00\x00\x01", 4) + _pps; h264_annexbtomp4(&avc, sps_pps.data(), (int)sps_pps.size(), NULL, 0, NULL, NULL); std::string extra_data; extra_data.resize(1024); auto extra_data_size = mpeg4_avc_decoder_configuration_record_save(&avc, (uint8_t *)extra_data.data(), extra_data.size()); if (extra_data_size == -1) { WarnL << "生成H264 extra_data 失败"; return nullptr; } extra_data.resize(extra_data_size); return std::make_shared(std::move(extra_data)); #else std::string extra_data; // AVCDecoderConfigurationRecord start extra_data.push_back(1); // version extra_data.push_back(_sps[1]); // profile extra_data.push_back(_sps[2]); // compat extra_data.push_back(_sps[3]); // level extra_data.push_back((char)0xff); // 6 bits reserved + 2 bits nal size length - 1 (11) extra_data.push_back((char)0xe1); // 3 bits reserved + 5 bits number of sps (00001) // sps uint16_t size = (uint16_t)_sps.size(); size = htons(size); extra_data.append((char *)&size, 2); extra_data.append(_sps); // pps extra_data.push_back(1); // version size = (uint16_t)_pps.size(); size = htons(size); extra_data.append((char *)&size, 2); extra_data.append(_pps); return std::make_shared(std::move(extra_data)); #endif } void H264Track::setExtraData(const uint8_t *data, size_t bytes) { #ifdef ENABLE_MP4 struct mpeg4_avc_t avc; memset(&avc, 0, sizeof(avc)); if (mpeg4_avc_decoder_configuration_record_load(data, bytes, &avc) > 0) { std::vector config(bytes * 2); int size = mpeg4_avc_to_nalu(&avc, config.data(), bytes * 2); if (size > 4) { splitH264((char *)config.data(), size, 4, [&](const char *ptr, size_t len, size_t prefix) { inputFrame_l(std::make_shared((char *)ptr, len, 0, 0, prefix)); }); update(); } } #else CHECK(bytes >= 8); // 6 + 2 size_t offset = 6; uint16_t sps_size = data[offset] << 8 | data[offset + 1]; auto sps_ptr = data + offset + 2; offset += (2 + sps_size); CHECK(bytes >= offset + 2); // + pps_size _sps.assign((char *)sps_ptr, sps_size); uint16_t pps_size = data[offset] << 8 | data[offset + 1]; auto pps_ptr = data + offset + 2; offset += (2 + pps_size); CHECK(bytes >= offset); _pps.assign((char *)pps_ptr, pps_size); update(); #endif } bool H264Track::update() { return getAVCInfo(_sps, _width, _height, _fps); } std::vector H264Track::getConfigFrames() const { if (!ready()) { return {}; } return { createConfigFrame(_sps, 0, getIndex()), createConfigFrame(_pps, 0, getIndex()) }; } Track::Ptr H264Track::clone() const { return std::make_shared(*this); } bool H264Track::inputFrame_l(const Frame::Ptr &frame) { int type = H264_TYPE(frame->data()[frame->prefixSize()]); if (type == H264Frame::NAL_AUD) { // AUD帧丢弃 return false; } bool ret = true; switch (type) { case H264Frame::NAL_SPS: { _sps = string(frame->data() + frame->prefixSize(), frame->size() - frame->prefixSize()); _latest_is_sps = true; ret = VideoTrack::inputFrame(frame); break; } case H264Frame::NAL_PPS: { _pps = string(frame->data() + frame->prefixSize(), frame->size() - frame->prefixSize()); _latest_is_pps = true; ret = VideoTrack::inputFrame(frame); break; } default: // 避免识别不出关键帧 [AUTO-TRANSLATED:8eb84679] // Avoid not being able to recognize keyframes if (latestIsConfigFrame() && !frame->dropAble()) { if (!frame->keyFrame()) { const_cast(frame) = std::make_shared(frame, true); } } // 判断是否是I帧, 并且如果是,那判断前面是否插入过config帧, 如果插入过就不插入了 [AUTO-TRANSLATED:40733cd8] // Determine if it is an I frame, and if it is, determine if a config frame has been inserted before, and if it has been inserted, do not insert it if (frame->keyFrame() && !latestIsConfigFrame()) { insertConfigFrame(frame); } if(!frame->dropAble()){ _latest_is_pps = false; _latest_is_sps = false; } ret = VideoTrack::inputFrame(frame); break; } if (_width == 0 && ready()) { update(); } return ret; } void H264Track::insertConfigFrame(const Frame::Ptr &frame) { if (!_sps.empty()) { VideoTrack::inputFrame(createConfigFrame(_sps, frame->dts(), frame->getIndex())); } if (!_pps.empty()) { VideoTrack::inputFrame(createConfigFrame(_pps, frame->dts(), frame->getIndex())); } } bool H264Track::latestIsConfigFrame(){ return _latest_is_sps && _latest_is_pps; } class H264Sdp : public Sdp { public: H264Sdp(const string &strSPS, const string &strPPS, int payload_type, int bitrate) : Sdp(90000, payload_type) { _printer << "m=video 0 RTP/AVP " << payload_type << "\r\n"; if (bitrate) { _printer << "b=AS:" << bitrate << "\r\n"; } _printer << "a=rtpmap:" << payload_type << " " << getCodecName(CodecH264) << "/" << 90000 << "\r\n"; /** Single NAI Unit Mode = 0. // Single NAI mode (Only nals from 1-23 are allowed) Non Interleaved Mode = 1,// Non-interleaved Mode: 1-23,24 (STAP-A),28 (FU-A) are allowed Interleaved Mode = 2, // 25 (STAP-B),26 (MTAP16),27 (MTAP24),28 (EU-A),and 29 (EU-B) are allowed. Single NAI Unit Mode = 0. // Single NAI mode (Only nals from 1-23 are allowed) Non Interleaved Mode = 1,// Non-interleaved Mode: 1-23,24 (STAP-A),28 (FU-A) are allowed Interleaved Mode = 2, // 25 (STAP-B),26 (MTAP16),27 (MTAP24),28 (EU-A),and 29 (EU-B) are allowed. * * [AUTO-TRANSLATED:6166738f] **/ GET_CONFIG(bool, h264_stap_a, Rtp::kH264StapA); _printer << "a=fmtp:" << payload_type << " packetization-mode=" << h264_stap_a << "; profile-level-id="; uint32_t profile_level_id = 0; if (strSPS.length() >= 4) { // sanity check profile_level_id = (uint8_t(strSPS[1]) << 16) | (uint8_t(strSPS[2]) << 8) | (uint8_t(strSPS[3])); // profile_idc|constraint_setN_flag|level_idc } char profile[8]; snprintf(profile, sizeof(profile), "%06X", profile_level_id); _printer << profile; _printer << "; sprop-parameter-sets="; _printer << encodeBase64(strSPS) << ","; _printer << encodeBase64(strPPS) << "\r\n"; } string getSdp() const { return _printer; } private: _StrPrinter _printer; }; Sdp::Ptr H264Track::getSdp(uint8_t payload_type) const { return std::make_shared(_sps, _pps, payload_type, getBitRate() >> 10); } namespace { CodecId getCodec() { return CodecH264; } Track::Ptr getTrackByCodecId(int sample_rate, int channels, int sample_bit) { return std::make_shared(); } Track::Ptr getTrackBySdp(const SdpTrack::Ptr &track) { //a=fmtp:96 packetization-mode=1;profile-level-id=42C01F;sprop-parameter-sets=Z0LAH9oBQBboQAAAAwBAAAAPI8YMqA==,aM48gA== auto map = Parser::parseArgs(track->_fmtp, ";", "="); auto sps_pps = map["sprop-parameter-sets"]; string base64_SPS = findSubString(sps_pps.data(), NULL, ","); string base64_PPS = findSubString(sps_pps.data(), ",", NULL); auto sps = decodeBase64(base64_SPS); auto pps = decodeBase64(base64_PPS); if (sps.empty() || pps.empty()) { // 如果sdp里面没有sps/pps,那么可能在后续的rtp里面恢复出sps/pps [AUTO-TRANSLATED:60f03d45] // If there is no sps/pps in the sdp, then it may be possible to recover the sps/pps in the subsequent rtp return std::make_shared(); } return std::make_shared(sps, pps, prefixSize(sps.data(), sps.size()), prefixSize(pps.data(), pps.size())); } RtpCodec::Ptr getRtpEncoderByCodecId(uint8_t pt) { return std::make_shared(); } RtpCodec::Ptr getRtpDecoderByCodecId() { return std::make_shared(); } RtmpCodec::Ptr getRtmpEncoderByTrack(const Track::Ptr &track) { return std::make_shared(track); } RtmpCodec::Ptr getRtmpDecoderByTrack(const Track::Ptr &track) { return std::make_shared(track); } Frame::Ptr getFrameFromPtr(const char *data, size_t bytes, uint64_t dts, uint64_t pts) { return std::make_shared((char *)data, bytes, dts, pts, prefixSize(data, bytes)); } } // namespace CodecPlugin h264_plugin = { getCodec, getTrackByCodecId, getTrackBySdp, getRtpEncoderByCodecId, getRtpDecoderByCodecId, getRtmpEncoderByTrack, getRtmpDecoderByTrack, getFrameFromPtr }; } // namespace mediakit