优化音视频同步

src/Record/MP4Muxer.cpp

@@ -134,6 +134,26 @@ static uint8_t getObject(CodecId codecId){
     }
 }
 
+void MP4Muxer::stampSync(){
+    if(_codec_to_trackid.size() < 2){
+        return;
+    }
+
+    Stamp *audio = nullptr, *video = nullptr;
+    for(auto &pr : _codec_to_trackid){
+        switch (getTrackType((CodecId) pr.first)){
+            case TrackAudio : audio = &pr.second.stamp; break;
+            case TrackVideo : video = &pr.second.stamp; break;
+            default : break;
+        }
+    }
+
+    if(audio && video){
+        //音频时间戳同步于视频，因为音频时间戳被修改后不影响播放
+        audio->syncTo(*video);
+    }
+}
+
 void MP4Muxer::addTrack(const Track::Ptr &track) {
     auto mp4_object = getObject(track->getCodecId());
     if (!mp4_object) {
@@ -261,6 +281,9 @@ void MP4Muxer::addTrack(const Track::Ptr &track) {
 
         default: WarnL << "MP4录制不支持该编码格式:" << track->getCodecName(); break;
     }
+
+    //尝试音视频同步
+    stampSync();
 }
 
 }//namespace mediakit

src/Record/MP4Muxer.h

@@ -45,13 +45,14 @@ public:
 private:
     void openMP4();
     void closeMP4();
+    void stampSync();
 
 private:
-    struct track_info{
+    struct track_info {
         int track_id = -1;
         Stamp stamp;
     };
-    unordered_map<int,track_info> _codec_to_trackid;
+    unordered_map<int, track_info> _codec_to_trackid;
     List<Frame::Ptr> _frameCached;
     bool _started = false;
     bool _have_video = false;

src/Record/TsMuxer.cpp

@@ -23,6 +23,26 @@ TsMuxer::~TsMuxer() {
     uninit();
 }
 
+void TsMuxer::stampSync(){
+    if(_codec_to_trackid.size() < 2){
+        return;
+    }
+
+    Stamp *audio = nullptr, *video = nullptr;
+    for(auto &pr : _codec_to_trackid){
+        switch (getTrackType((CodecId) pr.first)){
+            case TrackAudio : audio = &pr.second.stamp; break;
+            case TrackVideo : video = &pr.second.stamp; break;
+            default : break;
+        }
+    }
+
+    if(audio && video){
+        //音频时间戳同步于视频，因为音频时间戳被修改后不影响播放
+        audio->syncTo(*video);
+    }
+}
+
 void TsMuxer::addTrack(const Track::Ptr &track) {
     switch (track->getCodecId()) {
         case CodecH264: {
@@ -52,9 +72,11 @@ void TsMuxer::addTrack(const Track::Ptr &track) {
             break;
         }
 
-        default:
-            break;
+        default: WarnL << "mpeg-ts 不支持该编码格式,已忽略:" << track->getCodecName(); break;
     }
+
+    //尝试音视频同步
+    stampSync();
 }
 
 void TsMuxer::inputFrame(const Frame::Ptr &frame) {

src/Record/TsMuxer.h

@@ -17,37 +17,59 @@
 #include "Util/File.h"
 #include "Common/MediaSink.h"
 #include "Common/Stamp.h"
-
 using namespace toolkit;
-
 namespace mediakit {
 
+//该类用于产生MPEG-TS
 class TsMuxer : public MediaSinkInterface {
 public:
     TsMuxer();
     virtual ~TsMuxer();
+
+    /**
+     * 添加音视频轨道
+     */
     void addTrack(const Track::Ptr &track) override;
+
+    /**
+     * 重置音视频轨道
+     */
     void resetTracks() override;
+
+    /**
+     * 输入帧数据
+     */
     void inputFrame(const Frame::Ptr &frame) override;
+
 protected:
+    /**
+     * 输出mpegts数据回调
+     * @param packet mpegts数据
+     * @param bytes mpegts数据长度
+     * @param timestamp 时间戳，单位毫秒
+     * @param is_idr_fast_packet 是否为关键帧的第一个TS包，用于确保ts切片第一帧为关键帧
+     */
     virtual void onTs(const void *packet, int bytes,uint32_t timestamp,bool is_idr_fast_packet) = 0;
+
 private:
     void init();
     void uninit();
-private:
-    void  *_context = nullptr;
-    char *_tsbuf[188];
-    uint32_t _timestamp = 0;
+    //音视频时间戳同步用
+    void stampSync();
 
-    struct track_info{
+private:
+    void *_context = nullptr;
+    char _tsbuf[188];
+    uint32_t _timestamp = 0;
+    struct track_info {
         int track_id = -1;
         Stamp stamp;
     };
-    unordered_map<int,track_info> _codec_to_trackid;
+    unordered_map<int, track_info> _codec_to_trackid;
     List<Frame::Ptr> _frameCached;
     bool _is_idr_fast_packet = false;
     bool _have_video = false;
 };
 
 }//namespace mediakit
-#endif //TSMUXER_H
+#endif //TSMUXER_H