Mirror/ZLMediaKit
1
0
Fork 0
mirror of https://github.com/ZLMediaKit/ZLMediaKit.git synced 2026-07-01 23:57:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

Support multi audio/video track

Browse Source
This commit is contained in:
夏楚
2023-12-09 22:34:22 +08:00
committed by GitHub
parent bbe8f4a018
commit 64f15202de
36 changed files with 366 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/Record/MP4Demuxer.cpp
View File

@@ -61,7 +61,8 @@ void MP4Demuxer::onVideoTrack(uint32_t track, uint8_t object, int width, int hei
if (!video) {
return;
}
_track_to_codec.emplace(track, video);
video->setIndex(track);
_tracks.emplace(track, video);
if (extra && bytes) {
video->setExtraData((uint8_t *)extra, bytes);
}
@@ -72,7 +73,8 @@ void MP4Demuxer::onAudioTrack(uint32_t track, uint8_t object, int channel_count,
if (!audio) {
return;
}
_track_to_codec.emplace(track, audio);
audio->setIndex(track);
_tracks.emplace(track, audio);
if (extra && bytes) {
audio->setExtraData((uint8_t *)extra, bytes);
}
@@ -134,8 +136,8 @@ Frame::Ptr MP4Demuxer::readFrame(bool &keyFrame, bool &eof) {
}
Frame::Ptr MP4Demuxer::makeFrame(uint32_t track_id, const Buffer::Ptr &buf, int64_t pts, int64_t dts) {
auto it = _track_to_codec.find(track_id);
if (it == _track_to_codec.end()) {
auto it = _tracks.find(track_id);
if (it == _tracks.end()) {
return nullptr;
}
Frame::Ptr ret;
@@ -166,15 +168,16 @@ Frame::Ptr MP4Demuxer::makeFrame(uint32_t track_id, const Buffer::Ptr &buf, int6
}
}
if (ret) {
ret->setIndex(track_id);
it->second->inputFrame(ret);
}
return ret;
}
vector<Track::Ptr> MP4Demuxer::getTracks(bool trackReady) const {
vector<Track::Ptr> MP4Demuxer::getTracks(bool ready) const {
vector<Track::Ptr> ret;
for (auto &pr : _track_to_codec) {
if (trackReady && !pr.second->ready()) {
for (auto &pr : _tracks) {
if (ready && !pr.second->ready()) {
continue;
}
ret.push_back(pr.second);

2
src/Record/MP4Demuxer.h
View File

@@ -71,7 +71,7 @@ private:
MP4FileDisk::Ptr _mp4_file;
MP4FileDisk::Reader _mov_reader;
uint64_t _duration_ms = 0;
std::map<int, Track::Ptr> _track_to_codec;
std::unordered_map<int, Track::Ptr> _tracks;
toolkit::ResourcePool<toolkit::BufferRaw> _buffer_pool;
};

104
src/Record/MP4Muxer.cpp
View File

@@ -60,7 +60,7 @@ bool MP4MuxerInterface::haveVideo() const {
uint64_t MP4MuxerInterface::getDuration() const {
uint64_t ret = 0;
for (auto &pr : _codec_to_trackid) {
for (auto &pr : _tracks) {
if (pr.second.stamp.getRelativeStamp() > (int64_t)ret) {
ret = pr.second.stamp.getRelativeStamp();
}
@@ -72,61 +72,50 @@ void MP4MuxerInterface::resetTracks() {
_started = false;
_have_video = false;
_mov_writter = nullptr;
_frame_merger.clear();
_codec_to_trackid.clear();
_tracks.clear();
}
void MP4MuxerInterface::flush() {
_frame_merger.flush();
for (auto &pr : _tracks) {
pr.second.merger.flush();
}
}
bool MP4MuxerInterface::inputFrame(const Frame::Ptr &frame) {
auto it = _codec_to_trackid.find(frame->getCodecId());
if (it == _codec_to_trackid.end()) {
//该Track不存在或初始化失败
auto it = _tracks.find(frame->getIndex());
if (it == _tracks.end()) {
// 该Track不存在或初始化失败
return false;
}
if (!_started) {
//该逻辑确保含有视频时，第一帧为关键帧
// 该逻辑确保含有视频时，第一帧为关键帧
if (_have_video && !frame->keyFrame()) {
//含有视频，但是不是关键帧，那么前面的帧丢弃
// 含有视频，但是不是关键帧，那么前面的帧丢弃
return false;
}
//开始写文件
// 开始写文件
_started = true;
}
//mp4文件时间戳需要从0开始
auto &track_info = it->second;
// mp4文件时间戳需要从0开始
auto &track = it->second;
switch (frame->getCodecId()) {
case CodecH264:
case CodecH265: {
//这里的代码逻辑是让SPS、PPS、IDR这些时间戳相同的帧打包到一起当做一个帧处理，
_frame_merger.inputFrame(frame, [this, &track_info](uint64_t dts, uint64_t pts, const Buffer::Ptr &buffer, bool have_idr) {
// 这里的代码逻辑是让SPS、PPS、IDR这些时间戳相同的帧打包到一起当做一个帧处理，
track.merger.inputFrame(frame, [&](uint64_t dts, uint64_t pts, const Buffer::Ptr &buffer, bool have_idr) {
int64_t dts_out, pts_out;
track_info.stamp.revise(dts, pts, dts_out, pts_out);
mp4_writer_write(_mov_writter.get(),
track_info.track_id,
buffer->data(),
buffer->size(),
pts_out,
dts_out,
have_idr ? MOV_AV_FLAG_KEYFREAME : 0);
track.stamp.revise(dts, pts, dts_out, pts_out);
mp4_writer_write(_mov_writter.get(), track.track_id, buffer->data(), buffer->size(), pts_out, dts_out, have_idr ? MOV_AV_FLAG_KEYFREAME : 0);
});
break;
}
default: {
int64_t dts_out, pts_out;
track_info.stamp.revise(frame->dts(), frame->pts(), dts_out, pts_out);
mp4_writer_write(_mov_writter.get(),
track_info.track_id,
frame->data() + frame->prefixSize(),
frame->size() - frame->prefixSize(),
pts_out,
dts_out,
frame->keyFrame() ? MOV_AV_FLAG_KEYFREAME : 0);
track.stamp.revise(frame->dts(), frame->pts(), dts_out, pts_out);
mp4_writer_write(_mov_writter.get(), track.track_id, frame->data() + frame->prefixSize(), frame->size() - frame->prefixSize(), pts_out, dts_out, frame->keyFrame() ? MOV_AV_FLAG_KEYFREAME : 0);
break;
}
}
@@ -134,23 +123,14 @@ bool MP4MuxerInterface::inputFrame(const Frame::Ptr &frame) {
}
void MP4MuxerInterface::stampSync() {
if (_codec_to_trackid.size() < 2) {
return;
}
Stamp *audio = nullptr, *video = nullptr;
for(auto &pr : _codec_to_trackid){
switch (getTrackType((CodecId) pr.first)){
case TrackAudio : audio = &pr.second.stamp; break;
case TrackVideo : video = &pr.second.stamp; break;
default : break;
Stamp *first = nullptr;
for (auto &pr : _tracks) {
if (!first) {
first = &pr.second.stamp;
} else {
pr.second.stamp.syncTo(*first);
}
}
if (audio && video) {
//音频时间戳同步于视频，因为音频时间戳被修改后不影响播放
audio->syncTo(*video);
}
}
bool MP4MuxerInterface::addTrack(const Track::Ptr &track) {
@@ -164,7 +144,7 @@ bool MP4MuxerInterface::addTrack(const Track::Ptr &track) {
}
if (!track->ready()) {
WarnL << "Track[" << track->getCodecName() << "] unready";
WarnL << "Track " << track->getCodecName() << " unready";
return false;
}
@@ -175,36 +155,26 @@ bool MP4MuxerInterface::addTrack(const Track::Ptr &track) {
auto extra_size = extra ? extra->size() : 0;
if (track->getTrackType() == TrackVideo) {
auto video_track = dynamic_pointer_cast<VideoTrack>(track);
if (!video_track) {
WarnL << "不是VideoTrack";
return false;
}
CHECK(video_track);
auto track_id = mp4_writer_add_video(_mov_writter.get(), mp4_object, video_track->getVideoWidth(), video_track->getVideoHeight(), extra_data, extra_size);
if (track_id < 0) {
WarnL << "添加Video Track失败:" << video_track->getCodecName();
WarnL << "mp4_writer_add_video failed: " << video_track->getCodecName();
return false;
}
_codec_to_trackid[track->getCodecId()].track_id = track_id;
_tracks[track->getIndex()].track_id = track_id;
_have_video = true;
} else if (track->getTrackType() == TrackAudio) {
auto audio_track = dynamic_pointer_cast<AudioTrack>(track);
if (!audio_track) {
WarnL << "不是音频Track:" << track->getCodecName();
return false;
}
auto track_id = mp4_writer_add_audio(_mov_writter.get(), mp4_object, audio_track->getAudioChannel(),
audio_track->getAudioSampleBit() * audio_track->getAudioChannel(),
audio_track->getAudioSampleRate(), extra_data, extra_size);
CHECK(audio_track);
auto track_id = mp4_writer_add_audio(_mov_writter.get(), mp4_object, audio_track->getAudioChannel(), audio_track->getAudioSampleBit() * audio_track->getAudioChannel(), audio_track->getAudioSampleRate(), extra_data, extra_size);
if (track_id < 0) {
WarnL << "添加Track[" << track->getCodecName() << "]失败:" << track_id;
WarnL << "mp4_writer_add_audio failed: " << audio_track->getCodecName();
return false;
}
_codec_to_trackid[track->getCodecId()].track_id = track_id;
_tracks[track->getIndex()].track_id = track_id;
}
//尝试音视频同步
// 尝试音视频同步
stampSync();
return true;
}
@@ -236,7 +206,7 @@ void MP4MuxerMemory::resetTracks() {
bool MP4MuxerMemory::inputFrame(const Frame::Ptr &frame) {
if (_init_segment.empty()) {
//尚未生成init segment
// 尚未生成init segment
return false;
}
@@ -259,5 +229,5 @@ bool MP4MuxerMemory::inputFrame(const Frame::Ptr &frame) {
return MP4MuxerInterface::inputFrame(frame);
}
}//namespace mediakit
#endif //defined(ENABLE_MP4)
} // namespace mediakit
#endif // defined(ENABLE_MP4)

12
src/Record/MP4Muxer.h
View File

@@ -72,12 +72,18 @@ private:
bool _started = false;
bool _have_video = false;
MP4FileIO::Writer _mov_writter;
struct track_info {
class FrameMergerImp : public FrameMerger {
public:
FrameMergerImp() : FrameMerger(FrameMerger::mp4_nal_size) {}
};
struct MP4Track {
int track_id = -1;
Stamp stamp;
FrameMergerImp merger;
};
std::unordered_map<int, track_info> _codec_to_trackid;
FrameMerger _frame_merger { FrameMerger::mp4_nal_size };
std::unordered_map<int, MP4Track> _tracks;
};
class MP4Muxer : public MP4MuxerInterface{

3
src/Record/MP4Reader.cpp
View File

@@ -27,7 +27,8 @@ MP4Reader::MP4Reader(const std::string &vhost, const std::string &app, const std
option.enable_mp4 = false;
option.enable_hls = false;
option.enable_hls_fmp4 = false;
// mp4支持多track
option.max_track = 16;
setup(vhost, app, stream_id, file_path, option, std::move(poller));
}

28
src/Record/MPEG.cpp
View File

@@ -18,7 +18,7 @@
using namespace toolkit;
namespace mediakit{
namespace mediakit {
MpegMuxer::MpegMuxer(bool is_ps) {
_is_ps = is_ps;
@@ -40,27 +40,27 @@ bool MpegMuxer::addTrack(const Track::Ptr &track) {
if (track->getTrackType() == TrackVideo) {
_have_video = true;
}
_codec_to_trackid[track->getCodecId()] = mpeg_muxer_add_stream((::mpeg_muxer_t *)_context, mpeg_id, nullptr, 0);
_tracks[track->getIndex()].track_id = mpeg_muxer_add_stream((::mpeg_muxer_t *)_context, mpeg_id, nullptr, 0);
return true;
}
bool MpegMuxer::inputFrame(const Frame::Ptr &frame) {
auto it = _codec_to_trackid.find(frame->getCodecId());
if (it == _codec_to_trackid.end()) {
auto it = _tracks.find(frame->getIndex());
if (it == _tracks.end()) {
return false;
}
auto track_id = it->second;
auto &track = it->second;
_key_pos = !_have_video;
switch (frame->getCodecId()) {
case CodecH264:
case CodecH265: {
//这里的代码逻辑是让SPS、PPS、IDR这些时间戳相同的帧打包到一起当做一个帧处理，
return _frame_merger.inputFrame(frame,[this, track_id](uint64_t dts, uint64_t pts, const Buffer::Ptr &buffer, bool have_idr) {
// 这里的代码逻辑是让SPS、PPS、IDR这些时间戳相同的帧打包到一起当做一个帧处理，
return track.merger.inputFrame(frame, [&](uint64_t dts, uint64_t pts, const Buffer::Ptr &buffer, bool have_idr) {
_key_pos = have_idr;
//取视频时间戳为TS的时间戳
// 取视频时间戳为TS的时间戳
_timestamp = dts;
_max_cache_size = 512 + 1.2 * buffer->size();
mpeg_muxer_input((::mpeg_muxer_t *)_context, track_id, have_idr ? 0x0001 : 0, pts * 90LL,dts * 90LL, buffer->data(), buffer->size());
mpeg_muxer_input((::mpeg_muxer_t *)_context, track.track_id, have_idr ? 0x0001 : 0, pts * 90LL, dts * 90LL, buffer->data(), buffer->size());
flushCache();
});
}
@@ -80,7 +80,7 @@ bool MpegMuxer::inputFrame(const Frame::Ptr &frame) {
_timestamp = frame->dts();
}
_max_cache_size = 512 + 1.2 * frame->size();
mpeg_muxer_input((::mpeg_muxer_t *)_context, track_id, frame->keyFrame() ? 0x0001 : 0, frame->pts() * 90LL, frame->dts() * 90LL, frame->data(), frame->size());
mpeg_muxer_input((::mpeg_muxer_t *)_context, track.track_id, frame->keyFrame() ? 0x0001 : 0, frame->pts() * 90LL, frame->dts() * 90LL, frame->data(), frame->size());
flushCache();
return true;
}
@@ -103,7 +103,6 @@ void MpegMuxer::createContext() {
if (!thiz->_current_buffer
|| thiz->_current_buffer->size() + bytes > thiz->_current_buffer->getCapacity()) {
if (thiz->_current_buffer) {
//WarnL << "need realloc mpeg buffer" << thiz->_current_buffer->size() + bytes << " > " << thiz->_current_buffer->getCapacity();
thiz->flushCache();
}
thiz->_current_buffer = thiz->_buffer_pool.obtain2();
@@ -143,12 +142,13 @@ void MpegMuxer::releaseContext() {
mpeg_muxer_destroy((::mpeg_muxer_t *)_context);
_context = nullptr;
}
_codec_to_trackid.clear();
_frame_merger.clear();
_tracks.clear();
}
void MpegMuxer::flush() {
_frame_merger.flush();
for (auto &pr : _tracks) {
pr.second.merger.flush();
}
}
}//mediakit

13
src/Record/MPEG.h
View File

@@ -70,8 +70,17 @@ private:
uint32_t _max_cache_size = 0;
uint64_t _timestamp = 0;
struct mpeg_muxer_t *_context = nullptr;
std::unordered_map<int, int/*track_id*/> _codec_to_trackid;
FrameMerger _frame_merger{FrameMerger::h264_prefix};
class FrameMergerImp : public FrameMerger {
public:
FrameMergerImp() : FrameMerger(FrameMerger::h264_prefix) {}
};
struct MP4Track {
int track_id = -1;
FrameMergerImp merger;
};
std::unordered_map<int, MP4Track> _tracks;
toolkit::BufferRaw::Ptr _current_buffer;
toolkit::ResourcePool<toolkit::BufferRaw> _buffer_pool;
};