From 4a24057e732585efe922a55ce5c1fafbf7f8a06e Mon Sep 17 00:00:00 2001 From: YuLi Date: Wed, 22 Jul 2026 22:55:35 -0700 Subject: [PATCH] feat(http): decouple upload size limit from maxReqSize and harden file upload (#4780) - Add maxUploadSize config (default 1GB) for custom HttpBody streaming uploads; maxReqSize now only controls in-memory buffering threshold - Reject uploads exceeding maxUploadSize with 413 before writing any data - Delete incomplete files on HttpFileStorage destruction - Open HttpFileStorage in "wb" mode instead of "ab" to avoid stale data - Pass content_size to writeData() for integrity validation - Use uint64_t for content_len to support large file uploads - Wrap HttpBody _on_completed callback in try-catch to prevent unhandled exceptions - Make GET_CONFIG global variables read-only - Update ZLToolKit submodule --------- Co-authored-by: xiongziliang <771730766@qq.com> --- 3rdpart/ZLToolKit | 2 +- conf/config.ini | 4 +++ server/WebApi.cpp | 3 +- src/Common/config.cpp | 2 ++ src/Common/config.h | 17 +++++---- src/Http/HttpBody.cpp | 9 +++-- src/Http/HttpBody.h | 17 +++++---- src/Http/HttpSession.cpp | 78 +++++++++++++++++++++++++++++++--------- 8 files changed, 100 insertions(+), 32 deletions(-) diff --git a/3rdpart/ZLToolKit b/3rdpart/ZLToolKit index bb1a3826..00f56528 160000 --- a/3rdpart/ZLToolKit +++ b/3rdpart/ZLToolKit @@ -1 +1 @@ -Subproject commit bb1a382601e7cdeeb70d3bc978731c2648fd88b4 +Subproject commit 00f56528d28b5f4aaa94c175fe90ea9bdab4b17f diff --git a/conf/config.ini b/conf/config.ini index ec3a557a..72600377 100644 --- a/conf/config.ini +++ b/conf/config.ini @@ -401,6 +401,10 @@ keepAliveSecond=30 # http请求体最大字节数，如果post的body太大，则不适合缓存body在内存 # Maximum number of bytes for the HTTP request body. If the POST body is too large, it is not suitable to cache the body in memory. maxReqSize=40960 + +# 自定义HttpBody流式接收请求体的最大字节数；maxReqSize仍仅控制内存缓存阈值 +# Maximum request body size streamed into a custom HttpBody; maxReqSize remains an in-memory buffering threshold +maxUploadSize=1073741824 # 404网页内容，用户可以自定义404网页 # Custom 404 page content. Users can customize the 404 response page here. #notFound=404 Not Found

您访问的资源不存在！

ZLMediaKit-4.0
diff --git a/server/WebApi.cpp b/server/WebApi.cpp index 0dfb8303..7439f3e9 100755 --- a/server/WebApi.cpp +++ b/server/WebApi.cpp @@ -1488,7 +1488,8 @@ void installWebApi() { // Compatible with old version requests, the new version removes the only_audio parameter and adds the only_track parameter only_track = 1; } - GET_CONFIG(std::string, local_ip, General::kListenIP) + GET_CONFIG(std::string, s_local_ip, General::kListenIP) + auto local_ip = s_local_ip; if (!allArgs["local_ip"].empty()) { local_ip = allArgs["local_ip"]; } diff --git a/src/Common/config.cpp b/src/Common/config.cpp index 309dae69..8924a05b 100644 --- a/src/Common/config.cpp +++ b/src/Common/config.cpp @@ -191,6 +191,7 @@ namespace Http { #define HTTP_FIELD "http." const string kSendBufSize = HTTP_FIELD "sendBufSize"; const string kMaxReqSize = HTTP_FIELD "maxReqSize"; +const string kMaxUploadSize = HTTP_FIELD "maxUploadSize"; const string kKeepAliveSecond = HTTP_FIELD "keepAliveSecond"; const string kCharSet = HTTP_FIELD "charSet"; const string kRootPath = HTTP_FIELD "rootPath"; @@ -205,6 +206,7 @@ const string kAllowIPRange = HTTP_FIELD "allow_ip_range"; static onceToken token([]() { mINI::Instance()[kSendBufSize] = 64 * 1024; mINI::Instance()[kMaxReqSize] = 4 * 10240; + mINI::Instance()[kMaxUploadSize] = 1024ULL * 1024 * 1024; mINI::Instance()[kKeepAliveSecond] = 15; mINI::Instance()[kDirMenu] = true; mINI::Instance()[kVirtualPath] = ""; diff --git a/src/Common/config.h b/src/Common/config.h index 2e2ebeb6..8220e144 100644 --- a/src/Common/config.h +++ b/src/Common/config.h @@ -194,19 +194,21 @@ extern const std::string kBroadcastCreateMuxer; } while (0) #define GET_CONFIG(type, arg, key) \ - static type arg = ::toolkit::mINI::Instance()[key]; \ - LISTEN_RELOAD_KEY(arg, key, { RELOAD_KEY(arg, key); }); + static type arg##_storage_ = ::toolkit::mINI::Instance()[key]; \ + static const type &arg = arg##_storage_; \ + LISTEN_RELOAD_KEY(arg##_storage_, key, { RELOAD_KEY(arg##_storage_, key); }); #define GET_CONFIG_FUNC(type, arg, key, ...) \ - static type arg; \ + static type arg##_storage_; \ + static const type &arg = arg##_storage_; \ do { \ static ::toolkit::onceToken s_token_set([]() { \ static auto lam = __VA_ARGS__; \ static auto arg##_str = ::toolkit::mINI::Instance()[key]; \ - arg = lam(arg##_str); \ - LISTEN_RELOAD_KEY(arg, key, { \ + arg##_storage_ = lam(arg##_str); \ + LISTEN_RELOAD_KEY(arg##_str, key, { \ RELOAD_KEY(arg##_str, key); \ - arg = lam(arg##_str); \ + arg##_storage_ = lam(arg##_str); \ }); \ }); \ } while (0) @@ -356,6 +358,9 @@ extern const std::string kSendBufSize; // http 最大请求字节数 [AUTO-TRANSLATED:8239eb9c] // HTTP maximum request byte size extern const std::string kMaxReqSize; +// 自定义HttpBody流式接收请求体的最大字节数 +// Maximum request body size streamed into a custom HttpBody +extern const std::string kMaxUploadSize; // http keep-alive秒数 [AUTO-TRANSLATED:d4930c66] // HTTP keep-alive seconds extern const std::string kKeepAliveSecond; diff --git a/src/Http/HttpBody.cpp b/src/Http/HttpBody.cpp index fd57e742..c46af49a 100644 --- a/src/Http/HttpBody.cpp +++ b/src/Http/HttpBody.cpp @@ -405,7 +405,7 @@ Buffer::Ptr HttpBufferBody::readData(size_t size) { // ───────────────────────────────────────────────────────────────────────────── HttpFileStorage::HttpFileStorage(std::string file_path) { - _fp = fopen(file_path.data(), "ab"); + _fp = fopen(file_path.data(), "wb"); if (!_fp) { throw std::runtime_error("HttpFileStorage: failed to open file: " + file_path + ", err: " + toolkit::get_uv_errmsg()); } @@ -418,9 +418,14 @@ HttpFileStorage::~HttpFileStorage() { fclose(_fp); _fp = nullptr; } + if (_written != _content_size || !_written) { + // 删除不完整的文件 + File::delete_file(_path); + } } -void HttpFileStorage::writeData(const char *data, size_t size) { +void HttpFileStorage::writeData(const char *data, size_t size, uint64_t content_size) { + _content_size = content_size; if (!_fp) { throw std::runtime_error("HttpFileStorage: file not open"); } diff --git a/src/Http/HttpBody.h b/src/Http/HttpBody.h index deea1d22..0d46e4e1 100644 --- a/src/Http/HttpBody.h +++ b/src/Http/HttpBody.h @@ -35,7 +35,11 @@ public: using Ptr = std::shared_ptr; virtual ~HttpBody() { if (_on_completed) { - _on_completed(); + try { + _on_completed(); + } catch (std::exception &ex) { + ErrorL << ex.what(); + } } } @@ -97,7 +101,7 @@ public: * 写入数据，默认抛出异常表示不支持写入 * Write data, default throws exception indicating write not supported */ - virtual void writeData(const char *data, size_t size) { + virtual void writeData(const char *data, size_t size, uint64_t content_size) { throw std::runtime_error("HttpBody::writeData not supported"); } @@ -207,20 +211,21 @@ public: using Ptr = std::shared_ptr; /** - * @param file_path 文件路径，文件以二进制追加模式打开 - * @param file_path File path, opened in binary append mode + * @param file_path 文件路径，文件以二进制写入模式打开（覆盖已有内容） + * @param file_path File path, opened in binary write mode (truncates existing content) */ HttpFileStorage(std::string file_path); ~HttpFileStorage() override; - void writeData(const char *data, size_t size) override; + void writeData(const char *data, size_t size, uint64_t content_size) override; int64_t remainSize() override; toolkit::Buffer::Ptr readData(size_t size) override; const std::string& filePath() const; private: FILE *_fp = nullptr; - int64_t _written = 0; + uint64_t _written = 0; + uint64_t _content_size = 0; std::string _path; }; diff --git a/src/Http/HttpSession.cpp b/src/Http/HttpSession.cpp index 0323e521..2ef9b747 100644 --- a/src/Http/HttpSession.cpp +++ b/src/Http/HttpSession.cpp @@ -82,7 +82,7 @@ ssize_t HttpSession::onRecvHeader(const char *header, size_t len) { return 0; } - size_t content_len; + uint64_t content_len; auto &content_len_str = _parser["Content-Length"]; if (content_len_str.empty()) { if (it->first == "POST") { @@ -114,23 +114,75 @@ ssize_t HttpSession::onRecvHeader(const char *header, size_t len) { NOTICE_EMIT(BroadcastBeforeHttpRequestArgs, Broadcast::kBroadcastBeforeHttpRequest, _parser, body, *this); } - if (content_len > _max_req_size || body) { + // 自定义HttpBody直接消费网络分片，不经过HttpRequestSplitter的内存缓存；因此maxReqSize只负责选择缓存策略， + // 不能作为这里的上传上限，否则未携带Content-Length的正常流式上传会在默认40KB左右被错误拒绝。 + // A custom HttpBody consumes network fragments directly without HttpRequestSplitter buffering. Therefore maxReqSize + // only selects the buffering strategy and must not be used as the upload limit, or normal unknown-length uploads + // would be rejected at roughly the default 40KB threshold. + if (body) { + GET_CONFIG(uint64_t, max_upload_size_config, Http::kMaxUploadSize); + // 已知长度可以在接收body前判断是否超限，避免先向HttpBody写入数据再拒绝请求; + // 如果上传文件时不指定content-len，也直接拒绝，因为后续没法触发文件上传完毕事件 + if (content_len > max_upload_size_config) { + WarnL << "Http upload size is too huge or no content-len provided: " << content_len << " > " << max_upload_size_config + << ", please set " << Http::kMaxUploadSize << " in config.ini file."; + sendResponse(413, true); + _parser.clear(); + // 仍返回不定长body模式并丢弃连接关闭前可能到达的数据，防止splitter把body误当成下一条请求头。 + // Keep the splitter in variable-body mode and discard data arriving before close, so body bytes are not + // interpreted as another request header. + _on_recv_body = [](const char *, size_t) { return true; }; + return -1; + } + + size_t received = 0; + _on_recv_body = [this, received, content_len, body, it](const char *data, size_t len) mutable { + auto remain = content_len - received; + if (len > remain) { + // 告知HttpFileStorage，写入的数据长度超过content_len，触发文件删除操作 + body->writeData(data, len, content_len); + // 上传的数据超过声明的content-len， 直接拒绝 + sendResponse(413, true); + WarnL << "Upload file size larger than content_len: " << received + len << " > " << content_len; + return false; + } + + received += len; + body->writeData(data, len, content_len); + if (received < content_len) { + // 还没收满 [AUTO-TRANSLATED:cecc867e] + // Not yet received + return true; + } + // 收满了 [AUTO-TRANSLATED:0c9cebd7] + // Received full + setContentLen(0); + _parser.setBody(std::move(body)); + (this->*(it->second))(); + _parser.clear(); + return false; + }; + // 声明后续都是body；Http body在本对象缓冲，不通过HttpRequestSplitter保存 [AUTO-TRANSLATED:0012b6c1] + // Declare that the following is all body; Http body is buffered in this object, not saved through HttpRequestSplitter + return -1; + } + + // 未提供自定义HttpBody时保持原有语义：maxReqSize仅决定是否放弃内存整包缓存并改为分片回调， + // 不在本分支中把它扩展为普通请求体的硬上限。 + // Without a custom HttpBody, preserve the original behavior: maxReqSize only switches from whole-body buffering + // to fragment callbacks; it is not extended into a hard limit for ordinary request bodies in this branch. + if (content_len > _max_req_size) { // // 不定长body或超大body //// [AUTO-TRANSLATED:8d66ee77] // // Indefinite length body or oversized body //// - if (content_len != SIZE_MAX && !body) { + if (content_len != SIZE_MAX) { WarnL << "Http body size is too huge: " << content_len << " > " << _max_req_size << ", please set " << Http::kMaxReqSize << " in config.ini file."; } size_t received = 0; - _on_recv_body = [this, received, content_len, body, it](const char *data, size_t len) mutable { + _on_recv_body = [this, received, content_len](const char *data, size_t len) mutable { received += len; - if (body) { - body->writeData(data, len); - } else { - onRecvUnlimitedContent(_parser, data, len, content_len, received); - } - + onRecvUnlimitedContent(_parser, data, len, content_len, received); if (received < content_len) { // 还没收满 [AUTO-TRANSLATED:cecc867e] // Not yet received @@ -140,13 +192,7 @@ ssize_t HttpSession::onRecvHeader(const char *header, size_t len) { // 收满了 [AUTO-TRANSLATED:0c9cebd7] // Received full setContentLen(0); - - if (body) { - _parser.setBody(std::move(body)); - (this->*(it->second))(); - } _parser.clear(); - return false; }; // 声明后续都是body；Http body在本对象缓冲，不通过HttpRequestSplitter保存 [AUTO-TRANSLATED:0012b6c1]