fix: limit AMF decoder nesting depth

src/Rtmp/amf.cpp

@@ -452,6 +452,17 @@ const std::string& AMFEncoder::data() const {
 
 //////////////////Decoder//////////////////
 
+AMFDecoder::DepthGuard::DepthGuard(AMFDecoder &decoder) : _decoder(decoder) {
+    if (_decoder.depth >= AMFDecoder::kMaxDepth) {
+        throw std::runtime_error("Maximum AMF nesting depth exceeded");
+    }
+    ++_decoder.depth;
+}
+
+AMFDecoder::DepthGuard::~DepthGuard() {
+    --_decoder.depth;
+}
+
 uint8_t AMFDecoder::front() {
     if (pos >= buf.size()) {
         throw std::runtime_error("Not enough data");
@@ -625,6 +636,7 @@ std::string AMFDecoder::load_key() {
 }
 
 AMFValue AMFDecoder::load_object() {
+    DepthGuard depth_guard(*this);
     AMFValue object(AMF_OBJECT);
     if (pop_front() != AMF0_OBJECT) {
         throw std::runtime_error("Expected an object");
@@ -643,6 +655,7 @@ AMFValue AMFDecoder::load_object() {
 }
 
 AMFValue AMFDecoder::load_ecma() {
+    DepthGuard depth_guard(*this);
     /* ECMA array is the same as object, with 4 extra zero bytes */
     AMFValue object(AMF_ECMA_ARRAY);
     if (pop_front() != AMF0_ECMA_ARRAY) {
@@ -665,6 +678,7 @@ AMFValue AMFDecoder::load_ecma() {
     return object;
 }
 AMFValue AMFDecoder::load_arr() {
+    DepthGuard depth_guard(*this);
     /* ECMA array is the same as object, with 4 extra zero bytes */
     AMFValue object(AMF_STRICT_ARRAY);
     if (pop_front() != AMF0_STRICT_ARRAY) {
@@ -689,4 +703,3 @@ AMFValue AMFDecoder::load_arr() {
 AMFDecoder::AMFDecoder(const BufferLikeString &buf_in, size_t pos_in, int version_in) :
         buf(buf_in), pos(pos_in), version(version_in) {
 }
-

src/Rtmp/amf.h

@@ -89,6 +89,15 @@ public:
     TP load();
 
 private:
+    class DepthGuard {
+    public:
+        explicit DepthGuard(AMFDecoder &decoder);
+        ~DepthGuard();
+
+    private:
+        AMFDecoder &_decoder;
+    };
+
     std::string load_key();
     AMFValue load_object();
     AMFValue load_ecma();
@@ -100,6 +109,9 @@ private:
     const toolkit::BufferLikeString &buf;
     size_t pos;
     int version;
+    size_t depth = 0;
+
+    static constexpr size_t kMaxDepth = 64;
 };
 
 class AMFEncoder {

tests/test_amf.cpp

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
+ *
+ * This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
+ *
+ * Use of this source code is governed by MIT-like license that can be found in the
+ * LICENSE file in the root of the source tree. All contributing project authors
+ * may be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <cassert>
+#include <stdexcept>
+#include <string>
+
+#include "Network/Buffer.h"
+#include "Rtmp/amf.h"
+
+using namespace std;
+using namespace toolkit;
+
+static string make_nested_object(size_t depth) {
+    string result;
+    for (size_t i = 0; i < depth; ++i) {
+        result.append("\x03\x00\x01x", 4);
+    }
+    result.append("\x05", 1);
+    for (size_t i = 0; i < depth; ++i) {
+        result.append("\x00\x00\x09", 3);
+    }
+    return result;
+}
+
+int main() {
+    {
+        BufferLikeString input(make_nested_object(64));
+        AMFDecoder decoder(input, 0);
+        assert(decoder.load<AMFValue>().type() == AMF_OBJECT);
+    }
+
+    {
+        BufferLikeString input(make_nested_object(65));
+        AMFDecoder decoder(input, 0);
+        try {
+            decoder.load<AMFValue>();
+            assert(false);
+        } catch (const runtime_error &ex) {
+            assert(string(ex.what()) == "Maximum AMF nesting depth exceeded");
+        }
+    }
+
+    return 0;
+}