Abort transcoding if output size exceeds 30x input size

ffmpeg/encoder.c

@@ -296,6 +296,24 @@ int open_output(struct output_ctx *octx, struct input_ctx *ictx)
     if (ret < 0) LPMS_ERR(open_output_err, "Unable to open signature filter");
   }
 
+  // Get input file size for comparison with written output size
+  if (ictx->ic && ictx->ic->pb) {
+    octx->input_file_size = avio_size(ictx->ic->pb);
+  } else {
+    octx->input_file_size = 0;
+  }
+
+  // Set maximum output size to 3x input size or 1GB if input size unknown
+  octx->output_bytes_written = 0;
+  if (octx->input_file_size > 0) {
+    octx->max_output_size = octx->input_file_size * 30;
+    av_log(NULL, AV_LOG_DEBUG, "Setting output size limit to 30x input size: input_size=%lld, max_output_size=%lld\n",
+           (long long)octx->input_file_size, (long long)octx->max_output_size);
+  } else {
+    octx->max_output_size = 1024LL * 1024 * 1024;
+    av_log(NULL, AV_LOG_DEBUG, "Setting output size limit to 1GB (input size unknown)\n");
+  }
+
   octx->initialized = 1;
 
   return 0;
@@ -398,7 +416,11 @@ int encode(AVCodecContext* encoder, AVFrame *frame, struct output_ctx* octx, AVS
             pkt->pts = (int64_t)pkt->opaque; // already in filter timebase
             pkt->dts = pkt->pts - av_rescale_q(pts_dts, encoder->time_base, time_base);
           }
-          mux(pkt, time_base, octx, ost);
+          ret = mux(pkt, time_base, octx, ost);
+          if (ret < 0) {
+            av_packet_free(&pkt);
+            LPMS_ERR(encode_cleanup, "Error muxing packet during encoder flush");
+          }
         } else if (AVERROR_EOF != ret) {
           av_packet_free(&pkt);
           LPMS_ERR(encode_cleanup, "did not get eof");
@@ -517,6 +539,20 @@ int mux(AVPacket *pkt, AVRational tb, struct output_ctx *octx, AVStream *ost)
       octx->last_video_dts = pkt->dts;
   }
 
+  // Track output size and check for size limit before writing
+  if (pkt->size > 0) {
+    octx->output_bytes_written += pkt->size;
+    if (octx->output_bytes_written > octx->max_output_size) {
+      av_log(NULL, AV_LOG_ERROR, "Output size limit exceeded: %lld bytes written, limit is %lld bytes. "
+                                 "Input file size: %lld bytes. Output filename: %s.\n",
+             (long long)octx->output_bytes_written,
+             (long long)octx->max_output_size,
+             (long long)octx->input_file_size,
+             octx->fname ? octx->fname : "unknown");
+      return lpms_ERR_OUTPUT_SIZE;
+    }
+  }
+
   return av_interleaved_write_frame(octx->oc, pkt);
 }
 
@@ -626,6 +662,13 @@ int process_out(struct input_ctx *ictx, struct output_ctx *octx, AVCodecContext
       }
 
       ret = encode(encoder, frame, octx, ost);
+
+      // Abort further processing if output size limit is exceeded
+      if (ret == lpms_ERR_OUTPUT_SIZE) {
+        av_frame_unref(frame);
+        return ret;
+      }
+
 skip:
     av_frame_unref(frame);
     // For HW we keep the encoder open so will only get EAGAIN.

ffmpeg/ffmpeg.go

@@ -39,6 +39,7 @@ var ErrTranscoderFmt = errors.New("TranscoderUnrecognizedFormat")
 var ErrTranscoderPrf = errors.New("TranscoderUnrecognizedProfile")
 var ErrTranscoderGOP = errors.New("TranscoderInvalidGOP")
 var ErrTranscoderDev = errors.New("TranscoderIncompatibleDevices")
+var ErrTranscoderOutputSize = errors.New("TranscoderOutputSizeLimitExceeded")
 var ErrEmptyData = errors.New("EmptyData")
 var ErrSignCompare = errors.New("InvalidSignData")
 var ErrTranscoderPixelformat = errors.New("TranscoderInvalidPixelformat")

ffmpeg/ffmpeg_errors.go

@@ -20,6 +20,7 @@ var lpmsErrors = []struct {
 	{Code: C.lpms_ERR_INPUT_CODEC, Desc: "Unsupported input codec"},
 	{Code: C.lpms_ERR_INPUT_NOKF, Desc: "No keyframes in input"},
 	{Code: C.lpms_ERR_UNRECOVERABLE, Desc: "Unrecoverable state, restart process"},
+	{Code: C.lpms_ERR_OUTPUT_SIZE, Desc: "Output size limit exceeded"},
 }
 
 func error_map() map[int]error {
@@ -67,6 +68,7 @@ func non_retryable_errs() []string {
 	transcoderErrors := []error{
 		ErrTranscoderRes, ErrTranscoderVid, ErrTranscoderFmt,
 		ErrTranscoderPrf, ErrTranscoderGOP, ErrTranscoderDev,
+		ErrTranscoderOutputSize,
 	}
 	for _, v := range transcoderErrors {
 		errs = append(errs, v.Error())

ffmpeg/filter.h

@@ -79,6 +79,11 @@ struct output_ctx {
   int64_t clip_from, clip_to, clip_from_pts, clip_to_pts, clip_started, clip_start_pts, clip_start_pts_found; // for clipping
   int64_t clip_audio_from_pts, clip_audio_to_pts, clip_audio_start_pts, clip_audio_start_pts_found; // for clipping
 
+  // Output size monitoring
+  int64_t output_bytes_written;
+  int64_t input_file_size;
+  int64_t max_output_size;
+
   output_results  *res; // data to return for this output
   char *xcoderParams;
 };

ffmpeg/size_limit_test.go

@@ -0,0 +1,63 @@
+package ffmpeg
+
+import (
+	"os"
+	"testing"
+)
+
+// Tests that output size monitoring stops transcoding when output limit is exceeded.
+func TestOutputSizeLimit(t *testing.T) {
+	InitFFmpeg()
+
+	// Use any available test file
+	testFiles := []string{
+		"../data/bunny.mp4",
+		"../data/videotest.mp4",
+	}
+
+	var inputFile string
+	for _, file := range testFiles {
+		if _, err := os.Stat(file); err == nil {
+			inputFile = file
+			break
+		}
+	}
+
+	if inputFile == "" {
+		t.Skip("No test files found")
+	}
+
+	// Use high quality
+	options := []TranscodeOptions{
+		{
+			Oname:   "test_size.mp4",
+			Accel:   Software,
+			Profile: P720p30fps16x9,
+			VideoEncoder: ComponentOptions{
+				Name: "libx264",
+				Opts: map[string]string{"crf": "10"},
+			},
+			AudioEncoder: ComponentOptions{Name: "drop"},
+		},
+	}
+
+	_, err := Transcode3(&TranscodeOptionsIn{
+		Fname: inputFile,
+		Accel: Software,
+	}, options)
+
+	// Clean up regardless of result
+	os.Remove(options[0].Oname)
+
+	// Size limit error is expected/acceptable for this test
+	if err != nil {
+		errStr := err.Error()
+		if err == ErrTranscoderOutputSize ||
+			errStr == "TranscoderOutputSizeLimitExceeded" ||
+			errStr == "Output size limit exceeded" {
+			// This is expected - size limit protection worked
+			return
+		}
+		t.Fatal(err)
+	}
+}

ffmpeg/transcoder.c

@@ -20,6 +20,7 @@ const int lpms_ERR_PACKET_ONLY = FFERRTAG('P','K','O','N');
 const int lpms_ERR_FILTER_FLUSHED = FFERRTAG('F','L','F','L');
 const int lpms_ERR_OUTPUTS = FFERRTAG('O','U','T','P');
 const int lpms_ERR_UNRECOVERABLE = FFERRTAG('U', 'N', 'R', 'V');
+const int lpms_ERR_OUTPUT_SIZE = FFERRTAG('O','U','S','Z');
 
 //
 //  Notes on transcoder internals:
@@ -538,6 +539,10 @@ int transcode(struct transcode_thread *h,
         ret = process_out(ictx, octx, encoder, ost, filter, dframe);
       }
       if (AVERROR(EAGAIN) == ret || AVERROR_EOF == ret) continue;
+      else if (ret == lpms_ERR_OUTPUT_SIZE) {
+        // Muxer throws this error if it detects abnormal output size growth compared to input size
+        LPMS_ERR(transcode_cleanup, "Output size limit exceeded");
+      }
       else if (ret < 0) LPMS_ERR(transcode_cleanup, "Error encoding");
     }
 whileloop_end:

ffmpeg/transcoder.h

@@ -17,6 +17,7 @@ extern const int lpms_ERR_PACKET_ONLY;
 extern const int lpms_ERR_FILTER_FLUSHED;
 extern const int lpms_ERR_OUTPUTS;
 extern const int lpms_ERR_UNRECOVERABLE;
+extern const int lpms_ERR_OUTPUT_SIZE;
 
 struct transcode_thread;