From 0f47c998d4fa86518a453ebe41d5df7f71a5e73a Mon Sep 17 00:00:00 2001 From: Rick Ramsay <49293857+rickrams@users.noreply.github.com> Date: Fri, 17 Apr 2026 16:18:42 +0000 Subject: [PATCH] feat: Add FFmpeg movie from job output sample job bundle Add a new job bundle that downloads the output of a completed render job in the same queue and encodes the image sequence into an MP4 video using FFmpeg. Configurable parameters include frame rate, encoding preset, quality (CRF), pixel format, and resolution. The job uses the deadline.job_attachments Python API to download outputs and FFmpeg from conda-forge for video encoding. A pre-submission hook injects the queue's S3 bucket settings so no additional IAM permissions are needed on the queue role. Image format is auto-detected from the downloaded files. Signed-off-by: Rick Ramsay <49293857+rickrams@users.noreply.github.com> --- job_bundles/README.md | 7 + .../ffmpeg_movie_from_job_output/.gitignore | 1 + .../ffmpeg_movie_from_job_output/README.md | 98 ++++++ .../ffmpeg_movie_from_job_output/hooks.yaml | 5 + .../inject_s3_settings.py | 38 +++ .../template.yaml | 282 ++++++++++++++++++ 6 files changed, 431 insertions(+) create mode 100644 job_bundles/ffmpeg_movie_from_job_output/.gitignore create mode 100644 job_bundles/ffmpeg_movie_from_job_output/README.md create mode 100644 job_bundles/ffmpeg_movie_from_job_output/hooks.yaml create mode 100644 job_bundles/ffmpeg_movie_from_job_output/inject_s3_settings.py create mode 100644 job_bundles/ffmpeg_movie_from_job_output/template.yaml diff --git a/job_bundles/README.md b/job_bundles/README.md index e360502..c21c4e2 100644 --- a/job_bundles/README.md +++ b/job_bundles/README.md @@ -109,6 +109,13 @@ S3 prefix, then distributes the hashing and data copies across a number of worke uses content-addressed storage for data files, users that later submit jobs with these files attached will not have to upload them. +### FFmpeg movie from job output + +The [ffmpeg_movie_from_job_output](ffmpeg_movie_from_job_output) job bundle downloads the rendered output of another +completed job in the same queue and uses FFmpeg to encode the image sequence into an MP4 video. This is useful as a +post-processing utility — after a render job completes, submit this job with the source Job ID to automatically +assemble the frames into a movie with configurable frame rate, quality, and resolution settings. + ### SSH via SSM Managed Node The [ssh_to_smf](ssh_to_smf/README.md) job bundle registers a Deadline Cloud worker as an diff --git a/job_bundles/ffmpeg_movie_from_job_output/.gitignore b/job_bundles/ffmpeg_movie_from_job_output/.gitignore new file mode 100644 index 0000000..8ee0cf1 --- /dev/null +++ b/job_bundles/ffmpeg_movie_from_job_output/.gitignore @@ -0,0 +1 @@ +s3_settings.json diff --git a/job_bundles/ffmpeg_movie_from_job_output/README.md b/job_bundles/ffmpeg_movie_from_job_output/README.md new file mode 100644 index 0000000..bd27a2f --- /dev/null +++ b/job_bundles/ffmpeg_movie_from_job_output/README.md @@ -0,0 +1,98 @@ +# FFmpeg Movie from Job Output + +## Introduction + +This job bundle downloads the rendered output of another completed job in the same queue +and uses FFmpeg to encode the image sequence into an MP4 video file. It is useful as a +post-processing utility — for example, after a Blender or Maya render job completes, you +can submit this job to automatically assemble the frames into a movie. + +See also [ffmpeg_encode_video](../ffmpeg_encode_video) for a simpler sample that encodes +a local image sequence without downloading from another job. + +## How it works + +A [pre-submission hook](https://github.com/aws-deadline/deadline-cloud/blob/mainline/docs/submission-hooks.md) +(`inject_s3_settings.py`) runs at submission time on your workstation and looks up the +queue's job attachment S3 bucket configuration. It writes the settings to a JSON file that +gets uploaded as a job attachment, so the worker can access S3 without needing any +Deadline Cloud API permissions. + +On the worker, the job installs the `deadline` Python library via pip in a job environment, +then runs a single step that: + +1. Uses the `deadline.job_attachments` Python API to download the output files from the + source job's job attachments in S3. +2. Auto-detects the image format from the downloaded files, sorts them alphabetically, and + encodes them into an H.264 MP4 video using FFmpeg with BT.709 color space metadata. + +## Prerequisites + +### Software + +The job requires FFmpeg (from conda-forge) and the Deadline Cloud Python library (installed +via pip at runtime). On service-managed fleets, set the conda queue environment channel to +`conda-forge`. The job's `CondaPackages` parameter defaults to `ffmpeg`. + +### Submission hooks + +This job bundle uses a pre-submission hook to inject S3 settings. Enable bundle hooks +before submitting (one-time setup): + +```bash +deadline config set settings.allow_bundle_hooks true +``` + +The hook runs on your local machine at submission time using your existing AWS credentials. +No additional IAM permissions are needed on the queue role. + +### Source job requirements + +- The source job must have completed and produced output files via job attachments. +- Both jobs must be in the same queue (they share the same job attachments S3 bucket). + +## Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| Source Job ID | The Job ID of the completed source job | (required) | +| Source Step ID | Restrict download to a specific step's output | (empty = all) | +| Frame Rate | Video frame rate in fps | 24 | +| Pixel Format | Output pixel format (`yuv420p` or `yuv444p`) | yuv420p | +| Encoding Preset | FFmpeg speed/compression tradeoff | medium | +| Constant Rate Factor | H.264 quality (0 = lossless, 51 = worst, 17-18 ≈ visually lossless) | 18 | +| Output Resolution | Optional WIDTHxHEIGHT override (e.g. `1920x1080`) | (empty = source) | +| Output Filename | Name of the output video file | output.mp4 | +| Output Directory | Where to save the video | output | + +## Example submission + +```bash +# Enable bundle hooks (one-time setup) +deadline config set settings.allow_bundle_hooks true + +# Submit via GUI +deadline bundle gui-submit ffmpeg_movie_from_job_output/ + +# Submit via CLI +deadline bundle submit ffmpeg_movie_from_job_output/ \ + -p SourceJobId=job-0123456789abcdef0123456789abcdef \ + -p FrameRate=30 \ + -p OutputFilename=my_render.mp4 + +# Download only a specific step's output +deadline bundle submit ffmpeg_movie_from_job_output/ \ + -p SourceJobId=job-0123456789abcdef0123456789abcdef \ + -p SourceStepId=step-0123456789abcdef0123456789abcdef +``` + +## Typical workflow + +1. Submit a render job (e.g. Blender, Maya) to your queue. +2. Wait for the render job to complete. +3. Copy the Job ID from Deadline Cloud Monitor. +4. Submit this job bundle with the source Job ID. +5. Download the output video from Deadline Cloud Monitor. + +You can also automate this by scripting the submission after the render job completes +using `deadline job wait` followed by `deadline bundle submit`. diff --git a/job_bundles/ffmpeg_movie_from_job_output/hooks.yaml b/job_bundles/ffmpeg_movie_from_job_output/hooks.yaml new file mode 100644 index 0000000..5cd61ad --- /dev/null +++ b/job_bundles/ffmpeg_movie_from_job_output/hooks.yaml @@ -0,0 +1,5 @@ +version: "1.0" +preSubmission: + - command: python3 + args: [inject_s3_settings.py] + timeout: 30 diff --git a/job_bundles/ffmpeg_movie_from_job_output/inject_s3_settings.py b/job_bundles/ffmpeg_movie_from_job_output/inject_s3_settings.py new file mode 100644 index 0000000..104a219 --- /dev/null +++ b/job_bundles/ffmpeg_movie_from_job_output/inject_s3_settings.py @@ -0,0 +1,38 @@ +"""Pre-submission hook that writes job attachment S3 settings to a file in the bundle.""" +import json +import os +import sys + +from deadline.client import api + +metadata = json.load(sys.stdin) +farm_id = metadata["farmId"] +queue_id = metadata["queueId"] +bundle_dir = metadata["jobBundleDir"] + +print(f"Looking up job attachment settings for queue {queue_id}...", file=sys.stderr) +deadline = api.get_boto3_client("deadline") +queue = deadline.get_queue(farmId=farm_id, queueId=queue_id) +ja = queue.get("jobAttachmentSettings", {}) + +if not ja: + print("ERROR: Queue has no job attachment settings configured.", file=sys.stderr) + sys.exit(1) + +bucket = ja["s3BucketName"] +prefix = ja["rootPrefix"] +print(f"S3 bucket: {bucket}, prefix: {prefix}", file=sys.stderr) + +# Write settings file into the bundle so it gets uploaded as a job attachment +settings_path = os.path.join(bundle_dir, "s3_settings.json") +with open(settings_path, "w") as f: + json.dump({"s3BucketName": bucket, "rootPrefix": prefix}, f) + +# Output asset reference so the file gets uploaded +print(json.dumps({ + "attachments": { + "assetReferences": { + "inputFilenames": [settings_path] + } + } +})) diff --git a/job_bundles/ffmpeg_movie_from_job_output/template.yaml b/job_bundles/ffmpeg_movie_from_job_output/template.yaml new file mode 100644 index 0000000..42d1f32 --- /dev/null +++ b/job_bundles/ffmpeg_movie_from_job_output/template.yaml @@ -0,0 +1,282 @@ +specificationVersion: jobtemplate-2023-09 +name: FFmpeg Movie from Job Output +description: | + This job downloads the output of another completed job in the same queue and uses + FFmpeg to encode the image sequence into a video file. + + It requires FFmpeg from conda-forge and the Deadline Cloud Python library (installed + via pip at runtime). On service-managed fleets, use "conda-forge" as the CondaChannels + parameter for a conda queue environment. + +parameterDefinitions: +# Source Job +- name: SourceJobId + type: STRING + userInterface: + control: LINE_EDIT + label: Source Job ID + groupLabel: Source Job + description: > + The Job ID of a completed job in the same queue whose output images will be + assembled into a movie. Example: job-0123456789abcdef0123456789abcdef +- name: SourceStepId + type: STRING + userInterface: + control: LINE_EDIT + label: Source Step ID (optional) + groupLabel: Source Job + default: '' + description: > + Optionally restrict the download to a specific step ID from the source job. + Leave empty to download all outputs. + +# Movie Settings +- name: FrameRate + type: INT + userInterface: + control: SPIN_BOX + label: Frame Rate (fps) + groupLabel: Movie Settings + default: 24 + minValue: 1 + maxValue: 120 + description: The frame rate of the output video. +- name: PixelFormat + type: STRING + userInterface: + control: DROPDOWN_LIST + label: Pixel Format + groupLabel: Movie Settings + default: yuv420p + allowedValues: [yuv420p, yuv444p] + description: > + The pixel format for the output video. yuv420p is widely compatible; + yuv444p preserves more color detail. +- name: EncodingPreset + type: STRING + userInterface: + control: DROPDOWN_LIST + label: Encoding Preset + groupLabel: Movie Settings + default: medium + allowedValues: [ultrafast, superfast, veryfast, faster, fast, medium, slow, slower, veryslow] + description: Controls the encoding speed to compression ratio. +- name: ConstantRateFactor + type: INT + userInterface: + control: SPIN_BOX + label: Constant Rate Factor (CRF) + groupLabel: Movie Settings + default: 18 + minValue: 0 + maxValue: 51 + description: > + Quality setting for H.264 encoding. 0 is lossless, 51 is worst quality. + 17-18 is nearly visually lossless. +- name: Resolution + type: STRING + userInterface: + control: LINE_EDIT + label: Output Resolution (optional) + groupLabel: Movie Settings + default: '' + description: > + Optional output resolution as WIDTHxHEIGHT (e.g. 1920x1080). Leave empty + to use the source image resolution. +- name: OutputFilename + type: STRING + userInterface: + control: LINE_EDIT + label: Output Filename + groupLabel: Movie Settings + default: output.mp4 + description: The filename for the output video. + +# Output +- name: OutputDir + type: PATH + objectType: DIRECTORY + dataFlow: OUT + userInterface: + control: CHOOSE_DIRECTORY + label: Output Directory + groupLabel: Output + default: output + description: The directory where the output video will be saved. + +# Software Environment +- name: CondaPackages + type: STRING + userInterface: + control: HIDDEN + default: ffmpeg + description: > + Conda packages required by this job. Requires a conda queue environment. +- name: CondaChannels + type: STRING + userInterface: + control: HIDDEN + default: conda-forge + description: > + Conda channels to get packages from. Requires a conda queue environment. + +# Injected by pre-submission hook (inject_s3_settings.py) +- name: S3SettingsFile + type: PATH + objectType: FILE + dataFlow: IN + userInterface: + control: HIDDEN + default: s3_settings.json + description: > + JSON file with S3 bucket settings. Created by the pre-submission hook. + +jobEnvironments: +- name: InstallDeadline + description: Installs the Deadline Cloud Python library via pip. + script: + actions: + onEnter: + command: bash + args: ['{{Env.File.Setup}}'] + embeddedFiles: + - name: Setup + type: TEXT + runnable: true + data: | + #!/bin/env bash + set -euo pipefail + pip install deadline +- name: UnbufferedOutput + variables: + PYTHONUNBUFFERED: "True" + +steps: +- name: EncodeMovie + script: + actions: + onRun: + command: bash + args: ['{{Task.File.Run}}'] + embeddedFiles: + - name: Download + filename: download_outputs.py + type: TEXT + data: | + """Download output files from a source job using the job attachments API.""" + import json + import os + import sys + + from deadline.job_attachments.download import OutputDownloader + from deadline.job_attachments.models import ( + FileConflictResolution, + JobAttachmentS3Settings, + ) + + farm_id = os.environ["DEADLINE_FARM_ID"] + queue_id = os.environ["DEADLINE_QUEUE_ID"] + job_id = "{{Param.SourceJobId}}" + download_dir = sys.argv[1] + + with open("{{Param.S3SettingsFile}}") as f: + s3_cfg = json.load(f) + s3_settings = JobAttachmentS3Settings( + s3BucketName=s3_cfg["s3BucketName"], + rootPrefix=s3_cfg["rootPrefix"], + ) + + step_id = "{{Param.SourceStepId}}" or None + + downloader = OutputDownloader( + s3_settings=s3_settings, + farm_id=farm_id, + queue_id=queue_id, + job_id=job_id, + step_id=step_id, + ) + output_roots = list(downloader.get_output_paths_by_root().keys()) + for root in output_roots: + downloader.set_root_path(root, download_dir) + stats = downloader.download_job_output( + file_conflict_resolution=FileConflictResolution.OVERWRITE, + ) + print(f"Downloaded {stats.processed_files} files ({stats.processed_bytes} bytes)") + - name: Run + type: TEXT + runnable: true + data: | + #!/bin/env bash + set -xeuo pipefail + + DOWNLOAD_DIR="$(mktemp -d)" + mkdir -p '{{Param.OutputDir}}' + + # --- Download outputs from source job --- + echo "Downloading outputs from source job {{Param.SourceJobId}}..." + python3 '{{Task.File.Download}}' "$DOWNLOAD_DIR" + + # --- Find images --- + # Auto-detect the most common image extension + IMAGE_EXTS="png exr jpg jpeg tga tiff tif dpx hdr bmp" + BEST_EXT="" + BEST_COUNT=0 + for ext in $IMAGE_EXTS; do + COUNT=$(find "$DOWNLOAD_DIR" -type f -name "*.$ext" | wc -l) + if [ "$COUNT" -gt "$BEST_COUNT" ]; then + BEST_COUNT=$COUNT + BEST_EXT=$ext + fi + done + + if [ "$BEST_COUNT" -eq 0 ]; then + echo "ERROR: No image files found. Available files:" + find "$DOWNLOAD_DIR" -type f | head -20 + exit 1 + fi + echo "Detected $BEST_COUNT .$BEST_EXT files" + + # --- Build sorted file list and encode --- + CONCAT_FILE="$(mktemp)" + find "$DOWNLOAD_DIR" -type f -name "*.$BEST_EXT" | sort | while IFS= read -r img; do + echo "file '$img'" + done > "$CONCAT_FILE" + + echo "First images:"; head -3 "$CONCAT_FILE" + echo "Last images:"; tail -3 "$CONCAT_FILE" + + # Build scale filter + SCALE_FILTER="scale=in_color_matrix=bt709:out_color_matrix=bt709" + RESOLUTION='{{Param.Resolution}}' + if [ -n "$RESOLUTION" ]; then + WIDTH="${RESOLUTION%%x*}" + HEIGHT="${RESOLUTION##*x}" + SCALE_FILTER="scale=${WIDTH}:${HEIGHT}:in_color_matrix=bt709:out_color_matrix=bt709" + fi + + FFMPEG_ARGS=( + -y + -f concat -safe 0 + -r {{Param.FrameRate}} + -i "$CONCAT_FILE" + -pix_fmt {{Param.PixelFormat}} + -vf "$SCALE_FILTER" + -c:v libx264 + -preset {{Param.EncodingPreset}} + -crf {{Param.ConstantRateFactor}} + -color_range tv + -colorspace bt709 + -color_primaries bt709 + -color_trc iec61966-2-1 + -movflags faststart + '{{Param.OutputDir}}/{{Param.OutputFilename}}' + ) + + echo "Encoding video..." + ffmpeg "${FFMPEG_ARGS[@]}" + echo "Video saved to {{Param.OutputDir}}/{{Param.OutputFilename}}" + hostRequirements: + attributes: + - name: attr.worker.os.family + anyOf: + - linux