Unable to decode avif images encoded by ffmpeg via av1_qsv

[a15355447898a]

I got this avif image through this ffmpeg

[ayaya@arch-a380 avif压缩测试]$ find . -type f \( -iname '*.jpg' -o -iname '*.jpeg' -o -iname '*.png' -o -iname '*.tif' \) -print0 | while IFS= read -r -d '' f; do   out="${f%.*}.avif";   echo "Converting (Intel GPU): $f -> $out";    ffmpeg -nostdin -y -init_hw_device qsv=qsv:hw -filter_hw_device qsv -i "$f"     -vf "format=p010,hwupload=extra_hw_frames=64"     -c:v av1_qsv     -global_quality 25     -preset medium     "$out";  done
Converting (Intel GPU): ./neko.jpg -> ./neko.avif
ffmpeg version n8.0.1 Copyright (c) 2000-2025 the FFmpeg developers
  built with gcc 15.2.1 (GCC) 20251112
  configuration: --prefix=/usr --disable-debug --disable-static --disable-stripping --enable-amf --enable-avisynth --enable-cuda-llvm --enable-lto --enable-fontconfig --enable-frei0r --enable-gmp --enable-gnutls --enable-gpl --enable-ladspa --enable-libaom --enable-libass --enable-libbluray --enable-libbs2b --enable-libdav1d --enable-libdrm --enable-libdvdnav --enable-libdvdread --enable-libfreetype --enable-libfribidi --enable-libglslang --enable-libgsm --enable-libharfbuzz --enable-libiec61883 --enable-libjack --enable-libjxl --enable-libmodplug --enable-libmp3lame --enable-libopencore_amrnb --enable-libopencore_amrwb --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libplacebo --enable-libpulse --enable-librav1e --enable-librsvg --enable-librubberband --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libsrt --enable-libssh --enable-libsvtav1 --enable-libtheora --enable-libv4l2 --enable-libvidstab --enable-libvmaf --enable-libvorbis --enable-libvpl --enable-libvpx --enable-libwebp --enable-libx264 --enable-libx265 --enable-libxcb --enable-libxml2 --enable-libxvid --enable-libzimg --enable-libzmq --enable-nvdec --enable-nvenc --enable-opencl --enable-opengl --enable-shared --enable-vapoursynth --enable-version3 --enable-vulkan
  libavutil      60.  8.100 / 60.  8.100
  libavcodec     62. 11.100 / 62. 11.100
  libavformat    62.  3.100 / 62.  3.100
  libavdevice    62.  1.100 / 62.  1.100
  libavfilter    11.  4.100 / 11.  4.100
  libswscale      9.  1.100 /  9.  1.100
  libswresample   6.  1.100 /  6.  1.100
Input #0, image2, from './neko.jpg':
  Duration: 00:00:00.04, start: 0.000000, bitrate: 9547 kb/s
  Stream #0:0: Video: mjpeg (Baseline), yuvj420p(pc, bt470bg/unknown/unknown), 1091x681 [SAR 72:72 DAR 1091:681], 25 fps, 25 tbr, 25 tbn
Stream mapping:
  Stream #0:0 -> #0:0 (mjpeg (native) -> av1 (av1_qsv))
[swscaler @ 0x7d70c81079c0] deprecated pixel format used, make sure you did set range correctly
Output #0, avif, to './neko.avif':
  Metadata:
    encoder         : Lavf62.3.100
  Stream #0:0: Video: av1 (av01 / 0x31307661), qsv(pc, bt470bg/unknown/unknown, progressive), 1091x681 [SAR 1:1 DAR 1091:681], q=2-31, 25 fps, 12800 tbn (default)
    Metadata:
      encoder         : Lavc62.11.100 av1_qsv
[out#0/avif @ 0x5f54aed79cc0] video:42KiB audio:0KiB subtitle:0KiB other streams:0KiB global headers:0KiB muxing overhead: 0.654496%
frame=    1 fps=0.0 q=-0.0 Lsize=      42KiB time=00:00:00.00 bitrate=N/A speed=   0x elapsed=0:00:00.04   

But the image can't be parsed by avifdec

❯ avifdec neko-gpu.avif test_gpu.png
Decoding with codec 'dav1d' (20 worker threads), please wait...
ERROR: Failed to parse image: Not implemented
Diagnostics:
 * Box[pixi] contains unsupported plane count [0]

Images encoded with ffmpeg+svt-av1 can be parsed normally.

❯ find . -type f \( -iname '*.jpg' -o -iname '*.jpeg' -o -iname '*.png' -o -iname '*.tif' \) -print0 |
while IFS= read -r -d '' f; do
  out="${f%.*}.avif"
  echo "Converting: $f -> $out"

  ffmpeg -nostdin -y -i "$f" \
    -c:v libsvtav1 -crf 30 -preset 6 \
    -svtav1-params avif=1 -pix_fmt yuv420p10le -color_range pc \
    "$out"

done
Converting: ./neko.jpg -> ./neko.avif
ffmpeg version n8.0.1 Copyright (c) 2000-2025 the FFmpeg developers
  built with gcc 15.2.1 (GCC) 20251112
  configuration: --prefix=/usr --disable-debug --disable-static --disable-stripping --enable-amf --enable-avisynth --enable-cuda-llvm --enable-lto --enable-fontconfig --enable-frei0r --enable-gmp --enable-gnutls --enable-gpl --enable-ladspa --enable-libaom --enable-libass --enable-libbluray --enable-libbs2b --enable-libdav1d --enable-libdrm --enable-libdvdnav --enable-libdvdread --enable-libfreetype --enable-libfribidi --enable-libglslang --enable-libgsm --enable-libharfbuzz --enable-libiec61883 --enable-libjack --enable-libjxl --enable-libmodplug --enable-libmp3lame --enable-libopencore_amrnb --enable-libopencore_amrwb --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libplacebo --enable-libpulse --enable-librav1e --enable-librsvg --enable-librubberband --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libsrt --enable-libssh --enable-libsvtav1 --enable-libtheora --enable-libv4l2 --enable-libvidstab --enable-libvmaf --enable-libvorbis --enable-libvpl --enable-libvpx --enable-libwebp --enable-libx264 --enable-libx265 --enable-libxcb --enable-libxml2 --enable-libxvid --enable-libzimg --enable-libzmq --enable-nvdec --enable-nvenc --enable-opencl --enable-opengl --enable-shared --enable-vapoursynth --enable-version3 --enable-vulkan
  libavutil      60.  8.100 / 60.  8.100
  libavcodec     62. 11.100 / 62. 11.100
  libavformat    62.  3.100 / 62.  3.100
  libavdevice    62.  1.100 / 62.  1.100
  libavfilter    11.  4.100 / 11.  4.100
  libswscale      9.  1.100 /  9.  1.100
  libswresample   6.  1.100 /  6.  1.100
Input #0, image2, from './neko.jpg':
  Duration: 00:00:00.04, start: 0.000000, bitrate: 9547 kb/s
  Stream #0:0: Video: mjpeg (Baseline), yuvj420p(pc, bt470bg/unknown/unknown), 1091x681 [SAR 72:72 DAR 1091:681], 25 fps, 25 tbr, 25 tbn
Stream mapping:
  Stream #0:0 -> #0:0 (mjpeg (native) -> av1 (libsvtav1))
[swscaler @ 0x5586ec016ac0] deprecated pixel format used, make sure you did set range correctly
Svt[info]: -------------------------------------------
Svt[info]: SVT [version]:	SVT-AV1 Encoder Lib v3.1.2-dirty
Svt[info]: SVT [build]  :	GCC 15.2.1 20250813	 64 bit
Svt[info]: LIB Build date: Aug 28 2025 13:59:13
Svt[info]: -------------------------------------------
Svt[warn]: TPL is disabled for avif
Svt[info]: Level of Parallelism: 5
Svt[info]: Number of PPCS 2
Svt[info]: [asm level on system : up to avx2]
Svt[info]: [asm level selected : up to avx2]
Svt[info]: -------------------------------------------
Svt[info]: SVT [config]: main profile	tier (auto)	level (auto)
Svt[info]: SVT [config]: width / height / fps numerator / fps denominator 		: 1096 / 688 / 25 / 1
Svt[info]: SVT [config]: bit-depth / color format 					: 10 / YUV420
Svt[info]: SVT [config]: preset / tune / pred struct 					: 6 / PSNR / random access
Svt[info]: SVT [config]: gop size / mini-gop size / key-frame type 			: 161 / 32 / key frame
Svt[info]: SVT [config]: BRC mode / CQP Assignment 					: CQP / 30 
Svt[info]: SVT [config]: AQ mode / Variance Boost 					: 2 / 0
Svt[info]: SVT [config]: sharpness / luminance-based QP bias 				: 0 / 0
Svt[info]: -------------------------------------------
SvtMalloc[info]: SVT Memory Usage:
SvtMalloc[info]:     total allocated memory:       70.92 MB
SvtMalloc[info]:         malloced memory:          22.30 MB
SvtMalloc[info]:         callocated memory:        6.06 MB
SvtMalloc[info]:         allocated aligned memory: 42.56 MB
SvtMalloc[info]:     mutex count: 578
SvtMalloc[info]:     semaphore count: 207
SvtMalloc[info]:     thread count: 59
SvtMalloc[info]:     hash table fulless: 0.002747, hash bucket is healthy
SvtMalloc[info]: top 10 malloced memory locations:
SvtMalloc[info]: (3.81 MB): /usr/src/debug/svt-av1/SVT-AV1/Source/Lib/Codec/bitstream_unit.c:36
SvtMalloc[info]: (2.90 MB): /usr/src/debug/svt-av1/SVT-AV1/Source/Lib/Codec/coding_unit.c:109
SvtMalloc[info]: (2.71 MB): /usr/src/debug/svt-av1/SVT-AV1/Source/Lib/Codec/pcs.c:1088
SvtMalloc[info]: (1.98 MB): /usr/src/debug/svt-av1/SVT-AV1/Source/Lib/Codec/md_process.c:264
SvtMalloc[info]: (1.24 MB): /usr/src/debug/svt-av1/SVT-AV1/Source/Lib/Codec/md_process.c:307
SvtMalloc[info]: (1.24 MB): /usr/src/debug/svt-av1/SVT-AV1/Source/Lib/Codec/md_process.c:306
SvtMalloc[info]: (915.33 KB): /usr/src/debug/svt-av1/SVT-AV1/Source/Lib/Codec/md_process.c:348
SvtMalloc[info]: (830.06 KB): /usr/src/debug/svt-av1/SVT-AV1/Source/Lib/Codec/pcs.c:577
SvtMalloc[info]: (636.75 KB): /usr/src/debug/svt-av1/SVT-AV1/Source/Lib/Codec/md_process.c:310
SvtMalloc[info]: (636.75 KB): /usr/src/debug/svt-av1/SVT-AV1/Source/Lib/Codec/md_process.c:311
Output #0, avif, to './neko.avif':
  Metadata:
    encoder         : Lavf62.3.100
  Stream #0:0: Video: av1 (av01 / 0x31307661), yuv420p10le(pc, bt470bg/unknown/unknown, progressive), 1091x681 [SAR 1:1 DAR 1091:681], q=2-31, 25 fps, 12800 tbn (default)
    Metadata:
      encoder         : Lavc62.11.100 libsvtav1
[out#0/avif @ 0x55872e0e7b40] video:15KiB audio:0KiB subtitle:0KiB other streams:0KiB global headers:0KiB muxing overhead: 1.850181%
frame=    1 fps=0.0 q=30.0 Lsize=      15KiB time=00:00:00.00 bitrate=N/A speed=   0x elapsed=0:00:00.36    
SvtMalloc[info]: you have no memory leak

❯ avifdec neko-cpu.avif test_gpu.png
Decoding with codec 'dav1d' (20 worker threads), please wait...
Image decoded: neko-cpu.avif
 * Resolution     : 1091x681
 * Bit Depth      : 10
 * Format         : YUV420
 * Chroma Sam. Pos: 0
 * Alpha          : Absent
 * Range          : Full
 * Color Primaries: 2
 * Transfer Char. : 2
 * Matrix Coeffs. : 5
 * ICC Profile    : Absent
 * XMP Metadata   : Absent
 * Exif Metadata  : Absent
 * Transformations: None
 * Progressive    : Unavailable
 * Gain map       : Absent
 * 1 timescales per second, 1.00 seconds (1 timescales), 1 frame
 * Frame:
   * Decoded frame [0] [pts 0.00 (0 timescales)] [duration 1.00 (1 timescales)] [1091x681]
Wrote PNG: test_gpu.png

The test files are as follows
avif压缩测试.zip

[a15355447898a]

Also, those AVIF images encoded with the GPU can be opened directly with the native image viewers on both macOS and Windows. And on Linux, Firefox handles them just fine too.

[vrabaud]

That was changed in #2058. @y-guyon , HEIF does not seem to restrict num_channels to be > 0, does it ? (in 6.5.6.3 of 23008-12:2024)

[wantehchang]

Vincent: Thanks for looking into this. I uploaded neko-cpu.avif and neko-gpu.avif to the Compliance Warden AVIF validator for inspection:

https://gpac.github.io/ComplianceWarden-wasm/

It reported no errors for both files.

Here is the definition of the pixi box in HEIF ISO/IEC 23008-12:2022:

aligned(8) class PixelInformationProperty
extends ItemFullProperty('pixi', version = 0, flags = 0){
 unsigned int(8) num_channels;
 for (i=0; i<num_channels; i++) {
 unsigned int(8) bits_per_channel;
 }
}

If num_channels is equal to 0, the pixi box is almost empty. A zero channel count also doesn't make sense. I wonder if such a pix box is inserted just to satisfy some requirement that a pixi box is mandatory.

[y-guyon]

I suggest the following:

1. Be lenient in libavif and ignore empty pixi boxes instead of rejecting them, because such files are likely in the wild now
2. Open an issue in ffmpeg tracker asking for the channels to be written in the pixi box

What do others think?

[wantehchang]

Yannis: It would be good to first find out

• why ffmpeg via av1_qsv is inserting a pix box with num_channels equal to 0, and
• how long it has been doing that

before we take any action. It would also be good to get an opinion from the ComplianceWarden maintainers. Otherwise, I agree with your suggestion.

[wantehchang]

@jzern FYI

@a15355447898a ayaya: I am not familiar with ffmpeg. Is it possible for you to specify -pix_fmt p010le when you use ffmpeg with the av1_qsv encoder? That may resolve this issue.

Details:

I studied the ffmpeg source code. Here are the source files relevant to this issue.

libavformat/movenc: see mov_write_pixi_tag() and mov_write_ipco_tag(). In particular, mov_write_pixi_tag() has the following code:

    const AVPixFmtDescriptor *pixdesc =
        av_pix_fmt_desc_get(s->streams[stream_index]->codecpar->format);
    avio_wb32(pb, 0); /* size */
    ffio_wfourcc(pb, "pixi");
    avio_wb32(pb, 0); /* Version & flags */
    avio_w8(pb, pixdesc->nb_components); /* num_channels */

So the av_pix_fmt_desc_get() function is important. More on this below.

libavcodec/qsvenc_av1.c: defines the av1_qsv encoder. The pixel formats it suppors are:

FFCodec ff_av1_qsv_encoder = {
    .p.name           = "av1_qsv", 
    ...
    CODEC_PIXFMTS(AV_PIX_FMT_NV12, AV_PIX_FMT_P010, AV_PIX_FMT_QSV),

libavcodec/libsvtav1.c: defines the libsvtav1 encoder. The pixel formats it supports are:

const FFCodec ff_libsvtav1_encoder = {
    .p.name         = "libsvtav1",
    ...
    CODEC_PIXFMTS(AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P10),

libavutil/pixdesc.c: defines the av_pix_fmt_desc_get() function by using a lookup table:

const AVPixFmtDescriptor *av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
{                       
    if (pix_fmt < 0 || pix_fmt >= AV_PIX_FMT_NB)
        return NULL;
    return &av_pix_fmt_descriptors[pix_fmt];
}

Now we look for the pixel formats supported by the av1_qsv encoder in the av_pix_fmt_descriptors lookup table.

    [AV_PIX_FMT_NV12] = {
        .name = "nv12",
        .nb_components = 3,
        .log2_chroma_w = 1,
        .log2_chroma_h = 1,
        .comp = {
            { 0, 1, 0, 0, 8 },        /* Y */
            { 1, 2, 0, 0, 8 },        /* U */
            { 1, 2, 1, 0, 8 },        /* V */
        },  
        .flags = AV_PIX_FMT_FLAG_PLANAR,
    },

We assume we are on a little-endian computer:

    [AV_PIX_FMT_P010LE] = {
        .name = "p010le",
        .nb_components = 3,
        .log2_chroma_w = 1,
        .log2_chroma_h = 1,
        .comp = {
            { 0, 2, 0, 6, 10 },        /* Y */
            { 1, 4, 0, 6, 10 },        /* U */
            { 1, 4, 2, 6, 10 },        /* V */
        },  
        .flags = AV_PIX_FMT_FLAG_PLANAR,
    },

and finally,

    [AV_PIX_FMT_QSV] = {
        .name = "qsv",
        .flags = AV_PIX_FMT_FLAG_HWACCEL,
    },

Note that this lookup table entry leaves the nb_components unspecified, so it is initialized to 0.

[a15355447898a]

@jzern FYI

@a15355447898a ayaya: I am not familiar with ffmpeg. Is it possible for you to specify -pix_fmt p010le when you use ffmpeg with the av1_qsv encoder? That may resolve this issue.

Details:

I studied the ffmpeg source code. Here are the source files relevant to this issue.

libavformat/movenc: see mov_write_pixi_tag() and mov_write_ipco_tag(). In particular, mov_write_pixi_tag() has the following code:

    const AVPixFmtDescriptor *pixdesc =
        av_pix_fmt_desc_get(s->streams[stream_index]->codecpar->format);
    avio_wb32(pb, 0); /* size */
    ffio_wfourcc(pb, "pixi");
    avio_wb32(pb, 0); /* Version & flags */
    avio_w8(pb, pixdesc->nb_components); /* num_channels */

So the av_pix_fmt_desc_get() function is important. More on this below.

libavcodec/qsvenc_av1.c: defines the av1_qsv encoder. The pixel formats it suppors are:

FFCodec ff_av1_qsv_encoder = {
    .p.name           = "av1_qsv", 
    ...
    CODEC_PIXFMTS(AV_PIX_FMT_NV12, AV_PIX_FMT_P010, AV_PIX_FMT_QSV),

libavcodec/libsvtav1.c: defines the libsvtav1 encoder. The pixel formats it supports are:

const FFCodec ff_libsvtav1_encoder = {
    .p.name         = "libsvtav1",
    ...
    CODEC_PIXFMTS(AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P10),

libavutil/pixdesc.c: defines the av_pix_fmt_desc_get() function by using a lookup table:

const AVPixFmtDescriptor *av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
{                       
    if (pix_fmt < 0 || pix_fmt >= AV_PIX_FMT_NB)
        return NULL;
    return &av_pix_fmt_descriptors[pix_fmt];
}

Now we look for the pixel formats supported by the av1_qsv encoder in the av_pix_fmt_descriptors lookup table.

    [AV_PIX_FMT_NV12] = {
        .name = "nv12",
        .nb_components = 3,
        .log2_chroma_w = 1,
        .log2_chroma_h = 1,
        .comp = {
            { 0, 1, 0, 0, 8 },        /* Y */
            { 1, 2, 0, 0, 8 },        /* U */
            { 1, 2, 1, 0, 8 },        /* V */
        },  
        .flags = AV_PIX_FMT_FLAG_PLANAR,
    },

We assume we are on a little-endian computer:

    [AV_PIX_FMT_P010LE] = {
        .name = "p010le",
        .nb_components = 3,
        .log2_chroma_w = 1,
        .log2_chroma_h = 1,
        .comp = {
            { 0, 2, 0, 6, 10 },        /* Y */
            { 1, 4, 0, 6, 10 },        /* U */
            { 1, 4, 2, 6, 10 },        /* V */
        },  
        .flags = AV_PIX_FMT_FLAG_PLANAR,
    },

and finally,

    [AV_PIX_FMT_QSV] = {
        .name = "qsv",
        .flags = AV_PIX_FMT_FLAG_HWACCEL,
    },

Note that this lookup table entry leaves the nb_components unspecified, so it is initialized to 0.

According to what you said, the avif image obtained by using this command can be decoded normally

[ayaya@arch-a380 avif压缩测试]$ ffmpeg -nostdin -y \
  -i ./neko.jpg \
  -vf "format=p010le" \
  -c:v av1_qsv \
  -pix_fmt p010le \
  -color_range pc \
  -global_quality 25 \
  -preset medium \
  ./neko.avif
ffmpeg version n8.0.1 Copyright (c) 2000-2025 the FFmpeg developers
  built with gcc 15.2.1 (GCC) 20251112
  configuration: --prefix=/usr --disable-debug --disable-static --disable-stripping --enable-amf --enable-avisynth --enable-cuda-llvm --enable-lto --enable-fontconfig --enable-frei0r --enable-gmp --enable-gnutls --enable-gpl --enable-ladspa --enable-libaom --enable-libass --enable-libbluray --enable-libbs2b --enable-libdav1d --enable-libdrm --enable-libdvdnav --enable-libdvdread --enable-libfreetype --enable-libfribidi --enable-libglslang --enable-libgsm --enable-libharfbuzz --enable-libiec61883 --enable-libjack --enable-libjxl --enable-libmodplug --enable-libmp3lame --enable-libopencore_amrnb --enable-libopencore_amrwb --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libplacebo --enable-libpulse --enable-librav1e --enable-librsvg --enable-librubberband --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libsrt --enable-libssh --enable-libsvtav1 --enable-libtheora --enable-libv4l2 --enable-libvidstab --enable-libvmaf --enable-libvorbis --enable-libvpl --enable-libvpx --enable-libwebp --enable-libx264 --enable-libx265 --enable-libxcb --enable-libxml2 --enable-libxvid --enable-libzimg --enable-libzmq --enable-nvdec --enable-nvenc --enable-opencl --enable-opengl --enable-shared --enable-vapoursynth --enable-version3 --enable-vulkan
  libavutil      60.  8.100 / 60.  8.100
  libavcodec     62. 11.100 / 62. 11.100
  libavformat    62.  3.100 / 62.  3.100
  libavdevice    62.  1.100 / 62.  1.100
  libavfilter    11.  4.100 / 11.  4.100
  libswscale      9.  1.100 /  9.  1.100
  libswresample   6.  1.100 /  6.  1.100
Input #0, image2, from './neko.jpg':
  Duration: 00:00:00.04, start: 0.000000, bitrate: 9547 kb/s
  Stream #0:0: Video: mjpeg (Baseline), yuvj420p(pc, bt470bg/unknown/unknown), 1091x681 [SAR 72:72 DAR 1091:681], 25 fps, 25 tbr, 25 tbn
Stream mapping:
  Stream #0:0 -> #0:0 (mjpeg (native) -> av1 (av1_qsv))
[swscaler @ 0x72d1d4012d00] deprecated pixel format used, make sure you did set range correctly
Output #0, avif, to './neko.avif':
  Metadata:
    encoder         : Lavf62.3.100
  Stream #0:0: Video: av1 (av01 / 0x31307661), p010le(pc, bt470bg/unknown/unknown, progressive), 1091x681 [SAR 1:1 DAR 1091:681], q=2-31, 25 fps, 12800 tbn (default)
    Metadata:
      encoder         : Lavc62.11.100 av1_qsv
[out#0/avif @ 0x60869ee79ac0] video:42KiB audio:0KiB subtitle:0KiB other streams:0KiB global headers:0KiB muxing overhead: 0.670859%
frame=    1 fps=0.0 q=-0.0 Lsize=      42KiB time=00:00:00.00 bitrate=N/A speed=   0x elapsed=0:00:00.06 

[I]  avifdec neko.avif test_gpu.png
Decoding with codec 'dav1d' (20 worker threads), please wait...
Image decoded: neko.avif
 * Resolution     : 1091x681
 * Bit Depth      : 10
 * Format         : YUV420
 * Chroma Sam. Pos: 0
 * Alpha          : Absent
 * Range          : Full
 * Color Primaries: 2
 * Transfer Char. : 2
 * Matrix Coeffs. : 5
 * ICC Profile    : Absent
 * XMP Metadata   : Absent
 * Exif Metadata  : Absent
 * Transformations: None
 * Progressive    : Unavailable
 * Gain map       : Absent
 * 1 timescales per second, 1.00 seconds (1 timescales), 1 frame
 * Frame:
   * Decoded frame [0] [pts 0.00 (0 timescales)] [duration 1.00 (1 timescales)] [1091x681]
Wrote PNG: test_gpu.png

neko-gpu-new.zip

[wantehchang]

@a15355447898a avaya: Thank you very much for testing the av1_qsv encoder with -pix_fmt p010le. Glad to hear that it produced an AVIF file with a correct pixi box.

We need to verify that the hardware encoder is used when we specify -pix_fmt p010le. Do you know how to check that?

From the file sizes, that seems to be the case. Here are the file sizes:

• neko-gpu.avif: 43,061
• neko-gpu-new.avif: 43,068

Since the file sizes are very close, very likely the same method was used to encode the two files.

Note: A pixi box with num_channels=3 is 3 bytes larger than a pixi box with num_channels=0. But the file size increased by 7 bytes. So there is a 4-byte increase that I cannot explain.

I noticed that neko-gpu.avif (43,061) is much larger than neko-cpu.avif (15,799). Do you know why?

---

To my coworkers: With reasonable amount of work, we should be able to refactor ffmpeg's mov_write_ipco_tag() function so that it writes a correct pixi box even if the av_pix_fmt_descriptors lookup table entry is missing the nb_components and comp values -- it can get that information from the AV1 sequence header that the mov_write_av1c_tag() function needs to parse.

CrabbyAvif has the same checks in src/parser/mp4box.rs:

fn parse_pixi(stream: &mut IStream) -> AvifResult<ItemProperty> {
    // Section 6.5.6.2 of ISO/IEC 23008-12.
    let (_version, px_flags) = stream.read_and_enforce_version_and_flags(0)?;
    // unsigned int (8) num_channels;
    let num_channels = stream.read_u8()? as usize;
    if num_channels == 0 || num_channels > MAX_PLANE_COUNT {
        return AvifError::bmff_parse_failed(format!(
            "Invalid plane count {num_channels} in pixi box"
        ));
    }

Given our latest understanding of this issue, I think it is better to continue to require a correct pixi box.

[a15355447898a]

@a15355447898a avaya: Thank you very much for testing the av1_qsv encoder with -pix_fmt p010le. Glad to hear that it produced an AVIF file with a correct pixi box.

We need to verify that the hardware encoder is used when we specify -pix_fmt p010le. Do you know how to check that?

From the file sizes, that seems to be the case. Here are the file sizes:

* neko-gpu.avif: 43,061

* neko-gpu-new.avif: 43,068

Since the file sizes are very close, very likely the same method was used to encode the two files.

Note: A pixi box with num_channels=3 is 3 bytes larger than a pixi box with num_channels=0. But the file size increased by 7 bytes. So there is a 4-byte increase that I cannot explain.

I noticed that neko-gpu.avif (43,061) is much larger than neko-cpu.avif (15,799). Do you know why?

To my coworkers: With reasonable amount of work, we should be able to refactor ffmpeg's mov_write_ipco_tag() function so that it writes a correct pixi box even if the av_pix_fmt_descriptors lookup table entry is missing the nb_components and comp values -- it can get that information from the AV1 sequence header that the mov_write_av1c_tag() function needs to parse.

CrabbyAvif has the same checks in src/parser/mp4box.rs:

fn parse_pixi(stream: &mut IStream) -> AvifResult<ItemProperty> {
    // Section 6.5.6.2 of ISO/IEC 23008-12.
    let (_version, px_flags) = stream.read_and_enforce_version_and_flags(0)?;
    // unsigned int (8) num_channels;
    let num_channels = stream.read_u8()? as usize;
    if num_channels == 0 || num_channels > MAX_PLANE_COUNT {
        return AvifError::bmff_parse_failed(format!(
            "Invalid plane count {num_channels} in pixi box"
        ));
    }

Given our latest understanding of this issue, I think it is better to continue to require a correct pixi box.

Regarding whether a hardware encoder is used, I guess one can use intel_gpu_top to see

Regarding the four byte increase, I compared the raw binary contents of neko-gpu.avif and neko-gpu-new.avif

The difference in file size can only be explained by the performance difference between encoders

Use this script to compare different encoders

#!/bin/bash

INPUT="neko.jpg"

# -------------------------------------------------
# 1. 使用 libsvtav1
# -------------------------------------------------
OUTPUT_SVT="neko_svtav1.avif"
ffmpeg -nostdin -y -v error -i "$INPUT" \
  -c:v libsvtav1 -crf 30 -preset 8 \
  -pix_fmt yuv420p10le -color_range pc \
  -svtav1-params avif=1 \
  "$OUTPUT_SVT"
echo "-> 已生成: $OUTPUT_SVT"
echo ""


# -------------------------------------------------
# 2. 使用 libaom-av1
# -------------------------------------------------
OUTPUT_AOM="neko_aom.avif"
ffmpeg -nostdin -y -v error -i "$INPUT" \
  -c:v libaom-av1 -crf 30 -cpu-used 4 \
  -pix_fmt yuv420p10le -color_range pc \
  -still-picture 1 \
  "$OUTPUT_AOM"
echo "-> 已生成: $OUTPUT_AOM"
echo ""


# -------------------------------------------------
# 3. 使用 librav1e
# -------------------------------------------------
OUTPUT_RAV1E="neko_rav1e.avif"
ffmpeg -nostdin -y -v error -i "$INPUT" \
      -c:v librav1e -crf 30 -speed 6 \
      -pix_fmt yuv420p10le -color_range pc \
      "$OUTPUT_RAV1E"
echo "-> 已生成: $OUTPUT_RAV1E"

The script and the generated avif file are as follows

avif编码器比较.zip

[wantehchang]

@a15355447898a avaya: Thanks a lot for the detailed comparison. We are now certain -pix_fmt p010le is also using the hardware encoder, and the seven-byte file size increase consists of the following:

• the MA1B compatible brand in the ftyp box, and
• three bytes of 10 (0x0A) in the pixi box (for bits_per_channel)

The AV1 bitsreams are exactly the same.

So passing -pix_fmt p010le is a good solution to this issue. It not only fixes the pixi box but also improves the ftyp box.

[wantehchang]

I also looked into why the ftyp box doesn't have the MA1B compatible brand if the "qsv" pixel format is used with the av1_qsv encoder. See the mov_write_ftyp_tag() function in libavformat/movenc.c:

    } else if (mov->mode == MODE_AVIF) {
        const AVPixFmtDescriptor *pix_fmt_desc =
            av_pix_fmt_desc_get(s->streams[0]->codecpar->format);
        const int depth = pix_fmt_desc->comp[0].depth;
        if (mov->is_animated_avif) {
            // For animated AVIF, major brand is "avis". Add "avif" as a
            // compatible brand.
            ffio_wfourcc(pb, "avif");
            ffio_wfourcc(pb, "msf1");
            ffio_wfourcc(pb, "iso8");
        }
        ffio_wfourcc(pb, "mif1");
        ffio_wfourcc(pb, "miaf");
        if (depth == 8 || depth == 10) {
            // MA1B and MA1A brands are based on AV1 profile. Short hand for
            // computing that is based on chroma subsampling type. 420 chroma
            // subsampling is MA1B.  444 chroma subsampling is MA1A.
            if (!pix_fmt_desc->log2_chroma_w && !pix_fmt_desc->log2_chroma_h) {
                // 444 chroma subsampling.
                ffio_wfourcc(pb, "MA1A");
            } else {
                // 420 chroma subsampling.
                ffio_wfourcc(pb, "MA1B");
            }
        }
    } else if (mov->mode != MODE_MOV) {

This code has a bug that it uses pix_fmt_desc->comp[0] without checking if pix_fmt_desc->nb_components > 0. Since the pix_fmt_desc lookup table entry for the "qsv" pixel format leaves most fields initialized to 0:

    [AV_PIX_FMT_QSV] = {
        .name = "qsv",
        .flags = AV_PIX_FMT_FLAG_HWACCEL,
    },

pix_fmt_desc->comp[0].depth is equal to 0, so the code skips the if (depth == 8 || depth == 10) statement and doesn't write the "MA1A" or "MA1B" compatible brand.

This code has another bug: it assumes that if the chroma subsamping format is not 444, then it must be 420. But that ignores 422. So the innermost if-else statement should be rewritten as follows:

            if (!pix_fmt_desc->log2_chroma_w && !pix_fmt_desc->log2_chroma_h) {
                // 444 chroma subsampling.
                ffio_wfourcc(pb, "MA1A");
            } else if (pix_fmt_desc->log2_chroma_w && pix_fmt_desc->log2_chroma_h) {
                // 420 chroma subsampling.
                ffio_wfourcc(pb, "MA1B");
            }

Note: The most reliable way to determine whether the compatible brand "MA1A" or "MA1B" should be written is to check the profile and level in the AV1 sequence header OBU or equivalently the av1C box.