spike on testing a bunch of files from the MPEG file format set

.github/workflows/pr.yml

@@ -13,6 +13,9 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: true
+          lfs: true
 
       # Install Just to run CI scripts
       - uses: extractions/setup-just@v2

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "FileFormatConformance"]
+	path = FileFormatConformance
+	url = https://github.com/MPEGGroup/FileFormatConformance.git

src/any.rs

@@ -261,6 +261,8 @@ any! {
     Moov,
         Mvhd,
         Udta,
+            Cprt,
+            Kind,
             Skip,
         // Trak, // boxed to avoid large size differences between variants
             Tkhd,
@@ -304,6 +306,7 @@ any! {
                         Stts,
                         Stsc,
                         Stsz,
+                        Stz2,
                         Stss,
                         Stco,
                         Co64,
@@ -316,7 +319,9 @@ any! {
                         Saiz,
                     Dinf,
                         Dref,
+                    Nmhd,
                     Smhd,
+                    Sthd,
                     Vmhd,
             Edts,
                 Elst,

src/atom.rs

@@ -210,6 +210,56 @@ macro_rules! nested {
                 })
             }
 
+            fn encode_body<B: BufMut>(&self, buf: &mut B) -> Result<()> {
+                $( self.[<$required:lower>].encode(buf)?; )*
+                $( self.[<$optional:lower>].encode(buf)?; )*
+                $( self.[<$multiple:lower>].iter().map(|x| x.encode(buf)).collect::<Result<()>>()?; )*
+
+                Ok(())
+            }
+        }
+    };
+    (required: [$($required:ident),*$(,)?], optional: [$($optional:ident),*$(,)?], multiple: [$($multiple:ident),*$(,)?], post_parse: $some_fn:ident, ) => {
+        paste::paste! {
+            fn decode_body<B: Buf>(buf: &mut B) -> Result<Self> {
+                $( let mut [<$required:lower>] = None;)*
+                $( let mut [<$optional:lower>] = None;)*
+                $( let mut [<$multiple:lower>] = Vec::new();)*
+
+                while let Some(atom) = Any::decode_maybe(buf)? {
+                    match atom {
+                        $(Any::$required(atom) => {
+                            if [<$required:lower>].is_some() {
+                                return Err(Error::DuplicateBox($required::KIND));
+                            }
+                            [<$required:lower>] = Some(atom);
+                        },)*
+                        $(Any::$optional(atom) => {
+                            if [<$optional:lower>].is_some() {
+                                return Err(Error::DuplicateBox($optional::KIND));
+                            }
+                            [<$optional:lower>] = Some(atom);
+                        },)*
+                        $(Any::$multiple(atom) => {
+                            [<$multiple:lower>].push(atom.into());
+                        },)*
+                        Any::Skip(atom) => tracing::debug!(size = atom.zeroed.size, "skipping skip box"),
+                        Any::Free(atom) => tracing::debug!(size = atom.zeroed.size, "skipping free box"),
+                        unknown => Self::decode_unknown(&unknown)?,
+                    }
+                }
+
+                let result = Self {
+                    $([<$required:lower>]: [<$required:lower>].ok_or(Error::MissingBox($required::KIND))? ,)*
+                    $([<$optional:lower>],)*
+                    $([<$multiple:lower>],)*
+                };
+
+                result.[<$some_fn>]()?;
+
+                Ok(result)
+            }
+
             fn encode_body<B: BufMut>(&self, buf: &mut B) -> Result<()> {
                 $( self.[<$required:lower>].encode(buf)?; )*
                 $( self.[<$optional:lower>].encode(buf)?; )*

src/iso639.rs

@@ -0,0 +1,36 @@
+pub fn language_string(language: u16) -> String {
+    let mut lang: [u16; 3] = [0; 3];
+
+    lang[0] = ((language >> 10) & 0x1F) + 0x60;
+    lang[1] = ((language >> 5) & 0x1F) + 0x60;
+    lang[2] = ((language) & 0x1F) + 0x60;
+
+    // Decode utf-16 encoded bytes into a string.
+    String::from_utf16_lossy(&lang)
+}
+
+pub fn language_code(language: &str) -> u16 {
+    let mut lang = language.encode_utf16();
+    let mut code = (lang.next().unwrap_or(0) & 0x1F) << 10;
+    code += (lang.next().unwrap_or(0) & 0x1F) << 5;
+    code += lang.next().unwrap_or(0) & 0x1F;
+    code
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn test_language_code(lang: &str) {
+        let code = language_code(lang);
+        let lang2 = language_string(code);
+        assert_eq!(lang, lang2);
+    }
+
+    #[test]
+    fn test_language_codes() {
+        test_language_code("und");
+        test_language_code("eng");
+        test_language_code("kor");
+    }
+}

src/lib.rs

@@ -148,6 +148,7 @@ mod free;
 mod ftyp;
 mod header;
 mod io;
+mod iso639;
 mod mdat;
 mod meta;
 mod mfra;
@@ -169,6 +170,7 @@ pub use free::*;
 pub use ftyp::*;
 pub use header::*;
 pub use io::*;
+pub(crate) use iso639::*;
 pub use mdat::*;
 pub use meta::*;
 pub use mfra::*;

src/moov/mod.rs

@@ -180,7 +180,6 @@ mod test {
                                     blue: 0
                                 }
                             }),
-                            smhd: None,
                             dinf: Dinf {
                                 dref: Dref {
                                     urls: vec![Url::default()]
@@ -224,8 +223,10 @@ mod test {
                                     .into()],
                                 },
                                 stco: Some(Stco::default()),
+                                stsz: Some(Stsz::default()),
                                 ..Default::default()
-                            }
+                            },
+                            ..Default::default()
                         }
                     },
                     senc: None,

src/moov/trak/mdia/mdhd.rs

@@ -65,42 +65,10 @@ impl AtomExt for Mdhd {
     }
 }
 
-fn language_string(language: u16) -> String {
-    let mut lang: [u16; 3] = [0; 3];
-
-    lang[0] = ((language >> 10) & 0x1F) + 0x60;
-    lang[1] = ((language >> 5) & 0x1F) + 0x60;
-    lang[2] = ((language) & 0x1F) + 0x60;
-
-    // Decode utf-16 encoded bytes into a string.
-    String::from_utf16_lossy(&lang)
-}
-
-fn language_code(language: &str) -> u16 {
-    let mut lang = language.encode_utf16();
-    let mut code = (lang.next().unwrap_or(0) & 0x1F) << 10;
-    code += (lang.next().unwrap_or(0) & 0x1F) << 5;
-    code += lang.next().unwrap_or(0) & 0x1F;
-    code
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
 
-    fn test_language_code(lang: &str) {
-        let code = language_code(lang);
-        let lang2 = language_string(code);
-        assert_eq!(lang, lang2);
-    }
-
-    #[test]
-    fn test_language_codes() {
-        test_language_code("und");
-        test_language_code("eng");
-        test_language_code("kor");
-    }
-
     #[test]
     fn test_mdhd32() {
         let expected = Mdhd {

src/moov/trak/mdia/minf/mod.rs

@@ -1,11 +1,15 @@
 mod dinf;
+mod nmhd;
 mod smhd;
 mod stbl;
+mod sthd;
 mod vmhd;
 
 pub use dinf::*;
+pub use nmhd::*;
 pub use smhd::*;
 pub use stbl::*;
+pub use sthd::*;
 pub use vmhd::*;
 
 use crate::*;
@@ -15,6 +19,8 @@ use crate::*;
 pub struct Minf {
     pub vmhd: Option<Vmhd>,
     pub smhd: Option<Smhd>,
+    pub nmhd: Option<Nmhd>,
+    pub sthd: Option<Sthd>,
     pub dinf: Dinf,
     pub stbl: Stbl,
 }
@@ -24,7 +30,7 @@ impl Atom for Minf {
 
     nested! {
         required: [ Dinf, Stbl ],
-        optional: [ Vmhd, Smhd ],
+        optional: [ Vmhd, Smhd, Nmhd, Sthd ],
         multiple: [],
     }
 }

src/moov/trak/mdia/minf/nmhd.rs

@@ -0,0 +1,39 @@
+use crate::*;
+
+#[derive(Debug, Clone, PartialEq, Eq, Default)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+pub struct Nmhd {}
+
+impl AtomExt for Nmhd {
+    type Ext = ();
+
+    const KIND_EXT: FourCC = FourCC::new(b"nmhd");
+
+    fn decode_body_ext<B: Buf>(_buf: &mut B, _ext: ()) -> Result<Self> {
+        Ok(Nmhd {})
+    }
+
+    fn encode_body_ext<B: BufMut>(&self, _buf: &mut B) -> Result<()> {
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_nmhd() {
+        let nmhd = Nmhd {};
+        let mut buf = Vec::new();
+        nmhd.encode(&mut buf).unwrap();
+        assert_eq!(
+            buf,
+            vec![0x00, 0x00, 0x00, 0x0c, 0x6e, 0x6d, 0x68, 0x64, 0x00, 0x00, 0x00, 0x00]
+        );
+
+        let mut buf = buf.as_ref();
+        let decoded = Nmhd::decode(&mut buf).unwrap();
+        assert_eq!(decoded, nmhd);
+    }
+}

src/moov/trak/mdia/minf/stbl/mod.rs

@@ -10,6 +10,7 @@ mod stsd;
 mod stss;
 mod stsz;
 mod stts;
+mod stz2;
 mod subs;
 
 pub use co64::*;
@@ -24,6 +25,7 @@ pub use stsd::*;
 pub use stss::*;
 pub use stsz::*;
 pub use stts::*;
+pub use stz2::*;
 pub use subs::*;
 
 use crate::*;
@@ -36,7 +38,8 @@ pub struct Stbl {
     pub ctts: Option<Ctts>,
     pub stss: Option<Stss>,
     pub stsc: Stsc,
-    pub stsz: Stsz,
+    pub stsz: Option<Stsz>,
+    pub stz2: Option<Stz2>,
     pub stco: Option<Stco>,
     pub co64: Option<Co64>,
     pub sbgp: Vec<Sbgp>,
@@ -47,12 +50,28 @@ pub struct Stbl {
     pub cslg: Option<Cslg>,
 }
 
+impl Stbl {
+    fn do_validation(&self) -> Result<()> {
+        if self.stsz.is_none() && self.stz2.is_none() {
+            return Err(Error::MissingContent(
+                "one of stsz or stz2 is required in stbl box",
+            ));
+        }
+        if self.stsz.is_some() && self.stz2.is_some() {
+            // TODO: some kind of better error
+            return Err(Error::UnexpectedBox(Stz2::KIND));
+        }
+        Ok(())
+    }
+}
+
 impl Atom for Stbl {
     const KIND: FourCC = FourCC::new(b"stbl");
 
     nested! {
-        required: [ Stsd, Stts, Stsc, Stsz ],
-        optional: [ Ctts, Stss, Stco, Co64, Cslg ],
+        required: [ Stsd, Stts, Stsc ],
+        optional: [ Ctts, Stss, Stco, Co64, Cslg, Stsz, Stz2 ],
         multiple: [ Sbgp, Sgpd, Subs, Saiz, Saio ],
+        post_parse: do_validation,
     }
 }