feat: standardize pronunciation kinds to BCP 47

internal/src/enum_wrapper.rs

@@ -34,7 +34,7 @@ where
 #[cfg(test)]
 mod tests {
     use super::*;
-    use odict::schema::{FormKind, PartOfSpeech, PronunciationKind};
+    use odict::schema::{FormKind, PartOfSpeech};
 
     #[test]
     fn test_to_enum_wrapper_pos_custom() {
@@ -54,24 +54,6 @@ mod tests {
         assert_eq!(wrapper.value, "'kari' adjective (archaic)");
     }
 
-    #[test]
-    fn test_to_enum_wrapper_pronunciation_kind_custom() {
-        let wrapper = PronunciationKind::Other("Custom".to_string()).to_enum_wrapper();
-
-        assert_eq!(wrapper.name, "PronunciationKind");
-        assert_eq!(wrapper.variant, "other");
-        assert_eq!(wrapper.value, "Custom");
-    }
-
-    #[test]
-    fn test_to_enum_wrapper_pronunciation_kind() {
-        let wrapper = PronunciationKind::IPA.to_enum_wrapper();
-
-        assert_eq!(wrapper.name, "PronunciationKind");
-        assert_eq!(wrapper.variant, "ipa");
-        assert_eq!(wrapper.value, "ipa");
-    }
-
     #[test]
     fn test_to_enum_wrapper_form_kind() {
         let wrapper = FormKind::Conjugation.to_enum_wrapper();

lib/Cargo.toml

@@ -77,6 +77,8 @@ reqwest = { version = "0.12.24", optional = true, default-features = false, feat
   "http2",
   "stream",
 ] }
+strum = { version = "0.27.2", features = ["derive"] }
+icu_locale_core = { version = "2.0.0", features = ["serde"] }
 reqwest-middleware = { version = "0.4.2", optional = true }
 reqwest-retry = { version = "0.7.0", optional = true }
 futures-util = { version = "0.3", optional = true }
@@ -87,7 +89,7 @@ tantivy = { version = "0.25.0", optional = true }
 dirs = { version = "6.0.0", optional = true }
 # feature=sql
 sea-query = { version = "0.32.7", optional = true }
-strum = { version = "0.27.2", features = ["derive"] }
+rkyv_intern = { git = "https://github.com/rkyv/rkyv_intern.git", version = "0.1.0" }
 
 [dev-dependencies]
 criterion = { version = "0.7.0", features = ["async_tokio", "html_reports"] }

lib/src/format/json/pronunciation.rs

@@ -1,13 +1,14 @@
+use icu_locale_core::LanguageIdentifier;
 use serde::Serialize;
 use structural_convert::StructuralConvert;
 
 use super::media_url::MediaURLJSON;
-use crate::schema::{Pronunciation, PronunciationKind};
+use crate::schema::Pronunciation;
 
 #[derive(Serialize, PartialEq, Eq, StructuralConvert)]
 #[convert(from(Pronunciation))]
 pub struct PronunciationJSON {
-    pub kind: Option<PronunciationKind>,
+    pub kind: LanguageIdentifier,
 
     pub value: String,
 

lib/src/format/md/pronunciation.rs

@@ -5,23 +5,19 @@ pub fn write_pronunciation(
     pronunciation: &Pronunciation,
     _entry: &Entry, // Prefix with underscore to indicate intentional non-use
 ) -> crate::Result<()> {
-    if let Some(ref kind) = pronunciation.kind {
-        // Use the kind_str method to get the proper display string (including custom kinds)
-        let kind_str = kind.to_string();
+    // Use the kind_str method to get the proper display string (including custom kinds)
+    let kind_str = pronunciation.kind.to_string();
 
-        // Capitalize the first letter for display
-        let kind_display = kind_str
-            .chars()
-            .next()
-            .map(|c| c.to_uppercase().collect::<String>() + &kind_str[1..])
-            .unwrap_or_else(|| kind_str.to_string());
+    // Capitalize the first letter for display
+    let kind_display = kind_str
+        .chars()
+        .next()
+        .map(|c| c.to_uppercase().collect::<String>() + &kind_str[1..])
+        .unwrap_or_else(|| kind_str.to_string());
 
-        let pron_text = format!("*{}:* {}", kind_display, pronunciation.value);
+    let pron_text = format!("*{}:* {}", kind_display, pronunciation.value);
 
-        lines.push(pron_text);
-    } else {
-        lines.push(pronunciation.value.to_string());
-    }
+    lines.push(pron_text);
 
     // Add URLs as markdown links if they exist
     if !pronunciation.media.is_empty() {

lib/src/format/sql/pronunciations.rs

@@ -101,7 +101,7 @@ pub fn insert_pronunciation(
             ])
             .values([
                 id.as_str().into(),
-                pronunciation.kind.clone().map(|k| k.to_string()).into(),
+                pronunciation.kind.to_string().into(),
                 pronunciation.value.clone().into(),
                 etymology_id.into(),
             ])?,

lib/src/lib.rs

@@ -21,6 +21,7 @@ mod core;
 mod error;
 mod ext;
 mod odict;
+mod se;
 
 pub mod format;
 pub mod fs;

lib/src/schema/definition.rs

@@ -1,4 +1,5 @@
 use rkyv::with::{AsBox, MapNiche};
+use rkyv_intern::Intern;
 
 use crate::serializable;
 
@@ -13,6 +14,7 @@ serializable! {
     pub id: Option<String>,
 
     #[serde(rename = "@value")]
+    #[rkyv(with = Intern)]
     pub value: String,
 
     #[serde(default, rename="example")]

lib/src/schema/dictionary.rs

@@ -1,11 +1,12 @@
 use rkyv::{
-    deserialize, to_bytes,
+    deserialize,
     with::{AsBox, MapNiche},
 };
+
 use std::collections::HashSet;
 use std::str::FromStr;
 
-use crate::{error::Error, serializable};
+use crate::{error::Error, se::serialize_interned, serializable};
 
 use super::{entry::Entry, id::ID};
 
@@ -66,8 +67,7 @@ impl FromStr for Dictionary {
 
 impl Dictionary {
     pub(crate) fn serialize(&self) -> crate::Result<Vec<u8>> {
-        let bytes =
-            to_bytes::<rkyv::rancor::Error>(self).map_err(|e| Error::Serialize(e.to_string()))?;
+        let bytes = serialize_interned::<_, rkyv::rancor::Error>(self)?;
         Ok(bytes.to_vec())
     }
 }

lib/src/schema/entry.rs

@@ -1,13 +1,13 @@
 use rkyv::{
-    deserialize, to_bytes,
+    deserialize,
     with::{AsBox, MapNiche},
 };
 use std::{
     borrow::Borrow,
     hash::{Hash, Hasher},
 };
 
-use crate::{error::Error, schema::Etymology, serializable};
+use crate::{error::Error, schema::Etymology, se::serialize_interned, serializable};
 
 use super::{EntryRef, MediaURL};
 
@@ -68,9 +68,7 @@ impl Borrow<str> for ArchivedEntry {
 
 impl Entry {
     pub fn serialize(&self) -> crate::Result<Vec<u8>> {
-        let bytes =
-            to_bytes::<rkyv::rancor::Error>(self).map_err(|e| Error::Serialize(e.to_string()))?;
-
+        let bytes = serialize_interned::<_, rkyv::rancor::Error>(self)?;
         Ok(bytes.to_vec())
     }
 }

lib/src/schema/enums.rs

@@ -1,6 +1,6 @@
 use std::fmt::{Debug, Display};
 
-use super::{FormKind, PartOfSpeech, PronunciationKind};
+use super::{FormKind, PartOfSpeech};
 
 pub trait EnumIdentifier {
     fn id(&self) -> String;
@@ -30,4 +30,3 @@ macro_rules! impl_enum_identifier {
 
 impl_enum_identifier!(PartOfSpeech);
 impl_enum_identifier!(FormKind);
-impl_enum_identifier!(PronunciationKind);

lib/src/schema/example.rs

@@ -6,6 +6,7 @@ serializable! {
   #[serde(rename = "example")]
   pub struct Example {
     #[serde(rename = "@value")]
+    #[rkyv(with = rkyv_intern::Intern)]
     pub value: String,
 
     #[serde(default, rename = "translation")]

lib/src/schema/group.rs

@@ -14,6 +14,7 @@ serializable! {
     pub id: Option<String>,
 
     #[serde(rename = "@description")]
+    #[rkyv(with = rkyv_intern::Intern)]
     pub description: String,
 
     #[serde(default, rename = "definition")]

lib/src/schema/mod.rs

@@ -14,7 +14,6 @@ mod note;
 #[allow(non_camel_case_types)]
 mod pos;
 mod pronunciation;
-mod pronunciation_kind;
 mod sense;
 mod translation;
 
@@ -36,6 +35,6 @@ pub use self::media_url::*;
 pub use self::note::*;
 pub use self::pos::*;
 pub use self::pronunciation::*;
-pub use self::pronunciation_kind::*;
 pub use self::sense::*;
 pub use self::translation::*;
+pub use icu_locale_core::LanguageIdentifier;

lib/src/schema/note.rs

@@ -17,6 +17,7 @@ serializable! {
     pub id: Option<String>,
 
     #[serde(rename = "@value")]
+    #[rkyv(with = rkyv_intern::Intern)]
     pub value: String,
   }
 }

lib/src/schema/pronunciation.rs

@@ -1,13 +1,65 @@
-use rkyv::with::{AsBox, MapNiche};
+use icu_locale_core::LanguageIdentifier;
+use rkyv::{
+    rancor::{Fallible, Source},
+    with::{ArchiveWith, DeserializeWith, SerializeWith},
+    Archive, Archived, Deserialize, Place, Resolver, Serialize,
+};
 
-use super::{MediaURL, PronunciationKind};
+use super::MediaURL;
 use crate::serializable;
 
+// Custom wrapper for LanguageIdentifier to serialize as string and deserialize with try_from
+pub struct LanguageIdentifierAsString;
+
+impl ArchiveWith<LanguageIdentifier> for LanguageIdentifierAsString {
+    type Archived = Archived<String>;
+    type Resolver = Resolver<String>;
+
+    fn resolve_with(
+        field: &LanguageIdentifier,
+        resolver: Self::Resolver,
+        out: Place<Self::Archived>,
+    ) {
+        let string_repr = field.to_string();
+        string_repr.resolve(resolver, out);
+    }
+}
+
+impl<S: Fallible + ?Sized> SerializeWith<LanguageIdentifier, S> for LanguageIdentifierAsString
+where
+    String: Serialize<S>,
+{
+    fn serialize_with(
+        field: &LanguageIdentifier,
+        serializer: &mut S,
+    ) -> Result<Self::Resolver, S::Error> {
+        let string_repr = field.to_string();
+        string_repr.serialize(serializer)
+    }
+}
+
+impl<D> DeserializeWith<Archived<String>, LanguageIdentifier, D> for LanguageIdentifierAsString
+where
+    D: Fallible + ?Sized,
+    D::Error: Source,
+    String: Archive,
+    Archived<String>: Deserialize<String, D>,
+{
+    fn deserialize_with(
+        field: &Archived<String>,
+        deserializer: &mut D,
+    ) -> Result<LanguageIdentifier, D::Error> {
+        let string_repr: String = field.deserialize(deserializer)?;
+        LanguageIdentifier::try_from_str(string_repr.as_str())
+            .map_err(|e| <D::Error as Source>::new(e))
+    }
+}
+
 serializable! {
   pub struct Pronunciation {
     #[serde(rename = "@kind")]
-    #[rkyv(with = MapNiche<AsBox>)]
-    pub kind: Option<PronunciationKind>,
+    #[rkyv(with = LanguageIdentifierAsString)]
+    pub kind: LanguageIdentifier,
 
     #[serde(rename = "@value")]
     pub value: String,