Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .helix/languages.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
[language-server.rust-analyzer.config]
cargo.features = ["sqlite"]
10 changes: 10 additions & 0 deletions NEWS
Original file line number Diff line number Diff line change
@@ -1,7 +1,17 @@
What's New in libchewing (unreleased)
---------------------------------------------------------

* Features
- dict: loading user dictionary are now also controlled by enabled_dicts
in `chewing_new3()`.

* Bug Fixes
- dict: fixed parsing trie dictionary file with extension fields.

* Changes
- rust: breaking! renamed SystemDictionaryLoader to AssetLoader.
- rust: breaking! renamed UserDictionaryLoader to UserDictionaryManager.
- rust: Dictionary trait gained a new `set_usage()` method.
- conversion: adjust max output paths down to 10.

What's New in libchewing 0.11.0 (January 10, 2026)
Expand Down
4 changes: 2 additions & 2 deletions capi/src/io.rs
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ pub unsafe extern "C" fn chewing_new2(
chewing_new3(
syspath,
userpath,
c"word.dat,tsi.dat".as_ptr(),
c"word.dat,tsi.dat,chewing.dat".as_ptr(),
logger,
loggerdata,
)
Expand Down Expand Up @@ -237,7 +237,7 @@ pub unsafe extern "C" fn chewing_new3(
/// don't need to be freed.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn chewing_get_defaultDictionaryNames() -> *const c_char {
c"word.dat,tsi.dat".as_ptr()
c"word.dat,tsi.dat,chewing.dat".as_ptr()
}

/// Releases the resources used by the given Chewing IM instance.
Expand Down
111 changes: 60 additions & 51 deletions src/dictionary/layered.rs
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
use std::{
collections::{BTreeMap, btree_map::Entry},
iter,
};
use std::collections::{BTreeMap, btree_map::Entry};

use log::error;

use super::{Dictionary, DictionaryInfo, Entries, LookupStrategy, Phrase, UpdateDictionaryError};
use crate::zhuyin::Syllable;
use crate::{
dictionary::{DictionaryUsage, TrieBuf},
zhuyin::Syllable,
};

/// A collection of dictionaries that returns the union of the lookup results.
/// # Examples
Expand All @@ -29,7 +29,7 @@ use crate::zhuyin::Syllable;
/// vec![("策", 100), ("冊", 100)]
/// )]);
///
/// let dict = Layered::new(vec![Box::new(sys_dict)], Box::new(user_dict));
/// let dict = Layered::new(vec![Box::new(sys_dict), Box::new(user_dict)]);
/// assert_eq!(
/// [
/// ("側", 1, 0).into(),
Expand All @@ -48,20 +48,31 @@ use crate::zhuyin::Syllable;
/// ```
#[derive(Debug)]
pub struct Layered {
sys_dict: Vec<Box<dyn Dictionary>>,
user_dict: Box<dyn Dictionary>,
dicts: Vec<Box<dyn Dictionary>>,
user_dict_index: usize,
}

impl Layered {
/// Creates a new `Layered` with the list of dictionaries.
pub fn new(sys_dict: Vec<Box<dyn Dictionary>>, user_dict: Box<dyn Dictionary>) -> Layered {
pub fn new(mut dicts: Vec<Box<dyn Dictionary>>) -> Layered {
let user_dict_index = dicts.iter().enumerate().find_map(|d| {
if d.1.about().usage == DictionaryUsage::User {
Some(d.0)
} else {
None
}
});
if user_dict_index.is_none() {
dicts.push(Box::new(TrieBuf::new_in_memory()));
}
let user_dict_index = user_dict_index.unwrap_or(dicts.len() - 1);
Layered {
sys_dict,
user_dict,
dicts,
user_dict_index,
}
}
pub fn user_dict(&mut self) -> &mut dyn Dictionary {
self.user_dict.as_mut()
self.dicts[self.user_dict_index].as_mut()
}
}

Expand Down Expand Up @@ -89,44 +100,36 @@ impl Dictionary for Layered {
let mut sort_map: BTreeMap<String, usize> = BTreeMap::new();
let mut phrases: Vec<Phrase> = Vec::new();

self.sys_dict
.iter()
.chain(iter::once(&self.user_dict))
.for_each(|d| {
for phrase in d.lookup(syllables, strategy) {
debug_assert!(!phrase.as_str().is_empty());
match sort_map.entry(phrase.to_string()) {
Entry::Occupied(entry) => {
let index = *entry.get();
phrases[index].freq += phrase.freq;
phrases[index].last_used =
match (phrases[index].last_used, phrase.last_used) {
(Some(orig), Some(new)) => Some(u64::max(orig, new)),
(Some(orig), None) => Some(orig),
(None, Some(new)) => Some(new),
(None, None) => None,
};
}
Entry::Vacant(entry) => {
entry.insert(phrases.len());
phrases.push(phrase);
}
self.dicts.iter().for_each(|d| {
for phrase in d.lookup(syllables, strategy) {
debug_assert!(!phrase.as_str().is_empty());
match sort_map.entry(phrase.to_string()) {
Entry::Occupied(entry) => {
let index = *entry.get();
phrases[index].freq += phrase.freq;
phrases[index].last_used =
match (phrases[index].last_used, phrase.last_used) {
(Some(orig), Some(new)) => Some(u64::max(orig, new)),
(Some(orig), None) => Some(orig),
(None, Some(new)) => Some(new),
(None, None) => None,
};
}
Entry::Vacant(entry) => {
entry.insert(phrases.len());
phrases.push(phrase);
}
}
});
}
});
phrases
}

/// Returns all entries from all dictionaries.
///
/// **NOTE**: Duplicate entries are not removed.
fn entries(&self) -> Entries<'_> {
Box::new(
self.sys_dict
.iter()
.chain(iter::once(&self.user_dict))
.flat_map(|dict| dict.entries()),
)
Box::new(self.dicts.iter().flat_map(|dict| dict.entries()))
}

fn about(&self) -> DictionaryInfo {
Expand All @@ -140,12 +143,14 @@ impl Dictionary for Layered {
None
}

fn set_usage(&mut self, _usage: DictionaryUsage) {}

fn reopen(&mut self) -> Result<(), UpdateDictionaryError> {
self.user_dict.reopen()
self.user_dict().reopen()
}

fn flush(&mut self) -> Result<(), UpdateDictionaryError> {
self.user_dict.flush()
self.user_dict().flush()
}

fn add_phrase(
Expand All @@ -157,7 +162,7 @@ impl Dictionary for Layered {
error!("BUG! added phrase is empty");
return Ok(());
}
self.user_dict.add_phrase(syllables, phrase)
self.user_dict().add_phrase(syllables, phrase)
}

fn update_phrase(
Expand All @@ -171,7 +176,7 @@ impl Dictionary for Layered {
error!("BUG! added phrase is empty");
return Ok(());
}
self.user_dict
self.user_dict()
.update_phrase(syllables, phrase, user_freq, time)
}

Expand All @@ -180,7 +185,8 @@ impl Dictionary for Layered {
syllables: &[Syllable],
phrase_str: &str,
) -> Result<(), UpdateDictionaryError> {
self.user_dict.remove_phrase(syllables, phrase_str)
// TODO use exclude list
self.user_dict().remove_phrase(syllables, phrase_str)
}
}

Expand All @@ -194,7 +200,8 @@ mod tests {
use super::Layered;
use crate::{
dictionary::{
Dictionary, DictionaryBuilder, LookupStrategy, Phrase, Trie, TrieBuf, TrieBuilder,
Dictionary, DictionaryBuilder, DictionaryUsage, LookupStrategy, Phrase, Trie, TrieBuf,
TrieBuilder,
},
syl,
zhuyin::Bopomofo,
Expand All @@ -211,7 +218,7 @@ mod tests {
vec![("策", 100), ("冊", 100)],
)]);

let dict = Layered::new(vec![Box::new(sys_dict)], Box::new(user_dict));
let dict = Layered::new(vec![Box::new(sys_dict), Box::new(user_dict)]);
assert_eq!(
[
(
Expand Down Expand Up @@ -253,7 +260,7 @@ mod tests {
vec![("策", 100), ("冊", 100)],
)]);

let dict = Layered::new(vec![Box::new(sys_dict)], Box::new(user_dict));
let dict = Layered::new(vec![Box::new(sys_dict), Box::new(user_dict)]);
assert_eq!(
Some(("側", 1, 0).into()),
dict.lookup(
Expand Down Expand Up @@ -287,6 +294,7 @@ mod tests {
vec![("測", 1), ("冊", 1), ("側", 1)],
)]);
let mut builder = TrieBuilder::new();
builder.set_usage(DictionaryUsage::User);
builder.insert(
&[syl![Bopomofo::C, Bopomofo::E, Bopomofo::TONE4]],
("策", 100, 0).into(),
Expand All @@ -298,9 +306,10 @@ mod tests {
let mut cursor = Cursor::new(vec![]);
builder.write(&mut cursor)?;
cursor.rewind()?;
let user_dict = Trie::new(&mut cursor)?;
let mut user_dict = Trie::new(&mut cursor)?;
user_dict.set_usage(DictionaryUsage::User);

let mut dict = Layered::new(vec![Box::new(sys_dict)], Box::new(user_dict));
let mut dict = Layered::new(vec![Box::new(sys_dict), Box::new(user_dict)]);
assert_eq!(
Some(("側", 1, 0).into()),
dict.lookup(
Expand Down
Loading
Loading