Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions .github/workflows/periodic.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,17 @@ jobs:
platform:
- runner: ubuntu-latest
target: x86_64-unknown-linux-gnu
# - runner: macos-latest
# target: x86_64-apple-darwin
- runner: ubuntu-24.04-arm
target: aarch64-unknown-linux-gnu
- runner: macos-latest
target: aarch64-apple-darwin
- runner: macos-15-intel
target: x86_64-apple-darwin
- runner: windows-latest
target: x86_64-pc-windows-msvc
- runner: windows-latest
target: aarch64-pc-windows-msvc
skip_test_run: true
toolchain: [stable, beta, nightly]
runs-on: ${{ matrix.platform.runner }}
env:
Expand All @@ -59,4 +64,4 @@ jobs:
components: rustfmt, clippy

- name: Run test
run: cargo test --target "${{ matrix.platform.target }}" --all-features
run: cargo test --target "${{ matrix.platform.target }}" --all-features ${{ matrix.platform.skip_test_run == true && '--no-run' || '' }}
11 changes: 8 additions & 3 deletions .github/workflows/regression.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,12 +40,17 @@ jobs:
platform:
- runner: ubuntu-latest
target: x86_64-unknown-linux-gnu
# - runner: macos-latest
# target: x86_64-apple-darwin
- runner: ubuntu-24.04-arm
target: aarch64-unknown-linux-gnu
- runner: macos-latest
target: aarch64-apple-darwin
- runner: macos-15-intel
target: x86_64-apple-darwin
- runner: windows-latest
target: x86_64-pc-windows-msvc
- runner: windows-latest
target: aarch64-pc-windows-msvc
skip_test_run: true
toolchain: [stable]
runs-on: ${{ matrix.platform.runner }}
env:
Expand All @@ -62,4 +67,4 @@ jobs:
components: rustfmt, clippy

- name: Run test
run: cargo test --target "${{ matrix.platform.target }}" --all-features
run: cargo test --target "${{ matrix.platform.target }}" --all-features ${{ matrix.platform.skip_test_run == true && '--no-run' || '' }}
39 changes: 26 additions & 13 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,12 +42,17 @@ jobs:
platform:
- runner: ubuntu-latest
target: x86_64-unknown-linux-gnu
# - runner: macOS-latest
# target: x86_64-apple-darwin
- runner: macOS-latest
- runner: ubuntu-24.04-arm
target: aarch64-unknown-linux-gnu
- runner: macos-latest
target: aarch64-apple-darwin
- runner: macos-15-intel
target: x86_64-apple-darwin
- runner: windows-latest
target: x86_64-pc-windows-msvc
- runner: windows-latest
target: aarch64-pc-windows-msvc
skip_test_run: true
toolchain: [stable]
features: ["cjk"]
runs-on: ${{ matrix.platform.runner }}
Expand All @@ -65,7 +70,7 @@ jobs:
components: rustfmt, clippy

- name: Run test
run: cargo test --target "${{ matrix.platform.target }}" --all-features
run: cargo test --target "${{ matrix.platform.target }}" --all-features ${{ matrix.platform.skip_test_run == true && '--no-run' || '' }}

build:
name: Build
Expand All @@ -77,33 +82,41 @@ jobs:
target: x86_64-unknown-linux-gnu
archive: .zip
extension: ".so"
# - runner: macos-latest
# target: x86_64-apple-darwin
# archive: .zip
# extension: ".dylib"
- runner: ubuntu-24.04-arm
target: aarch64-unknown-linux-gnu
archive: .zip
extension: ".so"
- runner: macos-latest
target: aarch64-apple-darwin
archive: .zip
extension: ".dylib"
- runner: macos-15-intel
target: x86_64-apple-darwin
archive: .zip
extension: ".dylib"
- runner: windows-latest
target: x86_64-pc-windows-msvc
archive: .zip
extension: ".dll"
- runner: windows-latest
target: aarch64-pc-windows-msvc
archive: .zip
extension: ".dll"
toolchain: [stable]
features:
- value: "embedded-ipadic"
- value: "embed-ipadic"
package_name: "lindera-sqlite-ipadic"
package_description: "Python binding for Lindera with Japanese dictionary (IPADIC)"
- value: "embedded-unidic"
- value: "embed-unidic"
package_name: "lindera-sqlite-unidic"
package_description: "Python binding for Lindera with Japanese dictionary (Unidic)"
- value: "embedded-ko-dic"
- value: "embed-ko-dic"
package_name: "lindera-sqlite-ko-dic"
package_description: "Python binding for Lindera with Korean dictionary (ko-dic)"
- value: "embedded-cc-cedict"
- value: "embed-cc-cedict"
package_name: "lindera-sqlite-cc-cedict"
package_description: "Python binding for Lindera with CHinese dictionary (CC-CEDICT)"
- value: "embedded-cjk"
- value: "embed-cjk"
package_name: "lindera-sqlite"
package_description: "Python binding for Lindera with CJK dictionaries (IPADIC, ko-dic, CC-CEDICT)"
runs-on: ${{ matrix.platform.runner }}
Expand Down
34 changes: 17 additions & 17 deletions Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "lindera-sqlite"
version = "1.3.1"
version = "1.4.0"
edition = "2024"
description = "Lindera tokenizer for SQLite FTS5 extention"
documentation = "https://docs.rs/lindera-sqlite"
Expand All @@ -12,23 +12,23 @@ categories = ["text-processing"]
license = "MIT"

[features]
embedded-ipadic = [
"lindera/embedded-ipadic",
embed-ipadic = [
"lindera/embed-ipadic",
] # Include Japanese dictionary (IPADIC)
embedded-ipadic-neologd = [
"lindera/embedded-ipadic-neologd",
embed-ipadic-neologd = [
"lindera/embed-ipadic-neologd",
] # Include Japanese dictionary (IPADIC NEologd)
embedded-unidic = [
"lindera/embedded-unidic",
embed-unidic = [
"lindera/embed-unidic",
] # Include Japanese dictionary (UniDic)
embedded-ko-dic = [
"lindera/embedded-ko-dic",
embed-ko-dic = [
"lindera/embed-ko-dic",
] # Include Korean dictionary (ko-dic)
embedded-cc-cedict = [
"lindera/embedded-cc-cedict",
embed-cc-cedict = [
"lindera/embed-cc-cedict",
] # Include Chinese dictionary (CC-CEDICT)
embedded-cjk = [
"lindera/embedded-cjk",
embed-cjk = [
"lindera/embed-cjk",
] # Include CJK dictionary (Chinese, Japanese, Korean)
extension = []
default = ["extension"] # No directories included
Expand All @@ -46,14 +46,14 @@ panic = "abort" # Abort on panic for smaller binary and faster code

[dependencies]
# libc without `std`
libc = { version = "0.2.179", default-features = false, features = [] }
serde_json = "1.0.148"
sqlite-loadable = "0.0.5"
libc = { version = "0.2.180", default-features = false, features = [] }
serde_json = "1.0.149"
sqlite-loadable = "0.0.6-alpha.6"
sqlite3ext-sys = "0.0.1"
unicode-segmentation = "1.12.0"
unicode-normalization = "0.1.25"

lindera = "1.5.1"
lindera = "2.0.1"

[dev-dependencies]
criterion = "0.8.1"
Expand Down
8 changes: 4 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,16 @@ format: ## Format the code
cargo fmt

lint: ## Run linter
cargo clippy --features=embedded-cjk
cargo clippy --features=embed-cjk

build: ## Build the project
cargo build --release --features=embedded-cjk
cargo build --release --features=embed-cjk

test: ## Run tests
LINDERA_CONFIG_PATH=./resources/lindera.yml cargo test --features=embedded-cjk
LINDERA_CONFIG_PATH=./resources/lindera.yml cargo test --features=embed-cjk

bench: ## Run benchmarks
LINDERA_CONFIG_PATH=./resources/lindera.yml cargo bench --features=embedded-cjk
LINDERA_CONFIG_PATH=./resources/lindera.yml cargo bench --features=embed-cjk

tag: ## Make a new tag for the current version
git tag v$(LINDERA_SQLITE_VERSION)
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ When used as a custom FTS5 tokenizer this enables application to support Chinese
## Build extension

```sh
% cargo build --features=embedded-cjk
% cargo build --features=embed-cjk
```

## Set enviromment variable for Lindera configuration
Expand Down