Feature/3.13 support

.github/workflows/publish.yml

@@ -0,0 +1,69 @@
+name: Upload to PyPi
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  build_wheels:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+    name: Build on ${{ matrix.os }}
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+
+      - name: Install cibuildwheel
+        run: python -m pip install cibuildwheel==2.21.3
+
+      - name: Build wheels
+        run: python -m cibuildwheel --output-dir wheelhouse
+        env:
+          CIBW_SKIP: cp38-* pp*
+          CIBW_TEST_REQUIRES: pytest faker
+          CIBW_TEST_COMMAND: pytest {project}/tests
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
+          path: ./wheelhouse/*.whl
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build sdist
+        run: pipx run build --sdist
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-sdist
+          path: dist/*.tar.gz
+
+  upload_pypi:
+    needs: [build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+    environment: pypi
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          # unpacks all CIBW artifacts into dist/
+          pattern: cibw-*
+          path: dist
+          merge-multiple: true
+
+      - name: Publish package
+        uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_SIMSTRING_UPLOAD_TOKEN }}
+          skip_existing: true
+

.github/workflows/test.yml

@@ -0,0 +1,46 @@
+name: Test
+
+on:
+  push:
+    branches: 
+      - '*'
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        # os: [ubuntu-latest, windows-latest]
+        os: [ubuntu-latest]
+        python-version: [ '3.9', '3.10', '3.11', '3.12','3.13']
+    name: Python ${{ matrix.python-version }} on ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest hatch 
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+    - name: Test with hatch and pytest
+      run: |
+        hatch run test:cov
+
+    - name: Install module and test again
+      run: |
+        hatch run test:build
+        pip install --find-links=dist simstring-fast
+        hatch run test:cov

.gitignore

@@ -0,0 +1,17 @@
+env/
+cenv/
+senv/
+mypyenv/
+
+**/__pycache__/**
+*.egg-info/
+build/
+dist/
+
+dev/data/geo_address.csv
+dev/geo_matching.py
+*.so
+.coverage
+simstring/site/
+tmp*/
+addresses.csv

README.md

@@ -1,14 +1,15 @@
 # simstring
-[![PyPI - Status](https://img.shields.io/pypi/status/simstring-pure.svg)](https://pypi.org/project/simstring-pure/)
-[![PyPI version](https://badge.fury.io/py/simstring-pure.svg)](https://badge.fury.io/py/simstring-pure)
-[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/simstring-pure.svg)](https://pypi.org/project/simstring-pure/0.0.1/)
+[![PyPI - Status](https://img.shields.io/pypi/status/simstring-fast.svg)](https://pypi.org/project/simstring-fast/)
+[![PyPI version](https://badge.fury.io/py/simstring-fast.svg)](https://badge.fury.io/py/simstring-fast)
+![PyPI - Python Version](https://img.shields.io/pypi/pyversions/simstring-fast)
 [![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](LICENSE)
-[![CircleCI](https://circleci.com/gh/nullnull/simstring.svg?style=svg)](https://circleci.com/gh/nullnull/simstring)
-[![Maintainability](https://api.codeclimate.com/v1/badges/66eb2018262f03ece8a3/maintainability)](https://codeclimate.com/github/nullnull/simstring/maintainability)
 
+![icon](simstring/docs/strings_icon.png)
 
 A Python implementation of the [SimString](http://www.chokkan.org/software/simstring/index.html.en), a simple and efficient algorithm for approximate string matching.
 
+Docs are [here](https://banking-circle-advanced-analytics.github.io/simstring-fast/)
+
 ## Features
 With this library, you can extract strings/texts which has certain similarity from large amount of strings/texts. It will help you when you develop applications related to language processing.
 
@@ -20,14 +21,13 @@ SimString has the following features:
 * 100% exact retrieval. Although some algorithms allow misses (false positives) for faster query response, SimString is guaranteed to achieve 100% correct retrieval with fast query response.
 * Unicode support.
 * Extensibility. You can implement your own feature extractor easily.
-* Japanese support. [MeCab](http://taku910.github.io/mecab/)を使った形態素Nグラムをサポートしています。
-
+* no japanese support
 [Please see this paper for more details](http://www.aclweb.org/anthology/C10-1096).
 
 
 ## Install
 ```
-pip install simstring-pure
+pip install simstring-fast
 ```
 
 ## Usage
@@ -53,10 +53,10 @@ If you want to use other feature, measure, and database, simply replace these cl
 ```python
 from simstring.feature_extractor.word_ngram import WordNgramFeatureExtractor
 from simstring.measure.jaccard import JaccardMeasure
-from simstring.database.mongo import MongoDatabase
+from simstring.database.dict import DictDatabase
 from simstring.searcher import Searcher
 
-db = MongoDatabase(WordNgramFeatureExtractor(2))
+db = DictDatabase(WordNgramFeatureExtractor(2))
 db.add('You are so cool.')
 
 searcher = Searcher(db, JaccardMeasure())
@@ -68,83 +68,11 @@ print(results)
 - Cosine
 - Dice
 - Jaccard
+- Overlap
+- Left Overlap
 
-## Run Tests
-```
-docker-compose run main bash -c 'source activate simstring && python -m unittest discover tests'
-```
-
-## Benchmark
-* About 1ms to search strings from 5797 strings(company names).
-* About 14ms to search strings from 235544 strings(unabridged dictionary).
 
-#### search from `dev/data/company_names.txt`
-```
-$ python dev/benchmark.py
-benchmark for using dict as database
-## benchmarker:         release 4.0.1 (for python)
-## python version:      3.7.0
-## python compiler:     GCC 7.2.0
-## python platform:     Linux-4.9.87-linuxkit-aufs-x86_64-with-debian-9.4
-## python executable:   /opt/conda/envs/simstring/bin/python
-## cpu model:           Intel(R) Core(TM) i7-6567U CPU @ 3.30GHz  # 3300.000 MHz
-## parameters:          loop=1, cycle=1, extra=0
-
-##                        real    (total    = user    + sys)
-initialize database(5797 lines)    0.1227    0.1200    0.1200    0.0000
-search text(5797 times)    6.9719    6.9400    6.8900    0.0500
-
-## Ranking                real
-initialize database(5797 lines)    0.1227  (100.0) ********************
-search text(5797 times)    6.9719  (  1.8)
-
-## Matrix                 real    [01]    [02]
-[01] initialize database(5797 lines)    0.1227   100.0  5680.9
-[02] search text(5797 times)    6.9719     1.8   100.0
-
-benchmark for using Mongo as database
-## benchmarker:         release 4.0.1 (for python)
-## python version:      3.7.0
-## python compiler:     GCC 7.2.0
-## python platform:     Linux-4.9.87-linuxkit-aufs-x86_64-with-debian-9.4
-## python executable:   /opt/conda/envs/simstring/bin/python
-## cpu model:           Intel(R) Core(TM) i7-6567U CPU @ 3.30GHz  # 3300.000 MHz
-## parameters:          loop=1, cycle=1, extra=0
-
-##                        real    (total    = user    + sys)
-initialize database(5797 lines)    4.5762    2.4900    1.9200    0.5700
-search text(5797 times)  177.8401   60.9100   47.2500   13.6600
-
-## Ranking                real
-initialize database(5797 lines)    4.5762  (100.0) ********************
-search text(5797 times)  177.8401  (  2.6) *
-
-## Matrix                 real    [01]    [02]
-[01] initialize database(5797 lines)    4.5762   100.0  3886.2
-[02] search text(5797 times)  177.8401     2.6   100.0
-```
-
-#### search from `dev/data/unabridged_dictionary.txt`
-```
-$ python dev/benchmark.py
-benchmark for using dict as database
-## benchmarker:         release 4.0.1 (for python)
-## python version:      3.7.0
-## python compiler:     GCC 7.2.0
-## python platform:     Linux-4.9.87-linuxkit-aufs-x86_64-with-debian-9.4
-## python executable:   /opt/conda/envs/simstring/bin/python
-## cpu model:           Intel(R) Core(TM) i7-6567U CPU @ 3.30GHz  # 3300.000 MHz
-## parameters:          loop=1, cycle=1, extra=0
-
-##                        real    (total    = user    + sys)
-initialize database(235544 lines)    2.2576    2.2300    2.1200    0.1100
-search text(10000 times)  141.0302  140.6400  139.9600    0.6800
-
-## Ranking                real
-initialize database(235544 lines)    2.2576  (100.0) ********************
-search text(10000 times)  141.0302  (  1.6)
-
-## Matrix                 real    [01]    [02]
-[01] initialize database(235544 lines)    2.2576   100.0  6246.8
-[02] search text(10000 times)  141.0302     1.6   100.0
-```
+## Supported database backends
+- dictionary
+- diskcache (sqlite)
+- redis (in development #37)

dev/benchmark.py

@@ -1,40 +1,65 @@
 # coding: utf-8
 
+"""Module benchmarking
+
+This is code to benchmakr the performance of the module.
+
+Requires benchmarker as an additional dependency. Run from main folder with 'python dev/benchmark.py'
+
+    """
+
 import os, sys
+
 sys.path.append(os.getcwd())
 from benchmarker import Benchmarker
 
 from simstring.feature_extractor.character_ngram import CharacterNgramFeatureExtractor
 from simstring.measure.cosine import CosineMeasure
-from simstring.database.mongo import MongoDatabase
+from simstring.measure.overlap import OverlapMeasure, LeftOverlapMeasure
 from simstring.database.dict import DictDatabase
 from simstring.searcher import Searcher
+from time import time
+
+SEARCH_COUNT_LIMIT = 10**3
 
-SEARCH_COUNT_LIMIT = 10**4
 
 def output_similar_strings_of_each_line(path, Database):
     number_of_lines = len(open(path).readlines())
 
     with Benchmarker(width=20) as bench:
         db = Database(CharacterNgramFeatureExtractor(2))
+
         @bench("initialize database({0} lines)".format(number_of_lines))
         def _(bm):
-            with open(path, 'r') as lines:
+            with open(path, "r") as lines:
                 for line in lines:
-                    strings = line.rstrip('\r\n')
+                    strings = line.rstrip("\r\n")
                     db.add(strings)
 
-        @bench("search text({0} times)".format(min(number_of_lines, SEARCH_COUNT_LIMIT)))
+        @bench(
+            "search text({0} times)".format(min(number_of_lines, SEARCH_COUNT_LIMIT))
+        )
         def _(bm):
             searcher = Searcher(db, CosineMeasure())
-            with open(path, 'r') as lines:
+            with open(path, "r") as lines:
                 for i, line in enumerate(lines):
                     if i >= SEARCH_COUNT_LIMIT:
                         break
-                    strings = line.rstrip('\r\n')
+                    strings = line.rstrip("\r\n")
                     result = searcher.search(strings, 0.8)
 
-print('benchmark for using dict as database')
-output_similar_strings_of_each_line('./dev/data/company_names.txt', DictDatabase)
-print('benchmark for using Mongo as database')
-output_similar_strings_of_each_line('./dev/data/company_names.txt', MongoDatabase)
+
+print("benchmark for using dict as database")
+start = time()
+output_similar_strings_of_each_line("./dev/data/company_names.txt", DictDatabase)
+print(f"Benchmark took {time()-start:.2f}s.")
+
+try:
+    from simstring.database.mongo import MongoDatabase
+
+    print("benchmark for using Mongo as database")
+    start = time()
+    output_similar_strings_of_each_line("./dev/data/company_names.txt", MongoDatabase)
+    print(f"Benchmark took {time()-start:.2f}s.")
+except ModuleNotFoundError:
+    print("Pymongo not installed, won't benchmark against MongoDB")