From e08b59cebe4d47ddf4a1ec1e56ebf621c65ca7db Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Fri, 26 Sep 2025 13:43:32 -0600 Subject: [PATCH 01/10] update versions of dependencies to get build working --- .github/dependabot.yml | 26 ++++++++++++++++++++++++++ requirements.txt | 13 ++++++++----- 2 files changed, 34 insertions(+), 5 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..24534f5 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,26 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + groups: + dev-deps: + dependency-type: "development" + prod-deps: + dependency-type: "production" + open-pull-requests-limit: 5 + commit-message: + prefix: "deps" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + groups: + actions-deps: + patterns: + - "*" + open-pull-requests-limit: 5 + commit-message: + prefix: "ci" + diff --git a/requirements.txt b/requirements.txt index 314b66b..1d5f6d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,9 +2,11 @@ Django==2.2.28 django-bootstrap4==2.3.1 djangorestframework==3.11.2 djangorestframework-xml==1.0.1 -# Version of lxml has to match what is in testsuite -lxml==4.9.1 -psycopg2==2.8.6 + +# This won't install anymore, but not +# sure it needed? +# psycopg2==2.8.6 + semantic-version==2.10.0 stopit==1.1.2 xmlschema==1.0.15 @@ -14,7 +16,8 @@ xmltodict==0.12.0 jellyfish==0.7.1 # schematron runner from TestSuite repo -testsuite==0.1.3 +# This also sets the version of lxml to use +testsuite==0.1.5 # production deployment -uWSGI==2.0.22 +uWSGI==2.0.26 From f18a7a2124382eb77e30e5cd88adb60579902116 Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Fri, 26 Sep 2025 13:52:41 -0600 Subject: [PATCH 02/10] python 3.9 tests --- .github/workflows/ci.yml | 12 ++++---- .pre-commit-config.yaml | 61 ++++++++++++++++------------------------ requirements-test.txt | 1 + tox.ini | 3 -- 4 files changed, 32 insertions(+), 45 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72c3ad3..65c2b1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,9 +9,10 @@ on: jobs: test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: matrix: + python-version: ["3.9", "3.11"] tox_env: [python] #, precommit] services: postgres: @@ -28,16 +29,15 @@ jobs: ports: - 5432:5432 steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: "3.6.7" + python-version: ${{ matrix.python-version }} - name: Display system info run: python -c "import sys; print(sys.version)" - name: Setup Python run: | - apt-get update && apt-get install python-enchant -y - pip install tox + sudo apt-get update && sudo apt-get install python3-enchant -y pip install --upgrade pip pip install tox coveralls - name: Run tox diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d093996..a1062ae 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,53 +11,42 @@ exclude: | repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v5.0.0 hooks: - - id: trailing-whitespace - - id: check-added-large-files - args: ["--maxkb=5000"] - id: check-ast + - id: check-added-large-files + args: ["--maxkb=2000"] + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-toml + - id: check-yaml - id: check-json + - id: name-tests-test + args: ["--pytest-test-first"] + - id: fix-byte-order-marker + - id: check-case-conflict + - id: check-docstring-first + - id: check-executables-have-shebangs - id: check-merge-conflict - id: check-xml - - id: check-yaml - exclude: .*/templates - id: debug-statements - - id: end-of-file-fixer - exclude: seed/static/seed/locales/ - id: mixed-line-ending - exclude: seed/static/seed/locales/ - id: pretty-format-json args: ["--autofix", "--no-sort-keys"] - - repo: https://github.com/pre-commit/mirrors-isort - rev: v5.10.1 - hooks: - - id: isort - args: - [ - -m=VERTICAL_HANGING_INDENT, - --skip=seed/models/__init__.py, - --filter-files, - ] - - repo: https://github.com/PyCQA/autoflake - rev: v2.2.1 - hooks: - - id: autoflake - args: - [ - "--in-place", - "--recursive", - "--remove-all-unused-imports", - "--remove-unused-variable", - ] - - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 - hooks: - - id: flake8 - args: ["--ignore=F401,E402,E501,E731,W503,W504"] - - repo: https://github.com/pre-commit/mirrors-prettier + files: \.(json|template)$ + - repo: https://github.com/pre-commit/mirrors-prettier rev: v3.0.3 hooks: - id: prettier # for now ignoring html, javascript types_or: [yaml, markdown, css, scss] + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.11.7 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix, --output-format=full] + types_or: [python, pyi, jupyter] + - id: ruff-format + types_or: [python, pyi, jupyter] + + diff --git a/requirements-test.txt b/requirements-test.txt index e51e476..126ee76 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -3,4 +3,5 @@ autopep8==1.4.4 coveralls==2.2.0 coverage==5.0.1 +pre-commit==3.5.0 tox==3.14.1 diff --git a/tox.ini b/tox.ini index 87c8cb1..1a3ca53 100644 --- a/tox.ini +++ b/tox.ini @@ -13,9 +13,6 @@ envlist = passenv= DJANGO_LOG_LEVEL DJANGO_SETTINGS_MODULE - TRAVIS - TRAVIS_JOB_ID - TRAVIS_BRANCH commands = coverage run manage.py test deps = -r{toxinidir}/requirements-test.txt From 1b9ccfd37e04b0cb0b974f7d27cb3fe138bbb0fa Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Fri, 26 Sep 2025 13:57:22 -0600 Subject: [PATCH 03/10] add back in psycopg2 --- requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1d5f6d7..0a3ab94 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,9 +3,7 @@ django-bootstrap4==2.3.1 djangorestframework==3.11.2 djangorestframework-xml==1.0.1 -# This won't install anymore, but not -# sure it needed? -# psycopg2==2.8.6 +psycopg2-binary==2.9.7 semantic-version==2.10.0 stopit==1.1.2 From 0159fa053a7aa231663fc5fe37531999c1f469f5 Mon Sep 17 00:00:00 2001 From: Nicholas Long <1907354+nllong@users.noreply.github.com> Date: Fri, 26 Sep 2025 14:00:21 -0600 Subject: [PATCH 04/10] Update .pre-commit-config.yaml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a1062ae..21ade1b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,6 +15,7 @@ repos: hooks: - id: check-ast - id: check-added-large-files + # Reduced max file size from 5000KB to 2000KB to prevent large files from being added. args: ["--maxkb=2000"] - id: end-of-file-fixer - id: trailing-whitespace From 05ec7a737e8665437746de727bba668fce83f37a Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Fri, 26 Sep 2025 14:07:30 -0600 Subject: [PATCH 05/10] support python 3.9 --- bsyncviewer/lib/schema_parser.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/bsyncviewer/lib/schema_parser.py b/bsyncviewer/lib/schema_parser.py index c863ced..ac25829 100644 --- a/bsyncviewer/lib/schema_parser.py +++ b/bsyncviewer/lib/schema_parser.py @@ -199,7 +199,7 @@ def check_all_type_definitions(self): def _read_schema(self, root_element): full_schema = BuildingSyncSchemaRoot() - for child in root_element.getchildren(): + for child in root_element: if child.tag.endswith('element'): if 'name' in child.attrib: @@ -230,7 +230,7 @@ def _read_ref_element(self, parent_object): ref_element.ref_type = parent_object.attrib['ref'] - for child in parent_object.getchildren(): + for child in parent_object: if child.tag.endswith('attribute'): ref_element.attributes.append(self._read_attribute(child)) elif child.tag.endswith('annotation'): @@ -251,7 +251,7 @@ def _read_named_element(self, parent_object): named_element.min_occurs = parent_object.attrib['minOccurs'] if 'maxOccurs' in parent_object.attrib: named_element.max_occurs = parent_object.attrib['maxOccurs'] - for child in parent_object.getchildren(): + for child in parent_object: if child.tag.endswith('annotation'): named_element.annotations.append(self._read_annotation(child)) elif child.tag.endswith('complexType'): @@ -270,7 +270,7 @@ def _read_named_element(self, parent_object): @staticmethod def _read_annotation(parent_object): annotation = AnnotationElement() - for child in parent_object.getchildren(): + for child in parent_object: if child.tag.endswith('documentation'): annotation.documentation = child.text else: @@ -279,7 +279,7 @@ def _read_annotation(parent_object): def _read_attribute(self, parent_object): attribute = AttributeElement() - for child in parent_object.getchildren(): + for child in parent_object: if child.tag.endswith('simpleType'): attribute.simple_types.append(self._read_simple_type(child)) elif child.tag.endswith('annotation'): @@ -290,7 +290,7 @@ def _read_attribute(self, parent_object): def _read_complex_type(self, parent_object): this_complex_type = ComplexTypeElement() - for child in parent_object.getchildren(): + for child in parent_object: if child.tag.endswith('sequence'): this_complex_type.sequences.append(self._read_sequence(child)) elif child.tag.endswith('simpleContent'): @@ -312,7 +312,7 @@ def _read_complex_type(self, parent_object): def _read_sequence(self, parent_object): this_sequence = SequenceElement() - for child in parent_object.getchildren(): + for child in parent_object: if child.tag.endswith('element'): if 'name' in child.attrib: this_sequence.named_elements.append(self._read_named_element(child)) @@ -329,7 +329,7 @@ def _read_sequence(self, parent_object): def _read_simple_content(self, parent_object): this_simple_content = SimpleContentElement() - for child in parent_object.getchildren(): + for child in parent_object: if child.tag.endswith('extension'): this_simple_content.extensions.append(self._read_extension(child)) else: @@ -340,7 +340,7 @@ def _read_extension(self, parent_object): this_extension = ExtensionElement() if 'base' in parent_object.attrib: this_extension.base_type = parent_object.attrib['base'] - for child in parent_object.getchildren(): + for child in parent_object: if child.tag.endswith('attribute'): this_extension.attributes.append(self._read_attribute(child)) else: @@ -361,7 +361,7 @@ def _read_simple_type(self, parent_object): if parent_object.attrib and parent_object.attrib['name']: this_simple_content.name = parent_object.attrib['name'] - for child in parent_object.getchildren(): + for child in parent_object: if child.tag.endswith('restriction'): this_simple_content.restrictions.append(self._read_restriction(child)) elif child.tag.endswith('annotation'): @@ -378,7 +378,7 @@ def _read_simple_type(self, parent_object): @staticmethod def _read_restriction(parent_object): this_restriction = RestrictionElement() - for child in parent_object.getchildren(): + for child in parent_object: if child.tag.endswith('enumeration'): this_restriction.enumerations.append(child.attrib['value']) elif child.tag.endswith('minInclusive'): @@ -394,7 +394,7 @@ def _read_restriction(parent_object): def _read_choice(self, parent_object): this_choice = ChoiceElement() - for child in parent_object.getchildren(): + for child in parent_object: if child.tag.endswith('element'): if 'name' in child.attrib: this_choice.named_elements.append(self._read_named_element(child)) From b3a07d6f9aa15c6a9c5ac8f66e3c85300e61ffef Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Fri, 26 Sep 2025 14:15:19 -0600 Subject: [PATCH 06/10] use postgres 11 in ci --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65c2b1a..24e8ad8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: tox_env: [python] #, precommit] services: postgres: - image: postgres:9.6 + image: postgres:11.1 env: POSTGRES_PASSWORD: postgres POSTGRES_DB: bsync_validator From ba540fdf6fa767927122d3cb4d5ece3e94559b58 Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Fri, 26 Sep 2025 14:17:04 -0600 Subject: [PATCH 07/10] support mapping in >3.9 --- bsyncviewer/lib/bedes/bedes_parser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bsyncviewer/lib/bedes/bedes_parser.py b/bsyncviewer/lib/bedes/bedes_parser.py index 133bd85..d8e3a71 100644 --- a/bsyncviewer/lib/bedes/bedes_parser.py +++ b/bsyncviewer/lib/bedes/bedes_parser.py @@ -1,6 +1,7 @@ import json import os -from collections import Mapping, OrderedDict, defaultdict +from collections import OrderedDict, defaultdict +from collections.abc import Mapping import xmltodict From fab8f6fc5665f3b9797ddd7d702561d00f55efff Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Fri, 26 Sep 2025 14:28:10 -0600 Subject: [PATCH 08/10] support 3.11 --- bsyncviewer/management/commands/bedes.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bsyncviewer/management/commands/bedes.py b/bsyncviewer/management/commands/bedes.py index 986b695..1cc337b 100644 --- a/bsyncviewer/management/commands/bedes.py +++ b/bsyncviewer/management/commands/bedes.py @@ -89,6 +89,9 @@ def parse(self, bedes_version, schema_version): bsync_term = self.manual_mapping(attribute.name, manual_mappings) for bt in bedes.terms: + # Skip if bt['Term'] is None or not a string + if not bt.get('Term') or not isinstance(bt['Term'], str): + continue distance = jellyfish.jaro_winkler(bsync_term.lower(), bt['Term'].lower()) if distance >= 0.98: From 87b29fc24f3af0da9ab550bd5d734f4dd568c5de Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Fri, 26 Sep 2025 14:39:05 -0600 Subject: [PATCH 09/10] update jellyfish --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0a3ab94..653494d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ xmlschema==1.0.15 xmltodict==0.12.0 # string matching -jellyfish==0.7.1 +jellyfish==1.2.0 # schematron runner from TestSuite repo # This also sets the version of lxml to use From 3a78a37115f3f5155fa7e2a5732a8d0704ddac29 Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Fri, 26 Sep 2025 14:45:11 -0600 Subject: [PATCH 10/10] update jellyfish --- bsyncviewer/management/commands/bedes.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/bsyncviewer/management/commands/bedes.py b/bsyncviewer/management/commands/bedes.py index 1cc337b..7e58805 100644 --- a/bsyncviewer/management/commands/bedes.py +++ b/bsyncviewer/management/commands/bedes.py @@ -92,7 +92,7 @@ def parse(self, bedes_version, schema_version): # Skip if bt['Term'] is None or not a string if not bt.get('Term') or not isinstance(bt['Term'], str): continue - distance = jellyfish.jaro_winkler(bsync_term.lower(), bt['Term'].lower()) + distance = jellyfish.jaro_winkler_similarity(bsync_term.lower(), bt['Term'].lower()) if distance >= 0.98: results[attribute.id].append({ @@ -109,7 +109,7 @@ def parse(self, bedes_version, schema_version): if not results[attribute.id]: for be in bedes.enumerations: # .lower() function used to neutralize upper/lower case discrepancies (there are many in enumerations/list options) - distance = jellyfish.jaro_winkler(bsync_term.lower(), be['List-Option'].lower()) + distance = jellyfish.jaro_winkler_similarity(bsync_term.lower(), be['List-Option'].lower()) if distance >= 0.98: results[attribute.id].append({ @@ -165,7 +165,7 @@ def parse(self, bedes_version, schema_version): for word_group in word_groups.keys(): for bt in bedes.terms: - distance = jellyfish.jaro_winkler(word_group.lower(), bt['Term'].lower()) + distance = jellyfish.jaro_winkler_similarity(word_group.lower(), bt['Term'].lower()) if distance >= 0.98: @@ -186,7 +186,7 @@ def parse(self, bedes_version, schema_version): # if no matches found in BEDES terms, check list options for be in bedes.enumerations: - distance = jellyfish.jaro_winkler(word_group.lower(), be['List-Option'].lower()) + distance = jellyfish.jaro_winkler_similarity(word_group.lower(), be['List-Option'].lower()) if distance >= 0.98: for i in range(word_groups[word_group], word_groups[word_group] + number_of_words): @@ -370,7 +370,10 @@ def parse(self, bedes_version, schema_version): print(associated_attrs) for be in bedes.enumerations: - distance = jellyfish.jaro_winkler(enumeration.name, be['List-Option']) + # Skip if be['List-Option'] is None or not a string + if not be.get('List-Option') or not isinstance(be['List-Option'], str): + continue + distance = jellyfish.jaro_winkler_similarity(enumeration.name, be['List-Option']) if distance >= 0.95: results[enumeration.id].append({