From 233708b651766c8747b29f4ec3130ba826cc3fdb Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 7 Oct 2022 15:38:18 -0500 Subject: [PATCH 001/110] Update controller.py --- lib/biokbase/catalog/controller.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/biokbase/catalog/controller.py b/lib/biokbase/catalog/controller.py index f219f102..19fada0f 100644 --- a/lib/biokbase/catalog/controller.py +++ b/lib/biokbase/catalog/controller.py @@ -1226,6 +1226,7 @@ def is_admin(self, username, token): def version(self): return biokbase.catalog.version.CATALOG_VERSION + @log def log_exec_stats(self, username, token, user_id, app_module_name, app_id, func_module_name, func_name, git_commit_hash, creation_time, exec_start_time, finish_time, is_error, job_id): @@ -1236,6 +1237,10 @@ def log_exec_stats(self, username, token, user_id, app_module_name, app_id, func is_error, job_id) parts = datetime.fromtimestamp(creation_time).isocalendar() week_time_range = str(parts[0]) + "-W" + str(parts[1]) + + aesa1 = "".join(app_module_name, app_id, creation_time, exec_start_time, finish_time, is_error, "a", "*", "then", "w", week_time_range) + logging.info(aesa1) + self.db.add_exec_stats_apps(app_module_name, app_id, creation_time, exec_start_time, finish_time, is_error, "a", "*") self.db.add_exec_stats_apps(app_module_name, app_id, creation_time, exec_start_time, From 7661ea9a54946b3759b096cc7d3e56eb0f58faed Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 7 Oct 2022 15:38:55 -0500 Subject: [PATCH 002/110] Create pr_build.yml --- .github/workflows/pr_build.yml | 43 ++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/pr_build.yml diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml new file mode 100644 index 00000000..0fa1c464 --- /dev/null +++ b/.github/workflows/pr_build.yml @@ -0,0 +1,43 @@ +--- +name: Pull Request Build, Tag, & Push +on: + pull_request: + branches: + - develop + - main + - master + types: + - opened + - reopened + - synchronize + - closed +jobs: + build-develop-open: + if: github.base_ref == 'develop' && github.event.pull_request.merged == false + uses: kbase/.github/.github/workflows/reusable_build.yml@main + secrets: inherit + build-develop-merge: + if: github.base_ref == 'develop' && github.event.pull_request.merged == true + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}-develop' + tags: pr-${{ github.event.number }},latest + secrets: inherit + build-main-open: + if: (github.base_ref == 'main' || github.base_ref == 'master') && github.event.pull_request.merged == false + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}' + tags: pr-${{ github.event.number }} + secrets: inherit + build-main-merge: + if: (github.base_ref == 'main' || github.base_ref == 'master') && github.event.pull_request.merged == true + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}' + tags: pr-${{ github.event.number }},latest-rc + secrets: inherit + trivy-scans: + if: (github.base_ref == 'develop' || github.base_ref == 'main' || github.base_ref == 'master' ) && github.event.pull_request.merged == false + uses: kbase/.github/.github/workflows/reusable_trivy-scans.yml@main + secrets: inherit From cd61d0c114f7bcc34ec178e046a369be7cca63f8 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 7 Oct 2022 15:39:18 -0500 Subject: [PATCH 003/110] Create release-main.yml --- .github/workflows/release-main.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/release-main.yml diff --git a/.github/workflows/release-main.yml b/.github/workflows/release-main.yml new file mode 100644 index 00000000..a2546781 --- /dev/null +++ b/.github/workflows/release-main.yml @@ -0,0 +1,25 @@ +--- +name: Release - Build & Push Image +on: + release: + branches: + - main + - master + types: [ published ] +jobs: + check-source-branch: + uses: kbase/.github/.github/workflows/reusable_validate-branch.yml@main + with: + build_branch: '${{ github.event.release.target_commitish }}' + validate-release-tag: + needs: check-source-branch + uses: kbase/.github/.github/workflows/reusable_validate-release-tag.yml@main + with: + release_tag: '${{ github.event.release.tag_name }}' + build-push: + needs: validate-release-tag + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}' + tags: '${{ github.event.release.tag_name }},latest' + secrets: inherit From c701a39d49b801e17e148d3e99869f29cd05c4b0 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 7 Oct 2022 15:40:02 -0500 Subject: [PATCH 004/110] Create manual-build.yml --- .github/workflows/manual-build.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/workflows/manual-build.yml diff --git a/.github/workflows/manual-build.yml b/.github/workflows/manual-build.yml new file mode 100644 index 00000000..944f9035 --- /dev/null +++ b/.github/workflows/manual-build.yml @@ -0,0 +1,11 @@ +--- +name: Manual Build & Push +on: + workflow_dispatch: +jobs: + build-push: + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}-develop' + tags: br-${{ github.ref_name }} + secrets: inherit From f7df4ac7b257f7525f1ddff2e318f7bac8ca5d2d Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 7 Oct 2022 18:22:03 -0500 Subject: [PATCH 005/110] Update db.py --- lib/biokbase/catalog/db.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index d3ba841e..409a9bf0 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -1116,7 +1116,9 @@ def add_exec_stats_raw(self, user_id, app_module_name, app_id, func_module_name, def add_exec_stats_apps(self, app_module_name, app_id, creation_time, exec_start_time, finish_time, is_error, type, time_range): + print("Going to return if no app id") if not app_id: + print("Sorry, not going to add exec stats apps for " + app_module_name) return full_app_id = app_id if app_module_name: From 7d47f2fed8a5191396e1ea5cbdceb73cd5b74362 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Mon, 10 Oct 2022 12:39:41 -0500 Subject: [PATCH 006/110] Update controller.py --- lib/biokbase/catalog/controller.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/biokbase/catalog/controller.py b/lib/biokbase/catalog/controller.py index 19fada0f..6a419323 100644 --- a/lib/biokbase/catalog/controller.py +++ b/lib/biokbase/catalog/controller.py @@ -1238,9 +1238,6 @@ def log_exec_stats(self, username, token, user_id, app_module_name, app_id, func parts = datetime.fromtimestamp(creation_time).isocalendar() week_time_range = str(parts[0]) + "-W" + str(parts[1]) - aesa1 = "".join(app_module_name, app_id, creation_time, exec_start_time, finish_time, is_error, "a", "*", "then", "w", week_time_range) - logging.info(aesa1) - self.db.add_exec_stats_apps(app_module_name, app_id, creation_time, exec_start_time, finish_time, is_error, "a", "*") self.db.add_exec_stats_apps(app_module_name, app_id, creation_time, exec_start_time, From 21139c64ec6e1c93fced192a7358e76526a0a5d2 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Mon, 10 Oct 2022 12:40:18 -0500 Subject: [PATCH 007/110] Update db.py --- lib/biokbase/catalog/db.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 409a9bf0..d3ba841e 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -1116,9 +1116,7 @@ def add_exec_stats_raw(self, user_id, app_module_name, app_id, func_module_name, def add_exec_stats_apps(self, app_module_name, app_id, creation_time, exec_start_time, finish_time, is_error, type, time_range): - print("Going to return if no app id") if not app_id: - print("Sorry, not going to add exec stats apps for " + app_module_name) return full_app_id = app_id if app_module_name: From 7642a0c0737aa60057d552f2b0eb96ee43c2cd92 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 27 Dec 2022 14:23:33 -0600 Subject: [PATCH 008/110] Update pr_build.yml --- .github/workflows/pr_build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml index 0fa1c464..53b0d18d 100644 --- a/.github/workflows/pr_build.yml +++ b/.github/workflows/pr_build.yml @@ -37,7 +37,7 @@ jobs: name: '${{ github.event.repository.name }}' tags: pr-${{ github.event.number }},latest-rc secrets: inherit - trivy-scans: - if: (github.base_ref == 'develop' || github.base_ref == 'main' || github.base_ref == 'master' ) && github.event.pull_request.merged == false - uses: kbase/.github/.github/workflows/reusable_trivy-scans.yml@main - secrets: inherit +# trivy-scans: +# if: (github.base_ref == 'develop' || github.base_ref == 'main' || github.base_ref == 'master' ) && github.event.pull_request.merged == false +# uses: kbase/.github/.github/workflows/reusable_trivy-scans.yml@main +# secrets: inherit From 75769b38e3c8464d3b8ae47a18425ffd8284de3f Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 1 Mar 2024 21:27:40 -0600 Subject: [PATCH 009/110] Update controller.py --- lib/biokbase/catalog/controller.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/biokbase/catalog/controller.py b/lib/biokbase/catalog/controller.py index 6a419323..6272dbd5 100644 --- a/lib/biokbase/catalog/controller.py +++ b/lib/biokbase/catalog/controller.py @@ -116,7 +116,6 @@ def __init__(self, config): 'specified in the config') self.nms = NarrativeMethodStore(self.nms_url, token=self.nms_token) - @log def register_repo(self, params, username, token): if 'git_url' not in params: From 2054b0da894eb7cc523b7a74adc578beff467b51 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 1 Mar 2024 21:29:39 -0600 Subject: [PATCH 010/110] Update controller.py --- lib/biokbase/catalog/controller.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/biokbase/catalog/controller.py b/lib/biokbase/catalog/controller.py index 6272dbd5..f219f102 100644 --- a/lib/biokbase/catalog/controller.py +++ b/lib/biokbase/catalog/controller.py @@ -116,6 +116,7 @@ def __init__(self, config): 'specified in the config') self.nms = NarrativeMethodStore(self.nms_url, token=self.nms_token) + @log def register_repo(self, params, username, token): if 'git_url' not in params: @@ -1225,7 +1226,6 @@ def is_admin(self, username, token): def version(self): return biokbase.catalog.version.CATALOG_VERSION - @log def log_exec_stats(self, username, token, user_id, app_module_name, app_id, func_module_name, func_name, git_commit_hash, creation_time, exec_start_time, finish_time, is_error, job_id): @@ -1236,7 +1236,6 @@ def log_exec_stats(self, username, token, user_id, app_module_name, app_id, func is_error, job_id) parts = datetime.fromtimestamp(creation_time).isocalendar() week_time_range = str(parts[0]) + "-W" + str(parts[1]) - self.db.add_exec_stats_apps(app_module_name, app_id, creation_time, exec_start_time, finish_time, is_error, "a", "*") self.db.add_exec_stats_apps(app_module_name, app_id, creation_time, exec_start_time, From dd630ce109c915c05e4f97d752eb16b4ff7669ed Mon Sep 17 00:00:00 2001 From: Sijie Xiang Date: Mon, 16 Dec 2024 16:51:36 -0800 Subject: [PATCH 011/110] add GHA test.yml, codeql, and dependabot.yml (#136) --- .github/codeql.yml | 52 ++++++++++++++++++ .github/dependabot.yml | 14 +++++ .github/workflows/test.yml | 88 ++++++++++++++++++++++++++++++ .travis.yml | 40 -------------- lib/biokbase/catalog/registrar.py | 2 +- requirements.txt | 2 +- test/core_registration_test.py | 1 + test/local_function_module_test.py | 10 ++-- 8 files changed, 162 insertions(+), 47 deletions(-) create mode 100644 .github/codeql.yml create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/test.yml delete mode 100644 .travis.yml diff --git a/.github/codeql.yml b/.github/codeql.yml new file mode 100644 index 00000000..0f2f9828 --- /dev/null +++ b/.github/codeql.yml @@ -0,0 +1,52 @@ +name: "Code scanning - action" + +on: + push: + pull_request: + schedule: + - cron: '0 19 * * 0' + +jobs: + CodeQL-Build: + + # CodeQL runs on ubuntu-latest and windows-latest + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + # If this run was triggered by a pull request event, then checkout + # the head of the pull request instead of the merge commit. + - run: git checkout HEAD^2 + if: ${{ github.event_name == 'pull_request' }} + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + # Override language selection by uncommenting this and choosing your languages + # with: + # languages: go, javascript, csharp, python, cpp, java + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..fe381818 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,14 @@ +version: 2 +updates: +- package-ecosystem: docker + directory: "/" + schedule: + interval: weekly + time: '11:00' + open-pull-requests-limit: 10 +- package-ecosystem: pip + directory: "/" + schedule: + interval: weekly + time: '11:00' + open-pull-requests-limit: 10 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..6a4109be --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,88 @@ +name: KBase Catalog test + +on: + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review + push: + # run workflow when merging to main or develop + branches: + - main + - master + - develop + +jobs: + catalog_tests: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - python-version: '3.7' + mongo-version: '3.6' + # - python-version: '3.7' + # mongo-version: '7.0.4' + services: + mongo: + image: mongo:${{matrix.mongo-version}} + ports: + - 27017:27017 + options: --name mongo${{matrix.mongo-version}} + + steps: + - name: Repo checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{matrix.python-version}} + + - name: Install dependencies and set up test config + shell: bash + env: + KBASE_CI_TOKEN: ${{ secrets.KBASE_CI_TOKEN }} + ADMIN_USER: ${{ secrets.KBASE_BOT_USER_CI }} + + run: | + + # test mongo connection + curl http://localhost:27017 + returncode=$? + if [ $returncode != 0 ]; then exit $returncode; fi + + # set HOMEDIR + export HOMEDIR=`pwd` + + # move to parent dir to install binaries etc + cd .. + + # setup kb-sdk + mkdir -p $(pwd)/bin + docker run ghcr.io/kbase/kb_sdk_patch-develop:br-0.0.4 genscript > $(pwd)/bin/kb-sdk + chmod 755 $(pwd)/bin/kb-sdk + export PATH=$(pwd)/bin:$PATH + + # install catalog dependencies + cd $HOMEDIR + pip install -r requirements.txt + + # setup test config + cp -n test/test.cfg.example test/test.cfg + sed -i "s#^nms-admin-token.*#nms-admin-token=$KBASE_CI_TOKEN#" test/test.cfg + sed -i "s#^method-spec-admin-users.*#method-spec-admin-users=$ADMIN_USER#" test/test.cfg + + - name: Run tests + shell: bash + run: make test + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: true diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index beae4e42..00000000 --- a/.travis.yml +++ /dev/null @@ -1,40 +0,0 @@ -sudo: required -language: python -python: - - '3.6' -services: - - docker - -before_install: - #- docker run -d -p 5000:5000 --name registry registry:2 - - docker run -p 27017:27017 --name mongo3.6 -d mongo:3.6 - - sleep 3 - - mongo --eval "printjson(db.serverStatus())" - - sudo apt install ant - - mkdir $(pwd)/bin - - docker run kbase/kb-sdk genscript > $(pwd)/bin/kb-sdk - - chmod 755 $(pwd)/bin/kb-sdk - -install: - - pip install -r requirements.txt - - pip install python-coveralls - - pip install coverage - - cd .. - - git clone https://github.com/kbase/jars - - docker images - - export PATH=$(pwd)/bin:$PATH - - cd catalog - -script: - - cp -n test/test.cfg.example test/test.cfg - - sed -i 's/^nms-admin-toke.*/nms-admin-token = '$NMS_ADMIN_TOKEN'/' test/test.cfg - - make test - -after_success: - - mv test/.coverage . - - coveralls - -env: - global: - - secure: "o0xBDxPBVpqmezRtuDf4gdFYUIlFReH3kV1BOx75taCqzMvsvNnIDHzLITFTvGhNBzOT/hG/eEVDiKx/bnFoq0HJWsKYeKahOsiMlFavIR5jTIdMLq2zbZDkxV/+e0HoRRArWj53NV9wx0pWTv3DYxqbzPR1uu5letxdWnY3DQXggspcnJuAVYGn6LtgNM7MV+h3jdHe/E61LkwMff8KwpoBF+64AKbsZv1ddcFo0gbo9Z9CjLeBq4le2TwziLeuUYV885mPsRCvhToRMzpeVlbiNkzf7w5oh5g3FFsPOOYFeu22a+RrmKoM0hv9/pTTYG0G8/ylzOMIKWJvVrOJYptRvyQKmpmdcN7/CPYIQkf6ZBp0rP6G30Q2xt49+ST40ko1L0jhL42FfwXSmuNjklRMZAjpMNRZM2fIazEhDw3ND9l4T6HZnIwr/SQvKCs4rYBjqmSLo/dMISiFDLjWpMor5ew9fpZSLwJcPglGT0Anh4OIZUY2JGlO4TuOkq3m5kvXjjKLBc3Domd10c+DLfkldwwt6uqShsDECqFwSMf/liPEtLfi+03djrWvP26QeiEKL93ujAMS4oXAYxdquOy2Ib9ihd3XWTJzYc4s1H7furYqAbQDAhU0Dx3nvcyEsu5djdRAFe0eKUdjdAqsMAxe84h/EHZHrlE+KOOZo8s=" - - MONGODB_VERSION=2.7 diff --git a/lib/biokbase/catalog/registrar.py b/lib/biokbase/catalog/registrar.py index b10d158f..d8226cbc 100644 --- a/lib/biokbase/catalog/registrar.py +++ b/lib/biokbase/catalog/registrar.py @@ -252,7 +252,7 @@ def sanity_checks_and_parse(self, basedir, git_commit_hash): with codecs.open(os.path.join(basedir, yaml_filename), 'r', "utf-8", errors='ignore') as kb_yaml_file: kb_yaml_string = kb_yaml_file.read() - self.kb_yaml = yaml.load(kb_yaml_string) + self.kb_yaml = yaml.safe_load(kb_yaml_string) self.log('=====kbase.yaml parse:') self.log(pprint.pformat(self.kb_yaml)) self.log('=====end kbase.yaml') diff --git a/requirements.txt b/requirements.txt index e3190618..b94e0b90 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -pymongo==3.7 +pymongo==3.10 docker>=3.5 gitpython pyyaml diff --git a/test/core_registration_test.py b/test/core_registration_test.py index e222bb84..c1e63c72 100644 --- a/test/core_registration_test.py +++ b/test/core_registration_test.py @@ -37,6 +37,7 @@ def test_full_module_lifecycle(self): self.assertEqual(state['registration'], 'complete') # (3) check the log + sleep(3) # sleep to make sure the catalog db gets the final log messages parsed_log = self.catalog.get_parsed_build_log(self.cUtil.anonymous_ctx(), {'registration_id': registration_id})[0] self.assertEqual(parsed_log['registration'], 'complete') diff --git a/test/local_function_module_test.py b/test/local_function_module_test.py index bf510e51..0b173039 100644 --- a/test/local_function_module_test.py +++ b/test/local_function_module_test.py @@ -16,7 +16,7 @@ def test_local_function_module(self): # assume test user is already approved as a developer # (1) register the test repo giturl = self.cUtil.get_test_repo_1() - githash = 'a01e1a20b9c504a0136c75323b00b1cd4c7f7970' # branch local_method_module + githash = 'a8915afe6811de9199897d710348befad8f6f7ab' # branch local_method_module registration_id = self.catalog.register_repo(self.cUtil.user_ctx(), {'git_url': giturl, 'git_commit_hash': githash})[0] @@ -59,7 +59,7 @@ def test_local_function_module(self): self.assertEqual(len(specs), 1) info = specs[0]['info'] self.assertEqual(info['function_id'], 'powerpoint_to_genome') - self.assertEqual(info['git_commit_hash'], 'a01e1a20b9c504a0136c75323b00b1cd4c7f7970') + self.assertEqual(info['git_commit_hash'], 'a8915afe6811de9199897d710348befad8f6f7ab') self.assertEqual(info['module_name'], 'GenomeToPowerpointConverter') self.assertEqual(info['name'], 'Powerpoint to Genome') self.assertEqual(info['release_tag'], ['dev']) @@ -122,7 +122,7 @@ def test_local_function_module(self): self.assertEqual(len(specs), 1) info = specs[0]['info'] self.assertEqual(info['function_id'], 'powerpoint_to_genome') - self.assertEqual(info['git_commit_hash'], 'a01e1a20b9c504a0136c75323b00b1cd4c7f7970') + self.assertEqual(info['git_commit_hash'], 'a8915afe6811de9199897d710348befad8f6f7ab') self.assertEqual(info['module_name'], 'GenomeToPowerpointConverter') self.assertEqual(info['name'], 'Powerpoint to Genome') self.assertEqual(info['release_tag'], ['beta', 'dev']) @@ -179,12 +179,12 @@ def test_local_function_module(self): # make sure we can fetch it by commit hash specs = self.catalog.get_local_function_details(self.cUtil.user_ctx(), {'functions': [ {'module_name': 'GenomeTopowerpointConverter', 'function_id': 'powerpoint_to_genome', - 'git_commit_hash': 'a01e1a20b9c504a0136c75323b00b1cd4c7f7970'}]})[0] + 'git_commit_hash': 'a8915afe6811de9199897d710348befad8f6f7ab'}]})[0] self.assertEqual(len(specs), 1) info = specs[0]['info'] self.assertEqual(info['function_id'], 'powerpoint_to_genome') - self.assertEqual(info['git_commit_hash'], 'a01e1a20b9c504a0136c75323b00b1cd4c7f7970') + self.assertEqual(info['git_commit_hash'], 'a8915afe6811de9199897d710348befad8f6f7ab') self.assertEqual(info['module_name'], 'GenomeToPowerpointConverter') self.assertEqual(info['name'], 'Powerpoint to Genome') self.assertEqual(info['release_tag'], ['release', 'beta', 'dev']) From 6ec3d63c1fd278091fed18a59bb5c5292480c869 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 17 Dec 2024 14:49:37 -0800 Subject: [PATCH 012/110] mongo7 upgrade --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6a4109be..be7c4be6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,8 +23,8 @@ jobs: include: - python-version: '3.7' mongo-version: '3.6' - # - python-version: '3.7' - # mongo-version: '7.0.4' + - python-version: '3.7' + mongo-version: '7.0.4' services: mongo: image: mongo:${{matrix.mongo-version}} From 26b7d9e641eadcfb694b1921e3990e873e6adaf3 Mon Sep 17 00:00:00 2001 From: Sijie Date: Mon, 6 Jan 2025 10:10:35 -0800 Subject: [PATCH 013/110] upgrade pymongo to 4.7.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b94e0b90..e7d24fec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -pymongo==3.10 +pymongo==4.7.2 docker>=3.5 gitpython pyyaml From c5f12940c4c96d0e6a680db4b248b463d31f35df Mon Sep 17 00:00:00 2001 From: Sijie Date: Mon, 6 Jan 2025 10:36:00 -0800 Subject: [PATCH 014/110] update python to 3.9.19 --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index be7c4be6..327a224b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,9 +21,9 @@ jobs: fail-fast: false matrix: include: - - python-version: '3.7' + - python-version: '3.9.19' mongo-version: '3.6' - - python-version: '3.7' + - python-version: '3.9.19' mongo-version: '7.0.4' services: mongo: From 1e57094bc43ca00652e984c366a5ffcebc16c335 Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 9 Jan 2025 08:50:26 -0800 Subject: [PATCH 015/110] update Makefile and run_tests.sh scripts --- Makefile | 4 +++- test/run_tests.sh | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6be1ab4d..69d19e31 100644 --- a/Makefile +++ b/Makefile @@ -103,7 +103,9 @@ setup-tests: mkdir -p $(TESTDIR)/nms rsync -av lib/biokbase/* $(TESTLIB)/biokbase/. --exclude *.bak-* rsync -av kbapi_common/lib/biokbase/* $(TESTLIB)/biokbase/. - cd narrative_method_store; make; make build-classpath-list; + cd narrative_method_store + ./gradlew generateClasspathFile +# cd narrative_method_store; make; make build-classpath-list; # rsync -av narrative_method_store/lib/biokbase/* $(TESTLIB)/biokbase/. diff --git a/test/run_tests.sh b/test/run_tests.sh index e4d84c58..9fe48daf 100755 --- a/test/run_tests.sh +++ b/test/run_tests.sh @@ -11,7 +11,10 @@ # start the test NMS endpoint echo 'Starting NMS...' export KB_DEPLOYMENT_CONFIG=test.cfg -classpath=`cat ../narrative_method_store/dist/jar.classpath.txt` + +cat ../narrative_method_store/jar.classpath.txt + +classpath=`cat ../narrative_method_store/jar.classpath.txt` java -cp $classpath us.kbase.narrativemethodstore.NarrativeMethodStoreServer 7125 > nms/error.log 2>&1 & NMS_PID=$! From 47ff558bdb4c50473fb2a217d1116592a14fb694 Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 9 Jan 2025 08:51:49 -0800 Subject: [PATCH 016/110] update submodules --- jars | 2 +- narrative_method_store | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jars b/jars index 7a0edafa..06165dd6 160000 --- a/jars +++ b/jars @@ -1 +1 @@ -Subproject commit 7a0edafa4dd508eb5f38c1d7810c91e73a0f76d7 +Subproject commit 06165dd6f6f499ae8610b775a7eef80a4794e2c8 diff --git a/narrative_method_store b/narrative_method_store index fb85ae8b..3997f2ab 160000 --- a/narrative_method_store +++ b/narrative_method_store @@ -1 +1 @@ -Subproject commit fb85ae8b6bea39c4834676e7cfbd2596c7bb182a +Subproject commit 3997f2ab84a5a7950ac1740927e6ae5efce874ae From 4e1f8bfada6e350310d3b613cbc101236de8ed95 Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 9 Jan 2025 08:59:32 -0800 Subject: [PATCH 017/110] fix gradle not found --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 69d19e31..bab50ebf 100644 --- a/Makefile +++ b/Makefile @@ -103,8 +103,7 @@ setup-tests: mkdir -p $(TESTDIR)/nms rsync -av lib/biokbase/* $(TESTLIB)/biokbase/. --exclude *.bak-* rsync -av kbapi_common/lib/biokbase/* $(TESTLIB)/biokbase/. - cd narrative_method_store - ./gradlew generateClasspathFile + cd narrative_method_store; gradle generateClasspathFile # cd narrative_method_store; make; make build-classpath-list; # rsync -av narrative_method_store/lib/biokbase/* $(TESTLIB)/biokbase/. From 4e0c2dcff25c5b117de9bd041f0271fb9349513b Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 10 Jan 2025 16:17:50 -0800 Subject: [PATCH 018/110] start nms service with docker-compose --- Makefile | 1 - test/docker-compose_nms.yml | 47 +++++++++++++++++++++++++++++++++++++ test/run_tests.sh | 7 ++---- 3 files changed, 49 insertions(+), 6 deletions(-) create mode 100644 test/docker-compose_nms.yml diff --git a/Makefile b/Makefile index bab50ebf..7ce574a9 100644 --- a/Makefile +++ b/Makefile @@ -103,7 +103,6 @@ setup-tests: mkdir -p $(TESTDIR)/nms rsync -av lib/biokbase/* $(TESTLIB)/biokbase/. --exclude *.bak-* rsync -av kbapi_common/lib/biokbase/* $(TESTLIB)/biokbase/. - cd narrative_method_store; gradle generateClasspathFile # cd narrative_method_store; make; make build-classpath-list; # rsync -av narrative_method_store/lib/biokbase/* $(TESTLIB)/biokbase/. diff --git a/test/docker-compose_nms.yml b/test/docker-compose_nms.yml new file mode 100644 index 00000000..59b257a1 --- /dev/null +++ b/test/docker-compose_nms.yml @@ -0,0 +1,47 @@ +version: '3.4' + +services: + nms: + image: ghcr.io/kbase/narrative_method_store:v0.3.12 + platform: linux/amd64 + ports: + - "7125:7125" + depends_on: ["mongo"] + environment: + # see deployment/conf/.templates for more environment variables + - method_spec_git_repo=https://github.com/kbase/narrative_method_specs_ci + - method_spec_git_repo_branch=master + - method_spec_git_repo_local_dir=narrative_method_specs_recreated_at_startup + - method_spec_git_repo_refresh_rate=2 + - method_spec_cache_size=5000 + - method_spec_temp_dir=narrative_method_store_temp + - method_spec_mongo_host=mongo:27017 + - method_spec_mongo_dbname=method_store_repo_db + - method_spec_admin_users="" + - endpoint_host=https://ci.kbase.us + - endpoint_base=/services + - method_spec_default_tag=dev + - auth_service_url=https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login + - auth_service_url_allow_insecure=false + - service_port=7125 + command: + - "-wait" + - "tcp://mongo:27017" + - "-timeout" + - "120s" + - "-template" + - "/kb/deployment/conf/.templates/deployment.cfg.templ:/kb/deployment/conf/deployment.cfg" + - "-template" + - "/kb/deployment/conf/.templates/http.ini.templ:/kb/deployment/services/narrative_method_store/start.d/http.ini" + - "-template" + - "/kb/deployment/conf/.templates/server.ini.templ:/kb/deployment/services/narrative_method_store/start.d/server.ini" + - "-template" + - "/kb/deployment/conf/.templates/start_server.sh.templ:/kb/deployment/bin/start_server.sh" + - "-stdout" + - "/kb/deployment/services/narrative_method_store/logs/request.log" + - "/kb/deployment/bin/start_server.sh" + + mongo: + image: "mongo:7.0.4" + ports: + - "27017:27017" \ No newline at end of file diff --git a/test/run_tests.sh b/test/run_tests.sh index 9fe48daf..a6d86ed1 100755 --- a/test/run_tests.sh +++ b/test/run_tests.sh @@ -11,11 +11,7 @@ # start the test NMS endpoint echo 'Starting NMS...' export KB_DEPLOYMENT_CONFIG=test.cfg - -cat ../narrative_method_store/jar.classpath.txt - -classpath=`cat ../narrative_method_store/jar.classpath.txt` -java -cp $classpath us.kbase.narrativemethodstore.NarrativeMethodStoreServer 7125 > nms/error.log 2>&1 & +docker-compose -f docker-compose_nms.yml up -d NMS_PID=$! echo 'Starting Mock Auth API...' @@ -65,5 +61,6 @@ kill -9 $NMS_PID #stop Docker containers docker stop mock-auth docker stop registry +docker-compose -f docker-compose_nms.yml down exit ${TEST_RETURN_CODE} \ No newline at end of file From 8095c7fe29b03ac9e9a3465dafdd646ac24745db Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 10 Jan 2025 16:18:29 -0800 Subject: [PATCH 019/110] update nms submodule to the latest --- narrative_method_store | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/narrative_method_store b/narrative_method_store index 3997f2ab..6b371d36 160000 --- a/narrative_method_store +++ b/narrative_method_store @@ -1 +1 @@ -Subproject commit 3997f2ab84a5a7950ac1740927e6ae5efce874ae +Subproject commit 6b371d3697a8eae59e2899624f5ca5091e056340 From 6f22c74c7afe86bb23d65244e179dc8e893c91c1 Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 10 Jan 2025 16:37:46 -0800 Subject: [PATCH 020/110] install docker-compose in GHA --- .github/workflows/test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 327a224b..3097b262 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,6 +43,12 @@ jobs: with: python-version: ${{matrix.python-version}} + - name: Install Docker Compose + run: | + sudo curl -L "https://github.com/docker/compose/releases/download/v2.32.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + docker-compose --version + - name: Install dependencies and set up test config shell: bash env: From dff0b20ed6951e858fa0008e7b0b79a09fa19ec0 Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 10 Jan 2025 16:52:22 -0800 Subject: [PATCH 021/110] switch catalog mongo7 port to 27018 --- .github/workflows/test.yml | 2 +- test/test.cfg.example | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3097b262..6f00209b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,7 +29,7 @@ jobs: mongo: image: mongo:${{matrix.mongo-version}} ports: - - 27017:27017 + - 27018:27018 options: --name mongo${{matrix.mongo-version}} steps: diff --git a/test/test.cfg.example b/test/test.cfg.example index 75e3f505..c482967e 100644 --- a/test/test.cfg.example +++ b/test/test.cfg.example @@ -24,8 +24,8 @@ test-user-2 = wstester2 test-module-repo-1 = https://github.com/kbaseIncubator/catalog_test_module test-module-repo-2 = https://github.com/kbaseIncubator/catalog_test_module.git -# host where mongo lives, e.g. localhost:27017, for tests there should be not authentication required -mongodb-host = localhost:27017 +# host where mongo lives, e.g. localhost:27018, for tests there should be not authentication required +mongodb-host = localhost:27018 # docker registry host endpoint docker-registry-host = localhost:5000 From 76bff463252241ced6c7c91ab1aa8dabc8df5460 Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 10 Jan 2025 16:55:04 -0800 Subject: [PATCH 022/110] fix mongo port --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6f00209b..aa2a6643 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,7 +58,7 @@ jobs: run: | # test mongo connection - curl http://localhost:27017 + curl http://localhost:27018 returncode=$? if [ $returncode != 0 ]; then exit $returncode; fi From 0b7cd2a1a0a2894686dacc5e5800bf89d0f8cd8c Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 10 Jan 2025 17:15:22 -0800 Subject: [PATCH 023/110] fix mongo port mapping --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aa2a6643..320133de 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,7 +29,7 @@ jobs: mongo: image: mongo:${{matrix.mongo-version}} ports: - - 27018:27018 + - 27018:27017 options: --name mongo${{matrix.mongo-version}} steps: From 66f2485f9062bc8d1cc2597566f8c391d3d77543 Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 10 Jan 2025 17:55:38 -0800 Subject: [PATCH 024/110] set admin_users --- .github/workflows/test.yml | 3 +++ test/docker-compose_nms.yml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 320133de..5985a322 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -83,6 +83,9 @@ jobs: sed -i "s#^nms-admin-token.*#nms-admin-token=$KBASE_CI_TOKEN#" test/test.cfg sed -i "s#^method-spec-admin-users.*#method-spec-admin-users=$ADMIN_USER#" test/test.cfg + # setup env variables in docker-compose_nms.yml file + echo "ADMIN_USER=$ADMIN_USER" >> $GITHUB_ENV + - name: Run tests shell: bash run: make test diff --git a/test/docker-compose_nms.yml b/test/docker-compose_nms.yml index 59b257a1..aa5cd762 100644 --- a/test/docker-compose_nms.yml +++ b/test/docker-compose_nms.yml @@ -17,7 +17,7 @@ services: - method_spec_temp_dir=narrative_method_store_temp - method_spec_mongo_host=mongo:27017 - method_spec_mongo_dbname=method_store_repo_db - - method_spec_admin_users="" + - method_spec_admin_users=${ADMIN_USER} - endpoint_host=https://ci.kbase.us - endpoint_base=/services - method_spec_default_tag=dev From 85c65af89ae7502a28302e66c46e2c1c2b162eea Mon Sep 17 00:00:00 2001 From: Sijie Date: Mon, 13 Jan 2025 12:26:50 -0800 Subject: [PATCH 025/110] check admin users --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5985a322..e979e298 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -84,6 +84,7 @@ jobs: sed -i "s#^method-spec-admin-users.*#method-spec-admin-users=$ADMIN_USER#" test/test.cfg # setup env variables in docker-compose_nms.yml file + echo "ADMIN_USER=$ADMIN_USER" echo "ADMIN_USER=$ADMIN_USER" >> $GITHUB_ENV - name: Run tests From 9fda21f39e1c21c229f43ba4a61a3e8ee3c030ed Mon Sep 17 00:00:00 2001 From: Sijie Date: Mon, 13 Jan 2025 15:17:17 -0800 Subject: [PATCH 026/110] shut down NMS service properly --- test/run_tests.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/run_tests.sh b/test/run_tests.sh index a6d86ed1..1d8fa12b 100755 --- a/test/run_tests.sh +++ b/test/run_tests.sh @@ -56,11 +56,10 @@ echo "unit tests returned with error code=${TEST_RETURN_CODE}" #### SHUTDOWN stuff and exit # stop NMS -kill -9 $NMS_PID +docker-compose -f docker-compose_nms.yml down #stop Docker containers docker stop mock-auth docker stop registry -docker-compose -f docker-compose_nms.yml down exit ${TEST_RETURN_CODE} \ No newline at end of file From 91c72f76a8742e8420af0799d203ac233e56f171 Mon Sep 17 00:00:00 2001 From: Sijie Date: Mon, 13 Jan 2025 15:49:06 -0800 Subject: [PATCH 027/110] fix nms curl --- lib/biokbase/catalog/controller.py | 4 ++++ test/run_tests.sh | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/biokbase/catalog/controller.py b/lib/biokbase/catalog/controller.py index f219f102..d96102e2 100644 --- a/lib/biokbase/catalog/controller.py +++ b/lib/biokbase/catalog/controller.py @@ -1216,6 +1216,10 @@ def has_permission(self, username, token, owners): @log def is_admin(self, username, token): logging.info("URL:" + self.auth_api + '/api/V2/me') + print("------------------------------") + print(f"username is {username}") + print(f"token is {token}") + print("------------------------------") r = requests.get(self.auth_api + '/api/V2/me', headers={'Authorization': token}) logging.info(r.json()) roles = r.json().get('customroles', []) diff --git a/test/run_tests.sh b/test/run_tests.sh index 1d8fa12b..ad24f730 100755 --- a/test/run_tests.sh +++ b/test/run_tests.sh @@ -19,7 +19,7 @@ docker run -d --rm -v ${PWD}/mock_auth:/config -p 7777:5000 --name mock-auth moc echo 'Waiting for NMS to start...' sleep 25 -curl -d '{"id":"1","params":[],"method":"NarrativeMethodStore.ver","version":"1.1"}' http://localhost:7125 +curl -d '{"id":"1","params":[],"method":"NarrativeMethodStore.ver","version":"1.1"}' http://localhost:7125/rpc if [ $? -ne 0 ]; then kill -9 $NMS_PID echo 'NMS did not startup in time. Fail.' From dd1d2dad6d0f82b43860f541433f643bde7f0673 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 14 Jan 2025 14:53:35 -0800 Subject: [PATCH 028/110] update mock_auth service --- lib/biokbase/catalog/controller.py | 4 - test/mock_auth/server.py | 141 +++++++++++++++++++++++++++++ test/run_tests.sh | 2 +- 3 files changed, 142 insertions(+), 5 deletions(-) create mode 100644 test/mock_auth/server.py diff --git a/lib/biokbase/catalog/controller.py b/lib/biokbase/catalog/controller.py index d96102e2..f219f102 100644 --- a/lib/biokbase/catalog/controller.py +++ b/lib/biokbase/catalog/controller.py @@ -1216,10 +1216,6 @@ def has_permission(self, username, token, owners): @log def is_admin(self, username, token): logging.info("URL:" + self.auth_api + '/api/V2/me') - print("------------------------------") - print(f"username is {username}") - print(f"token is {token}") - print("------------------------------") r = requests.get(self.auth_api + '/api/V2/me', headers={'Authorization': token}) logging.info(r.json()) roles = r.json().get('customroles', []) diff --git a/test/mock_auth/server.py b/test/mock_auth/server.py new file mode 100644 index 00000000..959b642b --- /dev/null +++ b/test/mock_auth/server.py @@ -0,0 +1,141 @@ +import os +import sys +import json +import jsonschema +import traceback +import flask +from jsonschema.exceptions import ValidationError + +# Load the endpoints data, the schema, and validate the structure + +# For validating every config file +with open('endpoint_schema.json') as fd: + endpoint_schema = json.load(fd) + +if not os.path.exists('/config'): + sys.stderr.write('Path not found: /config\n') + sys.exit(1) + +endpoints = [] +catalog_mock_auth = ['auth_admin.json', 'auth_invalid.json', 'auth_missing.json', 'auth_non_admin.json'] +for path in catalog_mock_auth: + if path.endswith('.json'): + full_path = '/config/' + path + with open(full_path) as fd: + try: + endpoint = json.load(fd) + except ValueError as err: + sys.stderr.write(f'JSON parsing error:\n{err}') + sys.exit(1) + try: + jsonschema.validate(endpoint, endpoint_schema) + except ValidationError as err: + sys.stderr.write(f'JSON Schema validation Error for {path}:\n') + sys.stderr.write(str(err) + '\n') + sys.exit(1) + endpoints.append(endpoint) + +print(f'Loaded {len(endpoints)} mock endpoints') + +# Start the Flask app +app = flask.Flask(__name__) +methods = ['GET', 'POST', 'PUT', 'DELETE'] + + +@app.route('/', defaults={'path': ''}, methods=methods) +@app.route('/', methods=methods) +def handle_request(path): + """ + Catch-all: handle any request against the endpoints.json data. + """ + print('-' * 80) + path = '/' + path + req_body = flask.request.get_data().decode() or '' + method = flask.request.method + # Find the first endpoint that matches path, method, headers, and body + for endpoint in endpoints: + if endpoint['path'] == path: + print('Matched path:', path) + else: + continue + expected_methods = endpoint.get('methods', ['GET']) + if method in expected_methods: + print('Matched method') + else: + msg = f'Mismatch on method: {method} vs {expected_methods}' + print(msg) + continue + if match_headers(endpoint): + print('Matched headers') + else: + hs = dict(flask.request.headers) + expected_hs = endpoint.get('headers') + msg = f'Mismatch on headers:\n got: {hs}\n expected: {expected_hs}' + print(msg) + continue + expected_body = endpoint.get('body', '') + if isinstance(expected_body, dict): + expected_body_json = json.dumps(expected_body) + try: + given_body_json = json.dumps(json.loads(req_body)) + except Exception as err: + print('Error parsing json body:', str(err)) + continue + body_ok = expected_body_json == given_body_json + else: + body_ok = expected_body.strip() == req_body.strip() + if body_ok: + print('Matched body') + else: + msg = f'Mismatch on body:\n got: {req_body}\n expected: {expected_body}' + print(msg) + continue + print('Matched endpoint {} {}'.format(method, path)) + return mock_response(endpoint.get('response', {})) + raise Exception('Unable to match endpoint: %s %s' % (method, path)) + + +@app.errorhandler(Exception) +def any_exception(err): + """Catch any error with a JSON response.""" + class_name = err.__class__.__name__ + print(traceback.format_exc()) + resp = {'error': str(err), 'class': class_name} + return (flask.jsonify(resp), 500) + + +def match_headers(endpoint): + """ + Either check that there are no headers to match, or match that all headers + in the endpoint are present and equal in the request. + """ + if 'headers' not in endpoint and 'absent_headers' not in endpoint: + return True + headers = dict(flask.request.headers) + if 'headers' in endpoint: + for (key, val) in endpoint['headers'].items(): + if val != headers.get(key): + return False + # Enforce that certain headers must be absent + if 'absent_headers' in endpoint: + header_keys = set(key.lower() for key in headers.keys()) + print('headers are', headers) + for key in endpoint['absent_headers']: + print('checking absent', key) + if key.lower() in header_keys: + return False + return True + + +def mock_response(config): + """ + Create a mock flask response from the endpoints.json configuration + """ + resp_body = config.get('body') + if isinstance(resp_body, dict): + resp_body = json.dumps(resp_body) + resp = flask.Response(resp_body) + resp.status = config.get('status', '200') + for (header, val) in config.get('headers', {}).items(): + resp.headers[header] = val + return resp diff --git a/test/run_tests.sh b/test/run_tests.sh index ad24f730..0d14ef31 100755 --- a/test/run_tests.sh +++ b/test/run_tests.sh @@ -15,7 +15,7 @@ docker-compose -f docker-compose_nms.yml up -d NMS_PID=$! echo 'Starting Mock Auth API...' -docker run -d --rm -v ${PWD}/mock_auth:/config -p 7777:5000 --name mock-auth mockservices/mock_json_service +docker run -d --rm -v ${PWD}/mock_auth:/config -v ${PWD}/mock_auth/server.py:/server/server.py -p 7777:5000 --name mock-auth mockservices/mock_json_service echo 'Waiting for NMS to start...' sleep 25 From 2e840374c7e16f7acdbd24f99ca1bcd6d87aab39 Mon Sep 17 00:00:00 2001 From: Sijie Date: Mon, 27 Jan 2025 13:00:22 -0800 Subject: [PATCH 029/110] update the commit --- test/core_registration_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core_registration_test.py b/test/core_registration_test.py index c1e63c72..b8c48da8 100644 --- a/test/core_registration_test.py +++ b/test/core_registration_test.py @@ -17,7 +17,7 @@ def test_full_module_lifecycle(self): # assume test user is already approved as a developer # (1) register the test repo giturl = self.cUtil.get_test_repo_1() - githash = '4ada53f318f69a38276e82d0e841e685aa0c2362' # branch simple_good_repo + githash = 'f041513eeb9d8c556b1968969550a8466a6d3643' # branch simple_good_repo registration_id = self.catalog.register_repo(self.cUtil.user_ctx(), {'git_url': giturl, 'git_commit_hash': githash})[0] From 727753cd7de61b293c521a459e4cba65082cd0b4 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Mon, 27 Jan 2025 16:20:27 -0600 Subject: [PATCH 030/110] Update test.cfg.example --- test/test.cfg.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.cfg.example b/test/test.cfg.example index c482967e..4ac9c057 100644 --- a/test/test.cfg.example +++ b/test/test.cfg.example @@ -49,7 +49,7 @@ kbase-endpoint = https://ci.kbase.us/services [NarrativeMethodStore] -method-spec-mongo-host = localhost:27017 +method-spec-mongo-host = localhost:27017/rpc method-spec-admin-users = wstester4 method-spec-git-repo-local-dir = nms/local_narrative_method_specs From 77d161bdbd703b4e660bcf88691bba4e21b2e354 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Mon, 27 Jan 2025 16:20:56 -0600 Subject: [PATCH 031/110] Update test.cfg.example --- test/test.cfg.example | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test.cfg.example b/test/test.cfg.example index 4ac9c057..8572f8e1 100644 --- a/test/test.cfg.example +++ b/test/test.cfg.example @@ -39,7 +39,7 @@ docker-base-url = unix://var/run/docker.sock # Narrative Method Store configuration. Please provide a token. # If both are provided, the token is used. -nms-url = http://localhost:7125 +nms-url = http://localhost:7125/rpc nms-admin-token = # configs for reference data @@ -49,7 +49,7 @@ kbase-endpoint = https://ci.kbase.us/services [NarrativeMethodStore] -method-spec-mongo-host = localhost:27017/rpc +method-spec-mongo-host = localhost:27017 method-spec-admin-users = wstester4 method-spec-git-repo-local-dir = nms/local_narrative_method_specs From bca78a9590f9e8155c844a9cf2525aa5ca91c661 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Mon, 27 Jan 2025 16:27:23 -0600 Subject: [PATCH 032/110] Update core_registration_test.py --- test/core_registration_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core_registration_test.py b/test/core_registration_test.py index b8c48da8..ddbbd87c 100644 --- a/test/core_registration_test.py +++ b/test/core_registration_test.py @@ -124,7 +124,7 @@ def test_full_module_lifecycle(self): self.assertIsNone(info['release']) self.validate_basic_test_module_info_fields(info, giturl, module_name, owners) self.assertEqual(info['dev']['git_commit_hash'], githash) - self.assertEqual(info['dev']['git_commit_message'], 'added some basic things') + self.assertEqual(info['dev']['git_commit_message'], 'add test folder') self.assertEqual(info['dev']['narrative_methods'], ['test_method_1']) self.assertEqual(info['dev']['version'], '0.0.1') self.assertEqual(info['dev']['timestamp'], timestamp) From 178a63ca436787baa72357e8e9ec6cecabf22b82 Mon Sep 17 00:00:00 2001 From: Sijie Date: Mon, 27 Jan 2025 14:30:03 -0800 Subject: [PATCH 033/110] update message --- test/core_registration_test.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/core_registration_test.py b/test/core_registration_test.py index b8c48da8..a5300cac 100644 --- a/test/core_registration_test.py +++ b/test/core_registration_test.py @@ -124,7 +124,7 @@ def test_full_module_lifecycle(self): self.assertIsNone(info['release']) self.validate_basic_test_module_info_fields(info, giturl, module_name, owners) self.assertEqual(info['dev']['git_commit_hash'], githash) - self.assertEqual(info['dev']['git_commit_message'], 'added some basic things') + self.assertEqual(info['dev']['git_commit_message'], 'add test folder') self.assertEqual(info['dev']['narrative_methods'], ['test_method_1']) self.assertEqual(info['dev']['version'], '0.0.1') self.assertEqual(info['dev']['timestamp'], timestamp) @@ -179,7 +179,7 @@ def test_full_module_lifecycle(self): self.assertIsNone(info['release']) self.validate_basic_test_module_info_fields(info, giturl, module_name, owners) self.assertEqual(info['dev']['git_commit_hash'], githash) - self.assertEqual(info['dev']['git_commit_message'], 'added some basic things') + self.assertEqual(info['dev']['git_commit_message'], 'add test folder') self.assertEqual(info['dev']['narrative_methods'], ['test_method_1']) self.assertEqual(info['dev']['version'], '0.0.1') self.assertEqual(info['dev']['timestamp'], timestamp) @@ -187,7 +187,7 @@ def test_full_module_lifecycle(self): self.assertEqual(info['beta']['docker_img_name'].split('/')[1], 'kbase:' + module_name.lower() + '.' + githash) self.assertEqual(info['beta']['git_commit_hash'], githash) - self.assertEqual(info['beta']['git_commit_message'], 'added some basic things') + self.assertEqual(info['beta']['git_commit_message'], 'add test folder') self.assertEqual(info['beta']['narrative_methods'], ['test_method_1']) self.assertEqual(info['beta']['version'], '0.0.1') self.assertEqual(info['beta']['timestamp'], timestamp) @@ -275,7 +275,7 @@ def test_full_module_lifecycle(self): self.catalog.get_module_info(self.cUtil.anonymous_ctx(), {'module_name': module_name})[0] self.validate_basic_test_module_info_fields(info, giturl, module_name, owners) self.assertEqual(info['dev']['git_commit_hash'], githash) - self.assertEqual(info['dev']['git_commit_message'], 'added some basic things') + self.assertEqual(info['dev']['git_commit_message'], 'add test folder') self.assertEqual(info['dev']['narrative_methods'], ['test_method_1']) self.assertEqual(info['dev']['version'], '0.0.1') self.assertEqual(info['dev']['timestamp'], timestamp) @@ -283,7 +283,7 @@ def test_full_module_lifecycle(self): 'kbase:' + module_name.lower() + '.' + githash) self.assertEqual(info['beta']['git_commit_hash'], githash) - self.assertEqual(info['beta']['git_commit_message'], 'added some basic things') + self.assertEqual(info['beta']['git_commit_message'], 'add test folder') self.assertEqual(info['beta']['narrative_methods'], ['test_method_1']) self.assertEqual(info['beta']['version'], '0.0.1') self.assertEqual(info['beta']['timestamp'], timestamp) @@ -291,7 +291,7 @@ def test_full_module_lifecycle(self): 'kbase:' + module_name.lower() + '.' + githash) self.assertEqual(info['release']['git_commit_hash'], githash) - self.assertEqual(info['release']['git_commit_message'], 'added some basic things') + self.assertEqual(info['release']['git_commit_message'], 'add test folder') self.assertEqual(info['release']['narrative_methods'], ['test_method_1']) self.assertEqual(info['release']['version'], '0.0.1') self.assertEqual(info['release']['timestamp'], timestamp) @@ -304,7 +304,7 @@ def test_full_module_lifecycle(self): self.assertEqual(len(versions), 1) self.assertEqual(versions[0]['git_commit_hash'], githash) - self.assertEqual(versions[0]['git_commit_message'], 'added some basic things') + self.assertEqual(versions[0]['git_commit_message'], 'add test folder') self.assertEqual(versions[0]['narrative_methods'], ['test_method_1']) self.assertEqual(versions[0]['version'], '0.0.1') self.assertEqual(versions[0]['timestamp'], timestamp) @@ -366,7 +366,7 @@ def test_full_module_lifecycle(self): 'kbase:' + module_name.lower() + '.' + githash2) self.assertEqual(info['beta']['git_commit_hash'], githash) - self.assertEqual(info['beta']['git_commit_message'], 'added some basic things') + self.assertEqual(info['beta']['git_commit_message'], 'add test folder') self.assertEqual(info['beta']['narrative_methods'], ['test_method_1']) self.assertEqual(info['beta']['version'], '0.0.1') self.assertEqual(info['beta']['timestamp'], timestamp) @@ -374,7 +374,7 @@ def test_full_module_lifecycle(self): 'kbase:' + module_name.lower() + '.' + githash) self.assertEqual(info['release']['git_commit_hash'], githash) - self.assertEqual(info['release']['git_commit_message'], 'added some basic things') + self.assertEqual(info['release']['git_commit_message'], 'add test folder') self.assertEqual(info['release']['narrative_methods'], ['test_method_1']) self.assertEqual(info['release']['version'], '0.0.1') self.assertEqual(info['release']['timestamp'], timestamp) From 537908214d9b4f6b1909e7d681747b7c1bc226f9 Mon Sep 17 00:00:00 2001 From: Sijie Date: Mon, 27 Jan 2025 16:58:54 -0800 Subject: [PATCH 034/110] test method_spec_admin_users --- test/docker-compose_nms.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/docker-compose_nms.yml b/test/docker-compose_nms.yml index aa5cd762..59b257a1 100644 --- a/test/docker-compose_nms.yml +++ b/test/docker-compose_nms.yml @@ -17,7 +17,7 @@ services: - method_spec_temp_dir=narrative_method_store_temp - method_spec_mongo_host=mongo:27017 - method_spec_mongo_dbname=method_store_repo_db - - method_spec_admin_users=${ADMIN_USER} + - method_spec_admin_users="" - endpoint_host=https://ci.kbase.us - endpoint_base=/services - method_spec_default_tag=dev From 631b52f4b09e5b03315b1cf72fb20553c62aaee8 Mon Sep 17 00:00:00 2001 From: Sijie Date: Mon, 27 Jan 2025 17:29:52 -0800 Subject: [PATCH 035/110] test simple_good_repo old commit --- test/core_registration_test.py | 20 ++++++++++---------- test/docker-compose_nms.yml | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/test/core_registration_test.py b/test/core_registration_test.py index a5300cac..c1e63c72 100644 --- a/test/core_registration_test.py +++ b/test/core_registration_test.py @@ -17,7 +17,7 @@ def test_full_module_lifecycle(self): # assume test user is already approved as a developer # (1) register the test repo giturl = self.cUtil.get_test_repo_1() - githash = 'f041513eeb9d8c556b1968969550a8466a6d3643' # branch simple_good_repo + githash = '4ada53f318f69a38276e82d0e841e685aa0c2362' # branch simple_good_repo registration_id = self.catalog.register_repo(self.cUtil.user_ctx(), {'git_url': giturl, 'git_commit_hash': githash})[0] @@ -124,7 +124,7 @@ def test_full_module_lifecycle(self): self.assertIsNone(info['release']) self.validate_basic_test_module_info_fields(info, giturl, module_name, owners) self.assertEqual(info['dev']['git_commit_hash'], githash) - self.assertEqual(info['dev']['git_commit_message'], 'add test folder') + self.assertEqual(info['dev']['git_commit_message'], 'added some basic things') self.assertEqual(info['dev']['narrative_methods'], ['test_method_1']) self.assertEqual(info['dev']['version'], '0.0.1') self.assertEqual(info['dev']['timestamp'], timestamp) @@ -179,7 +179,7 @@ def test_full_module_lifecycle(self): self.assertIsNone(info['release']) self.validate_basic_test_module_info_fields(info, giturl, module_name, owners) self.assertEqual(info['dev']['git_commit_hash'], githash) - self.assertEqual(info['dev']['git_commit_message'], 'add test folder') + self.assertEqual(info['dev']['git_commit_message'], 'added some basic things') self.assertEqual(info['dev']['narrative_methods'], ['test_method_1']) self.assertEqual(info['dev']['version'], '0.0.1') self.assertEqual(info['dev']['timestamp'], timestamp) @@ -187,7 +187,7 @@ def test_full_module_lifecycle(self): self.assertEqual(info['beta']['docker_img_name'].split('/')[1], 'kbase:' + module_name.lower() + '.' + githash) self.assertEqual(info['beta']['git_commit_hash'], githash) - self.assertEqual(info['beta']['git_commit_message'], 'add test folder') + self.assertEqual(info['beta']['git_commit_message'], 'added some basic things') self.assertEqual(info['beta']['narrative_methods'], ['test_method_1']) self.assertEqual(info['beta']['version'], '0.0.1') self.assertEqual(info['beta']['timestamp'], timestamp) @@ -275,7 +275,7 @@ def test_full_module_lifecycle(self): self.catalog.get_module_info(self.cUtil.anonymous_ctx(), {'module_name': module_name})[0] self.validate_basic_test_module_info_fields(info, giturl, module_name, owners) self.assertEqual(info['dev']['git_commit_hash'], githash) - self.assertEqual(info['dev']['git_commit_message'], 'add test folder') + self.assertEqual(info['dev']['git_commit_message'], 'added some basic things') self.assertEqual(info['dev']['narrative_methods'], ['test_method_1']) self.assertEqual(info['dev']['version'], '0.0.1') self.assertEqual(info['dev']['timestamp'], timestamp) @@ -283,7 +283,7 @@ def test_full_module_lifecycle(self): 'kbase:' + module_name.lower() + '.' + githash) self.assertEqual(info['beta']['git_commit_hash'], githash) - self.assertEqual(info['beta']['git_commit_message'], 'add test folder') + self.assertEqual(info['beta']['git_commit_message'], 'added some basic things') self.assertEqual(info['beta']['narrative_methods'], ['test_method_1']) self.assertEqual(info['beta']['version'], '0.0.1') self.assertEqual(info['beta']['timestamp'], timestamp) @@ -291,7 +291,7 @@ def test_full_module_lifecycle(self): 'kbase:' + module_name.lower() + '.' + githash) self.assertEqual(info['release']['git_commit_hash'], githash) - self.assertEqual(info['release']['git_commit_message'], 'add test folder') + self.assertEqual(info['release']['git_commit_message'], 'added some basic things') self.assertEqual(info['release']['narrative_methods'], ['test_method_1']) self.assertEqual(info['release']['version'], '0.0.1') self.assertEqual(info['release']['timestamp'], timestamp) @@ -304,7 +304,7 @@ def test_full_module_lifecycle(self): self.assertEqual(len(versions), 1) self.assertEqual(versions[0]['git_commit_hash'], githash) - self.assertEqual(versions[0]['git_commit_message'], 'add test folder') + self.assertEqual(versions[0]['git_commit_message'], 'added some basic things') self.assertEqual(versions[0]['narrative_methods'], ['test_method_1']) self.assertEqual(versions[0]['version'], '0.0.1') self.assertEqual(versions[0]['timestamp'], timestamp) @@ -366,7 +366,7 @@ def test_full_module_lifecycle(self): 'kbase:' + module_name.lower() + '.' + githash2) self.assertEqual(info['beta']['git_commit_hash'], githash) - self.assertEqual(info['beta']['git_commit_message'], 'add test folder') + self.assertEqual(info['beta']['git_commit_message'], 'added some basic things') self.assertEqual(info['beta']['narrative_methods'], ['test_method_1']) self.assertEqual(info['beta']['version'], '0.0.1') self.assertEqual(info['beta']['timestamp'], timestamp) @@ -374,7 +374,7 @@ def test_full_module_lifecycle(self): 'kbase:' + module_name.lower() + '.' + githash) self.assertEqual(info['release']['git_commit_hash'], githash) - self.assertEqual(info['release']['git_commit_message'], 'add test folder') + self.assertEqual(info['release']['git_commit_message'], 'added some basic things') self.assertEqual(info['release']['narrative_methods'], ['test_method_1']) self.assertEqual(info['release']['version'], '0.0.1') self.assertEqual(info['release']['timestamp'], timestamp) diff --git a/test/docker-compose_nms.yml b/test/docker-compose_nms.yml index 59b257a1..aa5cd762 100644 --- a/test/docker-compose_nms.yml +++ b/test/docker-compose_nms.yml @@ -17,7 +17,7 @@ services: - method_spec_temp_dir=narrative_method_store_temp - method_spec_mongo_host=mongo:27017 - method_spec_mongo_dbname=method_store_repo_db - - method_spec_admin_users="" + - method_spec_admin_users=${ADMIN_USER} - endpoint_host=https://ci.kbase.us - endpoint_base=/services - method_spec_default_tag=dev From b4126723907854ba8eab20829dbd744988c3e0b0 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 28 Jan 2025 11:03:56 -0800 Subject: [PATCH 036/110] test register hack --- test/core_registration_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core_registration_test.py b/test/core_registration_test.py index c1e63c72..a7311187 100644 --- a/test/core_registration_test.py +++ b/test/core_registration_test.py @@ -737,7 +737,7 @@ def setUpClass(cls): # hack for testing!! remove when docker and NMS components can be tested from biokbase.catalog.registrar import Registrar - Registrar._TEST_WITHOUT_DOCKER = True + Registrar._TEST_WITHOUT_DOCKER = False cls.cUtil = CatalogTestUtil('.') # TODO: pass in test directory from outside cls.cUtil.setUp() From 80be97734674fe112edbc20700312549108d03f0 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 28 Jan 2025 12:26:51 -0800 Subject: [PATCH 037/110] remove jars nms submodules --- jars | 1 - narrative_method_store | 1 - test/core_registration_test.py | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) delete mode 160000 jars delete mode 160000 narrative_method_store diff --git a/jars b/jars deleted file mode 160000 index 06165dd6..00000000 --- a/jars +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 06165dd6f6f499ae8610b775a7eef80a4794e2c8 diff --git a/narrative_method_store b/narrative_method_store deleted file mode 160000 index 6b371d36..00000000 --- a/narrative_method_store +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6b371d3697a8eae59e2899624f5ca5091e056340 diff --git a/test/core_registration_test.py b/test/core_registration_test.py index a7311187..c1e63c72 100644 --- a/test/core_registration_test.py +++ b/test/core_registration_test.py @@ -737,7 +737,7 @@ def setUpClass(cls): # hack for testing!! remove when docker and NMS components can be tested from biokbase.catalog.registrar import Registrar - Registrar._TEST_WITHOUT_DOCKER = False + Registrar._TEST_WITHOUT_DOCKER = True cls.cUtil = CatalogTestUtil('.') # TODO: pass in test directory from outside cls.cUtil.setUp() From 7ead4f7c15ebe518481612c05e2ffb984f98b24e Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 28 Jan 2025 15:11:07 -0800 Subject: [PATCH 038/110] update Dockerfile --- Dockerfile | 22 +++++++++++++++++----- requirements.txt | 4 +++- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index f45f312d..5b4429b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,25 +1,37 @@ -FROM kbase/sdkbase2:python AS build +FROM python:3.9.19 AS build +RUN apt-get update && apt-get install -y rsync +RUN mkdir -p /kb/deployment/lib/biokbase COPY . /tmp/catalog RUN cd /tmp/catalog && make deploy-service deploy-server-control-scripts -FROM kbase/sdkbase2:python +FROM python:3.9.19 # These ARGs values are passed in via the docker build command ARG BUILD_DATE ARG VCS_REF ARG BRANCH +RUN apt-get update && apt-get install -y wget uwsgi + +# install dockerize +WORKDIR /opt +RUN wget -q https://github.com/kbase/dockerize/raw/master/dockerize-linux-amd64-v0.6.1.tar.gz \ + && tar xvzf dockerize-linux-amd64-v0.6.1.tar.gz \ + && rm dockerize-linux-amd64-v0.6.1.tar.gz +RUN mkdir -p /kb/deployment/bin/ +RUN ln -s /opt/dockerize /kb/deployment/bin/dockerize + ENV KB_DEPLOYMENT_CONFIG "/kb/deployment/conf/deploy.cfg" COPY --from=build /kb/deployment/lib/biokbase /kb/deployment/lib/biokbase COPY --from=build /kb/deployment/services /kb/deployment/services COPY --from=build /tmp/catalog/deployment/conf /kb/deployment/conf -SHELL ["/bin/bash", "-c"] +WORKDIR /tmp/catalog +RUN pip install --upgrade pip COPY requirements.txt requirements.txt -RUN source activate root && \ - pip install -r requirements.txt +RUN pip install -r requirements.txt LABEL org.label-schema.build-date=$BUILD_DATE \ org.label-schema.vcs-url="https://github.com/kbase/catalog.git" \ diff --git a/requirements.txt b/requirements.txt index e7d24fec..86d891ef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,6 @@ docker>=3.5 gitpython pyyaml semantic_version -coverage \ No newline at end of file +coverage +uwsgi==2.0.18 +JSONRPCBase==0.2.0 \ No newline at end of file From 01f1539fef86280832bbfea8b3f2d2e18698615d Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 28 Jan 2025 17:14:07 -0800 Subject: [PATCH 039/110] remove kbapi_common submodule --- Makefile | 2 +- kbapi_common | 1 - lib/biokbase/log.py | 371 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 372 insertions(+), 2 deletions(-) delete mode 160000 kbapi_common create mode 100644 lib/biokbase/log.py diff --git a/Makefile b/Makefile index 7ce574a9..41e971c8 100644 --- a/Makefile +++ b/Makefile @@ -102,7 +102,7 @@ setup-tests: mkdir -p $(TESTLIB)/biokbase mkdir -p $(TESTDIR)/nms rsync -av lib/biokbase/* $(TESTLIB)/biokbase/. --exclude *.bak-* - rsync -av kbapi_common/lib/biokbase/* $(TESTLIB)/biokbase/. +# rsync -av kbapi_common/lib/biokbase/* $(TESTLIB)/biokbase/. # cd narrative_method_store; make; make build-classpath-list; # rsync -av narrative_method_store/lib/biokbase/* $(TESTLIB)/biokbase/. diff --git a/kbapi_common b/kbapi_common deleted file mode 160000 index 65a6d746..00000000 --- a/kbapi_common +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 65a6d746b8c66a89f6c9bf52adf33d5725ee2011 diff --git a/lib/biokbase/log.py b/lib/biokbase/log.py new file mode 100644 index 00000000..59748cea --- /dev/null +++ b/lib/biokbase/log.py @@ -0,0 +1,371 @@ +""" +NAME + log + +DESCRIPTION + A library for sending logging messages to syslog. + +METHODS + log(string subsystem, hashref constraints): Initializes log. You + should call this at the beginning of your program. Constraints are + optional. + + log_message(int level, string message): sends log message to syslog. + + * level: (0-9) The logging level for this message is compared to + the logging level that has been set in log. If it is <= + the set logging level, the message will be sent to syslog, + otherwise it will be ignored. Logging level is set to 6 + if control API cannot be reached and the user does + not set the log level. Log level can also be entered as + string (e.g. 'DEBUG') + + * message: This is the log message. + + get_log_level(): Returns the current log level as an integer. + + set_log_level(integer level) : Sets the log level. Only use this if you + wish to override the log levels that are defined by the control API. + Can also be entered as string (e.g. 'DEBUG') + + * level : priority + + * 0 : EMERG - system is unusable + + * 1 : ALERT - component must be fixed immediately + + * 2 : CRIT - secondary component must be fixed immediately + + * 3 : ERR - non-urgent failure + + * 4 : WARNING - warning that an error will occur if no action + is taken + + * 5 : NOTICE - unusual but safe conditions + + * 6 : INFO - normal operational messages + + * 7 : DEBUG - lowest level of debug + + * 8 : DEBUG2 - second level of debug + + * 9 : DEBUG3 - highest level of debug + + set_log_msg_check_count(integer count): used to set the number the + messages that log will log before querying the control API for the + log level (default is 100 messages). + + set_log_msg_check_interval(integer seconds): used to set the interval, + in seconds, that will be allowed to pass before log will query the + control API for the log level (default is 300 seconds). + + update_api_log_level() : Checks the control API for the currently set + log level. + + use_api_log_level() : Removes the user-defined log level and tells log + to use the control API-defined log level. +""" + +# Copied from sample service, the log.py version in lib is python2 only +# TODO LOGGING rework this to just log to stdout, rethink logging in general + +import json as _json +import urllib.request as _urllib2 +import syslog as _syslog +import platform as _platform +import inspect as _inspect +import os as _os +import getpass as _getpass +import warnings as _warnings +from configparser import ConfigParser as _ConfigParser +import time + +MLOG_ENV_FILE = 'MLOG_CONFIG_FILE' +_GLOBAL = 'global' +MLOG_LOG_LEVEL = 'mlog_log_level' +MLOG_API_URL = 'mlog_api_url' +MLOG_LOG_FILE = 'mlog_log_file' + +DEFAULT_LOG_LEVEL = 6 +#MSG_CHECK_COUNT = 100 +#MSG_CHECK_INTERVAL = 300 # 300s = 5min +MSG_FACILITY = _syslog.LOG_LOCAL1 +EMERG_FACILITY = _syslog.LOG_LOCAL0 + +EMERG = 0 +ALERT = 1 +CRIT = 2 +ERR = 3 +WARNING = 4 +NOTICE = 5 +INFO = 6 +DEBUG = 7 +DEBUG2 = 8 +DEBUG3 = 9 +_MLOG_TEXT_TO_LEVEL = {'EMERG': EMERG, + 'ALERT': ALERT, + 'CRIT': CRIT, + 'ERR': ERR, + 'WARNING': WARNING, + 'NOTICE': NOTICE, + 'INFO': INFO, + 'DEBUG': DEBUG, + 'DEBUG2': DEBUG2, + 'DEBUG3': DEBUG3, + } +_MLOG_TO_SYSLOG = [_syslog.LOG_EMERG, _syslog.LOG_ALERT, _syslog.LOG_CRIT, + _syslog.LOG_ERR, _syslog.LOG_WARNING, _syslog.LOG_NOTICE, + _syslog.LOG_INFO, _syslog.LOG_DEBUG, _syslog.LOG_DEBUG, + _syslog.LOG_DEBUG] +#ALLOWED_LOG_LEVELS = set(_MLOG_TEXT_TO_LEVEL.values()) +_MLOG_LEVEL_TO_TEXT = {} +for k, v in _MLOG_TEXT_TO_LEVEL.items(): + _MLOG_LEVEL_TO_TEXT[v] = k +LOG_LEVEL_MIN = min(_MLOG_LEVEL_TO_TEXT.keys()) +LOG_LEVEL_MAX = max(_MLOG_LEVEL_TO_TEXT.keys()) +del k, v + + +class log(object): + """ + This class contains the methods necessary for sending log messages. + """ + + def __init__(self, subsystem, constraints=None, config=None, logfile=None, + ip_address=False, authuser=False, module=False, + method=False, call_id=False, changecallback=None): + if not subsystem: + raise ValueError("Subsystem must be supplied") + + self.user = _getpass.getuser() + self.parentfile = _os.path.abspath(_inspect.getfile( + _inspect.stack()[1][0])) + self.ip_address = ip_address + self.authuser = authuser + self.module = module + self.method = method + self.call_id = call_id + noop = lambda: None + self._callback = changecallback or noop + self._subsystem = str(subsystem) + self._mlog_config_file = config + if not self._mlog_config_file: + self._mlog_config_file = _os.environ.get(MLOG_ENV_FILE, None) + if self._mlog_config_file: + self._mlog_config_file = str(self._mlog_config_file) + self._user_log_level = -1 + self._config_log_level = -1 + self._user_log_file = logfile + self._config_log_file = None + self._api_log_level = -1 + self._msgs_since_config_update = 0 + self._time_at_config_update = time.time() + self.msg_count = 0 + self._recheck_api_msg = 100 + self._recheck_api_time = 300 # 5 mins + self._log_constraints = {} if not constraints else constraints + + self._init = True + self.update_config() + self._init = False + + def _get_time_since_start(self): + time_diff = time.time() - self._time_at_config_update + return time_diff + + def get_log_level(self): + if(self._user_log_level != -1): + return self._user_log_level + elif(self._config_log_level != -1): + return self._config_log_level + elif(self._api_log_level != -1): + return self._api_log_level + else: + return DEFAULT_LOG_LEVEL + + def _get_config_items(self, cfg, section): + cfgitems = {} + if cfg.has_section(section): + for k, v in cfg.items(section): + cfgitems[k] = v + return cfgitems + + def update_config(self): + loglevel = self.get_log_level() + logfile = self.get_log_file() + + self._api_log_level = -1 + self._msgs_since_config_update = 0 + self._time_at_config_update = time.time() + + # Retrieving the control API defined log level + api_url = None + if self._mlog_config_file and _os.path.isfile(self._mlog_config_file): + cfg = _ConfigParser() + cfg.read(self._mlog_config_file) + cfgitems = self._get_config_items(cfg, _GLOBAL) + cfgitems.update(self._get_config_items(cfg, self._subsystem)) + if MLOG_LOG_LEVEL in cfgitems: + try: + self._config_log_level = int(cfgitems[MLOG_LOG_LEVEL]) + except: + _warnings.warn( + 'Cannot parse log level {} from file {} to int'.format( + cfgitems[MLOG_LOG_LEVEL], self._mlog_config_file) + + '. Keeping current log level.') + if MLOG_API_URL in cfgitems: + api_url = cfgitems[MLOG_API_URL] + if MLOG_LOG_FILE in cfgitems: + self._config_log_file = cfgitems[MLOG_LOG_FILE] + elif self._mlog_config_file: + _warnings.warn('Cannot read config file ' + self._mlog_config_file) + + if (api_url): + subsystem_api_url = api_url + "/" + self._subsystem + try: + data = _json.load(_urllib2.urlopen(subsystem_api_url, + timeout=5)) + except _urllib2.URLError as e: + code_ = None + if hasattr(e, 'code'): + code_ = ' ' + str(e.code) + _warnings.warn( + 'Could not connect to mlog api server at ' + + '{}:{} {}. Using default log level {}.'.format( + subsystem_api_url, code_, str(e.reason), + str(DEFAULT_LOG_LEVEL))) + else: + max_matching_level = -1 + for constraint_set in data['log_levels']: + level = constraint_set['level'] + constraints = constraint_set['constraints'] + if level <= max_matching_level: + continue + + matches = 1 + for constraint in constraints: + if constraint not in self._log_constraints: + matches = 0 + elif (self._log_constraints[constraint] != + constraints[constraint]): + matches = 0 + + if matches == 1: + max_matching_level = level + + self._api_log_level = max_matching_level + if ((self.get_log_level() != loglevel or + self.get_log_file() != logfile) and not self._init): + self._callback() + + def _resolve_log_level(self, level): + if(level in _MLOG_TEXT_TO_LEVEL): + level = _MLOG_TEXT_TO_LEVEL[level] + elif(level not in _MLOG_LEVEL_TO_TEXT): + raise ValueError('Illegal log level') + return level + + def set_log_level(self, level): + self._user_log_level = self._resolve_log_level(level) + self._callback() + + def get_log_file(self): + if self._user_log_file: + return self._user_log_file + if self._config_log_file: + return self._config_log_file + return None + + def set_log_file(self, filename): + self._user_log_file = filename + self._callback() + + def set_log_msg_check_count(self, count): + count = int(count) + if count < 0: + raise ValueError('Cannot check a negative number of messages') + self._recheck_api_msg = count + + def set_log_msg_check_interval(self, interval): + interval = int(interval) + if interval < 0: + raise ValueError('interval must be positive') + self._recheck_api_time = interval + + def clear_user_log_level(self): + self._user_log_level = -1 + self._callback() + + def _get_ident(self, level, user, parentfile, ip_address, authuser, module, + method, call_id): + infos = [self._subsystem, _MLOG_LEVEL_TO_TEXT[level], + repr(time.time()), user, parentfile, str(_os.getpid())] + if self.ip_address: + infos.append(str(ip_address) if ip_address else '-') + if self.authuser: + infos.append(str(authuser) if authuser else '-') + if self.module: + infos.append(str(module) if module else '-') + if self.method: + infos.append(str(method) if method else '-') + if self.call_id: + infos.append(str(call_id) if call_id else '-') + return "[" + "] [".join(infos) + "]" + + def _syslog(self, facility, level, ident, message): + _syslog.openlog(ident, facility) + if isinstance(message, str): + _syslog.syslog(_MLOG_TO_SYSLOG[level], message) + else: + try: + for m in message: + _syslog.syslog(_MLOG_TO_SYSLOG[level], m) + except TypeError: + _syslog.syslog(_MLOG_TO_SYSLOG[level], str(message)) + _syslog.closelog() + + def _log(self, ident, message): + ident = ' '.join([str(time.strftime( + "%Y-%m-%d %H:%M:%S", time.localtime())), + _platform.node(), ident + ': ']) + try: + with open(self.get_log_file(), 'a') as log: + if isinstance(message, str): + log.write(ident + message + '\n') + else: + try: + for m in message: + log.write(ident + m + '\n') + except TypeError: + log.write(ident + str(message) + '\n') + except Exception as e: + err = 'Could not write to log file ' + str(self.get_log_file()) + \ + ': ' + str(e) + '.' + _warnings.warn(err) + + def log_message(self, level, message, ip_address=None, authuser=None, + module=None, method=None, call_id=None): +# message = str(message) + level = self._resolve_log_level(level) + + self.msg_count += 1 + self._msgs_since_config_update += 1 + + if(self._msgs_since_config_update >= self._recheck_api_msg + or self._get_time_since_start() >= self._recheck_api_time): + self.update_config() + + ident = self._get_ident(level, self.user, self.parentfile, ip_address, + authuser, module, method, call_id) + # If this message is an emergency, send a copy to the emergency + # facility first. + if(level == 0): + self._syslog(EMERG_FACILITY, level, ident, message) + + if(level <= self.get_log_level()): + self._syslog(MSG_FACILITY, level, ident, message) + if self.get_log_file(): + self._log(ident, message) + +if __name__ == '__main__': + pass From 35a18c45f81e72ff21b52698aa450c1571a100ba Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 29 Jan 2025 13:32:06 -0800 Subject: [PATCH 040/110] fix deprecated mongo auth --- lib/biokbase/catalog/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index d3ba841e..32066505 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -128,7 +128,7 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech # the MongoClient call failed # to do: add authMechanism as an argument if (mongo_user and mongo_psswd): - self.mongo[mongo_db].authenticate(mongo_user, mongo_psswd, mechanism=mongo_authMechanism) + self.mongo = MongoClient(f"mongodb://{mongo_user}:{mongo_psswd}@{mongo_host}/{mongo_db}?authMechanism={mongo_authMechanism}") # Grab a handle to the database and collections self.db = self.mongo[mongo_db] From c22843825a510956177f943be0809caa0f7e001b Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 29 Jan 2025 13:32:57 -0800 Subject: [PATCH 041/110] remove all submodules in catalog --- .github/workflows/test.yml | 2 -- .gitmodules | 12 ------------ DEPENDENCIES | 4 ---- 3 files changed, 18 deletions(-) delete mode 100644 .gitmodules delete mode 100644 DEPENDENCIES diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e979e298..87056952 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,8 +35,6 @@ jobs: steps: - name: Repo checkout uses: actions/checkout@v4 - with: - submodules: recursive - name: Setup Python uses: actions/setup-python@v5 diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 54a71942..00000000 --- a/.gitmodules +++ /dev/null @@ -1,12 +0,0 @@ -[submodule "jars"] - path = jars - url = https://github.com/kbase/jars -[submodule "auth"] - path = auth - url = https://github.com/kbase/auth -[submodule "kbapi_common"] - path = kbapi_common - url = https://github.com/kbase/kbapi_common -[submodule "narrative_method_store"] - path = narrative_method_store - url = https://github.com/kbase/narrative_method_store diff --git a/DEPENDENCIES b/DEPENDENCIES deleted file mode 100644 index a2c84014..00000000 --- a/DEPENDENCIES +++ /dev/null @@ -1,4 +0,0 @@ -kb_sdk -auth -kbapi_common -narrative_method_store From 91532354a03e70c14c43fefc0183f3009244df4b Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 30 Jan 2025 16:10:49 -0800 Subject: [PATCH 042/110] create MongoClient through passing in params --- lib/biokbase/catalog/db.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 32066505..4734701d 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -1,9 +1,11 @@ import copy import pprint +import traceback from pymongo import ASCENDING from pymongo import DESCENDING from pymongo import MongoClient +from pymongo.errors import ServerSelectionTimeoutError ''' @@ -120,15 +122,25 @@ class MongoCatalogDBI: def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMechanism): - # create the client - self.mongo = MongoClient('mongodb://' + mongo_host) - - # Try to authenticate, will throw an exception if the user/psswd is not valid for the db - # the pymongo docs say authenticate() is deprecated, but testing putting auth in - # the MongoClient call failed - # to do: add authMechanism as an argument if (mongo_user and mongo_psswd): - self.mongo = MongoClient(f"mongodb://{mongo_user}:{mongo_psswd}@{mongo_host}/{mongo_db}?authMechanism={mongo_authMechanism}") + self.mongo = MongoClient( + mongo_host, + username=mongo_user, + password=mongo_psswd, + authSource=mongo_db, + authMechanism=mongo_authMechanism, + ) + else: + self.mongo = MongoClient(mongo_host) + + try: + self.mongo.server_info() # force a call to server + except ServerSelectionTimeoutError as e: + error_msg = "Connot connect to Mongo server\n" + error_msg += "ERROR -- {}:\n{}".format( + e, "".join(traceback.format_exception(None, e, e.__traceback__)) + ) + raise ValueError(error_msg) # Grab a handle to the database and collections self.db = self.mongo[mongo_db] From 2344d64e3a87cc98f2b4c2ffdc9ff75fcfdb8847 Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 30 Jan 2025 16:21:47 -0800 Subject: [PATCH 043/110] add a comment for mock_auth server.py --- test/mock_auth/server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/mock_auth/server.py b/test/mock_auth/server.py index 959b642b..19d66e4f 100644 --- a/test/mock_auth/server.py +++ b/test/mock_auth/server.py @@ -17,6 +17,7 @@ sys.exit(1) endpoints = [] +# The order is preserved here to ensure routes capture requests correctly. catalog_mock_auth = ['auth_admin.json', 'auth_invalid.json', 'auth_missing.json', 'auth_non_admin.json'] for path in catalog_mock_auth: if path.endswith('.json'): From f14712087889be8bdb457eaf1878cc04ca2a6cd1 Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 31 Jan 2025 14:23:26 -0800 Subject: [PATCH 044/110] simplify code --- lib/biokbase/catalog/db.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 4734701d..d07b8ae5 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -123,18 +123,13 @@ class MongoCatalogDBI: def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMechanism): if (mongo_user and mongo_psswd): - self.mongo = MongoClient( - mongo_host, - username=mongo_user, - password=mongo_psswd, - authSource=mongo_db, - authMechanism=mongo_authMechanism, - ) + self.mongo = MongoClient(f"mongodb://{mongo_user}:{mongo_psswd}@{mongo_host}/{mongo_db}?authMechanism={mongo_authMechanism}") else: - self.mongo = MongoClient(mongo_host) + self.mongo = MongoClient(f"mongodb://{mongo_host}") try: self.mongo.server_info() # force a call to server + print("Connection successful!") except ServerSelectionTimeoutError as e: error_msg = "Connot connect to Mongo server\n" error_msg += "ERROR -- {}:\n{}".format( @@ -349,7 +344,7 @@ def list_builds(self, query, selection, skip=skip, limit=limit, - sort=[['timestamp', DESCENDING]])) + sort=[('timestamp', DESCENDING)])) # slice arg is used in the mongo query for getting lines. It is either a # pos int (get first n lines), neg int (last n lines), or array [skip, limit] @@ -943,16 +938,12 @@ def remove_favorite(self, module_name, app_id, username): def list_user_favorites(self, username): query = {'user': username} selection = {'_id': 0, 'module_name_lc': 1, 'id': 1, 'timestamp': 1} - return list(self.favorites.find( - query, selection, - sort=[['timestamp', DESCENDING]])) + return list(self.favorites.find(query, selection).sort('timestamp', DESCENDING)) def list_app_favorites(self, module_name, app_id): query = {'module_name_lc': module_name.strip().lower(), 'id': app_id.strip()} selection = {'_id': 0, 'user': 1, 'timestamp': 1} - return list(self.favorites.find( - query, selection, - sort=[['timestamp', DESCENDING]])) + return list(self.favorites.find(query, selection).sort('timestamp', DESCENDING)) def aggregate_favorites_over_apps(self, module_names_lc): ### WARNING! If we switch to Mongo 3.x, the result object will change and this will break From 7a5b354f7fa7f2f4ca2c889ce97bfafd1c333412 Mon Sep 17 00:00:00 2001 From: Sijie Date: Mon, 3 Feb 2025 14:45:26 -0800 Subject: [PATCH 045/110] adjust params in MongoClient --- lib/biokbase/catalog/db.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index d07b8ae5..9bb907c8 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -123,7 +123,11 @@ class MongoCatalogDBI: def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMechanism): if (mongo_user and mongo_psswd): - self.mongo = MongoClient(f"mongodb://{mongo_user}:{mongo_psswd}@{mongo_host}/{mongo_db}?authMechanism={mongo_authMechanism}") + self.mongo = MongoClient( + f"mongodb://{mongo_user}:{mongo_psswd}@{mongo_host}/{mongo_db}?authMechanism={mongo_authMechanism}", + maxPoolSize=500, + waitQueueTimeoutMS=3000 + ) else: self.mongo = MongoClient(f"mongodb://{mongo_host}") From 3e3673f4cc29003f6cda48a41138fe47208a61d4 Mon Sep 17 00:00:00 2001 From: Sijie Date: Mon, 3 Feb 2025 16:11:44 -0800 Subject: [PATCH 046/110] add debug log messages in list_user_favorites func --- lib/biokbase/catalog/db.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 9bb907c8..482c765d 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -5,7 +5,7 @@ from pymongo import ASCENDING from pymongo import DESCENDING from pymongo import MongoClient -from pymongo.errors import ServerSelectionTimeoutError +from pymongo.errors import ConnectionFailure ''' @@ -134,7 +134,7 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech try: self.mongo.server_info() # force a call to server print("Connection successful!") - except ServerSelectionTimeoutError as e: + except ConnectionFailure as e: error_msg = "Connot connect to Mongo server\n" error_msg += "ERROR -- {}:\n{}".format( e, "".join(traceback.format_exception(None, e, e.__traceback__)) @@ -940,6 +940,12 @@ def remove_favorite(self, module_name, app_id, username): return self._check_update_result(result) def list_user_favorites(self, username): + try: + self.mongo.server_info() + print("MongoDB is connected in list_user_favorites().") + except ConnectionFailure as e: + print(f"MongoDB connection failed list_user_favorites(): {e}") + query = {'user': username} selection = {'_id': 0, 'module_name_lc': 1, 'id': 1, 'timestamp': 1} return list(self.favorites.find(query, selection).sort('timestamp', DESCENDING)) From 093510eaa4fdcae4da28f900d5e69255b1c8b755 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 4 Feb 2025 11:31:35 -0800 Subject: [PATCH 047/110] try reinitiate mongoclient --- lib/biokbase/catalog/db.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 482c765d..314f23ca 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -122,12 +122,14 @@ class MongoCatalogDBI: def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMechanism): + self.mongo_host = mongo_host + self.mongo_db = mongo_db + self.mongo_user = mongo_user + self.mongo_psswd = mongo_psswd + self.mongo_authMechanism = mongo_authMechanism + if (mongo_user and mongo_psswd): - self.mongo = MongoClient( - f"mongodb://{mongo_user}:{mongo_psswd}@{mongo_host}/{mongo_db}?authMechanism={mongo_authMechanism}", - maxPoolSize=500, - waitQueueTimeoutMS=3000 - ) + self.mongo = MongoClient(f"mongodb://{mongo_user}:{mongo_psswd}@{mongo_host}/{mongo_db}?authMechanism={mongo_authMechanism}") else: self.mongo = MongoClient(f"mongodb://{mongo_host}") @@ -951,6 +953,18 @@ def list_user_favorites(self, username): return list(self.favorites.find(query, selection).sort('timestamp', DESCENDING)) def list_app_favorites(self, module_name, app_id): + try: + # Attempt to ping the server + self.mongo.admin.command('ping') + print("MongoDB is connected!") + except ConnectionFailure: + print("MongoDB connection failed!") + print("Reinitiating Mongo Client!") + if (self.mongo_user and self.mongo_psswd): + self.mongo = MongoClient(f"mongodb://{self.mongo_user}:{self.mongo_psswd}@{self.mongo_host}/{self.mongo_db}?authMechanism={self.mongo_authMechanism}") + else: + self.mongo = MongoClient(f"mongodb://{self.mongo_host}") + query = {'module_name_lc': module_name.strip().lower(), 'id': app_id.strip()} selection = {'_id': 0, 'user': 1, 'timestamp': 1} return list(self.favorites.find(query, selection).sort('timestamp', DESCENDING)) From ec3064b8115f12021fc903d852d1eb41fef0ff8b Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 5 Feb 2025 10:21:28 -0800 Subject: [PATCH 048/110] update __init__ in db.py file --- lib/biokbase/catalog/db.py | 281 +++++++++++++++++++++++-------------- 1 file changed, 177 insertions(+), 104 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 314f23ca..65fdda12 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -1,6 +1,7 @@ import copy import pprint import traceback +from collections import defaultdict from pymongo import ASCENDING from pymongo import DESCENDING @@ -128,135 +129,207 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech self.mongo_psswd = mongo_psswd self.mongo_authMechanism = mongo_authMechanism - if (mongo_user and mongo_psswd): - self.mongo = MongoClient(f"mongodb://{mongo_user}:{mongo_psswd}@{mongo_host}/{mongo_db}?authMechanism={mongo_authMechanism}") - else: - self.mongo = MongoClient(f"mongodb://{mongo_host}") + # MongoDB client and database initialization deferred + self._mongo_client_initialized = False + self.mongo = None + self.db = None + self.index_created = defaultdict(int) + + def _initiate_mongo_client(self): + """Initialize MongoDB client and collections lazily.""" + if not self._mongo_client_initialized: + try: + if self.mongo_user and self.mongo_psswd: + # Connection string with authentication + self.mongo = MongoClient( + f"mongodb://{self.mongo_user}:{self.mongo_psswd}@{self.mongo_host}/{self.mongo_db}?authMechanism={self.mongo_authMechanism}" + ) + else: + # Connection string without authentication + self.mongo = MongoClient(f"mongodb://{self.mongo_host}") - try: - self.mongo.server_info() # force a call to server - print("Connection successful!") - except ConnectionFailure as e: - error_msg = "Connot connect to Mongo server\n" - error_msg += "ERROR -- {}:\n{}".format( - e, "".join(traceback.format_exception(None, e, e.__traceback__)) - ) - raise ValueError(error_msg) + # Force a call to server to verify the connection + self.mongo.server_info() + print("Connection successful!") - # Grab a handle to the database and collections - self.db = self.mongo[mongo_db] - self.modules = self.db[MongoCatalogDBI._MODULES] - self.module_versions = self.db[MongoCatalogDBI._MODULE_VERSIONS] + # Grab a handle to the database + self.db = self.mongo[self.mongo_db] - self.local_functions = self.db[MongoCatalogDBI._LOCAL_FUNCTIONS] - self.developers = self.db[MongoCatalogDBI._DEVELOPERS] - self.build_logs = self.db[MongoCatalogDBI._BUILD_LOGS] - self.favorites = self.db[MongoCatalogDBI._FAVORITES] - self.client_groups = self.db[MongoCatalogDBI._CLIENT_GROUPS] - self.volume_mounts = self.db[MongoCatalogDBI._VOLUME_MOUNTS] + # Mark the client as initialized + self._mongo_client_initialized = True - self.exec_stats_raw = self.db[MongoCatalogDBI._EXEC_STATS_RAW] - self.exec_stats_apps = self.db[MongoCatalogDBI._EXEC_STATS_APPS] - self.exec_stats_users = self.db[MongoCatalogDBI._EXEC_STATS_USERS] + except ConnectionFailure as e: + error_msg = "Cannot connect to Mongo server\n" + error_msg += "ERROR -- {}:\n{}".format( + e, "".join(traceback.format_exception(None, e, e.__traceback__)) + ) + raise ValueError(error_msg) - self.secure_config_params = self.db[MongoCatalogDBI._SECURE_CONFIG_PARAMS] + def _get_collection(self, collection_name): + """Lazily load collections.""" + self._initialize_mongo_client() + if not self.index_created[collection_name]: + self._create_indexes(collection_name) + self.index_created[collection_name] = 1 + return self.db[collection_name] - # check the db schema - self.check_db_schema() + # Define getters for each collection + @property + def modules(self): + return self._get_collection(MongoCatalogDBI._MODULES) - # Make sure we have an index on module and git_repo_url - self.module_versions.create_index('module_name_lc', sparse=False) - self.module_versions.create_index('git_commit_hash', sparse=False) - self.module_versions.create_index([ - ('module_name_lc', ASCENDING), - ('git_commit_hash', ASCENDING)], - unique=True, sparse=False) + @property + def module_versions(self): + return self._get_collection(MongoCatalogDBI._MODULE_VERSIONS) - # Make sure we have a unique index on module_name_lc and git_commit_hash - self.local_functions.create_index('function_id') - self.local_functions.create_index([ - ('module_name_lc', ASCENDING), - ('function_id', ASCENDING), - ('git_commit_hash', ASCENDING)], - unique=True, sparse=False) + @property + def local_functions(self): + return self._get_collection(MongoCatalogDBI._LOCAL_FUNCTIONS) - # local function indecies - self.local_functions.create_index('module_name_lc') - self.local_functions.create_index('git_commit_hash') - self.local_functions.create_index('function_id') - self.local_functions.create_index([ - ('module_name_lc', ASCENDING), - ('function_id', ASCENDING), - ('git_commit_hash', ASCENDING)], - unique=True, sparse=False) + @property + def developers(self): + return self._get_collection(MongoCatalogDBI._DEVELOPERS) + + @property + def build_logs(self): + return self._get_collection(MongoCatalogDBI._BUILD_LOGS) + + @property + def favorites(self): + return self._get_collection(MongoCatalogDBI._FAVORITES) + + @property + def client_groups(self): + return self._get_collection(MongoCatalogDBI._CLIENT_GROUPS) + + @property + def volume_mounts(self): + return self._get_collection(MongoCatalogDBI._VOLUME_MOUNTS) + + @property + def exec_stats_raw(self): + return self._get_collection(MongoCatalogDBI._EXEC_STATS_RAW) + + @property + def exec_stats_apps(self): + return self._get_collection(MongoCatalogDBI._EXEC_STATS_APPS) + + @property + def exec_stats_users(self): + return self._get_collection(MongoCatalogDBI._EXEC_STATS_USERS) + + @property + def secure_config_params(self): + return self._get_collection(MongoCatalogDBI._SECURE_CONFIG_PARAMS) + + def _create_indexes(self, collection_name): + """Create indexes for the given collection lazily.""" + collection = self.db[collection_name] + + # Make sure we have an index on module and git_repo_url + if collection_name == MongoCatalogDBI._MODULE_VERSIONS: + collection.create_index('module_name_lc', sparse=False) + collection.create_index('git_commit_hash', sparse=False) + collection.create_index([ + ('module_name_lc', ASCENDING), + ('git_commit_hash', ASCENDING)], + unique=True, sparse=False) + + elif collection_name == MongoCatalogDBI._LOCAL_FUNCTIONS: + # Make sure we have a unique index on module_name_lc and git_commit_hash + collection.create_index('function_id') + collection.create_index([ + ('module_name_lc', ASCENDING), + ('function_id', ASCENDING), + ('git_commit_hash', ASCENDING)], + unique=True, sparse=False) + + # local function indecies + collection.create_index('module_name_lc') + collection.create_index('git_commit_hash') + collection.create_index('function_id') + collection.create_index([ + ('module_name_lc', ASCENDING), + ('function_id', ASCENDING), + ('git_commit_hash', ASCENDING)], + unique=True, sparse=False) # developers indecies - self.developers.create_index('kb_username', unique=True) + elif collection_name == MongoCatalogDBI._DEVELOPERS: + collection.create_index('kb_username', unique=True) - self.build_logs.create_index('registration_id', unique=True) - self.build_logs.create_index('module_name_lc') - self.build_logs.create_index('timestamp') - self.build_logs.create_index('registration') - self.build_logs.create_index('git_url') - self.build_logs.create_index('current_versions.release.release_timestamp') + elif collection_name == MongoCatalogDBI._BUILD_LOGS: + collection.create_index('registration_id', unique=True) + collection.create_index('module_name_lc') + collection.create_index('timestamp') + collection.create_index('registration') + collection.create_index('git_url') + collection.create_index('current_versions.release.release_timestamp') # for favorites - self.favorites.create_index('user') - self.favorites.create_index('module_name_lc') - self.favorites.create_index('id') - # you can only favorite a method once, so put a unique index on the triple - self.favorites.create_index([ - ('user', ASCENDING), - ('id', ASCENDING), - ('module_name_lc', ASCENDING)], - unique=True, sparse=False) + elif collection_name == MongoCatalogDBI._FAVORITES: + collection.create_index('user') + collection.create_index('module_name_lc') + collection.create_index('id') + # you can only favorite a method once, so put a unique index on the triple + collection.create_index([ + ('user', ASCENDING), + ('id', ASCENDING), + ('module_name_lc', ASCENDING)], + unique=True, sparse=False) # execution stats - self.exec_stats_raw.create_index('user_id', - unique=False, sparse=False) - self.exec_stats_raw.create_index([('app_module_name', ASCENDING), - ('app_id', ASCENDING)], - unique=False, sparse=True) - self.exec_stats_raw.create_index([('func_module_name', ASCENDING), - ('func_name', ASCENDING)], - unique=False, sparse=True) - self.exec_stats_raw.create_index('creation_time', - unique=False, sparse=False) - self.exec_stats_raw.create_index('finish_time', - unique=False, sparse=False) - - self.exec_stats_apps.create_index('module_name', - unique=False, sparse=True) - self.exec_stats_apps.create_index([('full_app_id', ASCENDING), - ('type', ASCENDING), - ('time_range', ASCENDING)], - unique=True, sparse=False) - self.exec_stats_apps.create_index([('type', ASCENDING), - ('time_range', ASCENDING)], - unique=False, sparse=False) - - self.exec_stats_users.create_index([('user_id', ASCENDING), + elif collection_name == MongoCatalogDBI._EXEC_STATS_RAW: + collection.create_index('user_id', + unique=False, sparse=False) + collection.create_index([('app_module_name', ASCENDING), + ('app_id', ASCENDING)], + unique=False, sparse=True) + collection.create_index([('func_module_name', ASCENDING), + ('func_name', ASCENDING)], + unique=False, sparse=True) + collection.create_index('creation_time', + unique=False, sparse=False) + collection.create_index('finish_time', + unique=False, sparse=False) + + elif collection_name == MongoCatalogDBI._EXEC_STATS_APPS: + collection.create_index('module_name', + unique=False, sparse=True) + collection.create_index([('full_app_id', ASCENDING), ('type', ASCENDING), ('time_range', ASCENDING)], - unique=True, sparse=False) + unique=True, sparse=False) + collection.create_index([('type', ASCENDING), + ('time_range', ASCENDING)], + unique=False, sparse=False) + + elif collection_name == MongoCatalogDBI._EXEC_STATS_USERS: + collection.create_index([('user_id', ASCENDING), + ('type', ASCENDING), + ('time_range', ASCENDING)], + unique=True, sparse=False) # client groups and volume mounts - self.client_groups.create_index([('module_name_lc', ASCENDING), + elif collection_name == MongoCatalogDBI._CLIENT_GROUPS: + collection.create_index([('module_name_lc', ASCENDING), ('function_name', ASCENDING)], unique=True, sparse=False) - self.volume_mounts.create_index([('client_group', ASCENDING), - ('module_name_lc', ASCENDING), - ('function_name', ASCENDING)], - unique=True, sparse=False) + elif collection_name == MongoCatalogDBI._VOLUME_MOUNTS: + collection.create_index([('client_group', ASCENDING), + ('module_name_lc', ASCENDING), + ('function_name', ASCENDING)], + unique=True, sparse=False) # hidden configuration parameters - self.secure_config_params.create_index('module_name_lc') - self.secure_config_params.create_index([ - ('module_name_lc', ASCENDING), - ('version', ASCENDING), - ('param_name', ASCENDING)], - unique=True, sparse=False) + elif collection_name == MongoCatalogDBI._SECURE_CONFIG_PARAMS: + collection.create_index('module_name_lc') + collection.create_index([ + ('module_name_lc', ASCENDING), + ('version', ASCENDING), + ('param_name', ASCENDING)], + unique=True, sparse=False) def is_registered(self, module_name='', git_url=''): if not module_name and not git_url: From 4aabfb90568925d397f4c63daf0cc2d6ce844ed6 Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 5 Feb 2025 10:31:03 -0800 Subject: [PATCH 049/110] fix typo --- lib/biokbase/catalog/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 65fdda12..9a038b11 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -135,7 +135,7 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech self.db = None self.index_created = defaultdict(int) - def _initiate_mongo_client(self): + def _initialize_mongo_client(self): """Initialize MongoDB client and collections lazily.""" if not self._mongo_client_initialized: try: From fe87438a5aab794c65a82dadc51fd28f3cce0ff8 Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 5 Feb 2025 10:47:49 -0800 Subject: [PATCH 050/110] add check_db_schema --- lib/biokbase/catalog/db.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 9a038b11..e464554f 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -155,6 +155,9 @@ def _initialize_mongo_client(self): # Grab a handle to the database self.db = self.mongo[self.mongo_db] + # Check DB schema + self.check_db_schema() + # Mark the client as initialized self._mongo_client_initialized = True From 5af0ccdc4bc39dc56730eedb4abf951216640127 Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 5 Feb 2025 11:32:50 -0800 Subject: [PATCH 051/110] add boolean flag for schema check --- lib/biokbase/catalog/db.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index e464554f..951fd65c 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -134,6 +134,7 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech self.mongo = None self.db = None self.index_created = defaultdict(int) + self._db_schema_checked = False def _initialize_mongo_client(self): """Initialize MongoDB client and collections lazily.""" @@ -161,6 +162,8 @@ def _initialize_mongo_client(self): # Mark the client as initialized self._mongo_client_initialized = True + self.check_db_schema() + except ConnectionFailure as e: error_msg = "Cannot connect to Mongo server\n" error_msg += "ERROR -- {}:\n{}".format( @@ -1377,6 +1380,12 @@ def get_secure_config_params(self, module_name): # another server is already starting an update, we can skip or abort def check_db_schema(self): + if self._db_schema_checked: + return + + # Mark that we've checked the schema + self._db_schema_checked = True + db_version = self.get_db_version() print('db_version=' + str(db_version)) From 93e00332aefaabaa4c165210cf1637761de8fddb Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 5 Feb 2025 11:55:45 -0800 Subject: [PATCH 052/110] collection direct call --- lib/biokbase/catalog/db.py | 55 +++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 951fd65c..cffa036d 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -1434,13 +1434,14 @@ def update_db_version(self, version): # version 1 kept released module versions in a map, version 2 updates that to a list # and adds dynamic service tags def update_db_1_to_2(self): - for m in self.modules.find({'release_versions': {'$exists': True}}): + modules_collection = self.db[MongoCatalogDBI._MODULES] + for m in modules_collection.find({'release_versions': {'$exists': True}}): release_version_list = [] for timestamp in m['release_versions']: m['release_versions'][timestamp]['dynamic_service'] = 0 release_version_list.append(m['release_versions'][timestamp]) - self.modules.update_one( + modules_collection.update_one( {'_id': m['_id']}, { '$unset': {'release_versions': ''}, @@ -1449,33 +1450,34 @@ def update_db_1_to_2(self): # make sure everything has the dynamic service flag if not 'dynamic_service' in m['info']: - self.modules.update_one( + modules_collection.update_one( {'_id': m['_id']}, {'$set': {'info.dynamic_service': 0}}) if m['current_versions']['release']: if not 'dynamic_service' in m['current_versions']['release']: - self.modules.update_one( + modules_collection.update_one( {'_id': m['_id']}, {'$set': {'current_versions.release.dynamic_service': 0}}) if m['current_versions']['beta']: if not 'dynamic_service' in m['current_versions']['beta']: - self.modules.update_one( + modules_collection.update_one( {'_id': m['_id']}, {'$set': {'current_versions.beta.dynamic_service': 0}}) if m['current_versions']['dev']: if not 'dynamic_service' in m['current_versions']['dev']: - self.modules.update_one( + modules_collection.update_one( {'_id': m['_id']}, {'$set': {'current_versions.dev.dynamic_service': 0}}) # also ensure the execution stats fields have correct names - self.exec_stats_apps.update_many({'avg_queue_time': {'$exists': True}}, + exec_stats_apps_colleciton = self.db[MongoCatalogDBI._EXEC_STATS_APPS] + exec_stats_apps_colleciton.update_many({'avg_queue_time': {'$exists': True}}, {'$rename': {'avg_queue_time': 'total_queue_time', 'avg_exec_time': 'total_exec_time'}}) - self.exec_stats_users.update_many({'avg_queue_time': {'$exists': True}}, + exec_stats_apps_colleciton.update_many({'avg_queue_time': {'$exists': True}}, {'$rename': {'avg_queue_time': 'total_queue_time', 'avg_exec_time': 'total_exec_time'}}) @@ -1483,15 +1485,17 @@ def update_db_1_to_2(self): # a separate module versions collection. def update_db_2_to_3(self): - self.module_versions.create_index('module_name_lc', sparse=False) - self.module_versions.create_index('git_commit_hash', sparse=False) - self.module_versions.create_index([ + module_versions_collection = self.db[MongoCatalogDBI._MODULE_VERSIONS] + module_versions_collection.create_index('module_name_lc', sparse=False) + module_versions_collection.create_index('git_commit_hash', sparse=False) + module_versions_collection.create_index([ ('module_name_lc', ASCENDING), ('git_commit_hash', ASCENDING)], unique=True, sparse=False) + modules_collection = self.db[MongoCatalogDBI._MODULES] # update all module versions - for m in self.modules.find({}): + for m in modules_collection.find({}): # skip modules that have not been properly registered, might want to delete these later if 'module_name' not in m or 'module_name_lc' not in m: @@ -1507,14 +1511,14 @@ def update_db_2_to_3(self): rVer['released'] = 1 self.prepare_version_doc_for_db_2_to_3_update(rVer, m) try: - self.module_versions.insert_one(rVer) + module_versions_collection.insert_one(rVer) except: print(' - Warning - ' + rVer['module_name'] + '.' + rVer[ 'git_commit_hash'] + ' already inserted, skipping.') new_release_version_list.append({ 'git_commit_hash': rVer['git_commit_hash'] }) - self.modules.update_one( + modules_collection.update_one( {'_id': m['_id']}, {'$set': {'release_version_list': new_release_version_list}} ) @@ -1527,19 +1531,19 @@ def update_db_2_to_3(self): self.prepare_version_doc_for_db_2_to_3_update(modVer, m) if modVer.get('git_commit_hash') is not None: try: - self.module_versions.insert_one(modVer) + module_versions_collection.insert_one(modVer) except Exception as e: # we expect this to happen for all 'release' tags and if, say, a # version still tagged as dev/beta has been released print(f" - Warning - {tag} ver of {modVer['module_name']}." f"{modVer['git_commit_hash']} already inserted, skipping.") - self.modules.update_one( + modules_collection.update_one( {'_id': m['_id']}, {'$set': {'current_versions.' + tag: { 'git_commit_hash': modVer['git_commit_hash']}}} ) else: - self.modules.update_one( + modules_collection.update_one( {'_id': m['_id']}, {'$set': {'current_versions.' + tag: None}} ) @@ -1567,20 +1571,23 @@ def prepare_version_doc_for_db_2_to_3_update(self, version, module): def update_db_3_to_4(self): # make sure we don't have any indecies on the collections - self.volume_mounts.drop_indexes() - self.client_groups.drop_indexes() + volume_mounts_collection = self.db[MongoCatalogDBI._VOLUME_MOUNTS] + client_groups_collection = self.db[MongoCatalogDBI._CLIENT_GROUPS] + + volume_mounts_collection.drop_indexes() + client_groups_collection.drop_indexes() # update the volume_mounts, just need to rename app_id to function_name - for vm in self.volume_mounts.find({}): + for vm in volume_mounts_collection.find({}): if 'app_id' in vm and 'function_name' not in vm: - self.volume_mounts.update_one( + volume_mounts_collection.update_one( {'_id': vm['_id']}, {'$set': {'function_name': vm['app_id']}, '$unset': {'app_id': 1}} ) - for cg in self.client_groups.find({}): + for cg in client_groups_collection.find({}): if 'app_id' in cg: - self.client_groups.delete_one({'_id': cg['_id']}) + client_groups_collection.delete_one({'_id': cg['_id']}) tokens = cg['app_id'].split('/') if len(tokens) != 2: print( @@ -1592,4 +1599,4 @@ def update_db_3_to_4(self): 'function_name': tokens[1], 'client_groups': cg['client_groups'] } - self.client_groups.insert_one(new_cg) + client_groups_collection.insert_one(new_cg) From 37f083da6c97972b81cd8a469bceaeadb272c5ac Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 5 Feb 2025 12:02:07 -0800 Subject: [PATCH 053/110] remove debugging message --- lib/biokbase/catalog/db.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index cffa036d..aa72c3cc 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -1021,29 +1021,11 @@ def remove_favorite(self, module_name, app_id, username): return self._check_update_result(result) def list_user_favorites(self, username): - try: - self.mongo.server_info() - print("MongoDB is connected in list_user_favorites().") - except ConnectionFailure as e: - print(f"MongoDB connection failed list_user_favorites(): {e}") - query = {'user': username} selection = {'_id': 0, 'module_name_lc': 1, 'id': 1, 'timestamp': 1} return list(self.favorites.find(query, selection).sort('timestamp', DESCENDING)) def list_app_favorites(self, module_name, app_id): - try: - # Attempt to ping the server - self.mongo.admin.command('ping') - print("MongoDB is connected!") - except ConnectionFailure: - print("MongoDB connection failed!") - print("Reinitiating Mongo Client!") - if (self.mongo_user and self.mongo_psswd): - self.mongo = MongoClient(f"mongodb://{self.mongo_user}:{self.mongo_psswd}@{self.mongo_host}/{self.mongo_db}?authMechanism={self.mongo_authMechanism}") - else: - self.mongo = MongoClient(f"mongodb://{self.mongo_host}") - query = {'module_name_lc': module_name.strip().lower(), 'id': app_id.strip()} selection = {'_id': 0, 'user': 1, 'timestamp': 1} return list(self.favorites.find(query, selection).sort('timestamp', DESCENDING)) From d0c2522b4836c8f8a379d591ec284b972b2b6ee2 Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 5 Feb 2025 13:16:19 -0800 Subject: [PATCH 054/110] display catalog version --- test/startup_test.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/startup_test.py b/test/startup_test.py index 61a30328..de336b35 100644 --- a/test/startup_test.py +++ b/test/startup_test.py @@ -13,19 +13,24 @@ def test_startups(self): # Test normal startup, should work self.cUtil.setUp() catalog = Catalog(self.cUtil.getCatalogConfig()) + print(f"1. output catalog.version is: {catalog.version(self.cUtil.anonymous_ctx())[0]}") self.assertTrue(semantic_version.validate(catalog.version(self.cUtil.anonymous_ctx())[0])) # Test empty startup without DB version should work self.cUtil.setUpEmpty() catalog = Catalog(self.cUtil.getCatalogConfig()) + print(f"2. output catalog.version is: {catalog.version(self.cUtil.anonymous_ctx())[0]}") self.assertTrue(semantic_version.validate(catalog.version(self.cUtil.anonymous_ctx())[0])) # Test empty startup with several different valid versions should work self.cUtil.setUpEmpty(db_version=3) catalog = Catalog(self.cUtil.getCatalogConfig()) + print(f"3. output catalog.version is: {catalog.version(self.cUtil.anonymous_ctx())[0]}") self.assertTrue(semantic_version.validate(catalog.version(self.cUtil.anonymous_ctx())[0])) + self.cUtil.setUpEmpty(db_version=4) catalog = Catalog(self.cUtil.getCatalogConfig()) + print(f"4. output catalog.version is: {catalog.version(self.cUtil.anonymous_ctx())[0]}") self.assertTrue(semantic_version.validate(catalog.version(self.cUtil.anonymous_ctx())[0])) # Startup with version that is too high should fail From 6b5fcdebc2063968cc92dadacc13be7ee39ea047 Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 5 Feb 2025 14:01:31 -0800 Subject: [PATCH 055/110] fix failed unit test in startup_test.py --- lib/biokbase/catalog/db.py | 4 +--- test/startup_test.py | 5 +---- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index aa72c3cc..1106e8cc 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -156,12 +156,10 @@ def _initialize_mongo_client(self): # Grab a handle to the database self.db = self.mongo[self.mongo_db] - # Check DB schema - self.check_db_schema() - # Mark the client as initialized self._mongo_client_initialized = True + # Check db schema self.check_db_schema() except ConnectionFailure as e: diff --git a/test/startup_test.py b/test/startup_test.py index de336b35..963c0a84 100644 --- a/test/startup_test.py +++ b/test/startup_test.py @@ -13,24 +13,20 @@ def test_startups(self): # Test normal startup, should work self.cUtil.setUp() catalog = Catalog(self.cUtil.getCatalogConfig()) - print(f"1. output catalog.version is: {catalog.version(self.cUtil.anonymous_ctx())[0]}") self.assertTrue(semantic_version.validate(catalog.version(self.cUtil.anonymous_ctx())[0])) # Test empty startup without DB version should work self.cUtil.setUpEmpty() catalog = Catalog(self.cUtil.getCatalogConfig()) - print(f"2. output catalog.version is: {catalog.version(self.cUtil.anonymous_ctx())[0]}") self.assertTrue(semantic_version.validate(catalog.version(self.cUtil.anonymous_ctx())[0])) # Test empty startup with several different valid versions should work self.cUtil.setUpEmpty(db_version=3) catalog = Catalog(self.cUtil.getCatalogConfig()) - print(f"3. output catalog.version is: {catalog.version(self.cUtil.anonymous_ctx())[0]}") self.assertTrue(semantic_version.validate(catalog.version(self.cUtil.anonymous_ctx())[0])) self.cUtil.setUpEmpty(db_version=4) catalog = Catalog(self.cUtil.getCatalogConfig()) - print(f"4. output catalog.version is: {catalog.version(self.cUtil.anonymous_ctx())[0]}") self.assertTrue(semantic_version.validate(catalog.version(self.cUtil.anonymous_ctx())[0])) # Startup with version that is too high should fail @@ -39,6 +35,7 @@ def test_startups(self): catalog = None with self.assertRaises(IOError) as e: catalog = Catalog(self.cUtil.getCatalogConfig()) + catalog.cc.db.check_db_schema() self.assertEqual(str(e.exception), 'Incompatible DB versions. Expecting DB V4, found DV V2525. You are ' 'probably running an old version of the service. Start up failed.') From 0143000dec84bcf2fa2c42226b42a2c513240c68 Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 5 Feb 2025 15:00:52 -0800 Subject: [PATCH 056/110] trigger lazy load collection --- test/startup_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/startup_test.py b/test/startup_test.py index 963c0a84..8a6fbd3e 100644 --- a/test/startup_test.py +++ b/test/startup_test.py @@ -33,9 +33,12 @@ def test_startups(self): self.cUtil.setUpEmpty(db_version=2525) catalog = None + ctx = self.cUtil.user_ctx() + user = ctx['user_id'] with self.assertRaises(IOError) as e: catalog = Catalog(self.cUtil.getCatalogConfig()) - catalog.cc.db.check_db_schema() + # Trigger the lazy load collection, which will call the check_db_schema() method. + catalog.list_favorites(ctx, user)[0] self.assertEqual(str(e.exception), 'Incompatible DB versions. Expecting DB V4, found DV V2525. You are ' 'probably running an old version of the service. Start up failed.') From d04e2eb9ba8df1bb6ce61ca8551a6ff9531922c0 Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 6 Feb 2025 12:54:39 -0800 Subject: [PATCH 057/110] clean up files --- Makefile | 5 ----- lib/biokbase/catalog/db.py | 4 ++-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 41e971c8..cbe4e165 100644 --- a/Makefile +++ b/Makefile @@ -27,11 +27,6 @@ LIB_DIR = lib PATH := kb_sdk/bin:$(PATH) -default: init - -init: - git submodule init - git submodule update compile-kb-module: kb-sdk compile $(SPEC_FILE) \ diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 1106e8cc..4729e9b8 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -133,7 +133,7 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech self._mongo_client_initialized = False self.mongo = None self.db = None - self.index_created = defaultdict(int) + self.index_created = defaultdict(bool) self._db_schema_checked = False def _initialize_mongo_client(self): @@ -174,7 +174,7 @@ def _get_collection(self, collection_name): self._initialize_mongo_client() if not self.index_created[collection_name]: self._create_indexes(collection_name) - self.index_created[collection_name] = 1 + self.index_created[collection_name] = True return self.db[collection_name] # Define getters for each collection From 59c653e0dd5719b9cd10b7bfc4e0b7c1e1304011 Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 7 Feb 2025 15:59:13 -0800 Subject: [PATCH 058/110] remove comments from Makefile --- .github/workflows/test.yml | 10 +++++----- Makefile | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 87056952..872c3931 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,11 +41,11 @@ jobs: with: python-version: ${{matrix.python-version}} - - name: Install Docker Compose - run: | - sudo curl -L "https://github.com/docker/compose/releases/download/v2.32.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose - docker-compose --version + # - name: Install Docker Compose + # run: | + # sudo curl -L "https://github.com/docker/compose/releases/download/v2.32.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + # sudo chmod +x /usr/local/bin/docker-compose + # docker-compose --version - name: Install dependencies and set up test config shell: bash diff --git a/Makefile b/Makefile index cbe4e165..f7db502c 100644 --- a/Makefile +++ b/Makefile @@ -97,7 +97,7 @@ setup-tests: mkdir -p $(TESTLIB)/biokbase mkdir -p $(TESTDIR)/nms rsync -av lib/biokbase/* $(TESTLIB)/biokbase/. --exclude *.bak-* -# rsync -av kbapi_common/lib/biokbase/* $(TESTLIB)/biokbase/. + # cd narrative_method_store; make; make build-classpath-list; # rsync -av narrative_method_store/lib/biokbase/* $(TESTLIB)/biokbase/. From 0859dbea1237f4d1cc938c35c2b6de3681b1d976 Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 7 Feb 2025 16:04:08 -0800 Subject: [PATCH 059/110] validate Install Docker Compose step in ci --- .github/workflows/test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 872c3931..87056952 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,11 +41,11 @@ jobs: with: python-version: ${{matrix.python-version}} - # - name: Install Docker Compose - # run: | - # sudo curl -L "https://github.com/docker/compose/releases/download/v2.32.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose - # sudo chmod +x /usr/local/bin/docker-compose - # docker-compose --version + - name: Install Docker Compose + run: | + sudo curl -L "https://github.com/docker/compose/releases/download/v2.32.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + docker-compose --version - name: Install dependencies and set up test config shell: bash From dda9eac0e3cf98690a9ef0d45b97b766bfef1810 Mon Sep 17 00:00:00 2001 From: Sijie Date: Mon, 10 Feb 2025 10:42:13 -0800 Subject: [PATCH 060/110] use docker compose --- .github/workflows/test.yml | 6 ------ test/run_tests.sh | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 87056952..e76f0841 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,12 +41,6 @@ jobs: with: python-version: ${{matrix.python-version}} - - name: Install Docker Compose - run: | - sudo curl -L "https://github.com/docker/compose/releases/download/v2.32.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose - docker-compose --version - - name: Install dependencies and set up test config shell: bash env: diff --git a/test/run_tests.sh b/test/run_tests.sh index 0d14ef31..0b9e03ff 100755 --- a/test/run_tests.sh +++ b/test/run_tests.sh @@ -11,7 +11,7 @@ # start the test NMS endpoint echo 'Starting NMS...' export KB_DEPLOYMENT_CONFIG=test.cfg -docker-compose -f docker-compose_nms.yml up -d +docker compose -f docker-compose_nms.yml up -d NMS_PID=$! echo 'Starting Mock Auth API...' @@ -56,7 +56,7 @@ echo "unit tests returned with error code=${TEST_RETURN_CODE}" #### SHUTDOWN stuff and exit # stop NMS -docker-compose -f docker-compose_nms.yml down +docker compose -f docker-compose_nms.yml down #stop Docker containers docker stop mock-auth From bbfc2dbc7ec1657c1a44c7842388b4dc3dc84a2b Mon Sep 17 00:00:00 2001 From: Sijie Date: Mon, 10 Feb 2025 11:49:27 -0800 Subject: [PATCH 061/110] map NMS to the local port 27018 --- .github/workflows/test.yml | 4 ++-- test/docker-compose_nms.yml | 2 +- test/test.cfg.example | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e76f0841..2831ba69 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,7 +29,7 @@ jobs: mongo: image: mongo:${{matrix.mongo-version}} ports: - - 27018:27017 + - 27017:27017 options: --name mongo${{matrix.mongo-version}} steps: @@ -50,7 +50,7 @@ jobs: run: | # test mongo connection - curl http://localhost:27018 + curl http://localhost:27017 returncode=$? if [ $returncode != 0 ]; then exit $returncode; fi diff --git a/test/docker-compose_nms.yml b/test/docker-compose_nms.yml index aa5cd762..906381f3 100644 --- a/test/docker-compose_nms.yml +++ b/test/docker-compose_nms.yml @@ -44,4 +44,4 @@ services: mongo: image: "mongo:7.0.4" ports: - - "27017:27017" \ No newline at end of file + - "27018:27017" \ No newline at end of file diff --git a/test/test.cfg.example b/test/test.cfg.example index 8572f8e1..2de74726 100644 --- a/test/test.cfg.example +++ b/test/test.cfg.example @@ -24,8 +24,8 @@ test-user-2 = wstester2 test-module-repo-1 = https://github.com/kbaseIncubator/catalog_test_module test-module-repo-2 = https://github.com/kbaseIncubator/catalog_test_module.git -# host where mongo lives, e.g. localhost:27018, for tests there should be not authentication required -mongodb-host = localhost:27018 +# host where mongo lives, e.g. localhost:27017, for tests there should be not authentication required +mongodb-host = localhost:27017 # docker registry host endpoint docker-registry-host = localhost:5000 From 293953cdcb5f9af3b347372e2c8e5c08505bc1e9 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 11 Feb 2025 12:50:20 -0800 Subject: [PATCH 062/110] Add a clearer comment for the environment variable in test.yml --- .github/workflows/test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2831ba69..d7ba4781 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,8 +75,7 @@ jobs: sed -i "s#^nms-admin-token.*#nms-admin-token=$KBASE_CI_TOKEN#" test/test.cfg sed -i "s#^method-spec-admin-users.*#method-spec-admin-users=$ADMIN_USER#" test/test.cfg - # setup env variables in docker-compose_nms.yml file - echo "ADMIN_USER=$ADMIN_USER" + # This env var is used in the catalog test docker compose file for starting NMS echo "ADMIN_USER=$ADMIN_USER" >> $GITHUB_ENV - name: Run tests From 7b4a495e433437e0f2c2bc431f273b07a5d70410 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 11 Feb 2025 13:29:59 -0800 Subject: [PATCH 063/110] move env to jobs level --- .github/workflows/test.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d7ba4781..d480d745 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,6 +32,11 @@ jobs: - 27017:27017 options: --name mongo${{matrix.mongo-version}} + env: + KBASE_CI_TOKEN: ${{ secrets.KBASE_CI_TOKEN }} + # This env var is also used in the catalog test docker compose file for starting NMS + ADMIN_USER: ${{ secrets.KBASE_BOT_USER_CI }} + steps: - name: Repo checkout uses: actions/checkout@v4 @@ -43,9 +48,6 @@ jobs: - name: Install dependencies and set up test config shell: bash - env: - KBASE_CI_TOKEN: ${{ secrets.KBASE_CI_TOKEN }} - ADMIN_USER: ${{ secrets.KBASE_BOT_USER_CI }} run: | @@ -75,9 +77,6 @@ jobs: sed -i "s#^nms-admin-token.*#nms-admin-token=$KBASE_CI_TOKEN#" test/test.cfg sed -i "s#^method-spec-admin-users.*#method-spec-admin-users=$ADMIN_USER#" test/test.cfg - # This env var is used in the catalog test docker compose file for starting NMS - echo "ADMIN_USER=$ADMIN_USER" >> $GITHUB_ENV - - name: Run tests shell: bash run: make test From 377f65108eea20bbbe4a9ac37fc15301ab484585 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 11 Feb 2025 14:11:48 -0800 Subject: [PATCH 064/110] add more comments in server.py --- test/mock_auth/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/mock_auth/server.py b/test/mock_auth/server.py index 19d66e4f..fc1b84ac 100644 --- a/test/mock_auth/server.py +++ b/test/mock_auth/server.py @@ -17,7 +17,8 @@ sys.exit(1) endpoints = [] -# The order is preserved here to ensure routes capture requests correctly. +# Mock server serves routes in a specific order, and this is the current order required for the tests to pass. +# Changing the order may cause the tests to fail, so it's important to maintain this sequence. catalog_mock_auth = ['auth_admin.json', 'auth_invalid.json', 'auth_missing.json', 'auth_non_admin.json'] for path in catalog_mock_auth: if path.endswith('.json'): From ae1b22445f28fc5a0705f36dc2dcc6cab1e936ba Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 11 Feb 2025 15:17:33 -0800 Subject: [PATCH 065/110] add more comments in db.py --- lib/biokbase/catalog/db.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 4729e9b8..4b030f27 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -129,7 +129,12 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech self.mongo_psswd = mongo_psswd self.mongo_authMechanism = mongo_authMechanism - # MongoDB client and database initialization deferred + # The MongoDB client and database initialization are deferred in lazy loading + # to prevent issues with forking processes. When the client is initialized before forking, + # it can cause race conditions or unsafe operations, triggering the error "MongoClient opened before fork." + # By deferring the initialization, the client is only created when needed, + # avoiding the potential conflicts with forking and ensuring safer operation in multi-process environments. + # https://pymongo.readthedocs.io/en/stable/faq.html#is-pymongo-fork-safe self._mongo_client_initialized = False self.mongo = None self.db = None @@ -140,6 +145,7 @@ def _initialize_mongo_client(self): """Initialize MongoDB client and collections lazily.""" if not self._mongo_client_initialized: try: + # This is only tested manually if self.mongo_user and self.mongo_psswd: # Connection string with authentication self.mongo = MongoClient( From 370c65e9d79a20015e0f22806aa717bf83b25f2d Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 13 Feb 2025 10:03:57 -0800 Subject: [PATCH 066/110] remove check_db_schema related functions --- lib/biokbase/catalog/db.py | 230 ------------------------------------- test/startup_test.py | 23 ---- 2 files changed, 253 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 4b030f27..af0d91d2 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -139,7 +139,6 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech self.mongo = None self.db = None self.index_created = defaultdict(bool) - self._db_schema_checked = False def _initialize_mongo_client(self): """Initialize MongoDB client and collections lazily.""" @@ -165,9 +164,6 @@ def _initialize_mongo_client(self): # Mark the client as initialized self._mongo_client_initialized = True - # Check db schema - self.check_db_schema() - except ConnectionFailure as e: error_msg = "Cannot connect to Mongo server\n" error_msg += "ERROR -- {}:\n{}".format( @@ -1360,229 +1356,3 @@ def get_secure_config_params(self, module_name): filter = {"module_name_lc": module_name.lower()} return list(self.secure_config_params.find(filter, selection)) - # DB version handling - - # todo: add 'in-progress' flag so if something goes done during an update, or if - # another server is already starting an update, we can skip or abort - def check_db_schema(self): - - if self._db_schema_checked: - return - - # Mark that we've checked the schema - self._db_schema_checked = True - - db_version = self.get_db_version() - print('db_version=' + str(db_version)) - - if db_version < 2: - print('Updating DB schema to V2...') - self.update_db_1_to_2() - self.update_db_version(2) - print('done.') - - if db_version < 3: - print('Updating DB schema to V3...') - self.update_db_2_to_3() - self.update_db_version(3) - print('done.') - - if db_version < 4: - print('Updating DB schema to V4...') - self.update_db_3_to_4() - self.update_db_version(4) - print('done.') - - if db_version > 4: - # bad version! - raise IOError( - 'Incompatible DB versions. Expecting DB V4, found DV V' + str(db_version) + - '. You are probably running an old version of the service. Start up failed.') - - def get_db_version(self): - # version is a collection that should only have a single - version_collection = self.db[MongoCatalogDBI._DB_VERSION] - ver = version_collection.find_one({}) - if (ver): - return ver['version'] - else: - # if there is no version document, then we are DB v1 - self.update_db_version(1) - return 1 - - def update_db_version(self, version): - # make sure we can't have two version documents - version_collection = self.db[MongoCatalogDBI._DB_VERSION] - version_collection.create_index('version_doc', unique=True, sparse=False) - version_collection.update_one({'version_doc': True}, {'$set': {'version': version}}, - upsert=True) - - # version 1 kept released module versions in a map, version 2 updates that to a list - # and adds dynamic service tags - def update_db_1_to_2(self): - modules_collection = self.db[MongoCatalogDBI._MODULES] - for m in modules_collection.find({'release_versions': {'$exists': True}}): - release_version_list = [] - for timestamp in m['release_versions']: - m['release_versions'][timestamp]['dynamic_service'] = 0 - release_version_list.append(m['release_versions'][timestamp]) - - modules_collection.update_one( - {'_id': m['_id']}, - { - '$unset': {'release_versions': ''}, - '$set': {'release_version_list': release_version_list} - }) - - # make sure everything has the dynamic service flag - if not 'dynamic_service' in m['info']: - modules_collection.update_one( - {'_id': m['_id']}, - {'$set': {'info.dynamic_service': 0}}) - - if m['current_versions']['release']: - if not 'dynamic_service' in m['current_versions']['release']: - modules_collection.update_one( - {'_id': m['_id']}, - {'$set': {'current_versions.release.dynamic_service': 0}}) - - if m['current_versions']['beta']: - if not 'dynamic_service' in m['current_versions']['beta']: - modules_collection.update_one( - {'_id': m['_id']}, - {'$set': {'current_versions.beta.dynamic_service': 0}}) - - if m['current_versions']['dev']: - if not 'dynamic_service' in m['current_versions']['dev']: - modules_collection.update_one( - {'_id': m['_id']}, - {'$set': {'current_versions.dev.dynamic_service': 0}}) - - # also ensure the execution stats fields have correct names - exec_stats_apps_colleciton = self.db[MongoCatalogDBI._EXEC_STATS_APPS] - exec_stats_apps_colleciton.update_many({'avg_queue_time': {'$exists': True}}, - {'$rename': {'avg_queue_time': 'total_queue_time', - 'avg_exec_time': 'total_exec_time'}}) - exec_stats_apps_colleciton.update_many({'avg_queue_time': {'$exists': True}}, - {'$rename': {'avg_queue_time': 'total_queue_time', - 'avg_exec_time': 'total_exec_time'}}) - - # version 3 moves the module version information out of the module document into - # a separate module versions collection. - def update_db_2_to_3(self): - - module_versions_collection = self.db[MongoCatalogDBI._MODULE_VERSIONS] - module_versions_collection.create_index('module_name_lc', sparse=False) - module_versions_collection.create_index('git_commit_hash', sparse=False) - module_versions_collection.create_index([ - ('module_name_lc', ASCENDING), - ('git_commit_hash', ASCENDING)], - unique=True, sparse=False) - - modules_collection = self.db[MongoCatalogDBI._MODULES] - # update all module versions - for m in modules_collection.find({}): - - # skip modules that have not been properly registered, might want to delete these later - if 'module_name' not in m or 'module_name_lc' not in m: - continue - - # skip modules that have not been properly registered, might want to delete these later - if 'info' not in m: - continue - - # first migrate over all released versions and update the module document - new_release_version_list = [] - for rVer in m['release_version_list']: - rVer['released'] = 1 - self.prepare_version_doc_for_db_2_to_3_update(rVer, m) - try: - module_versions_collection.insert_one(rVer) - except: - print(' - Warning - ' + rVer['module_name'] + '.' + rVer[ - 'git_commit_hash'] + ' already inserted, skipping.') - new_release_version_list.append({ - 'git_commit_hash': rVer['git_commit_hash'] - }) - modules_collection.update_one( - {'_id': m['_id']}, - {'$set': {'release_version_list': new_release_version_list}} - ) - - for tag in ['release', 'beta', 'dev']: - # we can skip release tags because that is already in the release_version_list - if m['current_versions'].get(tag) is not None: - modVer = m['current_versions'][tag] - modVer['released'] = 0 - self.prepare_version_doc_for_db_2_to_3_update(modVer, m) - if modVer.get('git_commit_hash') is not None: - try: - module_versions_collection.insert_one(modVer) - except Exception as e: - # we expect this to happen for all 'release' tags and if, say, a - # version still tagged as dev/beta has been released - print(f" - Warning - {tag} ver of {modVer['module_name']}." - f"{modVer['git_commit_hash']} already inserted, skipping.") - modules_collection.update_one( - {'_id': m['_id']}, - {'$set': {'current_versions.' + tag: { - 'git_commit_hash': modVer['git_commit_hash']}}} - ) - else: - modules_collection.update_one( - {'_id': m['_id']}, - {'$set': {'current_versions.' + tag: None}} - ) - print(' -> completed ' + m['module_name']) - - def prepare_version_doc_for_db_2_to_3_update(self, version, module): - version['module_name'] = module['module_name'] - version['module_name_lc'] = module['module_name_lc'] - version['module_description'] = module['info']['description'] - version['module_language'] = module['info']['language'] - version['notes'] = '' - if 'local_functions' not in version: - version['local_functions'] = [] - if 'compilation_report' not in version: - version['compilation_report'] = None - if 'narrative_methods' not in version: - version['narrative_methods'] = [] - if 'release_timestamp' not in version: - if version['released'] == 1: - version['release_timestamp'] = version['timestamp'] - else: - version['release_timestamp'] = None - - # version 4 performs a small modification to the client group structure and volume_mounts structure - def update_db_3_to_4(self): - - # make sure we don't have any indecies on the collections - volume_mounts_collection = self.db[MongoCatalogDBI._VOLUME_MOUNTS] - client_groups_collection = self.db[MongoCatalogDBI._CLIENT_GROUPS] - - volume_mounts_collection.drop_indexes() - client_groups_collection.drop_indexes() - - # update the volume_mounts, just need to rename app_id to function_name - for vm in volume_mounts_collection.find({}): - if 'app_id' in vm and 'function_name' not in vm: - volume_mounts_collection.update_one( - {'_id': vm['_id']}, - {'$set': {'function_name': vm['app_id']}, '$unset': {'app_id': 1}} - ) - - for cg in client_groups_collection.find({}): - if 'app_id' in cg: - client_groups_collection.delete_one({'_id': cg['_id']}) - tokens = cg['app_id'].split('/') - if len(tokens) != 2: - print( - ' When updating client groups, bad app_id found. Record will be lost:') - pprint.pprint(cg) - new_cg = { - 'module_name': tokens[0], - 'module_name_lc': tokens[0].lower(), - 'function_name': tokens[1], - 'client_groups': cg['client_groups'] - } - client_groups_collection.insert_one(new_cg) diff --git a/test/startup_test.py b/test/startup_test.py index 8a6fbd3e..0b1db4a1 100644 --- a/test/startup_test.py +++ b/test/startup_test.py @@ -20,29 +20,6 @@ def test_startups(self): catalog = Catalog(self.cUtil.getCatalogConfig()) self.assertTrue(semantic_version.validate(catalog.version(self.cUtil.anonymous_ctx())[0])) - # Test empty startup with several different valid versions should work - self.cUtil.setUpEmpty(db_version=3) - catalog = Catalog(self.cUtil.getCatalogConfig()) - self.assertTrue(semantic_version.validate(catalog.version(self.cUtil.anonymous_ctx())[0])) - - self.cUtil.setUpEmpty(db_version=4) - catalog = Catalog(self.cUtil.getCatalogConfig()) - self.assertTrue(semantic_version.validate(catalog.version(self.cUtil.anonymous_ctx())[0])) - - # Startup with version that is too high should fail - self.cUtil.setUpEmpty(db_version=2525) - - catalog = None - ctx = self.cUtil.user_ctx() - user = ctx['user_id'] - with self.assertRaises(IOError) as e: - catalog = Catalog(self.cUtil.getCatalogConfig()) - # Trigger the lazy load collection, which will call the check_db_schema() method. - catalog.list_favorites(ctx, user)[0] - self.assertEqual(str(e.exception), - 'Incompatible DB versions. Expecting DB V4, found DV V2525. You are ' - 'probably running an old version of the service. Start up failed.') - @classmethod def setUpClass(cls): print('++++++++++++ RUNNING startup_test.py +++++++++++') From 0015ac01c527e250be0c15a4e5086e593cf1af7c Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 13 Feb 2025 10:40:56 -0800 Subject: [PATCH 067/110] revert back changes --- lib/biokbase/catalog/db.py | 230 +++++++++++++++++++++++++++++++++++++ test/startup_test.py | 23 ++++ 2 files changed, 253 insertions(+) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index af0d91d2..90e40ff8 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -139,6 +139,7 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech self.mongo = None self.db = None self.index_created = defaultdict(bool) + self._db_schema_checked = False def _initialize_mongo_client(self): """Initialize MongoDB client and collections lazily.""" @@ -164,6 +165,9 @@ def _initialize_mongo_client(self): # Mark the client as initialized self._mongo_client_initialized = True + # Check db schema + self.check_db_schema() + except ConnectionFailure as e: error_msg = "Cannot connect to Mongo server\n" error_msg += "ERROR -- {}:\n{}".format( @@ -1356,3 +1360,229 @@ def get_secure_config_params(self, module_name): filter = {"module_name_lc": module_name.lower()} return list(self.secure_config_params.find(filter, selection)) + # DB version handling + + # todo: add 'in-progress' flag so if something goes done during an update, or if + # another server is already starting an update, we can skip or abort + def check_db_schema(self): + + if self._db_schema_checked: + return + + # Mark that we've checked the schema + self._db_schema_checked = True + + db_version = self.get_db_version() + print('db_version=' + str(db_version)) + + if db_version < 2: + print('Updating DB schema to V2...') + self.update_db_1_to_2() + self.update_db_version(2) + print('done.') + + if db_version < 3: + print('Updating DB schema to V3...') + self.update_db_2_to_3() + self.update_db_version(3) + print('done.') + + if db_version < 4: + print('Updating DB schema to V4...') + self.update_db_3_to_4() + self.update_db_version(4) + print('done.') + + if db_version > 4: + # bad version! + raise IOError( + 'Incompatible DB versions. Expecting DB V4, found DV V' + str(db_version) + + '. You are probably running an old version of the service. Start up failed.') + + def get_db_version(self): + # version is a collection that should only have a single + version_collection = self.db[MongoCatalogDBI._DB_VERSION] + ver = version_collection.find_one({}) + if (ver): + return ver['version'] + else: + # if there is no version document, then we are DB v1 + self.update_db_version(1) + return 1 + + def update_db_version(self, version): + # make sure we can't have two version documents + version_collection = self.db[MongoCatalogDBI._DB_VERSION] + version_collection.create_index('version_doc', unique=True, sparse=False) + version_collection.update_one({'version_doc': True}, {'$set': {'version': version}}, + upsert=True) + + # version 1 kept released module versions in a map, version 2 updates that to a list + # and adds dynamic service tags + def update_db_1_to_2(self): + modules_collection = self.db[MongoCatalogDBI._MODULES] + for m in modules_collection.find({'release_versions': {'$exists': True}}): + release_version_list = [] + for timestamp in m['release_versions']: + m['release_versions'][timestamp]['dynamic_service'] = 0 + release_version_list.append(m['release_versions'][timestamp]) + + modules_collection.update_one( + {'_id': m['_id']}, + { + '$unset': {'release_versions': ''}, + '$set': {'release_version_list': release_version_list} + }) + + # make sure everything has the dynamic service flag + if not 'dynamic_service' in m['info']: + modules_collection.update_one( + {'_id': m['_id']}, + {'$set': {'info.dynamic_service': 0}}) + + if m['current_versions']['release']: + if not 'dynamic_service' in m['current_versions']['release']: + modules_collection.update_one( + {'_id': m['_id']}, + {'$set': {'current_versions.release.dynamic_service': 0}}) + + if m['current_versions']['beta']: + if not 'dynamic_service' in m['current_versions']['beta']: + modules_collection.update_one( + {'_id': m['_id']}, + {'$set': {'current_versions.beta.dynamic_service': 0}}) + + if m['current_versions']['dev']: + if not 'dynamic_service' in m['current_versions']['dev']: + modules_collection.update_one( + {'_id': m['_id']}, + {'$set': {'current_versions.dev.dynamic_service': 0}}) + + # also ensure the execution stats fields have correct names + exec_stats_apps_colleciton = self.db[MongoCatalogDBI._EXEC_STATS_APPS] + exec_stats_apps_colleciton.update_many({'avg_queue_time': {'$exists': True}}, + {'$rename': {'avg_queue_time': 'total_queue_time', + 'avg_exec_time': 'total_exec_time'}}) + exec_stats_apps_colleciton.update_many({'avg_queue_time': {'$exists': True}}, + {'$rename': {'avg_queue_time': 'total_queue_time', + 'avg_exec_time': 'total_exec_time'}}) + + # version 3 moves the module version information out of the module document into + # a separate module versions collection. + def update_db_2_to_3(self): + + module_versions_collection = self.db[MongoCatalogDBI._MODULE_VERSIONS] + module_versions_collection.create_index('module_name_lc', sparse=False) + module_versions_collection.create_index('git_commit_hash', sparse=False) + module_versions_collection.create_index([ + ('module_name_lc', ASCENDING), + ('git_commit_hash', ASCENDING)], + unique=True, sparse=False) + + modules_collection = self.db[MongoCatalogDBI._MODULES] + # update all module versions + for m in modules_collection.find({}): + + # skip modules that have not been properly registered, might want to delete these later + if 'module_name' not in m or 'module_name_lc' not in m: + continue + + # skip modules that have not been properly registered, might want to delete these later + if 'info' not in m: + continue + + # first migrate over all released versions and update the module document + new_release_version_list = [] + for rVer in m['release_version_list']: + rVer['released'] = 1 + self.prepare_version_doc_for_db_2_to_3_update(rVer, m) + try: + module_versions_collection.insert_one(rVer) + except: + print(' - Warning - ' + rVer['module_name'] + '.' + rVer[ + 'git_commit_hash'] + ' already inserted, skipping.') + new_release_version_list.append({ + 'git_commit_hash': rVer['git_commit_hash'] + }) + modules_collection.update_one( + {'_id': m['_id']}, + {'$set': {'release_version_list': new_release_version_list}} + ) + + for tag in ['release', 'beta', 'dev']: + # we can skip release tags because that is already in the release_version_list + if m['current_versions'].get(tag) is not None: + modVer = m['current_versions'][tag] + modVer['released'] = 0 + self.prepare_version_doc_for_db_2_to_3_update(modVer, m) + if modVer.get('git_commit_hash') is not None: + try: + module_versions_collection.insert_one(modVer) + except Exception as e: + # we expect this to happen for all 'release' tags and if, say, a + # version still tagged as dev/beta has been released + print(f" - Warning - {tag} ver of {modVer['module_name']}." + f"{modVer['git_commit_hash']} already inserted, skipping.") + modules_collection.update_one( + {'_id': m['_id']}, + {'$set': {'current_versions.' + tag: { + 'git_commit_hash': modVer['git_commit_hash']}}} + ) + else: + modules_collection.update_one( + {'_id': m['_id']}, + {'$set': {'current_versions.' + tag: None}} + ) + print(' -> completed ' + m['module_name']) + + def prepare_version_doc_for_db_2_to_3_update(self, version, module): + version['module_name'] = module['module_name'] + version['module_name_lc'] = module['module_name_lc'] + version['module_description'] = module['info']['description'] + version['module_language'] = module['info']['language'] + version['notes'] = '' + if 'local_functions' not in version: + version['local_functions'] = [] + if 'compilation_report' not in version: + version['compilation_report'] = None + if 'narrative_methods' not in version: + version['narrative_methods'] = [] + if 'release_timestamp' not in version: + if version['released'] == 1: + version['release_timestamp'] = version['timestamp'] + else: + version['release_timestamp'] = None + + # version 4 performs a small modification to the client group structure and volume_mounts structure + def update_db_3_to_4(self): + + # make sure we don't have any indecies on the collections + volume_mounts_collection = self.db[MongoCatalogDBI._VOLUME_MOUNTS] + client_groups_collection = self.db[MongoCatalogDBI._CLIENT_GROUPS] + + volume_mounts_collection.drop_indexes() + client_groups_collection.drop_indexes() + + # update the volume_mounts, just need to rename app_id to function_name + for vm in volume_mounts_collection.find({}): + if 'app_id' in vm and 'function_name' not in vm: + volume_mounts_collection.update_one( + {'_id': vm['_id']}, + {'$set': {'function_name': vm['app_id']}, '$unset': {'app_id': 1}} + ) + + for cg in client_groups_collection.find({}): + if 'app_id' in cg: + client_groups_collection.delete_one({'_id': cg['_id']}) + tokens = cg['app_id'].split('/') + if len(tokens) != 2: + print( + ' When updating client groups, bad app_id found. Record will be lost:') + pprint.pprint(cg) + new_cg = { + 'module_name': tokens[0], + 'module_name_lc': tokens[0].lower(), + 'function_name': tokens[1], + 'client_groups': cg['client_groups'] + } + client_groups_collection.insert_one(new_cg) \ No newline at end of file diff --git a/test/startup_test.py b/test/startup_test.py index 0b1db4a1..8a6fbd3e 100644 --- a/test/startup_test.py +++ b/test/startup_test.py @@ -20,6 +20,29 @@ def test_startups(self): catalog = Catalog(self.cUtil.getCatalogConfig()) self.assertTrue(semantic_version.validate(catalog.version(self.cUtil.anonymous_ctx())[0])) + # Test empty startup with several different valid versions should work + self.cUtil.setUpEmpty(db_version=3) + catalog = Catalog(self.cUtil.getCatalogConfig()) + self.assertTrue(semantic_version.validate(catalog.version(self.cUtil.anonymous_ctx())[0])) + + self.cUtil.setUpEmpty(db_version=4) + catalog = Catalog(self.cUtil.getCatalogConfig()) + self.assertTrue(semantic_version.validate(catalog.version(self.cUtil.anonymous_ctx())[0])) + + # Startup with version that is too high should fail + self.cUtil.setUpEmpty(db_version=2525) + + catalog = None + ctx = self.cUtil.user_ctx() + user = ctx['user_id'] + with self.assertRaises(IOError) as e: + catalog = Catalog(self.cUtil.getCatalogConfig()) + # Trigger the lazy load collection, which will call the check_db_schema() method. + catalog.list_favorites(ctx, user)[0] + self.assertEqual(str(e.exception), + 'Incompatible DB versions. Expecting DB V4, found DV V2525. You are ' + 'probably running an old version of the service. Start up failed.') + @classmethod def setUpClass(cls): print('++++++++++++ RUNNING startup_test.py +++++++++++') From 1428c8525c5de69c137b836b80b23a2074dce7d6 Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 13 Feb 2025 10:57:03 -0800 Subject: [PATCH 068/110] move self._db_schema_checked flag outside function --- lib/biokbase/catalog/db.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 90e40ff8..a2a320a6 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -166,7 +166,10 @@ def _initialize_mongo_client(self): self._mongo_client_initialized = True # Check db schema - self.check_db_schema() + if not self._db_schema_checked: + self.check_db_schema() + # Mark that we've checked the schema + self._db_schema_checked = True except ConnectionFailure as e: error_msg = "Cannot connect to Mongo server\n" @@ -1366,12 +1369,6 @@ def get_secure_config_params(self, module_name): # another server is already starting an update, we can skip or abort def check_db_schema(self): - if self._db_schema_checked: - return - - # Mark that we've checked the schema - self._db_schema_checked = True - db_version = self.get_db_version() print('db_version=' + str(db_version)) From d106e96c901f57db2fe314edee57c71208516823 Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 13 Feb 2025 17:03:42 -0800 Subject: [PATCH 069/110] refactor _create_indexes function --- lib/biokbase/catalog/db.py | 120 ++++------------------------- lib/biokbase/catalog/db_indexes.py | 118 ++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 104 deletions(-) create mode 100644 lib/biokbase/catalog/db_indexes.py diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index a2a320a6..94906879 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -8,6 +8,8 @@ from pymongo import MongoClient from pymongo.errors import ConnectionFailure +from biokbase.catalog.db_indexes import DBIndexes + ''' 1) User registers git_repo git_url @@ -239,110 +241,20 @@ def _create_indexes(self, collection_name): """Create indexes for the given collection lazily.""" collection = self.db[collection_name] - # Make sure we have an index on module and git_repo_url - if collection_name == MongoCatalogDBI._MODULE_VERSIONS: - collection.create_index('module_name_lc', sparse=False) - collection.create_index('git_commit_hash', sparse=False) - collection.create_index([ - ('module_name_lc', ASCENDING), - ('git_commit_hash', ASCENDING)], - unique=True, sparse=False) - - elif collection_name == MongoCatalogDBI._LOCAL_FUNCTIONS: - # Make sure we have a unique index on module_name_lc and git_commit_hash - collection.create_index('function_id') - collection.create_index([ - ('module_name_lc', ASCENDING), - ('function_id', ASCENDING), - ('git_commit_hash', ASCENDING)], - unique=True, sparse=False) - - # local function indecies - collection.create_index('module_name_lc') - collection.create_index('git_commit_hash') - collection.create_index('function_id') - collection.create_index([ - ('module_name_lc', ASCENDING), - ('function_id', ASCENDING), - ('git_commit_hash', ASCENDING)], - unique=True, sparse=False) - - # developers indecies - elif collection_name == MongoCatalogDBI._DEVELOPERS: - collection.create_index('kb_username', unique=True) - - elif collection_name == MongoCatalogDBI._BUILD_LOGS: - collection.create_index('registration_id', unique=True) - collection.create_index('module_name_lc') - collection.create_index('timestamp') - collection.create_index('registration') - collection.create_index('git_url') - collection.create_index('current_versions.release.release_timestamp') - - # for favorites - elif collection_name == MongoCatalogDBI._FAVORITES: - collection.create_index('user') - collection.create_index('module_name_lc') - collection.create_index('id') - # you can only favorite a method once, so put a unique index on the triple - collection.create_index([ - ('user', ASCENDING), - ('id', ASCENDING), - ('module_name_lc', ASCENDING)], - unique=True, sparse=False) - - # execution stats - elif collection_name == MongoCatalogDBI._EXEC_STATS_RAW: - collection.create_index('user_id', - unique=False, sparse=False) - collection.create_index([('app_module_name', ASCENDING), - ('app_id', ASCENDING)], - unique=False, sparse=True) - collection.create_index([('func_module_name', ASCENDING), - ('func_name', ASCENDING)], - unique=False, sparse=True) - collection.create_index('creation_time', - unique=False, sparse=False) - collection.create_index('finish_time', - unique=False, sparse=False) - - elif collection_name == MongoCatalogDBI._EXEC_STATS_APPS: - collection.create_index('module_name', - unique=False, sparse=True) - collection.create_index([('full_app_id', ASCENDING), - ('type', ASCENDING), - ('time_range', ASCENDING)], - unique=True, sparse=False) - collection.create_index([('type', ASCENDING), - ('time_range', ASCENDING)], - unique=False, sparse=False) - - elif collection_name == MongoCatalogDBI._EXEC_STATS_USERS: - collection.create_index([('user_id', ASCENDING), - ('type', ASCENDING), - ('time_range', ASCENDING)], - unique=True, sparse=False) - - # client groups and volume mounts - elif collection_name == MongoCatalogDBI._CLIENT_GROUPS: - collection.create_index([('module_name_lc', ASCENDING), - ('function_name', ASCENDING)], - unique=True, sparse=False) - - elif collection_name == MongoCatalogDBI._VOLUME_MOUNTS: - collection.create_index([('client_group', ASCENDING), - ('module_name_lc', ASCENDING), - ('function_name', ASCENDING)], - unique=True, sparse=False) - - # hidden configuration parameters - elif collection_name == MongoCatalogDBI._SECURE_CONFIG_PARAMS: - collection.create_index('module_name_lc') - collection.create_index([ - ('module_name_lc', ASCENDING), - ('version', ASCENDING), - ('param_name', ASCENDING)], - unique=True, sparse=False) + # Get the indexes for the collection from the DBIndexes class + indexes = DBIndexes.get_indexes(collection_name) + + # Loop through and create indexes + for index in indexes: + if isinstance(index, tuple): + # Index with unique, and sparse flag + collection.create_index(index[0], unique=index[1], sparse=index[2]) + elif isinstance(index, list): + # index with multiple fields, unqie, and sparse flags + collection.create_index(index[:-2], unique=index[-2], sparse=index[-1]) + else: + # Single field index + collection.create_index(index) def is_registered(self, module_name='', git_url=''): if not module_name and not git_url: diff --git a/lib/biokbase/catalog/db_indexes.py b/lib/biokbase/catalog/db_indexes.py new file mode 100644 index 00000000..187d56d9 --- /dev/null +++ b/lib/biokbase/catalog/db_indexes.py @@ -0,0 +1,118 @@ +from pymongo import ASCENDING + + +class DBIndexes: + @staticmethod + def get_indexes(collection_name): + index_creation_map = { + "module_versions": [ + "module_name_lc", + "git_commit_hash", + [ + ("module_name_lc", ASCENDING), + ("git_commit_hash", ASCENDING), + True, + False, + ], + ], + "local_functions": [ + "function_id", + [ + ("module_name_lc", ASCENDING), + ("function_id", ASCENDING), + ("git_commit_hash", ASCENDING), + True, + False, + ], + "module_name_lc", + "git_commit_hash", + [ + ("module_name_lc", ASCENDING), + ("function_id", ASCENDING), + ("git_commit_hash", ASCENDING), + True, + False, + ], + ], + "developers": [("kb_username", True, False)], + "build_logs": [ + ("registration_id", True, False), + "module_name_lc", + "timestamp", + "registration", + "git_url", + "current_versions.release.release_timestamp", + ], + "favorites": [ + "user", + "module_name_lc", + "id", + [ + ("user", ASCENDING), + ("id", ASCENDING), + ("module_name_lc", ASCENDING), + True, + False, + ], + ], + "exec_stats_raw": [ + "user_id", + [("app_module_name", ASCENDING), ("app_id", ASCENDING), False, True], + [ + ("func_module_name", ASCENDING), + ("func_name", ASCENDING), + False, + True, + ], + "creation_time", + "finish_time", + ], + "exec_stats_apps": [ + ("module_name", False, True), + [ + ("full_app_id", ASCENDING), + ("type", ASCENDING), + ("time_range", ASCENDING), + True, + False, + ], + [("type", ASCENDING), ("time_range", ASCENDING), False, True], + ], + "exec_stats_users": [ + [ + ("user_id", ASCENDING), + ("type", ASCENDING), + ("time_range", ASCENDING), + True, + False, + ] + ], + "client_groups": [ + [ + ("module_name_lc", ASCENDING), + ("function_name", ASCENDING), + True, + False, + ] + ], + "volume_mounts": [ + [ + ("client_group", ASCENDING), + ("module_name_lc", ASCENDING), + ("function_name", ASCENDING), + True, + False, + ] + ], + "secure_config_params": [ + "module_name_lc", + [ + ("module_name_lc", ASCENDING), + ("version", ASCENDING), + ("param_name", ASCENDING), + True, + False, + ], + ], + } + return index_creation_map.get(collection_name, []) From 64f6fdb72b4a894d7233fb57a91fa3b7a01897af Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 14 Feb 2025 10:26:42 -0800 Subject: [PATCH 070/110] encapsulate create_indexes in the DBIndexes class --- lib/biokbase/catalog/db.py | 15 +-------------- lib/biokbase/catalog/db_indexes.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 94906879..6ac99a2b 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -240,21 +240,8 @@ def secure_config_params(self): def _create_indexes(self, collection_name): """Create indexes for the given collection lazily.""" collection = self.db[collection_name] - - # Get the indexes for the collection from the DBIndexes class indexes = DBIndexes.get_indexes(collection_name) - - # Loop through and create indexes - for index in indexes: - if isinstance(index, tuple): - # Index with unique, and sparse flag - collection.create_index(index[0], unique=index[1], sparse=index[2]) - elif isinstance(index, list): - # index with multiple fields, unqie, and sparse flags - collection.create_index(index[:-2], unique=index[-2], sparse=index[-1]) - else: - # Single field index - collection.create_index(index) + DBIndexes.create_indexes(collection, indexes) def is_registered(self, module_name='', git_url=''): if not module_name and not git_url: diff --git a/lib/biokbase/catalog/db_indexes.py b/lib/biokbase/catalog/db_indexes.py index 187d56d9..a633b9da 100644 --- a/lib/biokbase/catalog/db_indexes.py +++ b/lib/biokbase/catalog/db_indexes.py @@ -116,3 +116,17 @@ def get_indexes(collection_name): ], } return index_creation_map.get(collection_name, []) + + @staticmethod + def create_indexes(collection, indexes): + # Loop through and create indexes + for index in indexes: + if isinstance(index, tuple): + # Index with unique, and sparse flag + collection.create_index(index[0], unique=index[1], sparse=index[2]) + elif isinstance(index, list): + # Index with multiple fields, unique, and sparse flags + collection.create_index(index[:-2], unique=index[-2], sparse=index[-1]) + else: + # Single field index + collection.create_index(index) From d6bd55665bd6087ec50057c48349c6d770a50747 Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 14 Feb 2025 16:03:38 -0800 Subject: [PATCH 071/110] add comment at the top of server.py file --- test/mock_auth/server.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/mock_auth/server.py b/test/mock_auth/server.py index fc1b84ac..9a212a1d 100644 --- a/test/mock_auth/server.py +++ b/test/mock_auth/server.py @@ -1,3 +1,7 @@ +# We are replacing this file in the mock_auth server container so that the Mock server serves routes +# in a specific order, which will allow the test to pass. The original code uses os.listdir(), +# which does not guarantee preserving the order of the JSON files in the config directory. + import os import sys import json From 96a72e6d757d5580071e69f21286762607b0dae9 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 18 Feb 2025 16:27:50 -0800 Subject: [PATCH 072/110] refactor db.py file --- lib/biokbase/catalog/db.py | 331 +++++++++++++++++++---------- lib/biokbase/catalog/db_indexes.py | 132 ------------ 2 files changed, 223 insertions(+), 240 deletions(-) delete mode 100644 lib/biokbase/catalog/db_indexes.py diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 6ac99a2b..b90de177 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -1,15 +1,14 @@ import copy +from functools import wraps import pprint import traceback -from collections import defaultdict +import threading from pymongo import ASCENDING from pymongo import DESCENDING from pymongo import MongoClient from pymongo.errors import ConnectionFailure -from biokbase.catalog.db_indexes import DBIndexes - ''' 1) User registers git_repo git_url @@ -102,6 +101,13 @@ } ''' +def initialize_mongo_client(func): + """Decorator to ensure MongoDB client is initialized before calling the API function.""" + @wraps(func) + def wrapper(self, *args, **kwargs): + self._initialize_mongo_client() # Ensure MongoDB client is initialized before the method executes + return func(self, *args, **kwargs) + return wrapper class MongoCatalogDBI: # Collection Names @@ -131,118 +137,170 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech self.mongo_psswd = mongo_psswd self.mongo_authMechanism = mongo_authMechanism - # The MongoDB client and database initialization are deferred in lazy loading - # to prevent issues with forking processes. When the client is initialized before forking, - # it can cause race conditions or unsafe operations, triggering the error "MongoClient opened before fork." - # By deferring the initialization, the client is only created when needed, - # avoiding the potential conflicts with forking and ensuring safer operation in multi-process environments. - # https://pymongo.readthedocs.io/en/stable/faq.html#is-pymongo-fork-safe - self._mongo_client_initialized = False - self.mongo = None - self.db = None - self.index_created = defaultdict(bool) - self._db_schema_checked = False + self.lock = threading.Lock() + self.mongo_client = None + + # Initialize mongo client + self._initialize_mongo_client() + + # Create dbs and collections + self._create_collections() + + # Check the db schema + self.check_db_schema() + + # Create indexes + self._create_indexes() + + # Close the MongoDB client manually + self.mongo_client.close() + self.mongo_client = None def _initialize_mongo_client(self): - """Initialize MongoDB client and collections lazily.""" - if not self._mongo_client_initialized: - try: - # This is only tested manually - if self.mongo_user and self.mongo_psswd: - # Connection string with authentication - self.mongo = MongoClient( - f"mongodb://{self.mongo_user}:{self.mongo_psswd}@{self.mongo_host}/{self.mongo_db}?authMechanism={self.mongo_authMechanism}" - ) - else: - # Connection string without authentication - self.mongo = MongoClient(f"mongodb://{self.mongo_host}") + """Initialize MongoDB client with lock to prevent race conditions.""" + try: + # Use the lock to ensure only one thread initializes the mongo client at a time + with self.lock: + if self.mongo_client is None: # Check if mongo_client is already initialized + # This is only tested manually + if self.mongo_user and self.mongo_psswd: + # Connection string with authentication + self.mongo_client = MongoClient( + f"mongodb://{self.mongo_user}:{self.mongo_psswd}@{self.mongo_host}/{self.mongo_db}?authMechanism={self.mongo_authMechanism}" + ) + else: + # Connection string without authentication + self.mongo_client = MongoClient(f"mongodb://{self.mongo_host}") - # Force a call to server to verify the connection - self.mongo.server_info() - print("Connection successful!") + # Force a call to server to verify the connection + self.mongo_client.server_info() + print("Connection successful!") - # Grab a handle to the database - self.db = self.mongo[self.mongo_db] + except ConnectionFailure as e: + error_msg = "Cannot connect to Mongo server\n" + error_msg += "ERROR -- {}:\n{}".format( + e, "".join(traceback.format_exception(None, e, e.__traceback__)) + ) + raise ValueError(error_msg) + + def _create_collections(self): + """Grab a handle to the database and collections.""" + if not self.mongo_client: + self._initialize_mongo_client() + + self.db = self.mongo_client[self.mongo_db] + self.modules = self.db[MongoCatalogDBI._MODULES] + self.module_versions = self.db[MongoCatalogDBI._MODULE_VERSIONS] + + self.local_functions = self.db[MongoCatalogDBI._LOCAL_FUNCTIONS] + self.developers = self.db[MongoCatalogDBI._DEVELOPERS] + self.build_logs = self.db[MongoCatalogDBI._BUILD_LOGS] + self.favorites = self.db[MongoCatalogDBI._FAVORITES] + self.client_groups = self.db[MongoCatalogDBI._CLIENT_GROUPS] + self.volume_mounts = self.db[MongoCatalogDBI._VOLUME_MOUNTS] + + self.exec_stats_raw = self.db[MongoCatalogDBI._EXEC_STATS_RAW] + self.exec_stats_apps = self.db[MongoCatalogDBI._EXEC_STATS_APPS] + self.exec_stats_users = self.db[MongoCatalogDBI._EXEC_STATS_USERS] + + self.secure_config_params = self.db[MongoCatalogDBI._SECURE_CONFIG_PARAMS] + + def _create_indexes(self): + # Make sure we have an index on module and git_repo_url + self.module_versions.create_index('module_name_lc', sparse=False) + self.module_versions.create_index('git_commit_hash', sparse=False) + self.module_versions.create_index([ + ('module_name_lc', ASCENDING), + ('git_commit_hash', ASCENDING)], + unique=True, sparse=False) - # Mark the client as initialized - self._mongo_client_initialized = True + # Make sure we have a unique index on module_name_lc and git_commit_hash + self.local_functions.create_index('function_id') + self.local_functions.create_index([ + ('module_name_lc', ASCENDING), + ('function_id', ASCENDING), + ('git_commit_hash', ASCENDING)], + unique=True, sparse=False) - # Check db schema - if not self._db_schema_checked: - self.check_db_schema() - # Mark that we've checked the schema - self._db_schema_checked = True + # local function indecies + self.local_functions.create_index('module_name_lc') + self.local_functions.create_index('git_commit_hash') + self.local_functions.create_index('function_id') + self.local_functions.create_index([ + ('module_name_lc', ASCENDING), + ('function_id', ASCENDING), + ('git_commit_hash', ASCENDING)], + unique=True, sparse=False) - except ConnectionFailure as e: - error_msg = "Cannot connect to Mongo server\n" - error_msg += "ERROR -- {}:\n{}".format( - e, "".join(traceback.format_exception(None, e, e.__traceback__)) - ) - raise ValueError(error_msg) + # developers indecies + self.developers.create_index('kb_username', unique=True) + + self.build_logs.create_index('registration_id', unique=True) + self.build_logs.create_index('module_name_lc') + self.build_logs.create_index('timestamp') + self.build_logs.create_index('registration') + self.build_logs.create_index('git_url') + self.build_logs.create_index('current_versions.release.release_timestamp') + + # for favorites + self.favorites.create_index('user') + self.favorites.create_index('module_name_lc') + self.favorites.create_index('id') + # you can only favorite a method once, so put a unique index on the triple + self.favorites.create_index([ + ('user', ASCENDING), + ('id', ASCENDING), + ('module_name_lc', ASCENDING)], + unique=True, sparse=False) - def _get_collection(self, collection_name): - """Lazily load collections.""" - self._initialize_mongo_client() - if not self.index_created[collection_name]: - self._create_indexes(collection_name) - self.index_created[collection_name] = True - return self.db[collection_name] - - # Define getters for each collection - @property - def modules(self): - return self._get_collection(MongoCatalogDBI._MODULES) - - @property - def module_versions(self): - return self._get_collection(MongoCatalogDBI._MODULE_VERSIONS) - - @property - def local_functions(self): - return self._get_collection(MongoCatalogDBI._LOCAL_FUNCTIONS) - - @property - def developers(self): - return self._get_collection(MongoCatalogDBI._DEVELOPERS) - - @property - def build_logs(self): - return self._get_collection(MongoCatalogDBI._BUILD_LOGS) - - @property - def favorites(self): - return self._get_collection(MongoCatalogDBI._FAVORITES) - - @property - def client_groups(self): - return self._get_collection(MongoCatalogDBI._CLIENT_GROUPS) - - @property - def volume_mounts(self): - return self._get_collection(MongoCatalogDBI._VOLUME_MOUNTS) - - @property - def exec_stats_raw(self): - return self._get_collection(MongoCatalogDBI._EXEC_STATS_RAW) - - @property - def exec_stats_apps(self): - return self._get_collection(MongoCatalogDBI._EXEC_STATS_APPS) - - @property - def exec_stats_users(self): - return self._get_collection(MongoCatalogDBI._EXEC_STATS_USERS) - - @property - def secure_config_params(self): - return self._get_collection(MongoCatalogDBI._SECURE_CONFIG_PARAMS) - - def _create_indexes(self, collection_name): - """Create indexes for the given collection lazily.""" - collection = self.db[collection_name] - indexes = DBIndexes.get_indexes(collection_name) - DBIndexes.create_indexes(collection, indexes) + # execution stats + self.exec_stats_raw.create_index('user_id', + unique=False, sparse=False) + self.exec_stats_raw.create_index([('app_module_name', ASCENDING), + ('app_id', ASCENDING)], + unique=False, sparse=True) + self.exec_stats_raw.create_index([('func_module_name', ASCENDING), + ('func_name', ASCENDING)], + unique=False, sparse=True) + self.exec_stats_raw.create_index('creation_time', + unique=False, sparse=False) + self.exec_stats_raw.create_index('finish_time', + unique=False, sparse=False) + + self.exec_stats_apps.create_index('module_name', + unique=False, sparse=True) + self.exec_stats_apps.create_index([('full_app_id', ASCENDING), + ('type', ASCENDING), + ('time_range', ASCENDING)], + unique=True, sparse=False) + self.exec_stats_apps.create_index([('type', ASCENDING), + ('time_range', ASCENDING)], + unique=False, sparse=False) + + self.exec_stats_users.create_index([('user_id', ASCENDING), + ('type', ASCENDING), + ('time_range', ASCENDING)], + unique=True, sparse=False) + + # client groups and volume mounts + self.client_groups.create_index([('module_name_lc', ASCENDING), + ('function_name', ASCENDING)], + unique=True, sparse=False) + + self.volume_mounts.create_index([('client_group', ASCENDING), + ('module_name_lc', ASCENDING), + ('function_name', ASCENDING)], + unique=True, sparse=False) + + # hidden configuration parameters + self.secure_config_params.create_index('module_name_lc') + self.secure_config_params.create_index([ + ('module_name_lc', ASCENDING), + ('version', ASCENDING), + ('param_name', ASCENDING)], + unique=True, sparse=False) + @initialize_mongo_client def is_registered(self, module_name='', git_url=''): if not module_name and not git_url: return False @@ -252,6 +310,7 @@ def is_registered(self, module_name='', git_url=''): return True return False + @initialize_mongo_client def module_name_lc_exists(self, module_name_lc=''): if not module_name_lc: return False @@ -261,6 +320,7 @@ def module_name_lc_exists(self, module_name_lc=''): return False #### SET methods + @initialize_mongo_client def create_new_build_log(self, registration_id, timestamp, registration_state, git_url): build_log = { 'registration_id': registration_id, @@ -272,27 +332,32 @@ def create_new_build_log(self, registration_id, timestamp, registration_state, g } self.build_logs.insert_one(build_log) + @initialize_mongo_client def delete_build_log(self, registration_id): self.build_logs.delete_one({'registration_id': registration_id}) # new_lines is a list to objects, each representing a line # the object structure is : {'content':... 'error':True/False} + @initialize_mongo_client def append_to_build_log(self, registration_id, new_lines): result = self.build_logs.update_one({'registration_id': registration_id}, {'$push': {'log': {'$each': new_lines}}}) return self._check_update_result(result) + @initialize_mongo_client def set_build_log_state(self, registration_id, registration_state, error_message=''): result = self.build_logs.update_one({'registration_id': registration_id}, {'$set': {'registration': registration_state, 'error_message': error_message}}) return self._check_update_result(result) + @initialize_mongo_client def set_build_log_module_name(self, registration_id, module_name): result = self.build_logs.update_one({'registration_id': registration_id}, {'$set': {'module_name_lc': module_name.lower()}}) return self._check_update_result(result) + @initialize_mongo_client def list_builds(self, skip=0, limit=1000, @@ -339,6 +404,7 @@ def list_builds(self, # slice arg is used in the mongo query for getting lines. It is either a # pos int (get first n lines), neg int (last n lines), or array [skip, limit] + @initialize_mongo_client def get_parsed_build_log(self, registration_id, slice_arg=None): selection = { 'registration_id': 1, @@ -355,6 +421,7 @@ def get_parsed_build_log(self, registration_id, slice_arg=None): return self.build_logs.find_one({'registration_id': registration_id}, selection) + @initialize_mongo_client def register_new_module(self, git_url, username, timestamp, registration_state, registration_id): # get current time since epoch in ms in utc @@ -380,6 +447,7 @@ def register_new_module(self, git_url, username, timestamp, registration_state, # last_state is for concurency control. If set, it will match on state as well, and will fail # if the last_state does not match indicating another process changed the state + @initialize_mongo_client def set_module_registration_state(self, module_name='', git_url='', new_state=None, last_state=None, error_message=''): if new_state: @@ -391,6 +459,7 @@ def set_module_registration_state(self, module_name='', git_url='', new_state=No return self._check_update_result(result) return False + @initialize_mongo_client def set_module_release_state(self, module_name='', git_url='', new_state=None, last_state=None, review_message=''): if new_state: @@ -402,6 +471,7 @@ def set_module_release_state(self, module_name='', git_url='', new_state=None, l return self._check_update_result(result) return False + @initialize_mongo_client def push_beta_to_release(self, module_name='', git_url='', release_timestamp=None): current_versions = self.get_module_current_versions(module_name=module_name, @@ -430,6 +500,7 @@ def push_beta_to_release(self, module_name='', git_url='', release_timestamp=Non }}) return self._check_update_result(result) + @initialize_mongo_client def push_dev_to_beta(self, module_name='', git_url=''): current_versions = self.get_module_current_versions(module_name=module_name, git_url=git_url, @@ -440,6 +511,7 @@ def push_dev_to_beta(self, module_name='', git_url=''): return self._check_update_result(result) + @initialize_mongo_client def update_dev_version(self, version_info): if version_info: if 'git_commit_hash' in version_info and 'module_name_lc' in version_info: @@ -468,6 +540,7 @@ def update_dev_version(self, version_info): raise ValueError('git_commit_hash is required to register a new version') return False + @initialize_mongo_client def save_local_function_specs(self, local_functions): # just using insert doesn't accept a list of docs in mongo 2.6, so loop for now for l in local_functions: @@ -482,6 +555,7 @@ def save_local_function_specs(self, local_functions): return error return None + @initialize_mongo_client def lookup_module_versions(self, module_name, git_commit_hash=None, released=None, included_fields=[], excluded_fields=[]): @@ -502,6 +576,7 @@ def lookup_module_versions(self, module_name, git_commit_hash=None, released=Non return list(self.module_versions.find(query, selection)) + @initialize_mongo_client def list_local_function_info(self, release_tag=None, module_names=[]): git_commit_hash_list = [] @@ -574,6 +649,7 @@ def list_local_function_info(self, release_tag=None, module_names=[]): return returned_funcs + @initialize_mongo_client def get_local_function_spec(self, functions): result_list = [] @@ -672,6 +748,7 @@ def get_local_function_spec(self, functions): return result_list + @initialize_mongo_client def set_module_name(self, git_url, module_name): if not module_name: raise ValueError('module_name must be defined to set a module name') @@ -680,6 +757,7 @@ def set_module_name(self, git_url, module_name): '$set': {'module_name': module_name, 'module_name_lc': module_name.lower()}}) return self._check_update_result(result) + @initialize_mongo_client def set_module_info(self, info, module_name='', git_url=''): if not info: raise ValueError('info must be defined to set the info for a module') @@ -689,6 +767,7 @@ def set_module_info(self, info, module_name='', git_url=''): result = self.modules.update_one(query, {'$set': {'info': info}}) return self._check_update_result(result) + @initialize_mongo_client def set_module_owners(self, owners, module_name='', git_url=''): if not owners: raise ValueError('owners must be defined to set the owners for a module') @@ -699,16 +778,19 @@ def set_module_owners(self, owners, module_name='', git_url=''): return self._check_update_result(result) # active = True | False + @initialize_mongo_client def set_module_active_state(self, active, module_name='', git_url=''): query = self._get_mongo_query(git_url=git_url, module_name=module_name) result = self.modules.update_one(query, {'$set': {'state.active': active}}) return self._check_update_result(result) #### GET methods + @initialize_mongo_client def get_module_state(self, module_name='', git_url=''): query = self._get_mongo_query(module_name=module_name, git_url=git_url) return self.modules.find_one(query, ['state'])['state'] + @initialize_mongo_client def get_module_current_versions(self, module_name='', git_url='', substitute_versions=True): query = self._get_mongo_query(module_name=module_name, git_url=git_url) module_document = self.modules.find_one(query, ['module_name_lc', 'current_versions']) @@ -716,10 +798,12 @@ def get_module_current_versions(self, module_name='', git_url='', substitute_ver self.substitute_hashes_for_version_info([module_document]) return module_document['current_versions'] + @initialize_mongo_client def get_module_owners(self, module_name='', git_url=''): query = self._get_mongo_query(module_name=module_name, git_url=git_url) return self.modules.find_one(query, ['owners'])['owners'] + @initialize_mongo_client def get_module_details(self, module_name='', git_url='', substitute_versions=True): query = self._get_mongo_query(module_name=module_name, git_url=git_url) module_details = self.modules.find_one(query, ['module_name', 'module_name_lc', 'git_url', @@ -729,6 +813,7 @@ def get_module_details(self, module_name='', git_url='', substitute_versions=Tru self.substitute_hashes_for_version_info([module_details]) return module_details + @initialize_mongo_client def get_module_full_details(self, module_name='', git_url='', substitute_versions=True): query = self._get_mongo_query(module_name=module_name, git_url=git_url) module_document = self.modules.find_one(query) @@ -737,7 +822,7 @@ def get_module_full_details(self, module_name='', git_url='', substitute_version return module_document #### LIST / SEARCH methods - + @initialize_mongo_client def find_basic_module_info(self, query): selection = { '_id': 0, @@ -750,6 +835,7 @@ def find_basic_module_info(self, query): } return list(self.modules.find(query, selection)) + @initialize_mongo_client def find_current_versions_and_owners(self, query): result = list(self.modules.find(query, {'module_name': 1, 'module_name_lc': 1, 'git_url': 1, @@ -757,6 +843,7 @@ def find_current_versions_and_owners(self, query): self.substitute_hashes_for_version_info(result) return result + @initialize_mongo_client def substitute_hashes_for_version_info(self, module_list): # get all the version commit hashes @@ -804,6 +891,7 @@ def substitute_hashes_for_version_info(self, module_list): return module_list # tag should be one of dev, beta, release - do checking outside of this method + @initialize_mongo_client def list_service_module_versions_with_tag(self, tag): mods = list(self.modules.find({'info.dynamic_service': 1}, @@ -825,6 +913,7 @@ def list_service_module_versions_with_tag(self, tag): return result # all released service module versions + @initialize_mongo_client def list_all_released_service_module_versions(self): return list(self.module_versions.find( { @@ -840,19 +929,21 @@ def list_all_released_service_module_versions(self): })) #### developer check methods - + @initialize_mongo_client def approve_developer(self, developer): # if the developer is already on the list, just return if self.is_approved_developer([developer])[0]: return self.developers.insert_one({'kb_username': developer}) + @initialize_mongo_client def revoke_developer(self, developer): # if the developer is not on the list, throw an error (maybe a typo, so let's catch it) if not self.is_approved_developer([developer])[0]: raise ValueError('Cannot revoke "' + developer + '", that developer was not found.') self.developers.delete_one({'kb_username': developer}) + @initialize_mongo_client def is_approved_developer(self, usernames): # TODO: optimize, but I expect the list of usernames will be fairly small, so we can loop. Regardless, in # old mongo (2.x) I think this is even faster in most cases than using $in within a very large list @@ -865,9 +956,11 @@ def is_approved_developer(self, usernames): is_approved.append(False) return is_approved + @initialize_mongo_client def list_approved_developers(self): return list(self.developers.find({}, {'kb_username': 1, '_id': 0})) + @initialize_mongo_client def migrate_module_to_new_git_url(self, module_name, current_git_url, new_git_url): if not new_git_url.strip(): raise ValueError('New git url is required to migrate_module_to_new_git_url.') @@ -879,6 +972,7 @@ def migrate_module_to_new_git_url(self, module_name, current_git_url, new_git_ur result = self.modules.update_one(query, {'$set': {'git_url': new_git_url.strip()}}) return self._check_update_result(result) + @initialize_mongo_client def delete_module(self, module_name, git_url): if not module_name and not git_url: raise ValueError('Module name or git url is required to delete a module.') @@ -898,6 +992,7 @@ def delete_module(self, module_name, git_url): result = self.modules.delete_one({'_id': module_details['_id']}) return self._check_update_result(result) + @initialize_mongo_client def add_favorite(self, module_name, app_id, username, timestamp): favoriteAddition = { 'user': username, @@ -912,6 +1007,7 @@ def add_favorite(self, module_name, app_id, username, timestamp): favoriteAddition['timestamp'] = timestamp self.favorites.insert_one(favoriteAddition) + @initialize_mongo_client def remove_favorite(self, module_name, app_id, username): favoriteAddition = { 'user': username, @@ -926,16 +1022,19 @@ def remove_favorite(self, module_name, app_id, username): result = self.favorites.delete_one({'_id': found['_id']}) return self._check_update_result(result) + @initialize_mongo_client def list_user_favorites(self, username): query = {'user': username} selection = {'_id': 0, 'module_name_lc': 1, 'id': 1, 'timestamp': 1} return list(self.favorites.find(query, selection).sort('timestamp', DESCENDING)) + @initialize_mongo_client def list_app_favorites(self, module_name, app_id): query = {'module_name_lc': module_name.strip().lower(), 'id': app_id.strip()} selection = {'_id': 0, 'user': 1, 'timestamp': 1} return list(self.favorites.find(query, selection).sort('timestamp', DESCENDING)) + @initialize_mongo_client def aggregate_favorites_over_apps(self, module_names_lc): ### WARNING! If we switch to Mongo 3.x, the result object will change and this will break @@ -979,6 +1078,7 @@ def aggregate_favorites_over_apps(self, module_names_lc): return counts # DEPRECATED! temporary function until everything is migrated to new client group structure + @initialize_mongo_client def list_client_groups(self, app_ids): if app_ids is not None: selection = { @@ -1006,6 +1106,7 @@ def list_client_groups(self, app_ids): } return list(self.client_groups.find({}, selection)) + @initialize_mongo_client def set_client_group_config(self, config): config['module_name_lc'] = config['module_name'].lower() return self._check_update_result(self.client_groups.replace_one( @@ -1017,6 +1118,7 @@ def set_client_group_config(self, config): upsert=True )) + @initialize_mongo_client def remove_client_group_config(self, config): config['module_name_lc'] = config['module_name'].lower() return self._check_update_result(self.client_groups.delete_one( @@ -1026,6 +1128,7 @@ def remove_client_group_config(self, config): } )) + @initialize_mongo_client def list_client_group_configs(self, filter): selection = {"_id": 0, "module_name_lc": 0} if 'module_name' in filter: @@ -1033,6 +1136,7 @@ def list_client_group_configs(self, filter): del (filter['module_name']) return list(self.client_groups.find(filter, selection)) + @initialize_mongo_client def set_volume_mount(self, volume_mount): volume_mount['module_name_lc'] = volume_mount['module_name'].lower() return self._check_update_result(self.volume_mounts.replace_one( @@ -1045,6 +1149,7 @@ def set_volume_mount(self, volume_mount): upsert=True )) + @initialize_mongo_client def remove_volume_mount(self, volume_mount): volume_mount['module_name_lc'] = volume_mount['module_name'].lower() return self._check_update_result(self.volume_mounts.delete_one( @@ -1054,6 +1159,7 @@ def remove_volume_mount(self, volume_mount): 'client_group': volume_mount['client_group'] })) + @initialize_mongo_client def list_volume_mounts(self, filter): selection = {"_id": 0, "module_name_lc": 0} if 'module_name' in filter: @@ -1090,6 +1196,7 @@ def _check_update_result(self, result): return None return '{}' + @initialize_mongo_client def add_exec_stats_raw(self, user_id, app_module_name, app_id, func_module_name, func_name, git_commit_hash, creation_time, exec_start_time, finish_time, is_error, job_id): @@ -1108,6 +1215,7 @@ def add_exec_stats_raw(self, user_id, app_module_name, app_id, func_module_name, } self.exec_stats_raw.insert_one(stats) + @initialize_mongo_client def add_exec_stats_apps(self, app_module_name, app_id, creation_time, exec_start_time, finish_time, is_error, type, time_range): if not app_id: @@ -1130,6 +1238,7 @@ def add_exec_stats_apps(self, app_module_name, app_id, creation_time, exec_start {'full_app_id': full_app_id, 'type': type, 'time_range': time_range}, {'$setOnInsert': new_data, '$inc': inc_data}, upsert=True) + @initialize_mongo_client def add_exec_stats_users(self, user_id, creation_time, exec_start_time, finish_time, is_error, type, time_range): queue_time = exec_start_time - creation_time @@ -1144,6 +1253,7 @@ def add_exec_stats_users(self, user_id, creation_time, exec_start_time, {'user_id': user_id, 'type': type, 'time_range': time_range}, {'$inc': inc_data}, upsert=True) + @initialize_mongo_client def get_exec_stats_apps(self, full_app_ids, type, time_range): filter = {} if full_app_ids: @@ -1164,6 +1274,7 @@ def get_exec_stats_apps(self, full_app_ids, type, time_range): } return list(self.exec_stats_apps.find(filter, selection)) + @initialize_mongo_client def aggr_exec_stats_table(self, minTime, maxTime): # setup the query @@ -1218,6 +1329,7 @@ def aggr_exec_stats_table(self, minTime, maxTime): return counts + @initialize_mongo_client def get_exec_raw_stats(self, minTime, maxTime): filter = {} @@ -1233,6 +1345,7 @@ def get_exec_raw_stats(self, minTime, maxTime): return list(self.exec_stats_raw.find(filter, {'_id': 0})) + @initialize_mongo_client def set_secure_config_params(self, data_list): for param_data in data_list: param_data['module_name_lc'] = param_data['module_name'].lower() @@ -1246,6 +1359,7 @@ def set_secure_config_params(self, data_list): param_data, upsert=True) + @initialize_mongo_client def remove_secure_config_params(self, data_list): for param_data in data_list: param_data['module_name_lc'] = param_data['module_name'].lower() @@ -1257,6 +1371,7 @@ def remove_secure_config_params(self, data_list): 'param_name': param_data['param_name'] }) + @initialize_mongo_client def get_secure_config_params(self, module_name): selection = {"_id": 0, "module_name_lc": 0} filter = {"module_name_lc": module_name.lower()} diff --git a/lib/biokbase/catalog/db_indexes.py b/lib/biokbase/catalog/db_indexes.py deleted file mode 100644 index a633b9da..00000000 --- a/lib/biokbase/catalog/db_indexes.py +++ /dev/null @@ -1,132 +0,0 @@ -from pymongo import ASCENDING - - -class DBIndexes: - @staticmethod - def get_indexes(collection_name): - index_creation_map = { - "module_versions": [ - "module_name_lc", - "git_commit_hash", - [ - ("module_name_lc", ASCENDING), - ("git_commit_hash", ASCENDING), - True, - False, - ], - ], - "local_functions": [ - "function_id", - [ - ("module_name_lc", ASCENDING), - ("function_id", ASCENDING), - ("git_commit_hash", ASCENDING), - True, - False, - ], - "module_name_lc", - "git_commit_hash", - [ - ("module_name_lc", ASCENDING), - ("function_id", ASCENDING), - ("git_commit_hash", ASCENDING), - True, - False, - ], - ], - "developers": [("kb_username", True, False)], - "build_logs": [ - ("registration_id", True, False), - "module_name_lc", - "timestamp", - "registration", - "git_url", - "current_versions.release.release_timestamp", - ], - "favorites": [ - "user", - "module_name_lc", - "id", - [ - ("user", ASCENDING), - ("id", ASCENDING), - ("module_name_lc", ASCENDING), - True, - False, - ], - ], - "exec_stats_raw": [ - "user_id", - [("app_module_name", ASCENDING), ("app_id", ASCENDING), False, True], - [ - ("func_module_name", ASCENDING), - ("func_name", ASCENDING), - False, - True, - ], - "creation_time", - "finish_time", - ], - "exec_stats_apps": [ - ("module_name", False, True), - [ - ("full_app_id", ASCENDING), - ("type", ASCENDING), - ("time_range", ASCENDING), - True, - False, - ], - [("type", ASCENDING), ("time_range", ASCENDING), False, True], - ], - "exec_stats_users": [ - [ - ("user_id", ASCENDING), - ("type", ASCENDING), - ("time_range", ASCENDING), - True, - False, - ] - ], - "client_groups": [ - [ - ("module_name_lc", ASCENDING), - ("function_name", ASCENDING), - True, - False, - ] - ], - "volume_mounts": [ - [ - ("client_group", ASCENDING), - ("module_name_lc", ASCENDING), - ("function_name", ASCENDING), - True, - False, - ] - ], - "secure_config_params": [ - "module_name_lc", - [ - ("module_name_lc", ASCENDING), - ("version", ASCENDING), - ("param_name", ASCENDING), - True, - False, - ], - ], - } - return index_creation_map.get(collection_name, []) - - @staticmethod - def create_indexes(collection, indexes): - # Loop through and create indexes - for index in indexes: - if isinstance(index, tuple): - # Index with unique, and sparse flag - collection.create_index(index[0], unique=index[1], sparse=index[2]) - elif isinstance(index, list): - # Index with multiple fields, unique, and sparse flags - collection.create_index(index[:-2], unique=index[-2], sparse=index[-1]) - else: - # Single field index - collection.create_index(index) From a4025adf486c2959f8454080a0d0c353e119ebf1 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 18 Feb 2025 16:48:50 -0800 Subject: [PATCH 073/110] add flag for mongo client --- lib/biokbase/catalog/db.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index b90de177..61a285df 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -139,6 +139,7 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech self.lock = threading.Lock() self.mongo_client = None + self.client_initialized = False # Initialize mongo client self._initialize_mongo_client() @@ -154,14 +155,15 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech # Close the MongoDB client manually self.mongo_client.close() - self.mongo_client = None + self.client_initialized = False + print("MongoDB client closed.") def _initialize_mongo_client(self): """Initialize MongoDB client with lock to prevent race conditions.""" try: # Use the lock to ensure only one thread initializes the mongo client at a time with self.lock: - if self.mongo_client is None: # Check if mongo_client is already initialized + if not self.client_initialized or self.mongo_client is None: # This is only tested manually if self.mongo_user and self.mongo_psswd: # Connection string with authentication @@ -174,6 +176,9 @@ def _initialize_mongo_client(self): # Force a call to server to verify the connection self.mongo_client.server_info() + + # Mark client as initialized + self.client_initialized = True print("Connection successful!") except ConnectionFailure as e: From 46731b132530d4f301b863a48a0191edea6d553a Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 18 Feb 2025 17:06:54 -0800 Subject: [PATCH 074/110] update wrapper function --- lib/biokbase/catalog/db.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 61a285df..bc70d2ed 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -1,5 +1,4 @@ import copy -from functools import wraps import pprint import traceback import threading @@ -102,10 +101,10 @@ ''' def initialize_mongo_client(func): - """Decorator to ensure MongoDB client is initialized before calling the API function.""" - @wraps(func) + """Decorator to ensure MongoDB client is initialized before calling API functions.""" def wrapper(self, *args, **kwargs): - self._initialize_mongo_client() # Ensure MongoDB client is initialized before the method executes + # Ensure MongoClient is initialized before the API method is called + self._initialize_mongo_client() return func(self, *args, **kwargs) return wrapper @@ -160,9 +159,9 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech def _initialize_mongo_client(self): """Initialize MongoDB client with lock to prevent race conditions.""" - try: - # Use the lock to ensure only one thread initializes the mongo client at a time - with self.lock: + # Use the lock to ensure only one thread initializes the mongo client at a time + with self.lock: + try: if not self.client_initialized or self.mongo_client is None: # This is only tested manually if self.mongo_user and self.mongo_psswd: @@ -181,16 +180,16 @@ def _initialize_mongo_client(self): self.client_initialized = True print("Connection successful!") - except ConnectionFailure as e: - error_msg = "Cannot connect to Mongo server\n" - error_msg += "ERROR -- {}:\n{}".format( - e, "".join(traceback.format_exception(None, e, e.__traceback__)) - ) - raise ValueError(error_msg) + except ConnectionFailure as e: + error_msg = "Cannot connect to Mongo server\n" + error_msg += "ERROR -- {}:\n{}".format( + e, "".join(traceback.format_exception(None, e, e.__traceback__)) + ) + raise ValueError(error_msg) def _create_collections(self): """Grab a handle to the database and collections.""" - if not self.mongo_client: + if not self.client_initialized: self._initialize_mongo_client() self.db = self.mongo_client[self.mongo_db] From 9836af91574e43cf3debd5eca963768c2d5b59a6 Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 19 Feb 2025 13:05:58 -0800 Subject: [PATCH 075/110] add initialize_mongo_client function manually --- lib/biokbase/catalog/db.py | 203 +++++++++++++++++++++++-------------- 1 file changed, 127 insertions(+), 76 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index bc70d2ed..e986480d 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -100,13 +100,13 @@ } ''' -def initialize_mongo_client(func): - """Decorator to ensure MongoDB client is initialized before calling API functions.""" - def wrapper(self, *args, **kwargs): - # Ensure MongoClient is initialized before the API method is called - self._initialize_mongo_client() - return func(self, *args, **kwargs) - return wrapper +# def initialize_mongo_client(func): +# """Decorator to ensure MongoDB client is initialized before calling API functions.""" +# def wrapper(self, *args, **kwargs): +# # Ensure MongoClient is initialized before the API method is called +# self._initialize_mongo_client() +# return func(self, *args, **kwargs) +# return wrapper class MongoCatalogDBI: # Collection Names @@ -304,8 +304,9 @@ def _create_indexes(self): ('param_name', ASCENDING)], unique=True, sparse=False) - @initialize_mongo_client + #@initialize_mongo_client def is_registered(self, module_name='', git_url=''): + self._initialize_mongo_client() if not module_name and not git_url: return False query = self._get_mongo_query(module_name=module_name, git_url=git_url) @@ -314,8 +315,9 @@ def is_registered(self, module_name='', git_url=''): return True return False - @initialize_mongo_client + #@initialize_mongo_client def module_name_lc_exists(self, module_name_lc=''): + self._initialize_mongo_client() if not module_name_lc: return False module = self.modules.find_one({'module_name_lc': module_name_lc.lower()}, ['_id']) @@ -324,8 +326,9 @@ def module_name_lc_exists(self, module_name_lc=''): return False #### SET methods - @initialize_mongo_client + #@initialize_mongo_client def create_new_build_log(self, registration_id, timestamp, registration_state, git_url): + self._initialize_mongo_client() build_log = { 'registration_id': registration_id, 'timestamp': timestamp, @@ -336,32 +339,36 @@ def create_new_build_log(self, registration_id, timestamp, registration_state, g } self.build_logs.insert_one(build_log) - @initialize_mongo_client + #@initialize_mongo_client def delete_build_log(self, registration_id): + self._initialize_mongo_client() self.build_logs.delete_one({'registration_id': registration_id}) # new_lines is a list to objects, each representing a line # the object structure is : {'content':... 'error':True/False} - @initialize_mongo_client + #@initialize_mongo_client def append_to_build_log(self, registration_id, new_lines): + self._initialize_mongo_client() result = self.build_logs.update_one({'registration_id': registration_id}, {'$push': {'log': {'$each': new_lines}}}) return self._check_update_result(result) - @initialize_mongo_client + #@initialize_mongo_client def set_build_log_state(self, registration_id, registration_state, error_message=''): + self._initialize_mongo_client() result = self.build_logs.update_one({'registration_id': registration_id}, {'$set': {'registration': registration_state, 'error_message': error_message}}) return self._check_update_result(result) - @initialize_mongo_client + #@initialize_mongo_client def set_build_log_module_name(self, registration_id, module_name): + self._initialize_mongo_client() result = self.build_logs.update_one({'registration_id': registration_id}, {'$set': {'module_name_lc': module_name.lower()}}) return self._check_update_result(result) - @initialize_mongo_client + #@initialize_mongo_client def list_builds(self, skip=0, limit=1000, @@ -370,7 +377,7 @@ def list_builds(self, only_running=False, only_error=False, only_complete=False): - + self._initialize_mongo_client() query = {} registration_match = None @@ -408,8 +415,9 @@ def list_builds(self, # slice arg is used in the mongo query for getting lines. It is either a # pos int (get first n lines), neg int (last n lines), or array [skip, limit] - @initialize_mongo_client + #@initialize_mongo_client def get_parsed_build_log(self, registration_id, slice_arg=None): + self._initialize_mongo_client() selection = { 'registration_id': 1, 'timestamp': 1, @@ -425,9 +433,10 @@ def get_parsed_build_log(self, registration_id, slice_arg=None): return self.build_logs.find_one({'registration_id': registration_id}, selection) - @initialize_mongo_client + #@initialize_mongo_client def register_new_module(self, git_url, username, timestamp, registration_state, registration_id): + self._initialize_mongo_client() # get current time since epoch in ms in utc module = { 'info': {}, @@ -451,9 +460,10 @@ def register_new_module(self, git_url, username, timestamp, registration_state, # last_state is for concurency control. If set, it will match on state as well, and will fail # if the last_state does not match indicating another process changed the state - @initialize_mongo_client + #@initialize_mongo_client def set_module_registration_state(self, module_name='', git_url='', new_state=None, last_state=None, error_message=''): + self._initialize_mongo_client() if new_state: query = self._get_mongo_query(module_name=module_name, git_url=git_url) if last_state: @@ -463,9 +473,10 @@ def set_module_registration_state(self, module_name='', git_url='', new_state=No return self._check_update_result(result) return False - @initialize_mongo_client + #@initialize_mongo_client def set_module_release_state(self, module_name='', git_url='', new_state=None, last_state=None, review_message=''): + self._initialize_mongo_client() if new_state: query = self._get_mongo_query(module_name=module_name, git_url=git_url) if last_state: @@ -475,9 +486,9 @@ def set_module_release_state(self, module_name='', git_url='', new_state=None, l return self._check_update_result(result) return False - @initialize_mongo_client + #@initialize_mongo_client def push_beta_to_release(self, module_name='', git_url='', release_timestamp=None): - + self._initialize_mongo_client() current_versions = self.get_module_current_versions(module_name=module_name, git_url=git_url, substitute_versions=False) @@ -504,8 +515,9 @@ def push_beta_to_release(self, module_name='', git_url='', release_timestamp=Non }}) return self._check_update_result(result) - @initialize_mongo_client + #@initialize_mongo_client def push_dev_to_beta(self, module_name='', git_url=''): + self._initialize_mongo_client() current_versions = self.get_module_current_versions(module_name=module_name, git_url=git_url, substitute_versions=False) @@ -515,8 +527,9 @@ def push_dev_to_beta(self, module_name='', git_url=''): return self._check_update_result(result) - @initialize_mongo_client + #@initialize_mongo_client def update_dev_version(self, version_info): + self._initialize_mongo_client() if version_info: if 'git_commit_hash' in version_info and 'module_name_lc' in version_info: @@ -544,8 +557,9 @@ def update_dev_version(self, version_info): raise ValueError('git_commit_hash is required to register a new version') return False - @initialize_mongo_client + #@initialize_mongo_client def save_local_function_specs(self, local_functions): + self._initialize_mongo_client() # just using insert doesn't accept a list of docs in mongo 2.6, so loop for now for l in local_functions: matcher = {'module_name_lc': l['module_name_lc'], 'function_id': l['function_id'], @@ -559,10 +573,10 @@ def save_local_function_specs(self, local_functions): return error return None - @initialize_mongo_client + #@initialize_mongo_client def lookup_module_versions(self, module_name, git_commit_hash=None, released=None, included_fields=[], excluded_fields=[]): - + self._initialize_mongo_client() query = {'module_name_lc': module_name.strip().lower()} if git_commit_hash is not None: @@ -580,9 +594,9 @@ def lookup_module_versions(self, module_name, git_commit_hash=None, released=Non return list(self.module_versions.find(query, selection)) - @initialize_mongo_client + #@initialize_mongo_client def list_local_function_info(self, release_tag=None, module_names=[]): - + self._initialize_mongo_client() git_commit_hash_list = [] git_commit_hash_release_tag_map = {} @@ -653,9 +667,9 @@ def list_local_function_info(self, release_tag=None, module_names=[]): return returned_funcs - @initialize_mongo_client + #@initialize_mongo_client def get_local_function_spec(self, functions): - + self._initialize_mongo_client() result_list = [] # first lookup all the module info so we can figure out any tags, and make a quick dict @@ -752,8 +766,9 @@ def get_local_function_spec(self, functions): return result_list - @initialize_mongo_client + #@initialize_mongo_client def set_module_name(self, git_url, module_name): + self._initialize_mongo_client() if not module_name: raise ValueError('module_name must be defined to set a module name') query = self._get_mongo_query(git_url=git_url) @@ -761,8 +776,9 @@ def set_module_name(self, git_url, module_name): '$set': {'module_name': module_name, 'module_name_lc': module_name.lower()}}) return self._check_update_result(result) - @initialize_mongo_client + #@initialize_mongo_client def set_module_info(self, info, module_name='', git_url=''): + self._initialize_mongo_client() if not info: raise ValueError('info must be defined to set the info for a module') if type(info) is not dict: @@ -771,8 +787,9 @@ def set_module_info(self, info, module_name='', git_url=''): result = self.modules.update_one(query, {'$set': {'info': info}}) return self._check_update_result(result) - @initialize_mongo_client + #@initialize_mongo_client def set_module_owners(self, owners, module_name='', git_url=''): + self._initialize_mongo_client() if not owners: raise ValueError('owners must be defined to set the owners for a module') if type(owners) is not list: @@ -782,33 +799,38 @@ def set_module_owners(self, owners, module_name='', git_url=''): return self._check_update_result(result) # active = True | False - @initialize_mongo_client + #@initialize_mongo_client def set_module_active_state(self, active, module_name='', git_url=''): + self._initialize_mongo_client() query = self._get_mongo_query(git_url=git_url, module_name=module_name) result = self.modules.update_one(query, {'$set': {'state.active': active}}) return self._check_update_result(result) #### GET methods - @initialize_mongo_client + #@initialize_mongo_client def get_module_state(self, module_name='', git_url=''): + self._initialize_mongo_client() query = self._get_mongo_query(module_name=module_name, git_url=git_url) return self.modules.find_one(query, ['state'])['state'] - @initialize_mongo_client + #@initialize_mongo_client def get_module_current_versions(self, module_name='', git_url='', substitute_versions=True): + self._initialize_mongo_client() query = self._get_mongo_query(module_name=module_name, git_url=git_url) module_document = self.modules.find_one(query, ['module_name_lc', 'current_versions']) if substitute_versions and 'module_name_lc' in module_document: self.substitute_hashes_for_version_info([module_document]) return module_document['current_versions'] - @initialize_mongo_client + #@initialize_mongo_client def get_module_owners(self, module_name='', git_url=''): + self._initialize_mongo_client() query = self._get_mongo_query(module_name=module_name, git_url=git_url) return self.modules.find_one(query, ['owners'])['owners'] - @initialize_mongo_client + #@initialize_mongo_client def get_module_details(self, module_name='', git_url='', substitute_versions=True): + self._initialize_mongo_client() query = self._get_mongo_query(module_name=module_name, git_url=git_url) module_details = self.modules.find_one(query, ['module_name', 'module_name_lc', 'git_url', 'info', 'owners', 'state', @@ -817,8 +839,9 @@ def get_module_details(self, module_name='', git_url='', substitute_versions=Tru self.substitute_hashes_for_version_info([module_details]) return module_details - @initialize_mongo_client + #@initialize_mongo_client def get_module_full_details(self, module_name='', git_url='', substitute_versions=True): + self._initialize_mongo_client() query = self._get_mongo_query(module_name=module_name, git_url=git_url) module_document = self.modules.find_one(query) if substitute_versions and 'module_name_lc' in module_document: @@ -826,8 +849,9 @@ def get_module_full_details(self, module_name='', git_url='', substitute_version return module_document #### LIST / SEARCH methods - @initialize_mongo_client + #@initialize_mongo_client def find_basic_module_info(self, query): + self._initialize_mongo_client() selection = { '_id': 0, 'module_name': 1, @@ -839,17 +863,18 @@ def find_basic_module_info(self, query): } return list(self.modules.find(query, selection)) - @initialize_mongo_client + #@initialize_mongo_client def find_current_versions_and_owners(self, query): + self._initialize_mongo_client() result = list(self.modules.find(query, {'module_name': 1, 'module_name_lc': 1, 'git_url': 1, 'current_versions': 1, 'owners': 1, '_id': 0})) self.substitute_hashes_for_version_info(result) return result - @initialize_mongo_client + #@initialize_mongo_client def substitute_hashes_for_version_info(self, module_list): - + self._initialize_mongo_client() # get all the version commit hashes hash_list = [] for mod in module_list: @@ -895,9 +920,9 @@ def substitute_hashes_for_version_info(self, module_list): return module_list # tag should be one of dev, beta, release - do checking outside of this method - @initialize_mongo_client + #@initialize_mongo_client def list_service_module_versions_with_tag(self, tag): - + self._initialize_mongo_client() mods = list(self.modules.find({'info.dynamic_service': 1}, {'module_name_lc': 1, 'module_name': 1, 'current_versions.' + tag: 1})) @@ -917,8 +942,9 @@ def list_service_module_versions_with_tag(self, tag): return result # all released service module versions - @initialize_mongo_client + #@initialize_mongo_client def list_all_released_service_module_versions(self): + self._initialize_mongo_client() return list(self.module_versions.find( { 'dynamic_service': 1, @@ -933,22 +959,25 @@ def list_all_released_service_module_versions(self): })) #### developer check methods - @initialize_mongo_client + #@initialize_mongo_client def approve_developer(self, developer): + self._initialize_mongo_client() # if the developer is already on the list, just return if self.is_approved_developer([developer])[0]: return self.developers.insert_one({'kb_username': developer}) - @initialize_mongo_client + #@initialize_mongo_client def revoke_developer(self, developer): + self._initialize_mongo_client() # if the developer is not on the list, throw an error (maybe a typo, so let's catch it) if not self.is_approved_developer([developer])[0]: raise ValueError('Cannot revoke "' + developer + '", that developer was not found.') self.developers.delete_one({'kb_username': developer}) - @initialize_mongo_client + #@initialize_mongo_client def is_approved_developer(self, usernames): + self._initialize_mongo_client() # TODO: optimize, but I expect the list of usernames will be fairly small, so we can loop. Regardless, in # old mongo (2.x) I think this is even faster in most cases than using $in within a very large list is_approved = [] @@ -960,12 +989,14 @@ def is_approved_developer(self, usernames): is_approved.append(False) return is_approved - @initialize_mongo_client + #@initialize_mongo_client def list_approved_developers(self): + self._initialize_mongo_client() return list(self.developers.find({}, {'kb_username': 1, '_id': 0})) - @initialize_mongo_client + #@initialize_mongo_client def migrate_module_to_new_git_url(self, module_name, current_git_url, new_git_url): + self._initialize_mongo_client() if not new_git_url.strip(): raise ValueError('New git url is required to migrate_module_to_new_git_url.') query = self._get_mongo_query(module_name=module_name, git_url=current_git_url) @@ -976,8 +1007,9 @@ def migrate_module_to_new_git_url(self, module_name, current_git_url, new_git_ur result = self.modules.update_one(query, {'$set': {'git_url': new_git_url.strip()}}) return self._check_update_result(result) - @initialize_mongo_client + #@initialize_mongo_client def delete_module(self, module_name, git_url): + self._initialize_mongo_client() if not module_name and not git_url: raise ValueError('Module name or git url is required to delete a module.') query = self._get_mongo_query(module_name=module_name, git_url=git_url) @@ -996,8 +1028,9 @@ def delete_module(self, module_name, git_url): result = self.modules.delete_one({'_id': module_details['_id']}) return self._check_update_result(result) - @initialize_mongo_client + #@initialize_mongo_client def add_favorite(self, module_name, app_id, username, timestamp): + self._initialize_mongo_client() favoriteAddition = { 'user': username, 'module_name_lc': module_name.strip().lower(), @@ -1011,8 +1044,9 @@ def add_favorite(self, module_name, app_id, username, timestamp): favoriteAddition['timestamp'] = timestamp self.favorites.insert_one(favoriteAddition) - @initialize_mongo_client + #@initialize_mongo_client def remove_favorite(self, module_name, app_id, username): + self._initialize_mongo_client() favoriteAddition = { 'user': username, 'module_name_lc': module_name.strip().lower(), @@ -1026,20 +1060,23 @@ def remove_favorite(self, module_name, app_id, username): result = self.favorites.delete_one({'_id': found['_id']}) return self._check_update_result(result) - @initialize_mongo_client + #@initialize_mongo_client def list_user_favorites(self, username): + self._initialize_mongo_client() query = {'user': username} selection = {'_id': 0, 'module_name_lc': 1, 'id': 1, 'timestamp': 1} return list(self.favorites.find(query, selection).sort('timestamp', DESCENDING)) - @initialize_mongo_client + #@initialize_mongo_client def list_app_favorites(self, module_name, app_id): + self._initialize_mongo_client() query = {'module_name_lc': module_name.strip().lower(), 'id': app_id.strip()} selection = {'_id': 0, 'user': 1, 'timestamp': 1} return list(self.favorites.find(query, selection).sort('timestamp', DESCENDING)) - @initialize_mongo_client + #@initialize_mongo_client def aggregate_favorites_over_apps(self, module_names_lc): + self._initialize_mongo_client() ### WARNING! If we switch to Mongo 3.x, the result object will change and this will break # setup the query @@ -1082,8 +1119,9 @@ def aggregate_favorites_over_apps(self, module_names_lc): return counts # DEPRECATED! temporary function until everything is migrated to new client group structure - @initialize_mongo_client + #@initialize_mongo_client def list_client_groups(self, app_ids): + self._initialize_mongo_client() if app_ids is not None: selection = { '_id': 0, @@ -1110,8 +1148,9 @@ def list_client_groups(self, app_ids): } return list(self.client_groups.find({}, selection)) - @initialize_mongo_client + #@initialize_mongo_client def set_client_group_config(self, config): + self._initialize_mongo_client() config['module_name_lc'] = config['module_name'].lower() return self._check_update_result(self.client_groups.replace_one( { @@ -1122,8 +1161,9 @@ def set_client_group_config(self, config): upsert=True )) - @initialize_mongo_client + #@initialize_mongo_client def remove_client_group_config(self, config): + self._initialize_mongo_client() config['module_name_lc'] = config['module_name'].lower() return self._check_update_result(self.client_groups.delete_one( { @@ -1132,16 +1172,18 @@ def remove_client_group_config(self, config): } )) - @initialize_mongo_client + #@initialize_mongo_client def list_client_group_configs(self, filter): + self._initialize_mongo_client() selection = {"_id": 0, "module_name_lc": 0} if 'module_name' in filter: filter['module_name_lc'] = filter['module_name'].lower() del (filter['module_name']) return list(self.client_groups.find(filter, selection)) - @initialize_mongo_client + #@initialize_mongo_client def set_volume_mount(self, volume_mount): + self._initialize_mongo_client() volume_mount['module_name_lc'] = volume_mount['module_name'].lower() return self._check_update_result(self.volume_mounts.replace_one( { @@ -1153,8 +1195,9 @@ def set_volume_mount(self, volume_mount): upsert=True )) - @initialize_mongo_client + #@initialize_mongo_client def remove_volume_mount(self, volume_mount): + self._initialize_mongo_client() volume_mount['module_name_lc'] = volume_mount['module_name'].lower() return self._check_update_result(self.volume_mounts.delete_one( { @@ -1163,8 +1206,9 @@ def remove_volume_mount(self, volume_mount): 'client_group': volume_mount['client_group'] })) - @initialize_mongo_client + #@initialize_mongo_client def list_volume_mounts(self, filter): + self._initialize_mongo_client() selection = {"_id": 0, "module_name_lc": 0} if 'module_name' in filter: filter['module_name_lc'] = filter['module_name'].lower() @@ -1200,10 +1244,11 @@ def _check_update_result(self, result): return None return '{}' - @initialize_mongo_client + #@initialize_mongo_client def add_exec_stats_raw(self, user_id, app_module_name, app_id, func_module_name, func_name, git_commit_hash, creation_time, exec_start_time, finish_time, is_error, job_id): + self._initialize_mongo_client() stats = { 'user_id': user_id, 'app_module_name': app_module_name, @@ -1219,9 +1264,10 @@ def add_exec_stats_raw(self, user_id, app_module_name, app_id, func_module_name, } self.exec_stats_raw.insert_one(stats) - @initialize_mongo_client + #@initialize_mongo_client def add_exec_stats_apps(self, app_module_name, app_id, creation_time, exec_start_time, finish_time, is_error, type, time_range): + self._initialize_mongo_client() if not app_id: return full_app_id = app_id @@ -1242,9 +1288,10 @@ def add_exec_stats_apps(self, app_module_name, app_id, creation_time, exec_start {'full_app_id': full_app_id, 'type': type, 'time_range': time_range}, {'$setOnInsert': new_data, '$inc': inc_data}, upsert=True) - @initialize_mongo_client + #@initialize_mongo_client def add_exec_stats_users(self, user_id, creation_time, exec_start_time, finish_time, is_error, type, time_range): + self._initialize_mongo_client() queue_time = exec_start_time - creation_time exec_time = finish_time - exec_start_time inc_data = { @@ -1257,8 +1304,9 @@ def add_exec_stats_users(self, user_id, creation_time, exec_start_time, {'user_id': user_id, 'type': type, 'time_range': time_range}, {'$inc': inc_data}, upsert=True) - @initialize_mongo_client + #@initialize_mongo_client def get_exec_stats_apps(self, full_app_ids, type, time_range): + self._initialize_mongo_client() filter = {} if full_app_ids: filter['full_app_id'] = {'$in': full_app_ids} @@ -1278,9 +1326,9 @@ def get_exec_stats_apps(self, full_app_ids, type, time_range): } return list(self.exec_stats_apps.find(filter, selection)) - @initialize_mongo_client + #@initialize_mongo_client def aggr_exec_stats_table(self, minTime, maxTime): - + self._initialize_mongo_client() # setup the query aggParams = None group = { @@ -1333,9 +1381,9 @@ def aggr_exec_stats_table(self, minTime, maxTime): return counts - @initialize_mongo_client + #@initialize_mongo_client def get_exec_raw_stats(self, minTime, maxTime): - + self._initialize_mongo_client() filter = {} creationTimeFilter = {} if minTime is not None: @@ -1349,8 +1397,9 @@ def get_exec_raw_stats(self, minTime, maxTime): return list(self.exec_stats_raw.find(filter, {'_id': 0})) - @initialize_mongo_client + #@initialize_mongo_client def set_secure_config_params(self, data_list): + self._initialize_mongo_client() for param_data in data_list: param_data['module_name_lc'] = param_data['module_name'].lower() param_data['version'] = param_data.get('version', '') @@ -1363,8 +1412,9 @@ def set_secure_config_params(self, data_list): param_data, upsert=True) - @initialize_mongo_client + #@initialize_mongo_client def remove_secure_config_params(self, data_list): + self._initialize_mongo_client() for param_data in data_list: param_data['module_name_lc'] = param_data['module_name'].lower() param_data['version'] = param_data.get('version', '') @@ -1375,8 +1425,9 @@ def remove_secure_config_params(self, data_list): 'param_name': param_data['param_name'] }) - @initialize_mongo_client + #@initialize_mongo_client def get_secure_config_params(self, module_name): + self._initialize_mongo_client() selection = {"_id": 0, "module_name_lc": 0} filter = {"module_name_lc": module_name.lower()} return list(self.secure_config_params.find(filter, selection)) From 1f6024a3d6b9b084b005f72b50bf2325d1154f52 Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 19 Feb 2025 13:23:04 -0800 Subject: [PATCH 076/110] add _close_mongo_client function --- lib/biokbase/catalog/db.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index e986480d..1a2dbfee 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -153,9 +153,7 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech self._create_indexes() # Close the MongoDB client manually - self.mongo_client.close() - self.client_initialized = False - print("MongoDB client closed.") + self._close_mongo_client() def _initialize_mongo_client(self): """Initialize MongoDB client with lock to prevent race conditions.""" @@ -187,6 +185,14 @@ def _initialize_mongo_client(self): ) raise ValueError(error_msg) + def _close_mongo_client(self): + """Manually close the MongoDB client and mark it as not initialized.""" + if self.mongo_client: + self.mongo_client.close() + self.mongo_client = None # Ensure client is set to None to prevent invalid access + self.client_initialized = False # Mark client as closed + print("MongoDB client closed.") + def _create_collections(self): """Grab a handle to the database and collections.""" if not self.client_initialized: From 45110789d0ada0a0efe29b168d59f56aaf275564 Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 20 Feb 2025 08:57:21 -0800 Subject: [PATCH 077/110] try run tests without a lock --- lib/biokbase/catalog/db.py | 46 +++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 1a2dbfee..3f6b8dea 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -158,32 +158,32 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech def _initialize_mongo_client(self): """Initialize MongoDB client with lock to prevent race conditions.""" # Use the lock to ensure only one thread initializes the mongo client at a time - with self.lock: - try: - if not self.client_initialized or self.mongo_client is None: - # This is only tested manually - if self.mongo_user and self.mongo_psswd: - # Connection string with authentication - self.mongo_client = MongoClient( - f"mongodb://{self.mongo_user}:{self.mongo_psswd}@{self.mongo_host}/{self.mongo_db}?authMechanism={self.mongo_authMechanism}" - ) - else: - # Connection string without authentication - self.mongo_client = MongoClient(f"mongodb://{self.mongo_host}") + # with self.lock: + try: + if not self.client_initialized or self.mongo_client is None: + # This is only tested manually + if self.mongo_user and self.mongo_psswd: + # Connection string with authentication + self.mongo_client = MongoClient( + f"mongodb://{self.mongo_user}:{self.mongo_psswd}@{self.mongo_host}/{self.mongo_db}?authMechanism={self.mongo_authMechanism}" + ) + else: + # Connection string without authentication + self.mongo_client = MongoClient(f"mongodb://{self.mongo_host}") - # Force a call to server to verify the connection - self.mongo_client.server_info() + # Force a call to server to verify the connection + self.mongo_client.server_info() - # Mark client as initialized - self.client_initialized = True - print("Connection successful!") + # Mark client as initialized + self.client_initialized = True + print("Connection successful!") - except ConnectionFailure as e: - error_msg = "Cannot connect to Mongo server\n" - error_msg += "ERROR -- {}:\n{}".format( - e, "".join(traceback.format_exception(None, e, e.__traceback__)) - ) - raise ValueError(error_msg) + except ConnectionFailure as e: + error_msg = "Cannot connect to Mongo server\n" + error_msg += "ERROR -- {}:\n{}".format( + e, "".join(traceback.format_exception(None, e, e.__traceback__)) + ) + raise ValueError(error_msg) def _close_mongo_client(self): """Manually close the MongoDB client and mark it as not initialized.""" From 8806f22cdabed5e9cb7e8b43331dadcf02e12b29 Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 21 Feb 2025 10:53:18 -0800 Subject: [PATCH 078/110] revert self.lock changes --- lib/biokbase/catalog/db.py | 49 ++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 3f6b8dea..efe63360 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -158,32 +158,32 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech def _initialize_mongo_client(self): """Initialize MongoDB client with lock to prevent race conditions.""" # Use the lock to ensure only one thread initializes the mongo client at a time - # with self.lock: - try: - if not self.client_initialized or self.mongo_client is None: - # This is only tested manually - if self.mongo_user and self.mongo_psswd: - # Connection string with authentication - self.mongo_client = MongoClient( - f"mongodb://{self.mongo_user}:{self.mongo_psswd}@{self.mongo_host}/{self.mongo_db}?authMechanism={self.mongo_authMechanism}" - ) - else: - # Connection string without authentication - self.mongo_client = MongoClient(f"mongodb://{self.mongo_host}") + with self.lock: + try: + if not self.client_initialized or self.mongo_client is None: + # This is only tested manually + if self.mongo_user and self.mongo_psswd: + # Connection string with authentication + self.mongo_client = MongoClient( + f"mongodb://{self.mongo_user}:{self.mongo_psswd}@{self.mongo_host}/{self.mongo_db}?authMechanism={self.mongo_authMechanism}" + ) + else: + # Connection string without authentication + self.mongo_client = MongoClient(f"mongodb://{self.mongo_host}") - # Force a call to server to verify the connection - self.mongo_client.server_info() + # Force a call to server to verify the connection + self.mongo_client.server_info() - # Mark client as initialized - self.client_initialized = True - print("Connection successful!") + # Mark client as initialized + self.client_initialized = True + print("Connection successful!") - except ConnectionFailure as e: - error_msg = "Cannot connect to Mongo server\n" - error_msg += "ERROR -- {}:\n{}".format( - e, "".join(traceback.format_exception(None, e, e.__traceback__)) - ) - raise ValueError(error_msg) + except ConnectionFailure as e: + error_msg = "Cannot connect to Mongo server\n" + error_msg += "ERROR -- {}:\n{}".format( + e, "".join(traceback.format_exception(None, e, e.__traceback__)) + ) + raise ValueError(error_msg) def _close_mongo_client(self): """Manually close the MongoDB client and mark it as not initialized.""" @@ -195,9 +195,6 @@ def _close_mongo_client(self): def _create_collections(self): """Grab a handle to the database and collections.""" - if not self.client_initialized: - self._initialize_mongo_client() - self.db = self.mongo_client[self.mongo_db] self.modules = self.db[MongoCatalogDBI._MODULES] self.module_versions = self.db[MongoCatalogDBI._MODULE_VERSIONS] From 62bb424870254f299b261b9f2d18d24bf0ccbcd2 Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 21 Feb 2025 12:12:04 -0800 Subject: [PATCH 079/110] Recreate the database and collection handles after reinitializing the client --- lib/biokbase/catalog/db.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index efe63360..e5547a46 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -178,6 +178,9 @@ def _initialize_mongo_client(self): self.client_initialized = True print("Connection successful!") + # Recreate the database and collection handles after reinitializing the client + self._create_collections() + except ConnectionFailure as e: error_msg = "Cannot connect to Mongo server\n" error_msg += "ERROR -- {}:\n{}".format( From c919000a08afc97fee20b537a92a7e33eac2b1ca Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 21 Feb 2025 12:28:19 -0800 Subject: [PATCH 080/110] revert back tests --- test/startup_test.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/test/startup_test.py b/test/startup_test.py index 8a6fbd3e..25f03bf7 100644 --- a/test/startup_test.py +++ b/test/startup_test.py @@ -24,7 +24,6 @@ def test_startups(self): self.cUtil.setUpEmpty(db_version=3) catalog = Catalog(self.cUtil.getCatalogConfig()) self.assertTrue(semantic_version.validate(catalog.version(self.cUtil.anonymous_ctx())[0])) - self.cUtil.setUpEmpty(db_version=4) catalog = Catalog(self.cUtil.getCatalogConfig()) self.assertTrue(semantic_version.validate(catalog.version(self.cUtil.anonymous_ctx())[0])) @@ -33,12 +32,8 @@ def test_startups(self): self.cUtil.setUpEmpty(db_version=2525) catalog = None - ctx = self.cUtil.user_ctx() - user = ctx['user_id'] with self.assertRaises(IOError) as e: catalog = Catalog(self.cUtil.getCatalogConfig()) - # Trigger the lazy load collection, which will call the check_db_schema() method. - catalog.list_favorites(ctx, user)[0] self.assertEqual(str(e.exception), 'Incompatible DB versions. Expecting DB V4, found DV V2525. You are ' 'probably running an old version of the service. Start up failed.') @@ -50,4 +45,4 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - pass + pass \ No newline at end of file From dc79ddcafe98f9751180e6f33667656279326453 Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 21 Feb 2025 14:30:32 -0800 Subject: [PATCH 081/110] remove decorator related code --- lib/biokbase/catalog/db.py | 70 +------------------------------------- test/startup_test.py | 2 +- 2 files changed, 2 insertions(+), 70 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index e5547a46..7513d8a8 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -100,14 +100,6 @@ } ''' -# def initialize_mongo_client(func): -# """Decorator to ensure MongoDB client is initialized before calling API functions.""" -# def wrapper(self, *args, **kwargs): -# # Ensure MongoClient is initialized before the API method is called -# self._initialize_mongo_client() -# return func(self, *args, **kwargs) -# return wrapper - class MongoCatalogDBI: # Collection Names @@ -178,7 +170,7 @@ def _initialize_mongo_client(self): self.client_initialized = True print("Connection successful!") - # Recreate the database and collection handles after reinitializing the client + # Recreate the database and collection handles to avoid using a closed MongoClient self._create_collections() except ConnectionFailure as e: @@ -310,7 +302,6 @@ def _create_indexes(self): ('param_name', ASCENDING)], unique=True, sparse=False) - #@initialize_mongo_client def is_registered(self, module_name='', git_url=''): self._initialize_mongo_client() if not module_name and not git_url: @@ -321,7 +312,6 @@ def is_registered(self, module_name='', git_url=''): return True return False - #@initialize_mongo_client def module_name_lc_exists(self, module_name_lc=''): self._initialize_mongo_client() if not module_name_lc: @@ -332,7 +322,6 @@ def module_name_lc_exists(self, module_name_lc=''): return False #### SET methods - #@initialize_mongo_client def create_new_build_log(self, registration_id, timestamp, registration_state, git_url): self._initialize_mongo_client() build_log = { @@ -345,21 +334,18 @@ def create_new_build_log(self, registration_id, timestamp, registration_state, g } self.build_logs.insert_one(build_log) - #@initialize_mongo_client def delete_build_log(self, registration_id): self._initialize_mongo_client() self.build_logs.delete_one({'registration_id': registration_id}) # new_lines is a list to objects, each representing a line # the object structure is : {'content':... 'error':True/False} - #@initialize_mongo_client def append_to_build_log(self, registration_id, new_lines): self._initialize_mongo_client() result = self.build_logs.update_one({'registration_id': registration_id}, {'$push': {'log': {'$each': new_lines}}}) return self._check_update_result(result) - #@initialize_mongo_client def set_build_log_state(self, registration_id, registration_state, error_message=''): self._initialize_mongo_client() result = self.build_logs.update_one({'registration_id': registration_id}, @@ -367,14 +353,12 @@ def set_build_log_state(self, registration_id, registration_state, error_message 'error_message': error_message}}) return self._check_update_result(result) - #@initialize_mongo_client def set_build_log_module_name(self, registration_id, module_name): self._initialize_mongo_client() result = self.build_logs.update_one({'registration_id': registration_id}, {'$set': {'module_name_lc': module_name.lower()}}) return self._check_update_result(result) - #@initialize_mongo_client def list_builds(self, skip=0, limit=1000, @@ -421,7 +405,6 @@ def list_builds(self, # slice arg is used in the mongo query for getting lines. It is either a # pos int (get first n lines), neg int (last n lines), or array [skip, limit] - #@initialize_mongo_client def get_parsed_build_log(self, registration_id, slice_arg=None): self._initialize_mongo_client() selection = { @@ -439,7 +422,6 @@ def get_parsed_build_log(self, registration_id, slice_arg=None): return self.build_logs.find_one({'registration_id': registration_id}, selection) - #@initialize_mongo_client def register_new_module(self, git_url, username, timestamp, registration_state, registration_id): self._initialize_mongo_client() @@ -466,7 +448,6 @@ def register_new_module(self, git_url, username, timestamp, registration_state, # last_state is for concurency control. If set, it will match on state as well, and will fail # if the last_state does not match indicating another process changed the state - #@initialize_mongo_client def set_module_registration_state(self, module_name='', git_url='', new_state=None, last_state=None, error_message=''): self._initialize_mongo_client() @@ -479,7 +460,6 @@ def set_module_registration_state(self, module_name='', git_url='', new_state=No return self._check_update_result(result) return False - #@initialize_mongo_client def set_module_release_state(self, module_name='', git_url='', new_state=None, last_state=None, review_message=''): self._initialize_mongo_client() @@ -492,7 +472,6 @@ def set_module_release_state(self, module_name='', git_url='', new_state=None, l return self._check_update_result(result) return False - #@initialize_mongo_client def push_beta_to_release(self, module_name='', git_url='', release_timestamp=None): self._initialize_mongo_client() current_versions = self.get_module_current_versions(module_name=module_name, @@ -521,7 +500,6 @@ def push_beta_to_release(self, module_name='', git_url='', release_timestamp=Non }}) return self._check_update_result(result) - #@initialize_mongo_client def push_dev_to_beta(self, module_name='', git_url=''): self._initialize_mongo_client() current_versions = self.get_module_current_versions(module_name=module_name, @@ -533,7 +511,6 @@ def push_dev_to_beta(self, module_name='', git_url=''): return self._check_update_result(result) - #@initialize_mongo_client def update_dev_version(self, version_info): self._initialize_mongo_client() if version_info: @@ -563,7 +540,6 @@ def update_dev_version(self, version_info): raise ValueError('git_commit_hash is required to register a new version') return False - #@initialize_mongo_client def save_local_function_specs(self, local_functions): self._initialize_mongo_client() # just using insert doesn't accept a list of docs in mongo 2.6, so loop for now @@ -579,7 +555,6 @@ def save_local_function_specs(self, local_functions): return error return None - #@initialize_mongo_client def lookup_module_versions(self, module_name, git_commit_hash=None, released=None, included_fields=[], excluded_fields=[]): self._initialize_mongo_client() @@ -600,7 +575,6 @@ def lookup_module_versions(self, module_name, git_commit_hash=None, released=Non return list(self.module_versions.find(query, selection)) - #@initialize_mongo_client def list_local_function_info(self, release_tag=None, module_names=[]): self._initialize_mongo_client() git_commit_hash_list = [] @@ -673,7 +647,6 @@ def list_local_function_info(self, release_tag=None, module_names=[]): return returned_funcs - #@initialize_mongo_client def get_local_function_spec(self, functions): self._initialize_mongo_client() result_list = [] @@ -772,7 +745,6 @@ def get_local_function_spec(self, functions): return result_list - #@initialize_mongo_client def set_module_name(self, git_url, module_name): self._initialize_mongo_client() if not module_name: @@ -782,7 +754,6 @@ def set_module_name(self, git_url, module_name): '$set': {'module_name': module_name, 'module_name_lc': module_name.lower()}}) return self._check_update_result(result) - #@initialize_mongo_client def set_module_info(self, info, module_name='', git_url=''): self._initialize_mongo_client() if not info: @@ -793,7 +764,6 @@ def set_module_info(self, info, module_name='', git_url=''): result = self.modules.update_one(query, {'$set': {'info': info}}) return self._check_update_result(result) - #@initialize_mongo_client def set_module_owners(self, owners, module_name='', git_url=''): self._initialize_mongo_client() if not owners: @@ -805,7 +775,6 @@ def set_module_owners(self, owners, module_name='', git_url=''): return self._check_update_result(result) # active = True | False - #@initialize_mongo_client def set_module_active_state(self, active, module_name='', git_url=''): self._initialize_mongo_client() query = self._get_mongo_query(git_url=git_url, module_name=module_name) @@ -813,13 +782,11 @@ def set_module_active_state(self, active, module_name='', git_url=''): return self._check_update_result(result) #### GET methods - #@initialize_mongo_client def get_module_state(self, module_name='', git_url=''): self._initialize_mongo_client() query = self._get_mongo_query(module_name=module_name, git_url=git_url) return self.modules.find_one(query, ['state'])['state'] - #@initialize_mongo_client def get_module_current_versions(self, module_name='', git_url='', substitute_versions=True): self._initialize_mongo_client() query = self._get_mongo_query(module_name=module_name, git_url=git_url) @@ -828,13 +795,11 @@ def get_module_current_versions(self, module_name='', git_url='', substitute_ver self.substitute_hashes_for_version_info([module_document]) return module_document['current_versions'] - #@initialize_mongo_client def get_module_owners(self, module_name='', git_url=''): self._initialize_mongo_client() query = self._get_mongo_query(module_name=module_name, git_url=git_url) return self.modules.find_one(query, ['owners'])['owners'] - #@initialize_mongo_client def get_module_details(self, module_name='', git_url='', substitute_versions=True): self._initialize_mongo_client() query = self._get_mongo_query(module_name=module_name, git_url=git_url) @@ -845,7 +810,6 @@ def get_module_details(self, module_name='', git_url='', substitute_versions=Tru self.substitute_hashes_for_version_info([module_details]) return module_details - #@initialize_mongo_client def get_module_full_details(self, module_name='', git_url='', substitute_versions=True): self._initialize_mongo_client() query = self._get_mongo_query(module_name=module_name, git_url=git_url) @@ -855,7 +819,6 @@ def get_module_full_details(self, module_name='', git_url='', substitute_version return module_document #### LIST / SEARCH methods - #@initialize_mongo_client def find_basic_module_info(self, query): self._initialize_mongo_client() selection = { @@ -869,7 +832,6 @@ def find_basic_module_info(self, query): } return list(self.modules.find(query, selection)) - #@initialize_mongo_client def find_current_versions_and_owners(self, query): self._initialize_mongo_client() result = list(self.modules.find(query, @@ -878,7 +840,6 @@ def find_current_versions_and_owners(self, query): self.substitute_hashes_for_version_info(result) return result - #@initialize_mongo_client def substitute_hashes_for_version_info(self, module_list): self._initialize_mongo_client() # get all the version commit hashes @@ -926,7 +887,6 @@ def substitute_hashes_for_version_info(self, module_list): return module_list # tag should be one of dev, beta, release - do checking outside of this method - #@initialize_mongo_client def list_service_module_versions_with_tag(self, tag): self._initialize_mongo_client() mods = list(self.modules.find({'info.dynamic_service': 1}, @@ -948,7 +908,6 @@ def list_service_module_versions_with_tag(self, tag): return result # all released service module versions - #@initialize_mongo_client def list_all_released_service_module_versions(self): self._initialize_mongo_client() return list(self.module_versions.find( @@ -965,7 +924,6 @@ def list_all_released_service_module_versions(self): })) #### developer check methods - #@initialize_mongo_client def approve_developer(self, developer): self._initialize_mongo_client() # if the developer is already on the list, just return @@ -973,7 +931,6 @@ def approve_developer(self, developer): return self.developers.insert_one({'kb_username': developer}) - #@initialize_mongo_client def revoke_developer(self, developer): self._initialize_mongo_client() # if the developer is not on the list, throw an error (maybe a typo, so let's catch it) @@ -981,7 +938,6 @@ def revoke_developer(self, developer): raise ValueError('Cannot revoke "' + developer + '", that developer was not found.') self.developers.delete_one({'kb_username': developer}) - #@initialize_mongo_client def is_approved_developer(self, usernames): self._initialize_mongo_client() # TODO: optimize, but I expect the list of usernames will be fairly small, so we can loop. Regardless, in @@ -995,12 +951,10 @@ def is_approved_developer(self, usernames): is_approved.append(False) return is_approved - #@initialize_mongo_client def list_approved_developers(self): self._initialize_mongo_client() return list(self.developers.find({}, {'kb_username': 1, '_id': 0})) - #@initialize_mongo_client def migrate_module_to_new_git_url(self, module_name, current_git_url, new_git_url): self._initialize_mongo_client() if not new_git_url.strip(): @@ -1013,7 +967,6 @@ def migrate_module_to_new_git_url(self, module_name, current_git_url, new_git_ur result = self.modules.update_one(query, {'$set': {'git_url': new_git_url.strip()}}) return self._check_update_result(result) - #@initialize_mongo_client def delete_module(self, module_name, git_url): self._initialize_mongo_client() if not module_name and not git_url: @@ -1034,7 +987,6 @@ def delete_module(self, module_name, git_url): result = self.modules.delete_one({'_id': module_details['_id']}) return self._check_update_result(result) - #@initialize_mongo_client def add_favorite(self, module_name, app_id, username, timestamp): self._initialize_mongo_client() favoriteAddition = { @@ -1050,7 +1002,6 @@ def add_favorite(self, module_name, app_id, username, timestamp): favoriteAddition['timestamp'] = timestamp self.favorites.insert_one(favoriteAddition) - #@initialize_mongo_client def remove_favorite(self, module_name, app_id, username): self._initialize_mongo_client() favoriteAddition = { @@ -1066,21 +1017,18 @@ def remove_favorite(self, module_name, app_id, username): result = self.favorites.delete_one({'_id': found['_id']}) return self._check_update_result(result) - #@initialize_mongo_client def list_user_favorites(self, username): self._initialize_mongo_client() query = {'user': username} selection = {'_id': 0, 'module_name_lc': 1, 'id': 1, 'timestamp': 1} return list(self.favorites.find(query, selection).sort('timestamp', DESCENDING)) - #@initialize_mongo_client def list_app_favorites(self, module_name, app_id): self._initialize_mongo_client() query = {'module_name_lc': module_name.strip().lower(), 'id': app_id.strip()} selection = {'_id': 0, 'user': 1, 'timestamp': 1} return list(self.favorites.find(query, selection).sort('timestamp', DESCENDING)) - #@initialize_mongo_client def aggregate_favorites_over_apps(self, module_names_lc): self._initialize_mongo_client() ### WARNING! If we switch to Mongo 3.x, the result object will change and this will break @@ -1125,7 +1073,6 @@ def aggregate_favorites_over_apps(self, module_names_lc): return counts # DEPRECATED! temporary function until everything is migrated to new client group structure - #@initialize_mongo_client def list_client_groups(self, app_ids): self._initialize_mongo_client() if app_ids is not None: @@ -1154,7 +1101,6 @@ def list_client_groups(self, app_ids): } return list(self.client_groups.find({}, selection)) - #@initialize_mongo_client def set_client_group_config(self, config): self._initialize_mongo_client() config['module_name_lc'] = config['module_name'].lower() @@ -1167,7 +1113,6 @@ def set_client_group_config(self, config): upsert=True )) - #@initialize_mongo_client def remove_client_group_config(self, config): self._initialize_mongo_client() config['module_name_lc'] = config['module_name'].lower() @@ -1178,7 +1123,6 @@ def remove_client_group_config(self, config): } )) - #@initialize_mongo_client def list_client_group_configs(self, filter): self._initialize_mongo_client() selection = {"_id": 0, "module_name_lc": 0} @@ -1187,7 +1131,6 @@ def list_client_group_configs(self, filter): del (filter['module_name']) return list(self.client_groups.find(filter, selection)) - #@initialize_mongo_client def set_volume_mount(self, volume_mount): self._initialize_mongo_client() volume_mount['module_name_lc'] = volume_mount['module_name'].lower() @@ -1201,7 +1144,6 @@ def set_volume_mount(self, volume_mount): upsert=True )) - #@initialize_mongo_client def remove_volume_mount(self, volume_mount): self._initialize_mongo_client() volume_mount['module_name_lc'] = volume_mount['module_name'].lower() @@ -1212,7 +1154,6 @@ def remove_volume_mount(self, volume_mount): 'client_group': volume_mount['client_group'] })) - #@initialize_mongo_client def list_volume_mounts(self, filter): self._initialize_mongo_client() selection = {"_id": 0, "module_name_lc": 0} @@ -1250,7 +1191,6 @@ def _check_update_result(self, result): return None return '{}' - #@initialize_mongo_client def add_exec_stats_raw(self, user_id, app_module_name, app_id, func_module_name, func_name, git_commit_hash, creation_time, exec_start_time, finish_time, is_error, job_id): @@ -1270,7 +1210,6 @@ def add_exec_stats_raw(self, user_id, app_module_name, app_id, func_module_name, } self.exec_stats_raw.insert_one(stats) - #@initialize_mongo_client def add_exec_stats_apps(self, app_module_name, app_id, creation_time, exec_start_time, finish_time, is_error, type, time_range): self._initialize_mongo_client() @@ -1294,7 +1233,6 @@ def add_exec_stats_apps(self, app_module_name, app_id, creation_time, exec_start {'full_app_id': full_app_id, 'type': type, 'time_range': time_range}, {'$setOnInsert': new_data, '$inc': inc_data}, upsert=True) - #@initialize_mongo_client def add_exec_stats_users(self, user_id, creation_time, exec_start_time, finish_time, is_error, type, time_range): self._initialize_mongo_client() @@ -1310,7 +1248,6 @@ def add_exec_stats_users(self, user_id, creation_time, exec_start_time, {'user_id': user_id, 'type': type, 'time_range': time_range}, {'$inc': inc_data}, upsert=True) - #@initialize_mongo_client def get_exec_stats_apps(self, full_app_ids, type, time_range): self._initialize_mongo_client() filter = {} @@ -1332,7 +1269,6 @@ def get_exec_stats_apps(self, full_app_ids, type, time_range): } return list(self.exec_stats_apps.find(filter, selection)) - #@initialize_mongo_client def aggr_exec_stats_table(self, minTime, maxTime): self._initialize_mongo_client() # setup the query @@ -1387,7 +1323,6 @@ def aggr_exec_stats_table(self, minTime, maxTime): return counts - #@initialize_mongo_client def get_exec_raw_stats(self, minTime, maxTime): self._initialize_mongo_client() filter = {} @@ -1403,7 +1338,6 @@ def get_exec_raw_stats(self, minTime, maxTime): return list(self.exec_stats_raw.find(filter, {'_id': 0})) - #@initialize_mongo_client def set_secure_config_params(self, data_list): self._initialize_mongo_client() for param_data in data_list: @@ -1418,7 +1352,6 @@ def set_secure_config_params(self, data_list): param_data, upsert=True) - #@initialize_mongo_client def remove_secure_config_params(self, data_list): self._initialize_mongo_client() for param_data in data_list: @@ -1431,7 +1364,6 @@ def remove_secure_config_params(self, data_list): 'param_name': param_data['param_name'] }) - #@initialize_mongo_client def get_secure_config_params(self, module_name): self._initialize_mongo_client() selection = {"_id": 0, "module_name_lc": 0} diff --git a/test/startup_test.py b/test/startup_test.py index 25f03bf7..61a30328 100644 --- a/test/startup_test.py +++ b/test/startup_test.py @@ -45,4 +45,4 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - pass \ No newline at end of file + pass From 0c7b9065a63be779a2a72dc839af6cf9fe561645 Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 21 Feb 2025 14:45:05 -0800 Subject: [PATCH 082/110] add comment why need to manually close mongoclient --- lib/biokbase/catalog/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 7513d8a8..8ec2d00c 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -144,7 +144,7 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech # Create indexes self._create_indexes() - # Close the MongoDB client manually + # Close the MongoDB client manually before forking self._close_mongo_client() def _initialize_mongo_client(self): From 9ae4b7b6a3baf0c325a0a20f2be65f46200e2599 Mon Sep 17 00:00:00 2001 From: Sijie Date: Mon, 24 Feb 2025 13:55:07 -0800 Subject: [PATCH 083/110] update comments --- Dockerfile | 1 + Makefile | 4 ---- lib/biokbase/catalog/db.py | 49 +++++++++++++++++++------------------- 3 files changed, 26 insertions(+), 28 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5b4429b7..884f1268 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ FROM python:3.9.19 AS build +# The rsync installation is required for the Makefile RUN apt-get update && apt-get install -y rsync RUN mkdir -p /kb/deployment/lib/biokbase diff --git a/Makefile b/Makefile index f7db502c..53a2cd5b 100644 --- a/Makefile +++ b/Makefile @@ -98,10 +98,6 @@ setup-tests: mkdir -p $(TESTDIR)/nms rsync -av lib/biokbase/* $(TESTLIB)/biokbase/. --exclude *.bak-* -# cd narrative_method_store; make; make build-classpath-list; -# rsync -av narrative_method_store/lib/biokbase/* $(TESTLIB)/biokbase/. - - test: setup-tests -cp -n $(TESTDIR)/test.cfg.example $(TESTDIR)/test.cfg diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 8ec2d00c..0b708025 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -100,6 +100,9 @@ } ''' +# Module level lock +lock = threading.Lock() + class MongoCatalogDBI: # Collection Names @@ -128,9 +131,7 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech self.mongo_psswd = mongo_psswd self.mongo_authMechanism = mongo_authMechanism - self.lock = threading.Lock() self.mongo_client = None - self.client_initialized = False # Initialize mongo client self._initialize_mongo_client() @@ -149,29 +150,28 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech def _initialize_mongo_client(self): """Initialize MongoDB client with lock to prevent race conditions.""" + if self.mongo_client: + return + # Use the lock to ensure only one thread initializes the mongo client at a time - with self.lock: + with lock: try: - if not self.client_initialized or self.mongo_client is None: - # This is only tested manually - if self.mongo_user and self.mongo_psswd: - # Connection string with authentication - self.mongo_client = MongoClient( - f"mongodb://{self.mongo_user}:{self.mongo_psswd}@{self.mongo_host}/{self.mongo_db}?authMechanism={self.mongo_authMechanism}" - ) - else: - # Connection string without authentication - self.mongo_client = MongoClient(f"mongodb://{self.mongo_host}") - - # Force a call to server to verify the connection - self.mongo_client.server_info() + # This is only tested manually + if self.mongo_user and self.mongo_psswd: + # Connection string with authentication + self.mongo_client = MongoClient( + f"mongodb://{self.mongo_user}:{self.mongo_psswd}@{self.mongo_host}/{self.mongo_db}?authMechanism={self.mongo_authMechanism}" + ) + else: + # Connection string without authentication + self.mongo_client = MongoClient(f"mongodb://{self.mongo_host}") - # Mark client as initialized - self.client_initialized = True - print("Connection successful!") + # Force a call to server to verify the connection + self.mongo_client.server_info() + print("Connection successful!") - # Recreate the database and collection handles to avoid using a closed MongoClient - self._create_collections() + # Recreate the database and collection handles to avoid using a closed MongoClient + self._create_collections() except ConnectionFailure as e: error_msg = "Cannot connect to Mongo server\n" @@ -181,11 +181,10 @@ def _initialize_mongo_client(self): raise ValueError(error_msg) def _close_mongo_client(self): - """Manually close the MongoDB client and mark it as not initialized.""" + """Manually close the MongoDB client and mark it as None.""" if self.mongo_client: self.mongo_client.close() - self.mongo_client = None # Ensure client is set to None to prevent invalid access - self.client_initialized = False # Mark client as closed + self.mongo_client = None print("MongoDB client closed.") def _create_collections(self): @@ -819,6 +818,7 @@ def get_module_full_details(self, module_name='', git_url='', substitute_version return module_document #### LIST / SEARCH methods + def find_basic_module_info(self, query): self._initialize_mongo_client() selection = { @@ -924,6 +924,7 @@ def list_all_released_service_module_versions(self): })) #### developer check methods + def approve_developer(self, developer): self._initialize_mongo_client() # if the developer is already on the list, just return From 6c54bf0998cbb7f83988b7ae1a554f4c2b680440 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 25 Feb 2025 11:10:21 -0800 Subject: [PATCH 084/110] refactor the db.py and add comments for docker-compose file --- lib/biokbase/catalog/db.py | 325 +++++++++++++++++++----------------- test/docker-compose_nms.yml | 2 + 2 files changed, 171 insertions(+), 156 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 0b708025..408fbba6 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -134,58 +134,55 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech self.mongo_client = None # Initialize mongo client - self._initialize_mongo_client() - - # Create dbs and collections - self._create_collections() + mongo_client = self._initialize_mongo_client() # Check the db schema - self.check_db_schema() + self.check_db_schema(mongo_client) # Create indexes - self._create_indexes() + self._create_indexes(mongo_client) # Close the MongoDB client manually before forking - self._close_mongo_client() + mongo_client.close() + print("MongoDB client closed.") - def _initialize_mongo_client(self): - """Initialize MongoDB client with lock to prevent race conditions.""" - if self.mongo_client: - return + def _initialize_mongo_client(self): + """Initialize MongoDB client.""" # Use the lock to ensure only one thread initializes the mongo client at a time - with lock: - try: - # This is only tested manually - if self.mongo_user and self.mongo_psswd: - # Connection string with authentication - self.mongo_client = MongoClient( - f"mongodb://{self.mongo_user}:{self.mongo_psswd}@{self.mongo_host}/{self.mongo_db}?authMechanism={self.mongo_authMechanism}" - ) - else: - # Connection string without authentication - self.mongo_client = MongoClient(f"mongodb://{self.mongo_host}") + try: + # This is only tested manually + if self.mongo_user and self.mongo_psswd: + # Connection string with authentication + mongo_client = MongoClient( + f"mongodb://{self.mongo_user}:{self.mongo_psswd}@{self.mongo_host}/{self.mongo_db}?authMechanism={self.mongo_authMechanism}" + ) + else: + # Connection string without authentication + mongo_client = MongoClient(f"mongodb://{self.mongo_host}") - # Force a call to server to verify the connection - self.mongo_client.server_info() - print("Connection successful!") + # Force a call to server to verify the connection + mongo_client.server_info() + print("Connection successful!") - # Recreate the database and collection handles to avoid using a closed MongoClient - self._create_collections() + return mongo_client - except ConnectionFailure as e: - error_msg = "Cannot connect to Mongo server\n" - error_msg += "ERROR -- {}:\n{}".format( - e, "".join(traceback.format_exception(None, e, e.__traceback__)) - ) - raise ValueError(error_msg) - - def _close_mongo_client(self): - """Manually close the MongoDB client and mark it as None.""" - if self.mongo_client: - self.mongo_client.close() - self.mongo_client = None - print("MongoDB client closed.") + except ConnectionFailure as e: + error_msg = "Cannot connect to Mongo server\n" + error_msg += "ERROR -- {}:\n{}".format( + e, "".join(traceback.format_exception(None, e, e.__traceback__)) + ) + raise ValueError(error_msg) + + def _ensure_mongo_connection(self): + """Ensure the MongoDB connection is active.""" + # Don't enter the lock if we already have the client + if not self.mongo_client: + with lock: + if self.mongo_client: + return + self.mongo_client = self._initialize_mongo_client() + self._create_collections() def _create_collections(self): """Grab a handle to the database and collections.""" @@ -206,103 +203,116 @@ def _create_collections(self): self.secure_config_params = self.db[MongoCatalogDBI._SECURE_CONFIG_PARAMS] - def _create_indexes(self): + def _create_indexes(self, mongo_client): + db = mongo_client[self.mongo_db] + # Make sure we have an index on module and git_repo_url - self.module_versions.create_index('module_name_lc', sparse=False) - self.module_versions.create_index('git_commit_hash', sparse=False) - self.module_versions.create_index([ + module_versions = db[MongoCatalogDBI._MODULE_VERSIONS] + module_versions.create_index('module_name_lc', sparse=False) + module_versions.create_index('git_commit_hash', sparse=False) + module_versions.create_index([ ('module_name_lc', ASCENDING), ('git_commit_hash', ASCENDING)], unique=True, sparse=False) # Make sure we have a unique index on module_name_lc and git_commit_hash - self.local_functions.create_index('function_id') - self.local_functions.create_index([ + local_functions = db[MongoCatalogDBI._LOCAL_FUNCTIONS] + local_functions.create_index('function_id') + local_functions.create_index([ ('module_name_lc', ASCENDING), ('function_id', ASCENDING), ('git_commit_hash', ASCENDING)], unique=True, sparse=False) # local function indecies - self.local_functions.create_index('module_name_lc') - self.local_functions.create_index('git_commit_hash') - self.local_functions.create_index('function_id') - self.local_functions.create_index([ + local_functions.create_index('module_name_lc') + local_functions.create_index('git_commit_hash') + local_functions.create_index('function_id') + local_functions.create_index([ ('module_name_lc', ASCENDING), ('function_id', ASCENDING), ('git_commit_hash', ASCENDING)], unique=True, sparse=False) # developers indecies - self.developers.create_index('kb_username', unique=True) + developers = db[MongoCatalogDBI._DEVELOPERS] + developers.create_index('kb_username', unique=True) - self.build_logs.create_index('registration_id', unique=True) - self.build_logs.create_index('module_name_lc') - self.build_logs.create_index('timestamp') - self.build_logs.create_index('registration') - self.build_logs.create_index('git_url') - self.build_logs.create_index('current_versions.release.release_timestamp') + build_logs = db[MongoCatalogDBI._BUILD_LOGS] + build_logs.create_index('registration_id', unique=True) + build_logs.create_index('module_name_lc') + build_logs.create_index('timestamp') + build_logs.create_index('registration') + build_logs.create_index('git_url') + build_logs.create_index('current_versions.release.release_timestamp') # for favorites - self.favorites.create_index('user') - self.favorites.create_index('module_name_lc') - self.favorites.create_index('id') + favorites = db[MongoCatalogDBI._FAVORITES] + favorites.create_index('user') + favorites.create_index('module_name_lc') + favorites.create_index('id') # you can only favorite a method once, so put a unique index on the triple - self.favorites.create_index([ + favorites.create_index([ ('user', ASCENDING), ('id', ASCENDING), ('module_name_lc', ASCENDING)], unique=True, sparse=False) # execution stats - self.exec_stats_raw.create_index('user_id', + exec_stats_raw = db[MongoCatalogDBI._EXEC_STATS_RAW] + exec_stats_raw.create_index('user_id', unique=False, sparse=False) - self.exec_stats_raw.create_index([('app_module_name', ASCENDING), + exec_stats_raw.create_index([('app_module_name', ASCENDING), ('app_id', ASCENDING)], unique=False, sparse=True) - self.exec_stats_raw.create_index([('func_module_name', ASCENDING), + exec_stats_raw.create_index([('func_module_name', ASCENDING), ('func_name', ASCENDING)], unique=False, sparse=True) - self.exec_stats_raw.create_index('creation_time', + exec_stats_raw.create_index('creation_time', unique=False, sparse=False) - self.exec_stats_raw.create_index('finish_time', + exec_stats_raw.create_index('finish_time', unique=False, sparse=False) - self.exec_stats_apps.create_index('module_name', + exec_stats_apps = db[MongoCatalogDBI._EXEC_STATS_APPS] + exec_stats_apps.create_index('module_name', unique=False, sparse=True) - self.exec_stats_apps.create_index([('full_app_id', ASCENDING), + exec_stats_apps.create_index([('full_app_id', ASCENDING), ('type', ASCENDING), ('time_range', ASCENDING)], unique=True, sparse=False) - self.exec_stats_apps.create_index([('type', ASCENDING), + exec_stats_apps.create_index([('type', ASCENDING), ('time_range', ASCENDING)], unique=False, sparse=False) - self.exec_stats_users.create_index([('user_id', ASCENDING), + exec_stats_users = db[MongoCatalogDBI._EXEC_STATS_USERS] + exec_stats_users.create_index([('user_id', ASCENDING), ('type', ASCENDING), ('time_range', ASCENDING)], unique=True, sparse=False) # client groups and volume mounts - self.client_groups.create_index([('module_name_lc', ASCENDING), + client_groups = db[MongoCatalogDBI._CLIENT_GROUPS] + client_groups.create_index([('module_name_lc', ASCENDING), ('function_name', ASCENDING)], unique=True, sparse=False) - self.volume_mounts.create_index([('client_group', ASCENDING), + volume_mounts = db[MongoCatalogDBI._VOLUME_MOUNTS] + volume_mounts.create_index([('client_group', ASCENDING), ('module_name_lc', ASCENDING), ('function_name', ASCENDING)], unique=True, sparse=False) # hidden configuration parameters - self.secure_config_params.create_index('module_name_lc') - self.secure_config_params.create_index([ + secure_config_params = db[MongoCatalogDBI._SECURE_CONFIG_PARAMS] + secure_config_params.create_index('module_name_lc') + secure_config_params.create_index([ ('module_name_lc', ASCENDING), ('version', ASCENDING), ('param_name', ASCENDING)], unique=True, sparse=False) def is_registered(self, module_name='', git_url=''): - self._initialize_mongo_client() + self._ensure_mongo_connection() if not module_name and not git_url: return False query = self._get_mongo_query(module_name=module_name, git_url=git_url) @@ -312,7 +322,7 @@ def is_registered(self, module_name='', git_url=''): return False def module_name_lc_exists(self, module_name_lc=''): - self._initialize_mongo_client() + self._ensure_mongo_connection() if not module_name_lc: return False module = self.modules.find_one({'module_name_lc': module_name_lc.lower()}, ['_id']) @@ -322,7 +332,7 @@ def module_name_lc_exists(self, module_name_lc=''): #### SET methods def create_new_build_log(self, registration_id, timestamp, registration_state, git_url): - self._initialize_mongo_client() + self._ensure_mongo_connection() build_log = { 'registration_id': registration_id, 'timestamp': timestamp, @@ -334,26 +344,26 @@ def create_new_build_log(self, registration_id, timestamp, registration_state, g self.build_logs.insert_one(build_log) def delete_build_log(self, registration_id): - self._initialize_mongo_client() + self._ensure_mongo_connection() self.build_logs.delete_one({'registration_id': registration_id}) # new_lines is a list to objects, each representing a line # the object structure is : {'content':... 'error':True/False} def append_to_build_log(self, registration_id, new_lines): - self._initialize_mongo_client() + self._ensure_mongo_connection() result = self.build_logs.update_one({'registration_id': registration_id}, {'$push': {'log': {'$each': new_lines}}}) return self._check_update_result(result) def set_build_log_state(self, registration_id, registration_state, error_message=''): - self._initialize_mongo_client() + self._ensure_mongo_connection() result = self.build_logs.update_one({'registration_id': registration_id}, {'$set': {'registration': registration_state, 'error_message': error_message}}) return self._check_update_result(result) def set_build_log_module_name(self, registration_id, module_name): - self._initialize_mongo_client() + self._ensure_mongo_connection() result = self.build_logs.update_one({'registration_id': registration_id}, {'$set': {'module_name_lc': module_name.lower()}}) return self._check_update_result(result) @@ -366,7 +376,7 @@ def list_builds(self, only_running=False, only_error=False, only_complete=False): - self._initialize_mongo_client() + self._ensure_mongo_connection() query = {} registration_match = None @@ -405,7 +415,7 @@ def list_builds(self, # slice arg is used in the mongo query for getting lines. It is either a # pos int (get first n lines), neg int (last n lines), or array [skip, limit] def get_parsed_build_log(self, registration_id, slice_arg=None): - self._initialize_mongo_client() + self._ensure_mongo_connection() selection = { 'registration_id': 1, 'timestamp': 1, @@ -423,7 +433,7 @@ def get_parsed_build_log(self, registration_id, slice_arg=None): def register_new_module(self, git_url, username, timestamp, registration_state, registration_id): - self._initialize_mongo_client() + self._ensure_mongo_connection() # get current time since epoch in ms in utc module = { 'info': {}, @@ -449,7 +459,7 @@ def register_new_module(self, git_url, username, timestamp, registration_state, # if the last_state does not match indicating another process changed the state def set_module_registration_state(self, module_name='', git_url='', new_state=None, last_state=None, error_message=''): - self._initialize_mongo_client() + self._ensure_mongo_connection() if new_state: query = self._get_mongo_query(module_name=module_name, git_url=git_url) if last_state: @@ -461,7 +471,7 @@ def set_module_registration_state(self, module_name='', git_url='', new_state=No def set_module_release_state(self, module_name='', git_url='', new_state=None, last_state=None, review_message=''): - self._initialize_mongo_client() + self._ensure_mongo_connection() if new_state: query = self._get_mongo_query(module_name=module_name, git_url=git_url) if last_state: @@ -472,7 +482,7 @@ def set_module_release_state(self, module_name='', git_url='', new_state=None, l return False def push_beta_to_release(self, module_name='', git_url='', release_timestamp=None): - self._initialize_mongo_client() + self._ensure_mongo_connection() current_versions = self.get_module_current_versions(module_name=module_name, git_url=git_url, substitute_versions=False) @@ -500,7 +510,7 @@ def push_beta_to_release(self, module_name='', git_url='', release_timestamp=Non return self._check_update_result(result) def push_dev_to_beta(self, module_name='', git_url=''): - self._initialize_mongo_client() + self._ensure_mongo_connection() current_versions = self.get_module_current_versions(module_name=module_name, git_url=git_url, substitute_versions=False) @@ -511,7 +521,7 @@ def push_dev_to_beta(self, module_name='', git_url=''): return self._check_update_result(result) def update_dev_version(self, version_info): - self._initialize_mongo_client() + self._ensure_mongo_connection() if version_info: if 'git_commit_hash' in version_info and 'module_name_lc' in version_info: @@ -540,7 +550,7 @@ def update_dev_version(self, version_info): return False def save_local_function_specs(self, local_functions): - self._initialize_mongo_client() + self._ensure_mongo_connection() # just using insert doesn't accept a list of docs in mongo 2.6, so loop for now for l in local_functions: matcher = {'module_name_lc': l['module_name_lc'], 'function_id': l['function_id'], @@ -556,7 +566,7 @@ def save_local_function_specs(self, local_functions): def lookup_module_versions(self, module_name, git_commit_hash=None, released=None, included_fields=[], excluded_fields=[]): - self._initialize_mongo_client() + self._ensure_mongo_connection() query = {'module_name_lc': module_name.strip().lower()} if git_commit_hash is not None: @@ -575,7 +585,7 @@ def lookup_module_versions(self, module_name, git_commit_hash=None, released=Non return list(self.module_versions.find(query, selection)) def list_local_function_info(self, release_tag=None, module_names=[]): - self._initialize_mongo_client() + self._ensure_mongo_connection() git_commit_hash_list = [] git_commit_hash_release_tag_map = {} @@ -647,7 +657,7 @@ def list_local_function_info(self, release_tag=None, module_names=[]): return returned_funcs def get_local_function_spec(self, functions): - self._initialize_mongo_client() + self._ensure_mongo_connection() result_list = [] # first lookup all the module info so we can figure out any tags, and make a quick dict @@ -745,7 +755,7 @@ def get_local_function_spec(self, functions): return result_list def set_module_name(self, git_url, module_name): - self._initialize_mongo_client() + self._ensure_mongo_connection() if not module_name: raise ValueError('module_name must be defined to set a module name') query = self._get_mongo_query(git_url=git_url) @@ -754,7 +764,7 @@ def set_module_name(self, git_url, module_name): return self._check_update_result(result) def set_module_info(self, info, module_name='', git_url=''): - self._initialize_mongo_client() + self._ensure_mongo_connection() if not info: raise ValueError('info must be defined to set the info for a module') if type(info) is not dict: @@ -764,7 +774,7 @@ def set_module_info(self, info, module_name='', git_url=''): return self._check_update_result(result) def set_module_owners(self, owners, module_name='', git_url=''): - self._initialize_mongo_client() + self._ensure_mongo_connection() if not owners: raise ValueError('owners must be defined to set the owners for a module') if type(owners) is not list: @@ -775,19 +785,19 @@ def set_module_owners(self, owners, module_name='', git_url=''): # active = True | False def set_module_active_state(self, active, module_name='', git_url=''): - self._initialize_mongo_client() + self._ensure_mongo_connection() query = self._get_mongo_query(git_url=git_url, module_name=module_name) result = self.modules.update_one(query, {'$set': {'state.active': active}}) return self._check_update_result(result) #### GET methods def get_module_state(self, module_name='', git_url=''): - self._initialize_mongo_client() + self._ensure_mongo_connection() query = self._get_mongo_query(module_name=module_name, git_url=git_url) return self.modules.find_one(query, ['state'])['state'] def get_module_current_versions(self, module_name='', git_url='', substitute_versions=True): - self._initialize_mongo_client() + self._ensure_mongo_connection() query = self._get_mongo_query(module_name=module_name, git_url=git_url) module_document = self.modules.find_one(query, ['module_name_lc', 'current_versions']) if substitute_versions and 'module_name_lc' in module_document: @@ -795,12 +805,12 @@ def get_module_current_versions(self, module_name='', git_url='', substitute_ver return module_document['current_versions'] def get_module_owners(self, module_name='', git_url=''): - self._initialize_mongo_client() + self._ensure_mongo_connection() query = self._get_mongo_query(module_name=module_name, git_url=git_url) return self.modules.find_one(query, ['owners'])['owners'] def get_module_details(self, module_name='', git_url='', substitute_versions=True): - self._initialize_mongo_client() + self._ensure_mongo_connection() query = self._get_mongo_query(module_name=module_name, git_url=git_url) module_details = self.modules.find_one(query, ['module_name', 'module_name_lc', 'git_url', 'info', 'owners', 'state', @@ -810,7 +820,7 @@ def get_module_details(self, module_name='', git_url='', substitute_versions=Tru return module_details def get_module_full_details(self, module_name='', git_url='', substitute_versions=True): - self._initialize_mongo_client() + self._ensure_mongo_connection() query = self._get_mongo_query(module_name=module_name, git_url=git_url) module_document = self.modules.find_one(query) if substitute_versions and 'module_name_lc' in module_document: @@ -820,7 +830,7 @@ def get_module_full_details(self, module_name='', git_url='', substitute_version #### LIST / SEARCH methods def find_basic_module_info(self, query): - self._initialize_mongo_client() + self._ensure_mongo_connection() selection = { '_id': 0, 'module_name': 1, @@ -833,7 +843,7 @@ def find_basic_module_info(self, query): return list(self.modules.find(query, selection)) def find_current_versions_and_owners(self, query): - self._initialize_mongo_client() + self._ensure_mongo_connection() result = list(self.modules.find(query, {'module_name': 1, 'module_name_lc': 1, 'git_url': 1, 'current_versions': 1, 'owners': 1, '_id': 0})) @@ -841,7 +851,7 @@ def find_current_versions_and_owners(self, query): return result def substitute_hashes_for_version_info(self, module_list): - self._initialize_mongo_client() + self._ensure_mongo_connection() # get all the version commit hashes hash_list = [] for mod in module_list: @@ -888,7 +898,7 @@ def substitute_hashes_for_version_info(self, module_list): # tag should be one of dev, beta, release - do checking outside of this method def list_service_module_versions_with_tag(self, tag): - self._initialize_mongo_client() + self._ensure_mongo_connection() mods = list(self.modules.find({'info.dynamic_service': 1}, {'module_name_lc': 1, 'module_name': 1, 'current_versions.' + tag: 1})) @@ -909,7 +919,7 @@ def list_service_module_versions_with_tag(self, tag): # all released service module versions def list_all_released_service_module_versions(self): - self._initialize_mongo_client() + self._ensure_mongo_connection() return list(self.module_versions.find( { 'dynamic_service': 1, @@ -926,21 +936,21 @@ def list_all_released_service_module_versions(self): #### developer check methods def approve_developer(self, developer): - self._initialize_mongo_client() + self._ensure_mongo_connection() # if the developer is already on the list, just return if self.is_approved_developer([developer])[0]: return self.developers.insert_one({'kb_username': developer}) def revoke_developer(self, developer): - self._initialize_mongo_client() + self._ensure_mongo_connection() # if the developer is not on the list, throw an error (maybe a typo, so let's catch it) if not self.is_approved_developer([developer])[0]: raise ValueError('Cannot revoke "' + developer + '", that developer was not found.') self.developers.delete_one({'kb_username': developer}) def is_approved_developer(self, usernames): - self._initialize_mongo_client() + self._ensure_mongo_connection() # TODO: optimize, but I expect the list of usernames will be fairly small, so we can loop. Regardless, in # old mongo (2.x) I think this is even faster in most cases than using $in within a very large list is_approved = [] @@ -953,11 +963,11 @@ def is_approved_developer(self, usernames): return is_approved def list_approved_developers(self): - self._initialize_mongo_client() + self._ensure_mongo_connection() return list(self.developers.find({}, {'kb_username': 1, '_id': 0})) def migrate_module_to_new_git_url(self, module_name, current_git_url, new_git_url): - self._initialize_mongo_client() + self._ensure_mongo_connection() if not new_git_url.strip(): raise ValueError('New git url is required to migrate_module_to_new_git_url.') query = self._get_mongo_query(module_name=module_name, git_url=current_git_url) @@ -969,7 +979,7 @@ def migrate_module_to_new_git_url(self, module_name, current_git_url, new_git_ur return self._check_update_result(result) def delete_module(self, module_name, git_url): - self._initialize_mongo_client() + self._ensure_mongo_connection() if not module_name and not git_url: raise ValueError('Module name or git url is required to delete a module.') query = self._get_mongo_query(module_name=module_name, git_url=git_url) @@ -989,7 +999,7 @@ def delete_module(self, module_name, git_url): return self._check_update_result(result) def add_favorite(self, module_name, app_id, username, timestamp): - self._initialize_mongo_client() + self._ensure_mongo_connection() favoriteAddition = { 'user': username, 'module_name_lc': module_name.strip().lower(), @@ -1004,7 +1014,7 @@ def add_favorite(self, module_name, app_id, username, timestamp): self.favorites.insert_one(favoriteAddition) def remove_favorite(self, module_name, app_id, username): - self._initialize_mongo_client() + self._ensure_mongo_connection() favoriteAddition = { 'user': username, 'module_name_lc': module_name.strip().lower(), @@ -1019,19 +1029,19 @@ def remove_favorite(self, module_name, app_id, username): return self._check_update_result(result) def list_user_favorites(self, username): - self._initialize_mongo_client() + self._ensure_mongo_connection() query = {'user': username} selection = {'_id': 0, 'module_name_lc': 1, 'id': 1, 'timestamp': 1} return list(self.favorites.find(query, selection).sort('timestamp', DESCENDING)) def list_app_favorites(self, module_name, app_id): - self._initialize_mongo_client() + self._ensure_mongo_connection() query = {'module_name_lc': module_name.strip().lower(), 'id': app_id.strip()} selection = {'_id': 0, 'user': 1, 'timestamp': 1} return list(self.favorites.find(query, selection).sort('timestamp', DESCENDING)) def aggregate_favorites_over_apps(self, module_names_lc): - self._initialize_mongo_client() + self._ensure_mongo_connection() ### WARNING! If we switch to Mongo 3.x, the result object will change and this will break # setup the query @@ -1075,7 +1085,7 @@ def aggregate_favorites_over_apps(self, module_names_lc): # DEPRECATED! temporary function until everything is migrated to new client group structure def list_client_groups(self, app_ids): - self._initialize_mongo_client() + self._ensure_mongo_connection() if app_ids is not None: selection = { '_id': 0, @@ -1103,7 +1113,7 @@ def list_client_groups(self, app_ids): return list(self.client_groups.find({}, selection)) def set_client_group_config(self, config): - self._initialize_mongo_client() + self._ensure_mongo_connection() config['module_name_lc'] = config['module_name'].lower() return self._check_update_result(self.client_groups.replace_one( { @@ -1115,7 +1125,7 @@ def set_client_group_config(self, config): )) def remove_client_group_config(self, config): - self._initialize_mongo_client() + self._ensure_mongo_connection() config['module_name_lc'] = config['module_name'].lower() return self._check_update_result(self.client_groups.delete_one( { @@ -1125,7 +1135,7 @@ def remove_client_group_config(self, config): )) def list_client_group_configs(self, filter): - self._initialize_mongo_client() + self._ensure_mongo_connection() selection = {"_id": 0, "module_name_lc": 0} if 'module_name' in filter: filter['module_name_lc'] = filter['module_name'].lower() @@ -1133,7 +1143,7 @@ def list_client_group_configs(self, filter): return list(self.client_groups.find(filter, selection)) def set_volume_mount(self, volume_mount): - self._initialize_mongo_client() + self._ensure_mongo_connection() volume_mount['module_name_lc'] = volume_mount['module_name'].lower() return self._check_update_result(self.volume_mounts.replace_one( { @@ -1146,7 +1156,7 @@ def set_volume_mount(self, volume_mount): )) def remove_volume_mount(self, volume_mount): - self._initialize_mongo_client() + self._ensure_mongo_connection() volume_mount['module_name_lc'] = volume_mount['module_name'].lower() return self._check_update_result(self.volume_mounts.delete_one( { @@ -1156,7 +1166,7 @@ def remove_volume_mount(self, volume_mount): })) def list_volume_mounts(self, filter): - self._initialize_mongo_client() + self._ensure_mongo_connection() selection = {"_id": 0, "module_name_lc": 0} if 'module_name' in filter: filter['module_name_lc'] = filter['module_name'].lower() @@ -1195,7 +1205,7 @@ def _check_update_result(self, result): def add_exec_stats_raw(self, user_id, app_module_name, app_id, func_module_name, func_name, git_commit_hash, creation_time, exec_start_time, finish_time, is_error, job_id): - self._initialize_mongo_client() + self._ensure_mongo_connection() stats = { 'user_id': user_id, 'app_module_name': app_module_name, @@ -1213,7 +1223,7 @@ def add_exec_stats_raw(self, user_id, app_module_name, app_id, func_module_name, def add_exec_stats_apps(self, app_module_name, app_id, creation_time, exec_start_time, finish_time, is_error, type, time_range): - self._initialize_mongo_client() + self._ensure_mongo_connection() if not app_id: return full_app_id = app_id @@ -1236,7 +1246,7 @@ def add_exec_stats_apps(self, app_module_name, app_id, creation_time, exec_start def add_exec_stats_users(self, user_id, creation_time, exec_start_time, finish_time, is_error, type, time_range): - self._initialize_mongo_client() + self._ensure_mongo_connection() queue_time = exec_start_time - creation_time exec_time = finish_time - exec_start_time inc_data = { @@ -1250,7 +1260,7 @@ def add_exec_stats_users(self, user_id, creation_time, exec_start_time, {'$inc': inc_data}, upsert=True) def get_exec_stats_apps(self, full_app_ids, type, time_range): - self._initialize_mongo_client() + self._ensure_mongo_connection() filter = {} if full_app_ids: filter['full_app_id'] = {'$in': full_app_ids} @@ -1271,7 +1281,7 @@ def get_exec_stats_apps(self, full_app_ids, type, time_range): return list(self.exec_stats_apps.find(filter, selection)) def aggr_exec_stats_table(self, minTime, maxTime): - self._initialize_mongo_client() + self._ensure_mongo_connection() # setup the query aggParams = None group = { @@ -1325,7 +1335,7 @@ def aggr_exec_stats_table(self, minTime, maxTime): return counts def get_exec_raw_stats(self, minTime, maxTime): - self._initialize_mongo_client() + self._ensure_mongo_connection() filter = {} creationTimeFilter = {} if minTime is not None: @@ -1340,7 +1350,7 @@ def get_exec_raw_stats(self, minTime, maxTime): return list(self.exec_stats_raw.find(filter, {'_id': 0})) def set_secure_config_params(self, data_list): - self._initialize_mongo_client() + self._ensure_mongo_connection() for param_data in data_list: param_data['module_name_lc'] = param_data['module_name'].lower() param_data['version'] = param_data.get('version', '') @@ -1354,7 +1364,7 @@ def set_secure_config_params(self, data_list): upsert=True) def remove_secure_config_params(self, data_list): - self._initialize_mongo_client() + self._ensure_mongo_connection() for param_data in data_list: param_data['module_name_lc'] = param_data['module_name'].lower() param_data['version'] = param_data.get('version', '') @@ -1366,7 +1376,7 @@ def remove_secure_config_params(self, data_list): }) def get_secure_config_params(self, module_name): - self._initialize_mongo_client() + self._ensure_mongo_connection() selection = {"_id": 0, "module_name_lc": 0} filter = {"module_name_lc": module_name.lower()} return list(self.secure_config_params.find(filter, selection)) @@ -1375,27 +1385,28 @@ def get_secure_config_params(self, module_name): # todo: add 'in-progress' flag so if something goes done during an update, or if # another server is already starting an update, we can skip or abort - def check_db_schema(self): + def check_db_schema(self, mongo_client): + db = mongo_client[self.mongo_db] - db_version = self.get_db_version() + db_version = self.get_db_version(db) print('db_version=' + str(db_version)) if db_version < 2: print('Updating DB schema to V2...') - self.update_db_1_to_2() - self.update_db_version(2) + self.update_db_1_to_2(db) + self.update_db_version(2, db) print('done.') if db_version < 3: print('Updating DB schema to V3...') - self.update_db_2_to_3() - self.update_db_version(3) + self.update_db_2_to_3(db) + self.update_db_version(3, db) print('done.') if db_version < 4: print('Updating DB schema to V4...') - self.update_db_3_to_4() - self.update_db_version(4) + self.update_db_3_to_4(db) + self.update_db_version(4, db) print('done.') if db_version > 4: @@ -1404,28 +1415,30 @@ def check_db_schema(self): 'Incompatible DB versions. Expecting DB V4, found DV V' + str(db_version) + '. You are probably running an old version of the service. Start up failed.') - def get_db_version(self): + def get_db_version(self, db): + # version is a collection that should only have a single - version_collection = self.db[MongoCatalogDBI._DB_VERSION] + version_collection = db[MongoCatalogDBI._DB_VERSION] ver = version_collection.find_one({}) if (ver): return ver['version'] else: # if there is no version document, then we are DB v1 - self.update_db_version(1) + self.update_db_version(1, db) return 1 - def update_db_version(self, version): + def update_db_version(self, version, db): + # make sure we can't have two version documents - version_collection = self.db[MongoCatalogDBI._DB_VERSION] + version_collection = db[MongoCatalogDBI._DB_VERSION] version_collection.create_index('version_doc', unique=True, sparse=False) version_collection.update_one({'version_doc': True}, {'$set': {'version': version}}, upsert=True) # version 1 kept released module versions in a map, version 2 updates that to a list # and adds dynamic service tags - def update_db_1_to_2(self): - modules_collection = self.db[MongoCatalogDBI._MODULES] + def update_db_1_to_2(self, db): + modules_collection = db[MongoCatalogDBI._MODULES] for m in modules_collection.find({'release_versions': {'$exists': True}}): release_version_list = [] for timestamp in m['release_versions']: @@ -1464,7 +1477,7 @@ def update_db_1_to_2(self): {'$set': {'current_versions.dev.dynamic_service': 0}}) # also ensure the execution stats fields have correct names - exec_stats_apps_colleciton = self.db[MongoCatalogDBI._EXEC_STATS_APPS] + exec_stats_apps_colleciton = db[MongoCatalogDBI._EXEC_STATS_APPS] exec_stats_apps_colleciton.update_many({'avg_queue_time': {'$exists': True}}, {'$rename': {'avg_queue_time': 'total_queue_time', 'avg_exec_time': 'total_exec_time'}}) @@ -1474,9 +1487,9 @@ def update_db_1_to_2(self): # version 3 moves the module version information out of the module document into # a separate module versions collection. - def update_db_2_to_3(self): + def update_db_2_to_3(self, db): - module_versions_collection = self.db[MongoCatalogDBI._MODULE_VERSIONS] + module_versions_collection = db[MongoCatalogDBI._MODULE_VERSIONS] module_versions_collection.create_index('module_name_lc', sparse=False) module_versions_collection.create_index('git_commit_hash', sparse=False) module_versions_collection.create_index([ @@ -1484,7 +1497,7 @@ def update_db_2_to_3(self): ('git_commit_hash', ASCENDING)], unique=True, sparse=False) - modules_collection = self.db[MongoCatalogDBI._MODULES] + modules_collection = db[MongoCatalogDBI._MODULES] # update all module versions for m in modules_collection.find({}): @@ -1559,11 +1572,11 @@ def prepare_version_doc_for_db_2_to_3_update(self, version, module): version['release_timestamp'] = None # version 4 performs a small modification to the client group structure and volume_mounts structure - def update_db_3_to_4(self): + def update_db_3_to_4(self, db): # make sure we don't have any indecies on the collections - volume_mounts_collection = self.db[MongoCatalogDBI._VOLUME_MOUNTS] - client_groups_collection = self.db[MongoCatalogDBI._CLIENT_GROUPS] + volume_mounts_collection = db[MongoCatalogDBI._VOLUME_MOUNTS] + client_groups_collection = db[MongoCatalogDBI._CLIENT_GROUPS] volume_mounts_collection.drop_indexes() client_groups_collection.drop_indexes() diff --git a/test/docker-compose_nms.yml b/test/docker-compose_nms.yml index 906381f3..7e5e18c3 100644 --- a/test/docker-compose_nms.yml +++ b/test/docker-compose_nms.yml @@ -17,6 +17,8 @@ services: - method_spec_temp_dir=narrative_method_store_temp - method_spec_mongo_host=mongo:27017 - method_spec_mongo_dbname=method_store_repo_db + # For testing locally, either export the ADMIN_USER variable directly in the shell, + # or hardcode an admin user in the YAML file. - method_spec_admin_users=${ADMIN_USER} - endpoint_host=https://ci.kbase.us - endpoint_base=/services From 26d891066bf1f8e8fc9911a4b2aadcd70978f82d Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 25 Feb 2025 11:54:00 -0800 Subject: [PATCH 085/110] add a comment for _ensure_mongo_connection function --- lib/biokbase/catalog/db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 408fbba6..fab1ed12 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -179,6 +179,7 @@ def _ensure_mongo_connection(self): # Don't enter the lock if we already have the client if not self.mongo_client: with lock: + # Double-check if another thread has already initialized the client while we were waiting for the lock if self.mongo_client: return self.mongo_client = self._initialize_mongo_client() From e9db8f14c54e58eafa804f6b3ce6b874c7cc61ce Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 26 Feb 2025 12:11:15 -0800 Subject: [PATCH 086/110] Clarify the comments in the db.py and docker-compose.yml files --- lib/biokbase/catalog/db.py | 8 ++------ test/docker-compose_nms.yml | 5 +++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index fab1ed12..8580fd57 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -149,7 +149,6 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech def _initialize_mongo_client(self): """Initialize MongoDB client.""" - # Use the lock to ensure only one thread initializes the mongo client at a time try: # This is only tested manually if self.mongo_user and self.mongo_psswd: @@ -168,16 +167,13 @@ def _initialize_mongo_client(self): return mongo_client except ConnectionFailure as e: - error_msg = "Cannot connect to Mongo server\n" - error_msg += "ERROR -- {}:\n{}".format( - e, "".join(traceback.format_exception(None, e, e.__traceback__)) - ) - raise ValueError(error_msg) + raise ValueError(f"Cannot connect to Mongo server: {e}") from e def _ensure_mongo_connection(self): """Ensure the MongoDB connection is active.""" # Don't enter the lock if we already have the client if not self.mongo_client: + # Use the lock to ensure only one thread initializes the mongo client at a time with lock: # Double-check if another thread has already initialized the client while we were waiting for the lock if self.mongo_client: diff --git a/test/docker-compose_nms.yml b/test/docker-compose_nms.yml index 7e5e18c3..658b1343 100644 --- a/test/docker-compose_nms.yml +++ b/test/docker-compose_nms.yml @@ -17,9 +17,10 @@ services: - method_spec_temp_dir=narrative_method_store_temp - method_spec_mongo_host=mongo:27017 - method_spec_mongo_dbname=method_store_repo_db - # For testing locally, either export the ADMIN_USER variable directly in the shell, - # or hardcode an admin user in the YAML file. - method_spec_admin_users=${ADMIN_USER} + # For local testing, you can comment the above line and + # set method_spec_admin_users directly to your kbase_id, as shown in the example below: + # - method_spec_admin_users=sijiex - endpoint_host=https://ci.kbase.us - endpoint_base=/services - method_spec_default_tag=dev From 1c4ce2263d105eb32d346cabd4cc0aea5b79f865 Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 27 Feb 2025 16:54:27 -0800 Subject: [PATCH 087/110] remove NarrativeMethodStore section and update the test instructions in the test/test.cfg.example file --- README.md | 2 ++ test/catalog_test_util.py | 7 +++---- test/test.cfg.example | 26 ++------------------------ 3 files changed, 7 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index e3e84c69..d00b180f 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ KBase core service to manage app and module information, registration, and release. Administrators need to be set separately for the job stats page by being added to [deploy.cfg.](https://github.com/kbaseapps/kb_Metrics/blob/master/deploy.cfg) +Test: Please refer to the instructions at the top of `test/test.cfg.example` file. + Build status: master: [![Build Status](https://travis-ci.org/kbase/catalog.svg?branch=master)](https://travis-ci.org/kbase/catalog) staging: [![Build Status](https://travis-ci.org/kbase/catalog.svg?branch=staging)](https://travis-ci.org/kbase/catalog) diff --git a/test/catalog_test_util.py b/test/catalog_test_util.py index ce2a72e1..575f0b6a 100644 --- a/test/catalog_test_util.py +++ b/test/catalog_test_util.py @@ -42,11 +42,8 @@ def _setup_config(self): config = ConfigParser() config.read(os.path.join(self.test_dir, 'test.cfg')) self.test_cfg = {} - self.nms_test_cfg = {} for entry in config.items('catalog-test'): self.test_cfg[entry[0]] = entry[1] - for entry in config.items('NarrativeMethodStore'): - self.nms_test_cfg[entry[0]] = entry[1] self.log('test.cfg parse\n' + pformat(self.test_cfg)) # passwords not needed in tests yet @@ -230,7 +227,9 @@ def tearDown(self): self.dockerclient.remove_image(image['Id']) # make sure NMS is clean after each test - self.mongo.drop_database(self.nms_test_cfg['method-spec-mongo-dbname']) + # Drop the database set by the method_spec_mongo_dbname environment variable + # in the docker-compose_nms.yml file. + self.mongo.drop_database('method_store_repo_db') def log(self, mssg): # uncomment to debug test rig- warning: on travis this may print any passwords in your config diff --git a/test/test.cfg.example b/test/test.cfg.example index 2de74726..e2f3a32f 100644 --- a/test/test.cfg.example +++ b/test/test.cfg.example @@ -3,11 +3,8 @@ # TO RUN TESTS: # 1) copy this file to test.cfg -# 2) fill in the mongodb-host url in both [catalog-test] and [NarrativeMethodStore] -# 3) set the nms-admin-user and nms-admin-password or nms-admin-token to a -# kb user that matches the method-spec-admin-users list in -# the [NarrativeMethodStore section] -# +# 2) set the nms-admin-token to a kb user that matches the method-spec-admin-users list +# in the docker-compose_nms.yml file [catalog-test] @@ -38,28 +35,9 @@ mongodb-database = catalog-test docker-base-url = unix://var/run/docker.sock # Narrative Method Store configuration. Please provide a token. -# If both are provided, the token is used. nms-url = http://localhost:7125/rpc nms-admin-token = # configs for reference data ref-data-base = /kb/data kbase-endpoint = https://ci.kbase.us/services - - -[NarrativeMethodStore] - -method-spec-mongo-host = localhost:27017 -method-spec-admin-users = wstester4 - -method-spec-git-repo-local-dir = nms/local_narrative_method_specs -method-spec-temp-dir = nms/scratch - -method-spec-git-repo = https://github.com/kbase/narrative_method_specs -method-spec-git-repo-branch = develop -method-spec-git-repo-refresh-rate = 2000 -method-spec-cache-size = 5000 -method-spec-mongo-dbname = method_store_repo_db - -# The KBase auth server url. -auth-service-url=https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login From be5d794c8c7e54bf05b77258807105c98705e5ae Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 27 Feb 2025 17:29:08 -0800 Subject: [PATCH 088/110] expound on the explanation --- lib/biokbase/catalog/db.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 8580fd57..37d278df 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -125,6 +125,14 @@ class MongoCatalogDBI: def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMechanism): + # We're performing two MongoDB client initializations—one during the initial setup (on init), and + # another lazy initialization after the process is forked. This approach is necessary because MongoDB client + # connections are not fork-safe. When a process is forked, the child process inherits a copy of the parent’s memory, + # but the client connection doesn't transfer properly. This can cause issues if both the parent and child + # processes share the same connection. To prevent this, we close the MongoDB client connection before forking + # and reinitialize it in each process, ensuring that both the parent and child processes maintain + # their own independent, functional connections. + self.mongo_host = mongo_host self.mongo_db = mongo_db self.mongo_user = mongo_user From 925e94770917a5692d3de9949dc5b3afbbebba8c Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 28 Feb 2025 12:51:15 -0800 Subject: [PATCH 089/110] fix c&p error --- lib/biokbase/catalog/db.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 37d278df..93b7619b 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -1486,7 +1486,9 @@ def update_db_1_to_2(self, db): exec_stats_apps_colleciton.update_many({'avg_queue_time': {'$exists': True}}, {'$rename': {'avg_queue_time': 'total_queue_time', 'avg_exec_time': 'total_exec_time'}}) - exec_stats_apps_colleciton.update_many({'avg_queue_time': {'$exists': True}}, + + exec_stats_users_collection = db[MongoCatalogDBI._EXEC_STATS_USERS] + exec_stats_users_collection.update_many({'avg_queue_time': {'$exists': True}}, {'$rename': {'avg_queue_time': 'total_queue_time', 'avg_exec_time': 'total_exec_time'}}) From 14d3590eb34689bc5b597fb2ed9ad3666c925777 Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 28 Feb 2025 13:23:10 -0800 Subject: [PATCH 090/110] fix typos and clarify comments --- lib/biokbase/catalog/db.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 93b7619b..09d1fedd 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -127,7 +127,9 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech # We're performing two MongoDB client initializations—one during the initial setup (on init), and # another lazy initialization after the process is forked. This approach is necessary because MongoDB client - # connections are not fork-safe. When a process is forked, the child process inherits a copy of the parent’s memory, + # connections are not fork-safe and some servers, including uwsgi, fork workers from parent processes. + + # When a process is forked, the child process inherits a copy of the parent’s memory, # but the client connection doesn't transfer properly. This can cause issues if both the parent and child # processes share the same connection. To prevent this, we close the MongoDB client connection before forking # and reinitialize it in each process, ensuring that both the parent and child processes maintain @@ -1482,8 +1484,8 @@ def update_db_1_to_2(self, db): {'$set': {'current_versions.dev.dynamic_service': 0}}) # also ensure the execution stats fields have correct names - exec_stats_apps_colleciton = db[MongoCatalogDBI._EXEC_STATS_APPS] - exec_stats_apps_colleciton.update_many({'avg_queue_time': {'$exists': True}}, + exec_stats_apps_collection = db[MongoCatalogDBI._EXEC_STATS_APPS] + exec_stats_apps_collection.update_many({'avg_queue_time': {'$exists': True}}, {'$rename': {'avg_queue_time': 'total_queue_time', 'avg_exec_time': 'total_exec_time'}}) From de6d3341497dc4fdc403b15c7389a78a13ac6534 Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 28 Feb 2025 16:42:12 -0800 Subject: [PATCH 091/110] make sure NMS is clean after each test --- test/catalog_test_util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/catalog_test_util.py b/test/catalog_test_util.py index 575f0b6a..20b722fa 100644 --- a/test/catalog_test_util.py +++ b/test/catalog_test_util.py @@ -229,7 +229,8 @@ def tearDown(self): # make sure NMS is clean after each test # Drop the database set by the method_spec_mongo_dbname environment variable # in the docker-compose_nms.yml file. - self.mongo.drop_database('method_store_repo_db') + nms_mongo_client = MongoClient('mongodb://localhost:27018') + nms_mongo_client.drop_database('method_store_repo_db') def log(self, mssg): # uncomment to debug test rig- warning: on travis this may print any passwords in your config From b0af93300e8485cc16e3b38c2d6f44f502b9a6dc Mon Sep 17 00:00:00 2001 From: Sijie Date: Mon, 3 Mar 2025 14:09:26 -0800 Subject: [PATCH 092/110] remove uwsgi from dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 884f1268..7f267f60 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ ARG BUILD_DATE ARG VCS_REF ARG BRANCH -RUN apt-get update && apt-get install -y wget uwsgi +RUN apt-get update && apt-get install -y wget # install dockerize WORKDIR /opt From 7724d77c0510f6c6dfe443d63214179ec4e9f4e9 Mon Sep 17 00:00:00 2001 From: Sijie Date: Mon, 3 Mar 2025 15:08:24 -0800 Subject: [PATCH 093/110] add Pipenv and Pipenv.lock files --- .github/workflows/test.yml | 4 +++- Dockerfile | 11 ++++++++--- requirements.txt | 8 -------- 3 files changed, 11 insertions(+), 12 deletions(-) delete mode 100644 requirements.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d480d745..206a0ea9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -70,7 +70,9 @@ jobs: # install catalog dependencies cd $HOMEDIR - pip install -r requirements.txt + python -m pip install --upgrade pip + pip install pipenv + pipenv sync --system --dev # setup test config cp -n test/test.cfg.example test/test.cfg diff --git a/Dockerfile b/Dockerfile index 7f267f60..5768ed83 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,9 +30,14 @@ COPY --from=build /kb/deployment/services /kb/deployment/services COPY --from=build /tmp/catalog/deployment/conf /kb/deployment/conf WORKDIR /tmp/catalog -RUN pip install --upgrade pip -COPY requirements.txt requirements.txt -RUN pip install -r requirements.txt + +# install pipenv +RUN pip install --upgrade pip && \ + pip install pipenv + +# install deps +COPY Pipfile* ./ +RUN pipenv sync --system LABEL org.label-schema.build-date=$BUILD_DATE \ org.label-schema.vcs-url="https://github.com/kbase/catalog.git" \ diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 86d891ef..00000000 --- a/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -pymongo==4.7.2 -docker>=3.5 -gitpython -pyyaml -semantic_version -coverage -uwsgi==2.0.18 -JSONRPCBase==0.2.0 \ No newline at end of file From 8ec7c64a7505965d580d61d123d8ce9141e4bbf9 Mon Sep 17 00:00:00 2001 From: Sijie Date: Mon, 3 Mar 2025 15:12:50 -0800 Subject: [PATCH 094/110] upload Pipfiles --- Pipfile | 17 +++ Pipfile.lock | 346 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 363 insertions(+) create mode 100644 Pipfile create mode 100644 Pipfile.lock diff --git a/Pipfile b/Pipfile new file mode 100644 index 00000000..50b2f800 --- /dev/null +++ b/Pipfile @@ -0,0 +1,17 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +docker = "==7.1.0" +jsonrpcbase = "==0.2.0" +pymongo = "==4.7.2" +uwsgi = "==2.0.22" + +[dev-packages] +coverage = "==7.6.1" +semantic-version = "==2.10.0" + +[requires] +python_version = "3.9.19" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 00000000..037295b7 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,346 @@ +{ + "_meta": { + "hash": { + "sha256": "7172916256c731b2b8c17e008ed96a81d54ce2b90a702c7e25adfeb187b00c49" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9.19" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "certifi": { + "hashes": [ + "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651", + "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe" + ], + "markers": "python_version >= '3.6'", + "version": "==2025.1.31" + }, + "charset-normalizer": { + "hashes": [ + "sha256:0167ddc8ab6508fe81860a57dd472b2ef4060e8d378f0cc555707126830f2537", + "sha256:01732659ba9b5b873fc117534143e4feefecf3b2078b0a6a2e925271bb6f4cfa", + "sha256:01ad647cdd609225c5350561d084b42ddf732f4eeefe6e678765636791e78b9a", + "sha256:04432ad9479fa40ec0f387795ddad4437a2b50417c69fa275e212933519ff294", + "sha256:0907f11d019260cdc3f94fbdb23ff9125f6b5d1039b76003b5b0ac9d6a6c9d5b", + "sha256:0924e81d3d5e70f8126529951dac65c1010cdf117bb75eb02dd12339b57749dd", + "sha256:09b26ae6b1abf0d27570633b2b078a2a20419c99d66fb2823173d73f188ce601", + "sha256:09b5e6733cbd160dcc09589227187e242a30a49ca5cefa5a7edd3f9d19ed53fd", + "sha256:0af291f4fe114be0280cdd29d533696a77b5b49cfde5467176ecab32353395c4", + "sha256:0f55e69f030f7163dffe9fd0752b32f070566451afe180f99dbeeb81f511ad8d", + "sha256:1a2bc9f351a75ef49d664206d51f8e5ede9da246602dc2d2726837620ea034b2", + "sha256:22e14b5d70560b8dd51ec22863f370d1e595ac3d024cb8ad7d308b4cd95f8313", + "sha256:234ac59ea147c59ee4da87a0c0f098e9c8d169f4dc2a159ef720f1a61bbe27cd", + "sha256:2369eea1ee4a7610a860d88f268eb39b95cb588acd7235e02fd5a5601773d4fa", + "sha256:237bdbe6159cff53b4f24f397d43c6336c6b0b42affbe857970cefbb620911c8", + "sha256:28bf57629c75e810b6ae989f03c0828d64d6b26a5e205535585f96093e405ed1", + "sha256:2967f74ad52c3b98de4c3b32e1a44e32975e008a9cd2a8cc8966d6a5218c5cb2", + "sha256:2a75d49014d118e4198bcee5ee0a6f25856b29b12dbf7cd012791f8a6cc5c496", + "sha256:2bdfe3ac2e1bbe5b59a1a63721eb3b95fc9b6817ae4a46debbb4e11f6232428d", + "sha256:2d074908e1aecee37a7635990b2c6d504cd4766c7bc9fc86d63f9c09af3fa11b", + "sha256:2fb9bd477fdea8684f78791a6de97a953c51831ee2981f8e4f583ff3b9d9687e", + "sha256:311f30128d7d333eebd7896965bfcfbd0065f1716ec92bd5638d7748eb6f936a", + "sha256:329ce159e82018d646c7ac45b01a430369d526569ec08516081727a20e9e4af4", + "sha256:345b0426edd4e18138d6528aed636de7a9ed169b4aaf9d61a8c19e39d26838ca", + "sha256:363e2f92b0f0174b2f8238240a1a30142e3db7b957a5dd5689b0e75fb717cc78", + "sha256:3a3bd0dcd373514dcec91c411ddb9632c0d7d92aed7093b8c3bbb6d69ca74408", + "sha256:3bed14e9c89dcb10e8f3a29f9ccac4955aebe93c71ae803af79265c9ca5644c5", + "sha256:44251f18cd68a75b56585dd00dae26183e102cd5e0f9f1466e6df5da2ed64ea3", + "sha256:44ecbf16649486d4aebafeaa7ec4c9fed8b88101f4dd612dcaf65d5e815f837f", + "sha256:4532bff1b8421fd0a320463030c7520f56a79c9024a4e88f01c537316019005a", + "sha256:49402233c892a461407c512a19435d1ce275543138294f7ef013f0b63d5d3765", + "sha256:4c0907b1928a36d5a998d72d64d8eaa7244989f7aaaf947500d3a800c83a3fd6", + "sha256:4d86f7aff21ee58f26dcf5ae81a9addbd914115cdebcbb2217e4f0ed8982e146", + "sha256:5777ee0881f9499ed0f71cc82cf873d9a0ca8af166dfa0af8ec4e675b7df48e6", + "sha256:5df196eb874dae23dcfb968c83d4f8fdccb333330fe1fc278ac5ceeb101003a9", + "sha256:619a609aa74ae43d90ed2e89bdd784765de0a25ca761b93e196d938b8fd1dbbd", + "sha256:6e27f48bcd0957c6d4cb9d6fa6b61d192d0b13d5ef563e5f2ae35feafc0d179c", + "sha256:6ff8a4a60c227ad87030d76e99cd1698345d4491638dfa6673027c48b3cd395f", + "sha256:73d94b58ec7fecbc7366247d3b0b10a21681004153238750bb67bd9012414545", + "sha256:7461baadb4dc00fd9e0acbe254e3d7d2112e7f92ced2adc96e54ef6501c5f176", + "sha256:75832c08354f595c760a804588b9357d34ec00ba1c940c15e31e96d902093770", + "sha256:7709f51f5f7c853f0fb938bcd3bc59cdfdc5203635ffd18bf354f6967ea0f824", + "sha256:78baa6d91634dfb69ec52a463534bc0df05dbd546209b79a3880a34487f4b84f", + "sha256:7974a0b5ecd505609e3b19742b60cee7aa2aa2fb3151bc917e6e2646d7667dcf", + "sha256:7a4f97a081603d2050bfaffdefa5b02a9ec823f8348a572e39032caa8404a487", + "sha256:7b1bef6280950ee6c177b326508f86cad7ad4dff12454483b51d8b7d673a2c5d", + "sha256:7d053096f67cd1241601111b698f5cad775f97ab25d81567d3f59219b5f1adbd", + "sha256:804a4d582ba6e5b747c625bf1255e6b1507465494a40a2130978bda7b932c90b", + "sha256:807f52c1f798eef6cf26beb819eeb8819b1622ddfeef9d0977a8502d4db6d534", + "sha256:80ed5e856eb7f30115aaf94e4a08114ccc8813e6ed1b5efa74f9f82e8509858f", + "sha256:8417cb1f36cc0bc7eaba8ccb0e04d55f0ee52df06df3ad55259b9a323555fc8b", + "sha256:8436c508b408b82d87dc5f62496973a1805cd46727c34440b0d29d8a2f50a6c9", + "sha256:89149166622f4db9b4b6a449256291dc87a99ee53151c74cbd82a53c8c2f6ccd", + "sha256:8bfa33f4f2672964266e940dd22a195989ba31669bd84629f05fab3ef4e2d125", + "sha256:8c60ca7339acd497a55b0ea5d506b2a2612afb2826560416f6894e8b5770d4a9", + "sha256:91b36a978b5ae0ee86c394f5a54d6ef44db1de0815eb43de826d41d21e4af3de", + "sha256:955f8851919303c92343d2f66165294848d57e9bba6cf6e3625485a70a038d11", + "sha256:97f68b8d6831127e4787ad15e6757232e14e12060bec17091b85eb1486b91d8d", + "sha256:9b23ca7ef998bc739bf6ffc077c2116917eabcc901f88da1b9856b210ef63f35", + "sha256:9f0b8b1c6d84c8034a44893aba5e767bf9c7a211e313a9605d9c617d7083829f", + "sha256:aabfa34badd18f1da5ec1bc2715cadc8dca465868a4e73a0173466b688f29dda", + "sha256:ab36c8eb7e454e34e60eb55ca5d241a5d18b2c6244f6827a30e451c42410b5f7", + "sha256:b010a7a4fd316c3c484d482922d13044979e78d1861f0e0650423144c616a46a", + "sha256:b1ac5992a838106edb89654e0aebfc24f5848ae2547d22c2c3f66454daa11971", + "sha256:b7b2d86dd06bfc2ade3312a83a5c364c7ec2e3498f8734282c6c3d4b07b346b8", + "sha256:b97e690a2118911e39b4042088092771b4ae3fc3aa86518f84b8cf6888dbdb41", + "sha256:bc2722592d8998c870fa4e290c2eec2c1569b87fe58618e67d38b4665dfa680d", + "sha256:c0429126cf75e16c4f0ad00ee0eae4242dc652290f940152ca8c75c3a4b6ee8f", + "sha256:c30197aa96e8eed02200a83fba2657b4c3acd0f0aa4bdc9f6c1af8e8962e0757", + "sha256:c4c3e6da02df6fa1410a7680bd3f63d4f710232d3139089536310d027950696a", + "sha256:c75cb2a3e389853835e84a2d8fb2b81a10645b503eca9bcb98df6b5a43eb8886", + "sha256:c96836c97b1238e9c9e3fe90844c947d5afbf4f4c92762679acfe19927d81d77", + "sha256:d7f50a1f8c450f3925cb367d011448c39239bb3eb4117c36a6d354794de4ce76", + "sha256:d973f03c0cb71c5ed99037b870f2be986c3c05e63622c017ea9816881d2dd247", + "sha256:d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85", + "sha256:d9c3cdf5390dcd29aa8056d13e8e99526cda0305acc038b96b30352aff5ff2bb", + "sha256:dad3e487649f498dd991eeb901125411559b22e8d7ab25d3aeb1af367df5efd7", + "sha256:dccbe65bd2f7f7ec22c4ff99ed56faa1e9f785482b9bbd7c717e26fd723a1d1e", + "sha256:dd78cfcda14a1ef52584dbb008f7ac81c1328c0f58184bf9a84c49c605002da6", + "sha256:e218488cd232553829be0664c2292d3af2eeeb94b32bea483cf79ac6a694e037", + "sha256:e358e64305fe12299a08e08978f51fc21fac060dcfcddd95453eabe5b93ed0e1", + "sha256:ea0d8d539afa5eb2728aa1932a988a9a7af94f18582ffae4bc10b3fbdad0626e", + "sha256:eab677309cdb30d047996b36d34caeda1dc91149e4fdca0b1a039b3f79d9a807", + "sha256:eb8178fe3dba6450a3e024e95ac49ed3400e506fd4e9e5c32d30adda88cbd407", + "sha256:ecddf25bee22fe4fe3737a399d0d177d72bc22be6913acfab364b40bce1ba83c", + "sha256:eea6ee1db730b3483adf394ea72f808b6e18cf3cb6454b4d86e04fa8c4327a12", + "sha256:f08ff5e948271dc7e18a35641d2f11a4cd8dfd5634f55228b691e62b37125eb3", + "sha256:f30bf9fd9be89ecb2360c7d94a711f00c09b976258846efe40db3d05828e8089", + "sha256:fa88b843d6e211393a37219e6a1c1df99d35e8fd90446f1118f4216e307e48cd", + "sha256:fc54db6c8593ef7d4b2a331b58653356cf04f67c960f584edb7c3d8c97e8f39e", + "sha256:fd4ec41f914fa74ad1b8304bbc634b3de73d2a0889bd32076342a573e0779e00", + "sha256:ffc9202a29ab3920fa812879e95a9e78b2465fd10be7fcbd042899695d75e616" + ], + "markers": "python_version >= '3.7'", + "version": "==3.4.1" + }, + "dnspython": { + "hashes": [ + "sha256:5ef3b9680161f6fa89daf8ad451b5f1a33b18ae8a1c6778cdf4b43f08c0a6e50", + "sha256:e8f0f9c23a7b7cb99ded64e6c3a6f3e701d78f50c55e002b839dea7225cff7cc" + ], + "markers": "python_version >= '3.8'", + "version": "==2.6.1" + }, + "docker": { + "hashes": [ + "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c", + "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==7.1.0" + }, + "idna": { + "hashes": [ + "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", + "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3" + ], + "markers": "python_version >= '3.6'", + "version": "==3.10" + }, + "jsonrpcbase": { + "hashes": [ + "sha256:7ea67fc1a7c87756e9a876e18a342e431e80d0ef3ba867dfd6f3fac5bf3fcc0d" + ], + "index": "pypi", + "version": "==0.2.0" + }, + "pymongo": { + "hashes": [ + "sha256:02efd1bb3397e24ef2af45923888b41a378ce00cb3a4259c5f4fc3c70497a22f", + "sha256:0d833651f1ba938bb7501f13e326b96cfbb7d98867b2d545ca6d69c7664903e0", + "sha256:12c466e02133b7f8f4ff1045c6b5916215c5f7923bc83fd6e28e290cba18f9f6", + "sha256:12d1fef77d25640cb78893d07ff7d2fac4c4461d8eec45bd3b9ad491a1115d6e", + "sha256:194065c9d445017b3c82fb85f89aa2055464a080bde604010dc8eb932a6b3c95", + "sha256:1c78f156edc59b905c80c9003e022e1a764c54fd40ac4fea05b0764f829790e2", + "sha256:1e37faf298a37ffb3e0809e77fbbb0a32b6a2d18a83c59cfc2a7b794ea1136b0", + "sha256:25eeb2c18ede63891cbd617943dd9e6b9cbccc54f276e0b2e693a0cc40f243c5", + "sha256:268d8578c0500012140c5460755ea405cbfe541ef47c81efa9d6744f0f99aeca", + "sha256:2cb77d09bd012cb4b30636e7e38d00b5f9be5eb521c364bde66490c45ee6c4b4", + "sha256:347c49cf7f0ba49ea87c1a5a1984187ecc5516b7c753f31938bf7b37462824fd", + "sha256:35b3f0c7d49724859d4df5f0445818d525824a6cd55074c42573d9b50764df67", + "sha256:37e9ea81fa59ee9274457ed7d59b6c27f6f2a5fe8e26f184ecf58ea52a019cb8", + "sha256:47a1a4832ef2f4346dcd1a10a36ade7367ad6905929ddb476459abb4fd1b98cb", + "sha256:4bdb5ffe1cd3728c9479671a067ef44dacafc3743741d4dc700c377c4231356f", + "sha256:4ffd1519edbe311df73c74ec338de7d294af535b2748191c866ea3a7c484cd15", + "sha256:5239776633f7578b81207e5646245415a5a95f6ae5ef5dff8e7c2357e6264bfc", + "sha256:5239ef7e749f1326ea7564428bf861d5250aa39d7f26d612741b1b1273227062", + "sha256:56bf8b706946952acdea0fe478f8e44f1ed101c4b87f046859e6c3abe6c0a9f4", + "sha256:65b4c00dedbd333698b83cd2095a639a6f0d7c4e2a617988f6c65fb46711f028", + "sha256:6a87eef394039765679f75c6a47455a4030870341cb76eafc349c5944408c882", + "sha256:727ad07952c155cd20045f2ce91143c7dc4fb01a5b4e8012905a89a7da554b0c", + "sha256:730778b6f0964b164c187289f906bbc84cb0524df285b7a85aa355bbec43eb21", + "sha256:743552033c63f0afdb56b9189ab04b5c1dbffd7310cf7156ab98eebcecf24621", + "sha256:7e9d9d2c0aae73aa4369bd373ac2ac59f02c46d4e56c4b6d6e250cfe85f76802", + "sha256:82102e353be13f1a6769660dd88115b1da382447672ba1c2662a0fbe3df1d861", + "sha256:827611beb6c483260d520cfa6a49662d980dfa5368a04296f65fa39e78fccea7", + "sha256:84bc00200c3cbb6c98a2bb964c9e8284b641e4a33cf10c802390552575ee21de", + "sha256:87032f818bf5052ab742812c715eff896621385c43f8f97cdd37d15b5d394e95", + "sha256:87832d6076c2c82f42870157414fd876facbb6554d2faf271ffe7f8f30ce7bed", + "sha256:87bb453ac3eb44db95cb6d5a616fbc906c1c00661eec7f55696253a6245beb8a", + "sha256:9024e1661c6e40acf468177bf90ce924d1bc681d2b244adda3ed7b2f4c4d17d7", + "sha256:9349f0bb17a31371d4cacb64b306e4ca90413a3ad1fffe73ac7cd495570d94b5", + "sha256:9385654f01a90f73827af4db90c290a1519f7d9102ba43286e187b373e9a78e9", + "sha256:9a8bd37f5dabc86efceb8d8cbff5969256523d42d08088f098753dba15f3b37a", + "sha256:9d892fb91e81cccb83f507cdb2ea0aa026ec3ced7f12a1d60f6a5bf0f20f9c1f", + "sha256:a754e366c404d19ff3f077ddeed64be31e0bb515e04f502bf11987f1baa55a16", + "sha256:b48a5650ee5320d59f6d570bd99a8d5c58ac6f297a4e9090535f6561469ac32e", + "sha256:bcf337d1b252405779d9c79978d6ca15eab3cdaa2f44c100a79221bddad97c8a", + "sha256:c44efab10d9a3db920530f7bcb26af8f408b7273d2f0214081d3891979726328", + "sha256:c72d16fede22efe7cdd1f422e8da15760e9498024040429362886f946c10fe95", + "sha256:cb6e00a79dff22c9a72212ad82021b54bdb3b85f38a85f4fc466bde581d7d17a", + "sha256:ce1a374ea0e49808e0380ffc64284c0ce0f12bd21042b4bef1af3eb7bdf49054", + "sha256:cecd2df037249d1c74f0af86fb5b766104a5012becac6ff63d85d1de53ba8b98", + "sha256:cf17ea9cea14d59b0527403dd7106362917ced7c4ec936c4ba22bd36c912c8e0", + "sha256:cf28430ec1924af1bffed37b69a812339084697fd3f3e781074a0148e6475803", + "sha256:d1bcd58669e56c08f1e72c5758868b5df169fe267501c949ee83c418e9df9155", + "sha256:d275596f840018858757561840767b39272ac96436fcb54f5cac6d245393fd97", + "sha256:d2dcf608d35644e8d276d61bf40a93339d8d66a0e5f3e3f75b2c155a421a1b71", + "sha256:d4d59776f435564159196d971aa89422ead878174aff8fe18e06d9a0bc6d648c", + "sha256:d9b6cbc037108ff1a0a867e7670d8513c37f9bcd9ee3d2464411bfabf70ca002", + "sha256:db4380d1e69fdad1044a4b8f3bb105200542c49a0dde93452d938ff9db1d6d29", + "sha256:e004527ea42a6b99a8b8d5b42b42762c3bdf80f88fbdb5c3a9d47f3808495b86", + "sha256:e6eab12c6385526d386543d6823b07187fefba028f0da216506e00f0e1855119", + "sha256:eb0642e5f0dd7e86bb358749cc278e70b911e617f519989d346f742dc9520dfb", + "sha256:f91073049c43d14e66696970dd708d319b86ee57ef9af359294eee072abaac79", + "sha256:fadc6e8db7707c861ebe25b13ad6aca19ea4d2c56bf04a26691f46c23dadf6e4", + "sha256:fc5af24fcf5fc6f7f40d65446400d45dd12bea933d0299dc9e90c5b22197f1e9", + "sha256:fcaf8c911cb29316a02356f89dbc0e0dfcc6a712ace217b6b543805690d2aefd", + "sha256:ffd4d7cb2e6c6e100e2b39606d38a9ffc934e18593dc9bb326196afc7d93ce3d" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==4.7.2" + }, + "requests": { + "hashes": [ + "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", + "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6" + ], + "markers": "python_version >= '3.8'", + "version": "==2.32.3" + }, + "six": { + "hashes": [ + "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", + "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==1.17.0" + }, + "urllib3": { + "hashes": [ + "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac", + "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9" + ], + "markers": "python_version >= '3.8'", + "version": "==2.2.3" + }, + "uwsgi": { + "hashes": [ + "sha256:4cc4727258671ac5fa17ab422155e9aaef8a2008ebb86e4404b66deaae965db2" + ], + "index": "pypi", + "version": "==2.0.22" + } + }, + "develop": { + "coverage": { + "hashes": [ + "sha256:06a737c882bd26d0d6ee7269b20b12f14a8704807a01056c80bb881a4b2ce6ca", + "sha256:07e2ca0ad381b91350c0ed49d52699b625aab2b44b65e1b4e02fa9df0e92ad2d", + "sha256:0c0420b573964c760df9e9e86d1a9a622d0d27f417e1a949a8a66dd7bcee7bc6", + "sha256:0dbde0f4aa9a16fa4d754356a8f2e36296ff4d83994b2c9d8398aa32f222f989", + "sha256:1125ca0e5fd475cbbba3bb67ae20bd2c23a98fac4e32412883f9bcbaa81c314c", + "sha256:13b0a73a0896988f053e4fbb7de6d93388e6dd292b0d87ee51d106f2c11b465b", + "sha256:166811d20dfea725e2e4baa71fffd6c968a958577848d2131f39b60043400223", + "sha256:170d444ab405852903b7d04ea9ae9b98f98ab6d7e63e1115e82620807519797f", + "sha256:1f4aa8219db826ce6be7099d559f8ec311549bfc4046f7f9fe9b5cea5c581c56", + "sha256:225667980479a17db1048cb2bf8bfb39b8e5be8f164b8f6628b64f78a72cf9d3", + "sha256:260933720fdcd75340e7dbe9060655aff3af1f0c5d20f46b57f262ab6c86a5e8", + "sha256:2bdb062ea438f22d99cba0d7829c2ef0af1d768d1e4a4f528087224c90b132cb", + "sha256:2c09f4ce52cb99dd7505cd0fc8e0e37c77b87f46bc9c1eb03fe3bc9991085388", + "sha256:3115a95daa9bdba70aea750db7b96b37259a81a709223c8448fa97727d546fe0", + "sha256:3e0cadcf6733c09154b461f1ca72d5416635e5e4ec4e536192180d34ec160f8a", + "sha256:3f1156e3e8f2872197af3840d8ad307a9dd18e615dc64d9ee41696f287c57ad8", + "sha256:4421712dbfc5562150f7554f13dde997a2e932a6b5f352edcce948a815efee6f", + "sha256:44df346d5215a8c0e360307d46ffaabe0f5d3502c8a1cefd700b34baf31d411a", + "sha256:502753043567491d3ff6d08629270127e0c31d4184c4c8d98f92c26f65019962", + "sha256:547f45fa1a93154bd82050a7f3cddbc1a7a4dd2a9bf5cb7d06f4ae29fe94eaf8", + "sha256:5621a9175cf9d0b0c84c2ef2b12e9f5f5071357c4d2ea6ca1cf01814f45d2391", + "sha256:609b06f178fe8e9f89ef676532760ec0b4deea15e9969bf754b37f7c40326dbc", + "sha256:645786266c8f18a931b65bfcefdbf6952dd0dea98feee39bd188607a9d307ed2", + "sha256:6878ef48d4227aace338d88c48738a4258213cd7b74fd9a3d4d7582bb1d8a155", + "sha256:6a89ecca80709d4076b95f89f308544ec8f7b4727e8a547913a35f16717856cb", + "sha256:6db04803b6c7291985a761004e9060b2bca08da6d04f26a7f2294b8623a0c1a0", + "sha256:6e2cd258d7d927d09493c8df1ce9174ad01b381d4729a9d8d4e38670ca24774c", + "sha256:6e81d7a3e58882450ec4186ca59a3f20a5d4440f25b1cff6f0902ad890e6748a", + "sha256:702855feff378050ae4f741045e19a32d57d19f3e0676d589df0575008ea5004", + "sha256:78b260de9790fd81e69401c2dc8b17da47c8038176a79092a89cb2b7d945d060", + "sha256:7bb65125fcbef8d989fa1dd0e8a060999497629ca5b0efbca209588a73356232", + "sha256:7dea0889685db8550f839fa202744652e87c60015029ce3f60e006f8c4462c93", + "sha256:8284cf8c0dd272a247bc154eb6c95548722dce90d098c17a883ed36e67cdb129", + "sha256:877abb17e6339d96bf08e7a622d05095e72b71f8afd8a9fefc82cf30ed944163", + "sha256:8929543a7192c13d177b770008bc4e8119f2e1f881d563fc6b6305d2d0ebe9de", + "sha256:8ae539519c4c040c5ffd0632784e21b2f03fc1340752af711f33e5be83a9d6c6", + "sha256:8f59d57baca39b32db42b83b2a7ba6f47ad9c394ec2076b084c3f029b7afca23", + "sha256:9054a0754de38d9dbd01a46621636689124d666bad1936d76c0341f7d71bf569", + "sha256:953510dfb7b12ab69d20135a0662397f077c59b1e6379a768e97c59d852ee51d", + "sha256:95cae0efeb032af8458fc27d191f85d1717b1d4e49f7cb226cf526ff28179778", + "sha256:9bc572be474cafb617672c43fe989d6e48d3c83af02ce8de73fff1c6bb3c198d", + "sha256:9c56863d44bd1c4fe2abb8a4d6f5371d197f1ac0ebdee542f07f35895fc07f36", + "sha256:9e0b2df163b8ed01d515807af24f63de04bebcecbd6c3bfeff88385789fdf75a", + "sha256:a09ece4a69cf399510c8ab25e0950d9cf2b42f7b3cb0374f95d2e2ff594478a6", + "sha256:a1ac0ae2b8bd743b88ed0502544847c3053d7171a3cff9228af618a068ed9c34", + "sha256:a318d68e92e80af8b00fa99609796fdbcdfef3629c77c6283566c6f02c6d6704", + "sha256:a4acd025ecc06185ba2b801f2de85546e0b8ac787cf9d3b06e7e2a69f925b106", + "sha256:a6d3adcf24b624a7b778533480e32434a39ad8fa30c315208f6d3e5542aeb6e9", + "sha256:a78d169acd38300060b28d600344a803628c3fd585c912cacc9ea8790fe96862", + "sha256:a95324a9de9650a729239daea117df21f4b9868ce32e63f8b650ebe6cef5595b", + "sha256:abd5fd0db5f4dc9289408aaf34908072f805ff7792632250dcb36dc591d24255", + "sha256:b06079abebbc0e89e6163b8e8f0e16270124c154dc6e4a47b413dd538859af16", + "sha256:b43c03669dc4618ec25270b06ecd3ee4fa94c7f9b3c14bae6571ca00ef98b0d3", + "sha256:b48f312cca9621272ae49008c7f613337c53fadca647d6384cc129d2996d1133", + "sha256:b5d7b556859dd85f3a541db6a4e0167b86e7273e1cdc973e5b175166bb634fdb", + "sha256:b9f222de8cded79c49bf184bdbc06630d4c58eec9459b939b4a690c82ed05657", + "sha256:c3c02d12f837d9683e5ab2f3d9844dc57655b92c74e286c262e0fc54213c216d", + "sha256:c44fee9975f04b33331cb8eb272827111efc8930cfd582e0320613263ca849ca", + "sha256:cf4b19715bccd7ee27b6b120e7e9dd56037b9c0681dcc1adc9ba9db3d417fa36", + "sha256:d0c212c49b6c10e6951362f7c6df3329f04c2b1c28499563d4035d964ab8e08c", + "sha256:d3296782ca4eab572a1a4eca686d8bfb00226300dcefdf43faa25b5242ab8a3e", + "sha256:d85f5e9a5f8b73e2350097c3756ef7e785f55bd71205defa0bfdaf96c31616ff", + "sha256:da511e6ad4f7323ee5702e6633085fb76c2f893aaf8ce4c51a0ba4fc07580ea7", + "sha256:e05882b70b87a18d937ca6768ff33cc3f72847cbc4de4491c8e73880766718e5", + "sha256:e61c0abb4c85b095a784ef23fdd4aede7a2628478e7baba7c5e3deba61070a02", + "sha256:e6a08c0be454c3b3beb105c0596ebdc2371fab6bb90c0c0297f4e58fd7e1012c", + "sha256:e9a6e0eb86070e8ccaedfbd9d38fec54864f3125ab95419970575b42af7541df", + "sha256:ed37bd3c3b063412f7620464a9ac1314d33100329f39799255fb8d3027da50d3", + "sha256:f1adfc8ac319e1a348af294106bc6a8458a0f1633cc62a1446aebc30c5fa186a", + "sha256:f5796e664fe802da4f57a168c85359a8fbf3eab5e55cd4e4569fbacecc903959", + "sha256:fc5a77d0c516700ebad189b587de289a20a78324bc54baee03dd486f0855d234", + "sha256:fd21f6ae3f08b41004dfb433fa895d858f3f5979e7762d052b12aef444e29afc" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==7.6.1" + }, + "semantic-version": { + "hashes": [ + "sha256:bdabb6d336998cbb378d4b9db3a4b56a1e3235701dc05ea2690d9a997ed5041c", + "sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177" + ], + "index": "pypi", + "markers": "python_version >= '2.7'", + "version": "==2.10.0" + } + } +} From 71ba35eec053a5b8804fc068ca55d9aedabc047c Mon Sep 17 00:00:00 2001 From: Sijie Date: Mon, 3 Mar 2025 15:23:41 -0800 Subject: [PATCH 095/110] add missing lib pyyaml --- Pipfile | 1 + Pipfile.lock | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index 50b2f800..79768192 100644 --- a/Pipfile +++ b/Pipfile @@ -7,6 +7,7 @@ name = "pypi" docker = "==7.1.0" jsonrpcbase = "==0.2.0" pymongo = "==4.7.2" +pyyaml = "==6.0.2" uwsgi = "==2.0.22" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 037295b7..4b88045e 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "7172916256c731b2b8c17e008ed96a81d54ce2b90a702c7e25adfeb187b00c49" + "sha256": "acb8949b8d873ff356e54c019e5599a7e69831aaa65a0d4aa5178b6ce57a1f4e" }, "pipfile-spec": 6, "requires": { @@ -221,6 +221,66 @@ "markers": "python_version >= '3.7'", "version": "==4.7.2" }, + "pyyaml": { + "hashes": [ + "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff", + "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", + "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086", + "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e", + "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", + "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", + "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", + "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", + "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", + "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68", + "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a", + "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf", + "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99", + "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8", + "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", + "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19", + "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", + "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a", + "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", + "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", + "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", + "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631", + "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d", + "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", + "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", + "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", + "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b", + "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", + "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", + "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706", + "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", + "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237", + "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", + "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083", + "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180", + "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", + "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e", + "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f", + "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725", + "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", + "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", + "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", + "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", + "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", + "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5", + "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d", + "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290", + "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", + "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed", + "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", + "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", + "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12", + "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==6.0.2" + }, "requests": { "hashes": [ "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", From 0c8e6c0ff152efa74bc9dde6349c60e8f832a285 Mon Sep 17 00:00:00 2001 From: Sijie Date: Mon, 3 Mar 2025 15:41:38 -0800 Subject: [PATCH 096/110] move semantic_version to packages --- Pipfile | 2 +- Pipfile.lock | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Pipfile b/Pipfile index 79768192..90bcf0c7 100644 --- a/Pipfile +++ b/Pipfile @@ -8,11 +8,11 @@ docker = "==7.1.0" jsonrpcbase = "==0.2.0" pymongo = "==4.7.2" pyyaml = "==6.0.2" +semantic-version = "==2.10.0" uwsgi = "==2.0.22" [dev-packages] coverage = "==7.6.1" -semantic-version = "==2.10.0" [requires] python_version = "3.9.19" diff --git a/Pipfile.lock b/Pipfile.lock index 4b88045e..23d4a415 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "acb8949b8d873ff356e54c019e5599a7e69831aaa65a0d4aa5178b6ce57a1f4e" + "sha256": "e69e1bd51bda138e86072b2ad7a04f6a8afcd5d8c701a878db6cec814ff595b0" }, "pipfile-spec": 6, "requires": { @@ -289,6 +289,15 @@ "markers": "python_version >= '3.8'", "version": "==2.32.3" }, + "semantic-version": { + "hashes": [ + "sha256:bdabb6d336998cbb378d4b9db3a4b56a1e3235701dc05ea2690d9a997ed5041c", + "sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177" + ], + "index": "pypi", + "markers": "python_version >= '2.7'", + "version": "==2.10.0" + }, "six": { "hashes": [ "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", @@ -392,15 +401,6 @@ "index": "pypi", "markers": "python_version >= '3.8'", "version": "==7.6.1" - }, - "semantic-version": { - "hashes": [ - "sha256:bdabb6d336998cbb378d4b9db3a4b56a1e3235701dc05ea2690d9a997ed5041c", - "sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177" - ], - "index": "pypi", - "markers": "python_version >= '2.7'", - "version": "==2.10.0" } } } From ef31a9c516882984def136cc89e6b19e1c7b023b Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 4 Mar 2025 15:43:52 -0800 Subject: [PATCH 097/110] fix mongo_authMechanism name --- lib/biokbase/catalog/db.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 09d1fedd..949f4197 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -123,7 +123,7 @@ class MongoCatalogDBI: _EXEC_STATS_USERS = 'exec_stats_users' _SECURE_CONFIG_PARAMS = 'secure_config_params' - def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMechanism): + def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_auth_mechanism): # We're performing two MongoDB client initializations—one during the initial setup (on init), and # another lazy initialization after the process is forked. This approach is necessary because MongoDB client @@ -139,7 +139,7 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_authMech self.mongo_db = mongo_db self.mongo_user = mongo_user self.mongo_psswd = mongo_psswd - self.mongo_authMechanism = mongo_authMechanism + self.mongo_auth_mechanism = mongo_auth_mechanism self.mongo_client = None @@ -164,7 +164,7 @@ def _initialize_mongo_client(self): if self.mongo_user and self.mongo_psswd: # Connection string with authentication mongo_client = MongoClient( - f"mongodb://{self.mongo_user}:{self.mongo_psswd}@{self.mongo_host}/{self.mongo_db}?authMechanism={self.mongo_authMechanism}" + f"mongodb://{self.mongo_user}:{self.mongo_psswd}@{self.mongo_host}/{self.mongo_db}?authMechanism={self.mongo_auth_mechanism}" ) else: # Connection string without authentication From b9d056a1c8f94513d7ca4008c8b7e4376e027aba Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 5 Mar 2025 13:20:33 -0800 Subject: [PATCH 098/110] add retrywrites param --- deploy.cfg | 4 ++++ deployment/conf/.templates/deploy.cfg.templ | 3 +++ lib/biokbase/catalog/controller.py | 10 +++++++++- lib/biokbase/catalog/db.py | 16 ++++++++++++---- test/catalog_test_util.py | 1 + test/test.cfg.example | 4 ++++ 6 files changed, 33 insertions(+), 5 deletions(-) diff --git a/deploy.cfg b/deploy.cfg index d4759445..9e4dd1ea 100644 --- a/deploy.cfg +++ b/deploy.cfg @@ -12,6 +12,10 @@ mongodb-database = catalog # password for the account #mongodb-pwd = add password here +# Whether to enable ('true') the MongoDB retryWrites parameter or not (anything other than 'true'). +# See https://www.mongodb.com/docs/manual/core/retryable-writes/ +mongodb-retrywrites=false + # The KBase auth server url. auth-service-url = https://kbase.us/services/authorization/Sessions/Login admin-roles = KBASE_ADMIN,CATALOG_ADMIN diff --git a/deployment/conf/.templates/deploy.cfg.templ b/deployment/conf/.templates/deploy.cfg.templ index 0e35e725..f0b91095 100644 --- a/deployment/conf/.templates/deploy.cfg.templ +++ b/deployment/conf/.templates/deploy.cfg.templ @@ -15,6 +15,9 @@ mongodb-pwd = {{ default .Env.mongodb_pwd "" }} # auth mechanism mongodb-authmechanism = {{ default .Env.mongodb_authMechanism "DEFAULT" }} +# Whether to enable the MongoDB retryWrites parameter or not +mongodb-retrywrites={{ default .Env.db_retrywrites "false" }} + # The KBase auth server url. auth-service-url = {{ default .Env.auth_service_url "https://kbase.us/services/authorization/Sessions/Login" }} # The KBase auth API. diff --git a/lib/biokbase/catalog/controller.py b/lib/biokbase/catalog/controller.py index f219f102..11032f40 100644 --- a/lib/biokbase/catalog/controller.py +++ b/lib/biokbase/catalog/controller.py @@ -56,13 +56,21 @@ def __init__(self, config): warnings.warn('"mongodb-authmechanism" is not set in config of CatalogController, using DEFAULT.') config['mongodb-authmechanism'] = 'DEFAULT' + if 'mongodb-retrywrites' not in config: # pragma: no cover + warnings.warn('"mongodb-retrywrites" is not set in config of CatalogController, using False.') + config['mongodb-retrywrites'] = False + else: + config['mongodb-retrywrites'] = config['mongodb-retrywrites'] == "true" + # instantiate the mongo client self.db = MongoCatalogDBI( config['mongodb-host'], config['mongodb-database'], config['mongodb-user'], config['mongodb-pwd'], - config['mongodb-authmechanism']) + config['mongodb-authmechanism'], + config['mongodb-retrywrites'] + ) # check for the temp directory and make sure it exists if 'temp-dir' not in config: # pragma: no cover diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 949f4197..6ca5ff28 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -1,6 +1,5 @@ import copy import pprint -import traceback import threading from pymongo import ASCENDING @@ -123,7 +122,7 @@ class MongoCatalogDBI: _EXEC_STATS_USERS = 'exec_stats_users' _SECURE_CONFIG_PARAMS = 'secure_config_params' - def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_auth_mechanism): + def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_auth_mechanism, mongo_retry_writes): # We're performing two MongoDB client initializations—one during the initial setup (on init), and # another lazy initialization after the process is forked. This approach is necessary because MongoDB client @@ -140,6 +139,7 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_auth_mec self.mongo_user = mongo_user self.mongo_psswd = mongo_psswd self.mongo_auth_mechanism = mongo_auth_mechanism + self.mongo_retry_writes = mongo_retry_writes self.mongo_client = None @@ -164,11 +164,19 @@ def _initialize_mongo_client(self): if self.mongo_user and self.mongo_psswd: # Connection string with authentication mongo_client = MongoClient( - f"mongodb://{self.mongo_user}:{self.mongo_psswd}@{self.mongo_host}/{self.mongo_db}?authMechanism={self.mongo_auth_mechanism}" + self.mongo_host, + username=self.mongo_user, + password=self.mongo_psswd, + authSource=self.mongo_db, + authMechanism=self.mongo_auth_mechanism, + retryWrites=self.mongo_retry_writes ) else: # Connection string without authentication - mongo_client = MongoClient(f"mongodb://{self.mongo_host}") + mongo_client = MongoClient( + self.mongo_host, + retryWrites=self.mongo_retry_writes + ) # Force a call to server to verify the connection mongo_client.server_info() diff --git a/test/catalog_test_util.py b/test/catalog_test_util.py index 20b722fa..6d161032 100644 --- a/test/catalog_test_util.py +++ b/test/catalog_test_util.py @@ -63,6 +63,7 @@ def _setup_config(self): 'admin-users': self.test_user_2, 'mongodb-host': self.test_cfg['mongodb-host'], 'mongodb-database': self.test_cfg['mongodb-database'], + 'mongodb-retrywrites': self.test_cfg['mongodb-retrywrites'], 'temp-dir': self.scratch_dir, 'docker-base-url': self.test_cfg['docker-base-url'], 'docker-registry-host': self.test_cfg['docker-registry-host'], diff --git a/test/test.cfg.example b/test/test.cfg.example index e2f3a32f..0d47d5b2 100644 --- a/test/test.cfg.example +++ b/test/test.cfg.example @@ -31,6 +31,10 @@ docker-registry-host = localhost:5000 # WARNING: this will get wiped! Don't run tests against something that you may want to keep! mongodb-database = catalog-test +# Whether to enable ('true') the MongoDB retryWrites parameter or not (anything other than 'true'). +# See https://www.mongodb.com/docs/manual/core/retryable-writes/ +mongodb-retrywrites=false + # Path to docker socket/host - you can leave this blank if you have the appropriate docker env variables defined docker-base-url = unix://var/run/docker.sock From 9b1d4b9f794ab5b89923ad0a74890eea30085452 Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 5 Mar 2025 14:44:26 -0800 Subject: [PATCH 099/110] move release notes from README.md to RELEASE_NOTES.md --- README.md | 71 ---------------------------- RELEASE_NOTES.md | 83 +++++++++++++++++++++++++++++++++ lib/biokbase/catalog/version.py | 2 +- 3 files changed, 84 insertions(+), 72 deletions(-) create mode 100644 RELEASE_NOTES.md diff --git a/README.md b/README.md index d00b180f..972fa2e7 100644 --- a/README.md +++ b/README.md @@ -13,74 +13,3 @@ develop: [![Build Status](https://travis-ci.org/kbase/catalog.svg?branch=develop Code coverage: (develop branch) [![Coverage Status](https://coveralls.io/repos/github/kbase/catalog/badge.svg?branch=develop)](https://coveralls.io/github/kbase/catalog?branch=develop) -#### v2.2.4 - 7/10/2020 - - Use auth role for list_approved_developers() - -#### v2.2.0 - 1/23/19 - - Update code to run on Python 3 - - Use Auth Roles for Catalog Admin - -#### v2.1.3 - 11/16/18 - - Update docker-py client code to current 3.x API - - Get Travis-CI tests working again - - Convert to dockerhub image builds - - Change start script to keep service running in foreground - -#### v2.1.2 - 3/16/18 - - Pull a new base image if possible each time a module is registered - - Fix the logic that allows additional html files to be passed from a method's - ui specification directory to the narrative method service during method validation - -#### v2.1.1 - 6/26/17 - - Bugfix for change in docker build log - -#### v2.1.0 - 4/13/17 - - No change from 2.0.7, but upgraded minor version number because many new features - now exist since the initial 2.0.x release. - -#### v2.0.7 - 3/28/17 - - Added job_id field to raw execution statistics - - Support for hidden configuration parameters - -#### v2.0.6 - 12/7/16 - - Bug is fixed in module registration related to docker client timeout happening - for long reference-data stage - -#### v2.0.5 - 9/12/16 - - Added volume mount configuration - - Modified client group configurations so that functions are specified, not app_ids - - Allow admin users to register modules - - Initial porting to new KBase authentication clients - -#### v2.0.3 - 5/31/16 - - Major release to support storage of local functions and dynamic services information, - including methods to query/filter/fetch local function and dynamic service info - - Improved methods for fetching module versions by semantic version matching - - All old module versions are now preserved and can be retrieved by git commit hash - - Module descriptions are now attached to specific module versions instead of to - the module itself, so are effectively versioned - - Tests extended to cover docker steps in registration in Travis, and added to coveralls - -#### v1.0.4 - 2/26/16 - - Fix for bug with accessible dev-version after registration failure - -#### v1.0.3 - 2/24/16 - - Method to generate usage stats for admins - -#### v1.0.2 - 2/18/16 - - Allow specification of client groups - - Method to check for admin status - -#### v1.0.1 - 2/17/16 - - Prevent reregistration of inactive modules - -#### v1.0.0 - 2/11/16 - - First release, all features are new - - Dynamic KBase SDK module registration - - Management of the module release process (dev->beta->release) - - Versioning of all release versions - - Basic query and search of modules - - Management of approved KBase developers - - Management of favorite Apps - - Tracking and query of SDK module run statistics - - Admin methods for approving modules/developers, updating module state diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md new file mode 100644 index 00000000..8847d0ce --- /dev/null +++ b/RELEASE_NOTES.md @@ -0,0 +1,83 @@ +# Catalog Service release notes + +## v2.3.0 - 3/5/2025 + - Removed all submodules(jars, kbapi_common, nms) as part of repository clean up. + - The MongoDB clients have been updated to the most recent version and the service tested against Mongo 7. + - Added the mongo-retrywrites configuration setting in deployment.cfg.templ, defaulting to false. + - Added the docker-compose_nms.yml file to start a narrative method store server in test mode. + - Added pipenv to handle dependencies. + - Removed .travis.yml and added test.yml in .github/workflows for updated CI configuration. + - Updated Python to 3.9.19. + - Implemented two MongoDB client initializations in db.py: one during the init setup and another for lazy initializtion after process forking, preventing the "MongoClient opened before fork" error. + +## v2.2.4 - 7/10/2020 + - Use auth role for list_approved_developers() + +## v2.2.0 - 1/23/19 + - Update code to run on Python 3 + - Use Auth Roles for Catalog Admin + +## v2.1.3 - 11/16/18 + - Update docker-py client code to current 3.x API + - Get Travis-CI tests working again + - Convert to dockerhub image builds + - Change start script to keep service running in foreground + +## v2.1.2 - 3/16/18 + - Pull a new base image if possible each time a module is registered + - Fix the logic that allows additional html files to be passed from a method's + ui specification directory to the narrative method service during method validation + +## v2.1.1 - 6/26/17 + - Bugfix for change in docker build log + +## v2.1.0 - 4/13/17 + - No change from 2.0.7, but upgraded minor version number because many new features + now exist since the initial 2.0.x release. + +## v2.0.7 - 3/28/17 + - Added job_id field to raw execution statistics + - Support for hidden configuration parameters + +## v2.0.6 - 12/7/16 + - Bug is fixed in module registration related to docker client timeout happening + for long reference-data stage + +## v2.0.5 - 9/12/16 + - Added volume mount configuration + - Modified client group configurations so that functions are specified, not app_ids + - Allow admin users to register modules + - Initial porting to new KBase authentication clients + +## v2.0.3 - 5/31/16 + - Major release to support storage of local functions and dynamic services information, + including methods to query/filter/fetch local function and dynamic service info + - Improved methods for fetching module versions by semantic version matching + - All old module versions are now preserved and can be retrieved by git commit hash + - Module descriptions are now attached to specific module versions instead of to + the module itself, so are effectively versioned + - Tests extended to cover docker steps in registration in Travis, and added to coveralls + +## v1.0.4 - 2/26/16 + - Fix for bug with accessible dev-version after registration failure + +## v1.0.3 - 2/24/16 + - Method to generate usage stats for admins + +## v1.0.2 - 2/18/16 + - Allow specification of client groups + - Method to check for admin status + +## v1.0.1 - 2/17/16 + - Prevent reregistration of inactive modules + +## v1.0.0 - 2/11/16 + - First release, all features are new + - Dynamic KBase SDK module registration + - Management of the module release process (dev->beta->release) + - Versioning of all release versions + - Basic query and search of modules + - Management of approved KBase developers + - Management of favorite Apps + - Tracking and query of SDK module run statistics + - Admin methods for approving modules/developers, updating module state \ No newline at end of file diff --git a/lib/biokbase/catalog/version.py b/lib/biokbase/catalog/version.py index ccef70f8..006d3cff 100644 --- a/lib/biokbase/catalog/version.py +++ b/lib/biokbase/catalog/version.py @@ -1,2 +1,2 @@ # File that simply defines version information -CATALOG_VERSION = '2.2.4' +CATALOG_VERSION = '2.3.0' From 0f576bf1c26b0e084d28ec0b46631519098513ae Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 5 Mar 2025 18:56:00 -0800 Subject: [PATCH 100/110] removed mongodb-retrywrites from test.cfg.example --- deployment/conf/.templates/deploy.cfg.templ | 2 +- test/catalog_test_util.py | 1 - test/test.cfg.example | 4 ---- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/deployment/conf/.templates/deploy.cfg.templ b/deployment/conf/.templates/deploy.cfg.templ index f0b91095..295d6edd 100644 --- a/deployment/conf/.templates/deploy.cfg.templ +++ b/deployment/conf/.templates/deploy.cfg.templ @@ -16,7 +16,7 @@ mongodb-pwd = {{ default .Env.mongodb_pwd "" }} mongodb-authmechanism = {{ default .Env.mongodb_authMechanism "DEFAULT" }} # Whether to enable the MongoDB retryWrites parameter or not -mongodb-retrywrites={{ default .Env.db_retrywrites "false" }} +mongodb-retrywrites={{ default .Env.mongodb_retrywrites "false" }} # The KBase auth server url. auth-service-url = {{ default .Env.auth_service_url "https://kbase.us/services/authorization/Sessions/Login" }} diff --git a/test/catalog_test_util.py b/test/catalog_test_util.py index 6d161032..20b722fa 100644 --- a/test/catalog_test_util.py +++ b/test/catalog_test_util.py @@ -63,7 +63,6 @@ def _setup_config(self): 'admin-users': self.test_user_2, 'mongodb-host': self.test_cfg['mongodb-host'], 'mongodb-database': self.test_cfg['mongodb-database'], - 'mongodb-retrywrites': self.test_cfg['mongodb-retrywrites'], 'temp-dir': self.scratch_dir, 'docker-base-url': self.test_cfg['docker-base-url'], 'docker-registry-host': self.test_cfg['docker-registry-host'], diff --git a/test/test.cfg.example b/test/test.cfg.example index 0d47d5b2..e2f3a32f 100644 --- a/test/test.cfg.example +++ b/test/test.cfg.example @@ -31,10 +31,6 @@ docker-registry-host = localhost:5000 # WARNING: this will get wiped! Don't run tests against something that you may want to keep! mongodb-database = catalog-test -# Whether to enable ('true') the MongoDB retryWrites parameter or not (anything other than 'true'). -# See https://www.mongodb.com/docs/manual/core/retryable-writes/ -mongodb-retrywrites=false - # Path to docker socket/host - you can leave this blank if you have the appropriate docker env variables defined docker-base-url = unix://var/run/docker.sock From fa3fbf543c7cc5ecf953e7dfe5c339686695e571 Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 7 Mar 2025 12:51:07 -0800 Subject: [PATCH 101/110] add tests for retryWrites --- test/catalog_config_test.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 test/catalog_config_test.py diff --git a/test/catalog_config_test.py b/test/catalog_config_test.py new file mode 100644 index 00000000..7529a98e --- /dev/null +++ b/test/catalog_config_test.py @@ -0,0 +1,29 @@ +import unittest + +from biokbase.catalog.Impl import Catalog +from catalog_test_util import CatalogTestUtil + +_RETRY_WRITES = "mongodb-retrywrites" + +class CatalogConfigTest(unittest.TestCase): + + def test_catalog_without_retryWrites(self): + self.catalog_cfg.pop(_RETRY_WRITES) + catalog = Catalog(self.catalog_cfg) + self.assertFalse(catalog.cc.db.mongo_retry_writes) + + def test_catalog_with_retryWrites_is_true(self): + self.catalog_cfg[_RETRY_WRITES] = "true" + catalog = Catalog(self.catalog_cfg) + self.assertTrue(catalog.cc.db.mongo_retry_writes) + + @classmethod + def setUpClass(cls): + print('++++++++++++ RUNNING catalog_config_test.py +++++++++++') + cls.cUtil = CatalogTestUtil('.') # TODO: pass in test directory from outside + cls.cUtil.setUp() + cls.catalog_cfg = cls.cUtil.getCatalogConfig() + + @classmethod + def tearDownClass(cls): + cls.cUtil.tearDown() \ No newline at end of file From 3111c53828ca46ebf27891dd6d838cf99794e4b4 Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 7 Mar 2025 12:57:56 -0800 Subject: [PATCH 102/110] fix format --- test/catalog_config_test.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/catalog_config_test.py b/test/catalog_config_test.py index 7529a98e..79a754f2 100644 --- a/test/catalog_config_test.py +++ b/test/catalog_config_test.py @@ -5,10 +5,11 @@ _RETRY_WRITES = "mongodb-retrywrites" -class CatalogConfigTest(unittest.TestCase): - + +class CatalogConfigTest(unittest.TestCase): + def test_catalog_without_retryWrites(self): - self.catalog_cfg.pop(_RETRY_WRITES) + self.catalog_cfg.pop(_RETRY_WRITES, None) catalog = Catalog(self.catalog_cfg) self.assertFalse(catalog.cc.db.mongo_retry_writes) @@ -19,11 +20,11 @@ def test_catalog_with_retryWrites_is_true(self): @classmethod def setUpClass(cls): - print('++++++++++++ RUNNING catalog_config_test.py +++++++++++') - cls.cUtil = CatalogTestUtil('.') # TODO: pass in test directory from outside + print("++++++++++++ RUNNING catalog_config_test.py +++++++++++") + cls.cUtil = CatalogTestUtil(".") # TODO: pass in test directory from outside cls.cUtil.setUp() cls.catalog_cfg = cls.cUtil.getCatalogConfig() @classmethod def tearDownClass(cls): - cls.cUtil.tearDown() \ No newline at end of file + cls.cUtil.tearDown() From 91b5c743a876133f0a3b9b14c175dd67b2d6d727 Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 7 Mar 2025 13:42:50 -0800 Subject: [PATCH 103/110] add print meesage to verify retryWrites setting --- lib/biokbase/catalog/db.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index 6ca5ff28..f09e4815 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -141,6 +141,10 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_auth_mec self.mongo_auth_mechanism = mongo_auth_mechanism self.mongo_retry_writes = mongo_retry_writes + print("*" * 50) + print(f"self.mongo_retry_writes is: {self.mongo_retry_writes}") + print("*" * 50) + self.mongo_client = None # Initialize mongo client From 63f2287f91068efe22395876cc3a93646a2f990d Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 7 Mar 2025 14:06:28 -0800 Subject: [PATCH 104/110] remove print --- lib/biokbase/catalog/db.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/biokbase/catalog/db.py b/lib/biokbase/catalog/db.py index f09e4815..6ca5ff28 100644 --- a/lib/biokbase/catalog/db.py +++ b/lib/biokbase/catalog/db.py @@ -141,10 +141,6 @@ def __init__(self, mongo_host, mongo_db, mongo_user, mongo_psswd, mongo_auth_mec self.mongo_auth_mechanism = mongo_auth_mechanism self.mongo_retry_writes = mongo_retry_writes - print("*" * 50) - print(f"self.mongo_retry_writes is: {self.mongo_retry_writes}") - print("*" * 50) - self.mongo_client = None # Initialize mongo client From d6f3deda389b169e50d362c7bc0e04d9f7e09537 Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 7 Mar 2025 15:23:22 -0800 Subject: [PATCH 105/110] fix typo in README.md --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 8847d0ce..b498eeb9 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -8,7 +8,7 @@ - Added pipenv to handle dependencies. - Removed .travis.yml and added test.yml in .github/workflows for updated CI configuration. - Updated Python to 3.9.19. - - Implemented two MongoDB client initializations in db.py: one during the init setup and another for lazy initializtion after process forking, preventing the "MongoClient opened before fork" error. + - Implemented two MongoDB client initializations in db.py: one during the init setup and another for lazy initialization after process forking, preventing the "MongoClient opened before fork" error. ## v2.2.4 - 7/10/2020 - Use auth role for list_approved_developers() From 8b5f518379f1def1395a6f8823a7f1a8c1e0ebc8 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 11 Mar 2025 11:56:27 -0700 Subject: [PATCH 106/110] uncomment trivy-scan --- .github/workflows/pr_build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml index 53b0d18d..0fa1c464 100644 --- a/.github/workflows/pr_build.yml +++ b/.github/workflows/pr_build.yml @@ -37,7 +37,7 @@ jobs: name: '${{ github.event.repository.name }}' tags: pr-${{ github.event.number }},latest-rc secrets: inherit -# trivy-scans: -# if: (github.base_ref == 'develop' || github.base_ref == 'main' || github.base_ref == 'master' ) && github.event.pull_request.merged == false -# uses: kbase/.github/.github/workflows/reusable_trivy-scans.yml@main -# secrets: inherit + trivy-scans: + if: (github.base_ref == 'develop' || github.base_ref == 'main' || github.base_ref == 'master' ) && github.event.pull_request.merged == false + uses: kbase/.github/.github/workflows/reusable_trivy-scans.yml@main + secrets: inherit From bbac4a4cfe97c8ba71c68c40ee51f63c174c118b Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 12 Jun 2025 16:07:55 -0700 Subject: [PATCH 107/110] add gha to dependabot.yml --- .github/dependabot.yml | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index fe381818..6f0521da 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,14 +1,26 @@ version: 2 updates: -- package-ecosystem: docker - directory: "/" - schedule: - interval: weekly - time: '11:00' - open-pull-requests-limit: 10 -- package-ecosystem: pip - directory: "/" - schedule: - interval: weekly - time: '11:00' - open-pull-requests-limit: 10 + + # Docker base image updates + - package-ecosystem: docker + directory: "/" + schedule: + interval: weekly + time: "11:00" + open-pull-requests-limit: 10 + + # Python (pip) dependencies + - package-ecosystem: pip + directory: "/" + schedule: + interval: weekly + time: "11:00" + open-pull-requests-limit: 10 + + # GitHub Actions dependencies + - package-ecosystem: "github-actions" + directory: ".github/workflows" + schedule: + interval: weekly + time: "11:00" + open-pull-requests-limit: 10 From 8975a961f25cfb6afe68a9432a49e24b28c759ea Mon Sep 17 00:00:00 2001 From: Sijie Date: Mon, 16 Jun 2025 12:47:01 -0700 Subject: [PATCH 108/110] change schedule.interval to monthly --- .github/dependabot.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6f0521da..f04d6f91 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,22 +5,22 @@ updates: - package-ecosystem: docker directory: "/" schedule: - interval: weekly + interval: monthly time: "11:00" - open-pull-requests-limit: 10 + open-pull-requests-limit: 25 # Python (pip) dependencies - package-ecosystem: pip directory: "/" schedule: - interval: weekly + interval: monthly time: "11:00" - open-pull-requests-limit: 10 + open-pull-requests-limit: 25 # GitHub Actions dependencies - package-ecosystem: "github-actions" directory: ".github/workflows" schedule: - interval: weekly + interval: monthly time: "11:00" - open-pull-requests-limit: 10 + open-pull-requests-limit: 25 From 7cb46a6d3e0c051097e339561f90a600958b6fc2 Mon Sep 17 00:00:00 2001 From: MrCreosote Date: Wed, 3 Sep 2025 17:05:47 -0700 Subject: [PATCH 109/110] Add JITPack java client build --- .gitattributes | 12 + .gitignore | 7 + Catalog.html | 1 + KIDLspec.css | 65 +++++ Makefile | 17 +- RELEASE_NOTES.md | 3 + build.gradle | 58 ++++ gradle.properties | 5 + gradle/libs.versions.toml | 2 + gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 43764 bytes gradle/wrapper/gradle-wrapper.properties | 7 + gradlew | 251 ++++++++++++++++++ lib/biokbase/catalog/Impl.py | 6 +- lib/biokbase/catalog/version.py | 2 +- .../us/kbase/catalog/CurrentRepoParams.java | 82 ------ .../us/kbase/catalog/GetRawStatsParams.java | 80 ------ .../us/kbase/catalog/HistoryRepoParams.java | 121 --------- lib/java/us/kbase/catalog/Icon.java | 42 --- .../us/kbase/catalog/ListReposParams.java | 63 ----- lib/java/us/kbase/catalog/RepoDetails.java | 228 ---------------- lib/java/us/kbase/catalog/RepoVersion.java | 98 ------- .../us/kbase/catalog/SelectModuleParams.java | 101 ------- .../us/kbase/catalog/SetRepoStateParams.java | 119 --------- settings.gradle | 8 + .../java/us/kbase/catalog/AppClientGroup.java | 0 .../us/kbase/catalog/BasicModuleInfo.java | 0 .../kbase/catalog/BasicModuleVersionInfo.java | 0 .../java/us/kbase/catalog/BuildInfo.java | 0 .../main}/java/us/kbase/catalog/BuildLog.java | 0 .../java/us/kbase/catalog/BuildLogLine.java | 0 .../java/us/kbase/catalog/CatalogClient.java | 2 + .../us/kbase/catalog/ClientGroupConfig.java | 0 .../us/kbase/catalog/ClientGroupFilter.java | 0 .../us/kbase/catalog/CompilationReport.java | 0 .../java/us/kbase/catalog/ExecAggrStats.java | 0 .../us/kbase/catalog/ExecAggrTableParams.java | 0 .../java/us/kbase/catalog/FavoriteCount.java | 0 .../java/us/kbase/catalog/FavoriteItem.java | 0 .../java/us/kbase/catalog/FavoriteUser.java | 0 .../main}/java/us/kbase/catalog/Function.java | 0 .../java/us/kbase/catalog/FunctionPlace.java | 0 .../us/kbase/catalog/GetBuildLogParams.java | 0 .../kbase/catalog/GetClientGroupParams.java | 0 .../kbase/catalog/GetExecAggrStatsParams.java | 0 .../kbase/catalog/GetExecRawStatsParams.java | 0 .../catalog/GetLocalFunctionDetails.java | 0 .../catalog/GetSecureConfigParamsInput.java | 0 .../main}/java/us/kbase/catalog/IOTags.java | 0 .../us/kbase/catalog/ListBuildParams.java | 0 .../us/kbase/catalog/ListFavoriteCounts.java | 0 .../catalog/ListLocalFunctionParams.java | 0 .../us/kbase/catalog/ListModuleParams.java | 0 .../catalog/ListServiceModuleParams.java | 0 .../kbase/catalog/LocalFunctionDetails.java | 0 .../us/kbase/catalog/LocalFunctionInfo.java | 0 .../us/kbase/catalog/LocalFunctionTags.java | 0 .../us/kbase/catalog/LogExecStatsParams.java | 0 .../ModifySecureConfigParamsInput.java | 0 .../java/us/kbase/catalog/ModuleInfo.java | 0 .../java/us/kbase/catalog/ModuleState.java | 0 .../java/us/kbase/catalog/ModuleVersion.java | 0 .../us/kbase/catalog/ModuleVersionInfo.java | 0 .../catalog/ModuleVersionLookupParams.java | 0 .../java/us/kbase/catalog/Parameter.java | 0 .../us/kbase/catalog/RegisterRepoParams.java | 0 .../java/us/kbase/catalog/ReleaseReview.java | 0 .../kbase/catalog/RequestedReleaseInfo.java | 0 .../kbase/catalog/SecureConfigParameter.java | 0 .../us/kbase/catalog/SelectModuleVersion.java | 0 .../catalog/SelectModuleVersionParams.java | 0 .../kbase/catalog/SelectOneLocalFunction.java | 0 .../kbase/catalog/SelectOneModuleParams.java | 0 .../catalog/SetRegistrationStateParams.java | 0 .../main}/java/us/kbase/catalog/SpecFile.java | 0 .../us/kbase/catalog/UpdateGitUrlParams.java | 0 .../us/kbase/catalog/VersionCommitInfo.java | 0 .../java/us/kbase/catalog/VolumeMount.java | 0 .../us/kbase/catalog/VolumeMountConfig.java | 0 .../us/kbase/catalog/VolumeMountFilter.java | 0 79 files changed, 433 insertions(+), 947 deletions(-) create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 Catalog.html create mode 100644 KIDLspec.css create mode 100644 build.gradle create mode 100644 gradle.properties create mode 100644 gradle/libs.versions.toml create mode 100644 gradle/wrapper/gradle-wrapper.jar create mode 100644 gradle/wrapper/gradle-wrapper.properties create mode 100755 gradlew delete mode 100644 lib/java/us/kbase/catalog/CurrentRepoParams.java delete mode 100644 lib/java/us/kbase/catalog/GetRawStatsParams.java delete mode 100644 lib/java/us/kbase/catalog/HistoryRepoParams.java delete mode 100644 lib/java/us/kbase/catalog/Icon.java delete mode 100644 lib/java/us/kbase/catalog/ListReposParams.java delete mode 100644 lib/java/us/kbase/catalog/RepoDetails.java delete mode 100644 lib/java/us/kbase/catalog/RepoVersion.java delete mode 100644 lib/java/us/kbase/catalog/SelectModuleParams.java delete mode 100644 lib/java/us/kbase/catalog/SetRepoStateParams.java create mode 100644 settings.gradle rename {lib => src/main}/java/us/kbase/catalog/AppClientGroup.java (100%) rename {lib => src/main}/java/us/kbase/catalog/BasicModuleInfo.java (100%) rename {lib => src/main}/java/us/kbase/catalog/BasicModuleVersionInfo.java (100%) rename {lib => src/main}/java/us/kbase/catalog/BuildInfo.java (100%) rename {lib => src/main}/java/us/kbase/catalog/BuildLog.java (100%) rename {lib => src/main}/java/us/kbase/catalog/BuildLogLine.java (100%) rename {lib => src/main}/java/us/kbase/catalog/CatalogClient.java (99%) rename {lib => src/main}/java/us/kbase/catalog/ClientGroupConfig.java (100%) rename {lib => src/main}/java/us/kbase/catalog/ClientGroupFilter.java (100%) rename {lib => src/main}/java/us/kbase/catalog/CompilationReport.java (100%) rename {lib => src/main}/java/us/kbase/catalog/ExecAggrStats.java (100%) rename {lib => src/main}/java/us/kbase/catalog/ExecAggrTableParams.java (100%) rename {lib => src/main}/java/us/kbase/catalog/FavoriteCount.java (100%) rename {lib => src/main}/java/us/kbase/catalog/FavoriteItem.java (100%) rename {lib => src/main}/java/us/kbase/catalog/FavoriteUser.java (100%) rename {lib => src/main}/java/us/kbase/catalog/Function.java (100%) rename {lib => src/main}/java/us/kbase/catalog/FunctionPlace.java (100%) rename {lib => src/main}/java/us/kbase/catalog/GetBuildLogParams.java (100%) rename {lib => src/main}/java/us/kbase/catalog/GetClientGroupParams.java (100%) rename {lib => src/main}/java/us/kbase/catalog/GetExecAggrStatsParams.java (100%) rename {lib => src/main}/java/us/kbase/catalog/GetExecRawStatsParams.java (100%) rename {lib => src/main}/java/us/kbase/catalog/GetLocalFunctionDetails.java (100%) rename {lib => src/main}/java/us/kbase/catalog/GetSecureConfigParamsInput.java (100%) rename {lib => src/main}/java/us/kbase/catalog/IOTags.java (100%) rename {lib => src/main}/java/us/kbase/catalog/ListBuildParams.java (100%) rename {lib => src/main}/java/us/kbase/catalog/ListFavoriteCounts.java (100%) rename {lib => src/main}/java/us/kbase/catalog/ListLocalFunctionParams.java (100%) rename {lib => src/main}/java/us/kbase/catalog/ListModuleParams.java (100%) rename {lib => src/main}/java/us/kbase/catalog/ListServiceModuleParams.java (100%) rename {lib => src/main}/java/us/kbase/catalog/LocalFunctionDetails.java (100%) rename {lib => src/main}/java/us/kbase/catalog/LocalFunctionInfo.java (100%) rename {lib => src/main}/java/us/kbase/catalog/LocalFunctionTags.java (100%) rename {lib => src/main}/java/us/kbase/catalog/LogExecStatsParams.java (100%) rename {lib => src/main}/java/us/kbase/catalog/ModifySecureConfigParamsInput.java (100%) rename {lib => src/main}/java/us/kbase/catalog/ModuleInfo.java (100%) rename {lib => src/main}/java/us/kbase/catalog/ModuleState.java (100%) rename {lib => src/main}/java/us/kbase/catalog/ModuleVersion.java (100%) rename {lib => src/main}/java/us/kbase/catalog/ModuleVersionInfo.java (100%) rename {lib => src/main}/java/us/kbase/catalog/ModuleVersionLookupParams.java (100%) rename {lib => src/main}/java/us/kbase/catalog/Parameter.java (100%) rename {lib => src/main}/java/us/kbase/catalog/RegisterRepoParams.java (100%) rename {lib => src/main}/java/us/kbase/catalog/ReleaseReview.java (100%) rename {lib => src/main}/java/us/kbase/catalog/RequestedReleaseInfo.java (100%) rename {lib => src/main}/java/us/kbase/catalog/SecureConfigParameter.java (100%) rename {lib => src/main}/java/us/kbase/catalog/SelectModuleVersion.java (100%) rename {lib => src/main}/java/us/kbase/catalog/SelectModuleVersionParams.java (100%) rename {lib => src/main}/java/us/kbase/catalog/SelectOneLocalFunction.java (100%) rename {lib => src/main}/java/us/kbase/catalog/SelectOneModuleParams.java (100%) rename {lib => src/main}/java/us/kbase/catalog/SetRegistrationStateParams.java (100%) rename {lib => src/main}/java/us/kbase/catalog/SpecFile.java (100%) rename {lib => src/main}/java/us/kbase/catalog/UpdateGitUrlParams.java (100%) rename {lib => src/main}/java/us/kbase/catalog/VersionCommitInfo.java (100%) rename {lib => src/main}/java/us/kbase/catalog/VolumeMount.java (100%) rename {lib => src/main}/java/us/kbase/catalog/VolumeMountConfig.java (100%) rename {lib => src/main}/java/us/kbase/catalog/VolumeMountFilter.java (100%) diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..f91f6460 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,12 @@ +# +# https://help.github.com/articles/dealing-with-line-endings/ +# +# Linux start script should use lf +/gradlew text eol=lf + +# These are Windows script files and should use crlf +*.bat text eol=crlf + +# Binary files should be left untouched +*.jar binary + diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..2c40bade --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +/bin/ + +# Ignore Gradle project-specific cache directory +.gradle + +# Ignore Gradle build output directory +build diff --git a/Catalog.html b/Catalog.html new file mode 100644 index 00000000..36f17388 --- /dev/null +++ b/Catalog.html @@ -0,0 +1 @@ +Catalog
/*
*Service for managing, registering, and building KBase Modules using the KBase SDK.
*/
moduleCatalog{

/*
*@range[0,1]
*/
typedefintboolean;

/*
*Get the version of the deployed catalog service endpoint.
*/
funcdefversion()returns(stringversion)authenticationnone;

/*
*Describes how to find a single module/repository.
*module_name - name of module defined in kbase.yaml file;
*git_url - the url used to register the module
*/
typedefstructure{
stringmodule_name;
stringgit_url;
}SelectOneModuleParams;

/*
*returns true (1) if the module exists, false (2) otherwise
*/
funcdefis_registered(SelectOneModuleParamsparams)returns(boolean)authenticationnone;

typedefstructure{
stringgit_url;
stringgit_commit_hash;
}RegisterRepoParams;

/*
*allow/require developer to supply git branch/git commit tag?
*if this is a new module, creates the initial registration with the authenticated user as
*the sole owner, then launches a build to update the dev version of the module. You can check
*the state of this build with the 'get_module_state' method passing in the git_url. If the module
*already exists, then you must be an owner to reregister. That will immediately overwrite your
*dev version of the module (old dev versions are not stored, but you can always reregister an old
*version from the repo) and start a build.
*/
funcdefregister_repo(RegisterRepoParamsparams)returns(stringregistration_id)authenticationrequired;

/*
*immediately updates the beta tag to what is currently in dev, whatever is currently in beta
*is discarded. Will fail if a release request is active and has not been approved/denied
*/
funcdefpush_dev_to_beta(SelectOneModuleParamsparams)returns()authenticationrequired;

/*
*requests a push from beta to release version; must be approved be a kbase Admin
*/
funcdefrequest_release(SelectOneModuleParamsparams)returns()authenticationrequired;

typedefstructure{
stringmodule_name;
stringgit_url;
stringgit_commit_hash;
stringgit_commit_message;
inttimestamp;
list<string>owners;
}RequestedReleaseInfo;

funcdeflist_requested_releases()returns(list<RequestedReleaseInfo>requested_releases)authenticationnone;

/*
*decision - approved | denied
*review_message -
*/
typedefstructure{
stringmodule_name;
stringgit_url;
stringdecision;
stringreview_message;
}ReleaseReview;

funcdefreview_release_request(ReleaseReviewreview)returns()authenticationrequired;

/*
*Describes how to filter repositories.
*include_released - optional flag indicated modules that are released are included (default:true)
*include_unreleased - optional flag indicated modules that are not released are included (default:false)
*with_disabled - optional flag indicating disabled repos should be included (default:false).
*include_modules_with_no_name_set - default to 0, if set return modules that were never
*registered successfully (first registration failed, never
*got a module name, but there is a git_url)
*/
typedefstructure{
list<string>owners;
booleaninclude_released;
booleaninclude_unreleased;
booleaninclude_disabled;
booleaninclude_modules_with_no_name_set;
}ListModuleParams;

typedefstructure{
stringgit_commit_hash;
}VersionCommitInfo;

/*
*git_url is always returned. Every other field
*may or may not exist depending on what has been registered or if
*certain registrations have failed
*/
typedefstructure{
stringmodule_name;
stringgit_url;
stringlanguage;
booleandynamic_service;
list<string>owners;
list<VersionCommitInfo>released_version_list;
}BasicModuleInfo;

funcdeflist_basic_module_info(ListModuleParamsparams)returns(list<BasicModuleInfo>info_list)authenticationnone;

/*
*FAVORITES!!
*/
typedefstructure{
stringmodule_name;
stringid;
}FavoriteItem;

funcdefadd_favorite(FavoriteItemparams)returns()authenticationrequired;

funcdefremove_favorite(FavoriteItemparams)returns()authenticationrequired;

funcdeflist_favorites(stringusername)returns(list<FavoriteItem>favorites)authenticationnone;

typedefstructure{
stringusername;
stringtimestamp;
}FavoriteUser;

funcdeflist_app_favorites(FavoriteItemitem)returns(list<FavoriteUser>users)authenticationnone;

/*
*if favorite item is given, will return stars just for that item. If a module
*name is given, will return stars for all methods in that module. If none of
*those are given, then will return stars for every method that there is info on
*
*parameters to add:
*list<FavoriteItem> items;
*/
typedefstructure{
list<string>modules;
}ListFavoriteCounts;

typedefstructure{
stringmodule_name;
stringapp_id;
intcount;
}FavoriteCount;

funcdeflist_favorite_counts(ListFavoriteCountsparams)returns(list<FavoriteCount>counts)authenticationnone;

typedefstructure{
intstart_line;
intend_line;
}FunctionPlace;

typedefstructure{
stringtype;
stringcomment;
}Parameter;

typedefstructure{
stringname;
stringcomment;
list<Parameter>input;
list<Parameter>output;
}Function;

typedefstructure{
stringfile_name;
stringcontent;
booleanis_main;
}SpecFile;

typedefstructure{
stringmodule_name;
stringsdk_version;
stringsdk_git_commit;
stringimpl_file_path;
mapping<string,FunctionPlace>function_places;
mapping<string,Function>functions;
list<SpecFile>spec_files;
}CompilationReport;

/*
*data_folder - optional field representing unique module name (like <module_name> transformed to
*lower cases) used for reference data purposes (see description for data_version field). This
*value will be treated as part of file system path relative to the base that comes from the
*config (currently base is supposed to be "/kb/data" defined in "ref-data-base" parameter).
*data_version - optional field, reflects version of data defined in kbase.yml (see "data-version"
*key). In case this field is set data folder with path "/kb/data/<data_folder>/<data_version>"
*should be initialized by running docker image with "init" target from catalog. And later when
*async methods are run it should be mounted on AWE worker machine into "/data" folder inside
*docker container by execution engine.
*/
typedefstructure{
inttimestamp;
stringregistration_id;
stringversion;
stringgit_commit_hash;
stringgit_commit_message;
booleandynamic_service;
list<string>narrative_method_ids;
list<string>local_function_ids;
stringdocker_img_name;
stringdata_folder;
stringdata_version;
CompilationReportcompilation_report;
}ModuleVersionInfo;

typedefstructure{
stringmodule_name;
stringgit_url;
stringdescription;
stringlanguage;
list<string>owners;
}ModuleInfo;

funcdefget_module_info(SelectOneModuleParamsselection)returns(ModuleInfoinfo)authenticationnone;

/*
*only required: module_name or git_url, the rest are optional selectors
*If no selectors given, returns current release version
*version is one of: release | beta | dev
*old release versions can only be retrieved individually by timestamp or git_commit_hash
*
*Note: this method isn't particularly smart or effecient yet, because it pulls the info for a particular
*module first, then searches in code for matches to the relevant query. Instead, this should be
*performed on the database side through queries. Will optimize when this becomes an issue.
*
*In the future, this will be extended so that you can retrieve version info by only
*timestamp, git commit, etc, but the necessary indicies have not been setup yet. In general, we will
*need to add better search capabilities
*/
typedefstructure{
stringmodule_name;
stringgit_url;
inttimestamp;
stringgit_commit_hash;
stringversion;
}SelectModuleVersionParams;

/*
*DEPRECATED!!! use get_module_version
*/
funcdefget_version_info(SelectModuleVersionParamsparams)returns(ModuleVersionInfoversion)authenticationnone;

funcdeflist_released_module_versions(SelectOneModuleParamsparams)returns(list<ModuleVersionInfo>versions)authenticationnone;

/*
*module_name - the name of the module
*module_description - (optionally returned) html description in KBase YAML of this module
*git_url - the git url of the source for this module
*
*released - 1 if this version has been released, 0 otherwise
*release_tags - list of strings of: 'dev', 'beta', or 'release', or empty list
*this is a list because the same commit version may be the version in multiple release states
*release_timestamp - time in ms since epoch when this module was approved and moved to release, null otherwise
*note that a module was released before v1.0.0, the release timestamp may not have been
*recorded and will default to the registration timestamp
*
*timestamp - time in ms since epoch when the registration for this version was started
*registration_id - id of the last registration for this version, used for fetching registration logs and state
*
*version - validated semantic version number as indicated in the KBase YAML of this version
*semantic versions are unique among released versions of this module
*
*git_commit_hash - the full git commit hash of the source for this module
*git_commit_message - the message attached to this git commit
*
*dynamic_service - 1 if this version is available as a web service, 0 otherwise
*
*narrative_app_ids - list of Narrative App ids registered with this module version
*local_function_ids - list of Local Function ids registered with this module version
*
*docker_img_name - name of the docker image for this module created on registration
*data_folder - name of the data folder used
*
*compilation_report - (optionally returned) summary of the KIDL specification compilation
*/
typedefstructure{
stringmodule_name;
stringmodule_description;
stringgit_url;
booleanreleased;
list<string>release_tags;
inttimestamp;
stringregistration_id;
stringversion;
stringgit_commit_hash;
stringgit_commit_message;
booleandynamic_service;
list<string>narrative_app_ids;
list<string>local_function_ids;
stringdocker_img_name;
stringdata_folder;
stringdata_version;
CompilationReportcompilation_report;
}ModuleVersion;

/*
*Get a specific module version.
*
*Requires either a module_name or git_url. If both are provided, they both must match.
*
*If no other options are specified, then the latest 'release' version is returned. If
*the module has not been released, then the latest 'beta' or 'dev' version is returned.
*You can check in the returned object if the version has been released (see is_released)
*and what release tags are pointing to this version (see release_tags).
*
*Optionally, a 'version' parameter can be provided that can be either:
*1) release tag: 'dev' | 'beta' | 'release'
*
*2) specific semantic version of a released version (you cannot pull dev/beta or other
*unreleased versions by semantic version)
*- e.g. 2.0.1
*
*3) semantic version requirement specification, see: https://pypi.python.org/pypi/semantic_version/
*which will return the latest released version that matches the criteria. You cannot pull
*dev/beta or other unreleased versions this way.
*- e.g.:
*- '>1.0.0'
*- '>=2.1.1,<3.3.0'
*- '!=0.2.4-alpha,<0.3.0'
*
*4) specific full git commit hash
*
*include_module_description - set to 1 to include the module description in the YAML file of this version;
*default is 0
*include_compilation_report - set to 1 to include the module compilation report, default is 0
*/
typedefstructure{
stringmodule_name;
stringgit_url;
stringversion;
booleaninclude_module_description;
booleaninclude_compilation_report;
}SelectModuleVersion;

funcdefget_module_version(SelectModuleVersionselection)returns(ModuleVersionversion)authenticationnone;

/*
*Local Function Listing Support
*/
typedefstructure{
list<string>file_types;
list<string>kb_types;
}IOTags;

typedefstructure{
list<string>categories;
IOTagsinput;
IOTagsoutput;
}LocalFunctionTags;

/*
*todo: switch release_tag to release_tags
*/
typedefstructure{
stringmodule_name;
stringfunction_id;
stringgit_commit_hash;
stringversion;
list<string>release_tag;
stringname;
stringshort_description;
}LocalFunctionInfo;

typedefstructure{
stringlong_description;
}LocalFunctionDetails;

/*
*Allows various ways to filter.
*Release tag = dev/beta/release, default is release
*module_names = only include modules in the list; if empty or not
*provided then include everything
*/
typedefstructure{
stringrelease_tag;
list<string>module_names;
}ListLocalFunctionParams;

funcdeflist_local_functions(ListLocalFunctionParamsparams)returns(list<LocalFunctionInfo>info_list)authenticationnone;

/*
*release_tag = dev | beta | release, if it doesn't exist and git_commit_hash isn't set, we default to release
*and will not return anything if the function is not released
*/
typedefstructure{
stringmodule_name;
stringfunction_id;
stringrelease_tag;
stringgit_commit_hash;
}SelectOneLocalFunction;

typedefstructure{
list<SelectOneLocalFunction>functions;
}GetLocalFunctionDetails;

funcdefget_local_function_details(GetLocalFunctionDetailsparams)returns(list<LocalFunctionDetails>detail_list)authenticationnone;

/*
*DYNAMIC SERVICES SUPPORT Methods
*/
typedefstructure{
stringmodule_name;
stringversion;
stringgit_commit_hash;
stringdocker_img_name;
}BasicModuleVersionInfo;

/*
*module_name - required for module lookup
*lookup - a lookup string, if empty will get the latest released module
*1) version tag = dev | beta | release
*2) semantic version match identifiier
*not supported yet: 3) exact commit hash
*not supported yet: 4) exact timestamp
*only_service_versions - 1/0, default is 1
*/
typedefstructure{
stringmodule_name;
stringlookup;
booleanonly_service_versions;
}ModuleVersionLookupParams;

funcdefmodule_version_lookup(ModuleVersionLookupParamsselection)returns(BasicModuleVersionInfo)authenticationnone;

/*
*tag = dev | beta | release
*if tag is not set, all release versions are returned
*/
typedefstructure{
stringtag;
}ListServiceModuleParams;

funcdeflist_service_modules(ListServiceModuleParamsfilter)returns(list<BasicModuleVersionInfo>service_modules)authenticationnone;

/*
*End Dynamic Services Support Methods
*/
typedefstructure{
stringmodule_name;
stringgit_url;
stringregistration_state;
stringerror_message;
}SetRegistrationStateParams;

funcdefset_registration_state(SetRegistrationStateParamsparams)returns()authenticationrequired;

/*
*active: True | False,
*release_approval: approved | denied | under_review | not_requested, (all releases require approval)
*review_message: str, (optional)
*registration: complete | error | (build state status),
*error_message: str (optional)
*/
typedefstructure{
booleanactive;
booleanreleased;
stringrelease_approval;
stringreview_message;
stringregistration;
stringerror_message;
}ModuleState;

funcdefget_module_state(SelectOneModuleParamsparams)returns(ModuleStatestate)authenticationnone;

/*
*must specify skip & limit, or first_n, or last_n. If none given, this gets last 5000 lines
*/
typedefstructure{
stringregistration_id;
intskip;
intlimit;
intfirst_n;
intlast_n;
}GetBuildLogParams;

typedefstructure{
stringcontent;
booleanerror;
}BuildLogLine;

typedefstructure{
stringregistration_id;
stringtimestamp;
stringmodule_name_lc;
stringgit_url;
stringerror;
stringregistration;
list<BuildLogLine>log;
}BuildLog;

funcdefget_build_log(stringregistration_id)returns(string)authenticationnone;

/*
*given the registration_id returned from the register method, you can check the build log with this method
*/
funcdefget_parsed_build_log(GetBuildLogParamsparams)returns(BuildLogbuild_log)authenticationnone;

typedefstructure{
stringtimestamp;
stringregistration_id;
stringregistration;
stringerror_message;
stringmodule_name_lc;
stringgit_url;
}BuildInfo;

/*
*Always sorted by time, oldest builds are last.
*
*only one of these can be set to true:
*only_running - if true, only show running builds
*only_error - if true, only show builds that ended in an error
*only_complete - if true, only show builds that are complete
*skip - skip these first n records, default 0
*limit - limit result to the most recent n records, default 1000
*
*modules - only include builds from these modules based on names/git_urls
*/
typedefstructure{
booleanonly_runnning;
booleanonly_error;
booleanonly_complete;
intskip;
intlimit;
list<SelectOneModuleParams>modules;
}ListBuildParams;

funcdeflist_builds(ListBuildParamsparams)returns(list<BuildInfo>builds)authenticationnone;

/*
*all fields are required to make sure you update the right one
*/
typedefstructure{
stringmodule_name;
stringcurrent_git_url;
stringnew_git_url;
}UpdateGitUrlParams;

/*
*admin method to delete a module, will only work if the module has not been released
*/
funcdefdelete_module(SelectOneModuleParamsparams)returns()authenticationrequired;

/*
*admin method to move the git url for a module, should only be used if the exact same code has migrated to
*a new URL. It should not be used as a way to change ownership, get updates from a new source, or get a new
*module name for an existing git url because old versions are retained and git commits saved will no longer
*be correct.
*/
funcdefmigrate_module_to_new_git_url(UpdateGitUrlParamsparams)returns()authenticationrequired;

/*
*admin methods to turn on/off modules
*/
funcdefset_to_active(SelectOneModuleParamsparams)returns()authenticationrequired;

funcdefset_to_inactive(SelectOneModuleParamsparams)returns()authenticationrequired;

/*
*temporary developer approval, should be moved to more mature user profile service
*/
funcdefis_approved_developer(list<string>usernames)returns(list<boolean>is_approved)authenticationnone;

funcdeflist_approved_developers()returns(list<string>usernames)authenticationrequired;

funcdefapprove_developer(stringusername)returns()authenticationrequired;

funcdefrevoke_developer(stringusername)returns()authenticationrequired;

/*
*user_id - GlobusOnline login of invoker,
*app_module_name - optional module name of registered repo (could be absent of null for
*old fashioned services) where app_id comes from,
*app_id - optional method-spec id without module_name prefix (could be absent or null
*in case original execution was started through API call without app ID defined),
*func_module_name - optional module name of registered repo (could be absent of null for
*old fashioned services) where func_name comes from,
*func_name - name of function in KIDL-spec without module_name prefix,
*git_commit_hash - optional service version (in case of dynamically registered repo),
*creation_time, exec_start_time and finish_time - defined in seconds since Epoch (POSIX),
*is_error - indicates whether execution was finished with error or not.
*/
typedefstructure{
stringuser_id;
stringapp_module_name;
stringapp_id;
stringfunc_module_name;
stringfunc_name;
stringgit_commit_hash;
floatcreation_time;
floatexec_start_time;
floatfinish_time;
booleanis_error;
stringjob_id;
}LogExecStatsParams;

/*
*Request from Execution Engine for adding statistics about each method run. It could be done
*using catalog admin credentials only.
*/
funcdeflog_exec_stats(LogExecStatsParamsparams)returns()authenticationrequired;

/*
*full_app_ids - list of fully qualified app IDs (including module_name prefix followed by
*slash in case of dynamically registered repo).
*per_week - optional flag switching results to weekly data rather than one row per app for
*all time (default value is false)
*/
typedefstructure{
list<string>full_app_ids;
booleanper_week;
}GetExecAggrStatsParams;

/*
*full_app_id - optional fully qualified method-spec id including module_name prefix followed
*by slash in case of dynamically registered repo (it could be absent or null in case
*original execution was started through API call without app ID defined),
*time_range - one of supported time ranges (currently it could be either '*' for all time
*or ISO-encoded week like "2016-W01")
*total_queue_time - summarized time difference between exec_start_time and creation_time moments
*defined in seconds since Epoch (POSIX),
*total_exec_time - summarized time difference between finish_time and exec_start_time moments
*defined in seconds since Epoch (POSIX).
*/
typedefstructure{
stringfull_app_id;
stringtime_range;
intnumber_of_calls;
intnumber_of_errors;
floattotal_queue_time;
floattotal_exec_time;
}ExecAggrStats;

funcdefget_exec_aggr_stats(GetExecAggrStatsParamsparams)returns(list<ExecAggrStats>)authenticationnone;

/*
*Get aggregated usage metrics; available only to Admins.
*/
typedefstructure{
intbegin;
intend;
}ExecAggrTableParams;

funcdefget_exec_aggr_table(ExecAggrTableParamsparams)returns(UnspecifiedObjecttable)authenticationrequired;

/*
*Get raw usage metrics; available only to Admins.
*/
typedefstructure{
intbegin;
intend;
}GetExecRawStatsParams;

funcdefget_exec_raw_stats(GetExecRawStatsParamsparams)returns(list<UnspecifiedObject>records)authenticationrequired;

/*
*app_id = full app id; if module name is used it will be case insensitive
*this will overwrite all existing client groups (it won't just push what's on the list)
*If client_groups is empty or set to null, then the client_group mapping will be removed.
*/
typedefstructure{
stringapp_id;
list<string>client_groups;
}AppClientGroup;

/*
*if app_ids is empty or null, all client groups are returned
*/
typedefstructure{}GetClientGroupParams;

/*
*@deprecatedlist_client_group_configs
*/
funcdefget_client_groups(GetClientGroupParamsparams)returns(list<AppClientGroup>groups)authenticationnone;

typedefstructure{
stringmodule_name;
stringfunction_name;
list<string>client_groups;
}ClientGroupConfig;

funcdefset_client_group_config(ClientGroupConfigconfig)returns()authenticationrequired;

funcdefremove_client_group_config(ClientGroupConfigconfig)returns()authenticationrequired;

typedefstructure{
stringmodule_name;
stringfunction_name;
}ClientGroupFilter;

funcdeflist_client_group_configs(ClientGroupFilterfilter)returns(list<ClientGroupConfig>groups)authenticationnone;

typedefstructure{
stringhost_dir;
stringcontainer_dir;
booleanread_only;
}VolumeMount;

/*
*for a module, function, and client group, set mount configurations
*/
typedefstructure{
stringmodule_name;
stringfunction_name;
stringclient_group;
list<VolumeMount>volume_mounts;
}VolumeMountConfig;

/*
*must specify all properties of the VolumeMountConfig
*/
funcdefset_volume_mount(VolumeMountConfigconfig)returns()authenticationrequired;

/*
*must specify module_name, function_name, client_group and this method will delete any configured mounts
*/
funcdefremove_volume_mount(VolumeMountConfigconfig)returns()authenticationrequired;

/*
*Parameters for listing VolumeMountConfigs. If nothing is set, everything is
*returned. Otherwise, will return everything that matches all fields set. For
*instance, if only module_name is set, will return everything for that module. If
*they are all set, will return the specific module/app/client group config. Returns
*nothing if no matches are found.
*/
typedefstructure{
stringmodule_name;
stringfunction_name;
stringclient_group;
}VolumeMountFilter;

funcdeflist_volume_mounts(VolumeMountFilterfilter)returns(list<VolumeMountConfig>volume_mount_configs)authenticationrequired;

/*
*returns true (1) if the user is an admin, false (0) otherwise.
*NOTE: username is now ignored (it checks the token) but retained for back compatibility
*/
funcdefis_admin(stringusername)returns(boolean)authenticationoptional;

/*
*version - optional version (commit hash, tag or semantic one) of module, if not set
*then default "" value is used which means parameter is applied to any version;
*is_password - optional flag meaning to hide this parameter's value in UI.
*/
typedefstructure{
stringmodule_name;
stringversion;
stringparam_name;
booleanis_password;
stringparam_value;
}SecureConfigParameter;

typedefstructure{}ModifySecureConfigParamsInput;

/*
*Only admins can use this function.
*/
funcdefset_secure_config_params(ModifySecureConfigParamsInputparams)returns()authenticationrequired;

/*
*Only admins can use this function.
*/
funcdefremove_secure_config_params(ModifySecureConfigParamsInputparams)returns()authenticationrequired;

/*
*version - optional version (commit hash, tag or semantic one) of module, if
*not set then default "release" value is used;
*load_all_versions - optional flag indicating that all parameter versions
*should be loaded (version filter is not applied), default value is 0.
*/
typedefstructure{
stringmodule_name;
stringversion;
booleanload_all_versions;
}GetSecureConfigParamsInput;

/*
*Only admins can use this function.
*/
funcdefget_secure_config_params(GetSecureConfigParamsInputparams)returns(list<SecureConfigParameter>)authenticationrequired;
};

Function Index

Type Index

\ No newline at end of file diff --git a/KIDLspec.css b/KIDLspec.css new file mode 100644 index 00000000..4d2a3e3a --- /dev/null +++ b/KIDLspec.css @@ -0,0 +1,65 @@ +html, body { + height: 100%; +} +html { + display: table; + margin: auto; +} +body { + background-color: white; + color: #000; + font-family: Menlo, Monaco, Consolas, "Courier New", monospace; + font-weight: normal; + font-size: 12px; + margin: 0; + padding: 20px; + display: table-cell; + vertical-align: middle; +} +span.space { + display: inline-block; + width: 7px; +} +span.tab { + display: inline-block; + width: 30px; +} +span.keyword { + font-weight: bold; + color: #008; +} +span.name { + color: #000; !important +} +span.deprecated { + text-decoration: line-through; +} +span.annotation { + color: #303030; +} +span.primitive { + font-weight: bold; + color: #066; +} +div.body { + background-color: #ffffff; + color: #3e4349; + padding: 0 30px; +} +div.comment { + color: #A0A0A0; +} +a { + color: #004b6b; + text-decoration: none; +} +a:hover { + color: #6d4100; + text-decoration: underline; +} +:target { + background-color: #ffa; +} +div.body p, div.body dd, div.body li { + line-height: 1.4em; +} diff --git a/Makefile b/Makefile index 53a2cd5b..ce536ce5 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,6 @@ SERVICE_CAPS = Catalog #$(shell perl server_scripts/get_deploy_cfg.pm $(SERVICE_CAPS).port) SERVICE_PORT = 5000 SPEC_FILE = catalog.spec -URL = https://kbase.us/services/catalog/rpc #End of user defined variables @@ -29,18 +28,18 @@ PATH := kb_sdk/bin:$(PATH) compile-kb-module: + kb-sdk compile $(SPEC_FILE) \ + --out . \ + --html kb-sdk compile $(SPEC_FILE) \ --out $(LIB_DIR) \ - --plclname Bio::KBase::$(SERVICE_CAPS)::Client \ - --jsclname javascript/Client \ --pyclname biokbase.$(SERVICE).Client \ - --javasrc java \ - --java \ --pysrvname biokbase.$(SERVICE).Server \ - --pyimplname biokbase.$(SERVICE).Impl; - touch $(LIB_DIR)/biokbase/__init__.py - touch $(LIB_DIR)/biokbase/$(SERVICE)/__init__.py - + --pyimplname biokbase.$(SERVICE).Impl + kb-sdk compile $(SPEC_FILE) \ + --out . \ + --java \ + --javasrc src/main/java # start/stop the service running out of THIS directory build-local-server-control-scripts: diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index b498eeb9..95eb6038 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,8 @@ # Catalog Service release notes +## v2.3.1 - 9/3/2025 + - Added a JITPack build for the catalog client + ## v2.3.0 - 3/5/2025 - Removed all submodules(jars, kbapi_common, nms) as part of repository clean up. - The MongoDB clients have been updated to the most recent version and the service tested against Mongo 7. diff --git a/build.gradle b/build.gradle new file mode 100644 index 00000000..81035700 --- /dev/null +++ b/build.gradle @@ -0,0 +1,58 @@ +/* + * This file was generated by the Gradle 'init' task. + */ + +plugins { + id 'java' + id 'maven-publish' +} + +group = 'com.github.kbase' + +var VER_AUTH2_CLIENT = "0.5.0" +var VER_JAVA_COMMON = "0.3.0" + +repositories { + mavenCentral() + maven { + name = "JitPack" + url = 'https://jitpack.io' + } +} + +compileJava { + options.release = 11 +} + +java { + withSourcesJar() + withJavadocJar() +} + +javadoc { + options { + // I don't know why this isn't working, but it's not worth spending time on right now + links "https://docs.oracle.com/javase/11/docs/api/" + links "https://javadoc.jitpack.io/com/github/kbase/auth2_client_java/$VER_AUTH2_CLIENT/javadoc/" + links "https://javadoc.jitpack.io/com/github/kbase/java_common/$VER_JAVA_COMMON/javadoc/" + } +} + +publishing { + publications { + maven(MavenPublication) { + from components.java + } + } +} + +dependencies { + + // using older dependencies to not force upgrades on services that might not be able to + // handle them. Need to upgrade the services and then upgrade here + implementation "com.github.kbase:java_common:$VER_JAVA_COMMON" + implementation "com.fasterxml.jackson.core:jackson-databind:2.5.4" + implementation "com.github.kbase:auth2_client_java:$VER_AUTH2_CLIENT" + implementation 'javax.annotation:javax.annotation-api:1.3.2' + +} diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 00000000..377538c9 --- /dev/null +++ b/gradle.properties @@ -0,0 +1,5 @@ +# This file was generated by the Gradle 'init' task. +# https://docs.gradle.org/current/userguide/build_environment.html#sec:gradle_configuration_properties + +org.gradle.configuration-cache=true + diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml new file mode 100644 index 00000000..58ad57ac --- /dev/null +++ b/gradle/libs.versions.toml @@ -0,0 +1,2 @@ +# This file was generated by the Gradle 'init' task. +# https://docs.gradle.org/current/userguide/platforms.html#sub::toml-dependencies-format \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..1b33c55baabb587c669f562ae36f953de2481846 GIT binary patch literal 43764 zcma&OWmKeVvL#I6?i3D%6z=Zs?ofE*?rw#G$eqJB ziT4y8-Y@s9rkH0Tz>ll(^xkcTl)CY?rS&9VNd66Yc)g^6)JcWaY(5$5gt z8gr3SBXUTN;~cBgz&})qX%#!Fxom2Yau_`&8)+6aSN7YY+pS410rRUU*>J}qL0TnJ zRxt*7QeUqTh8j)Q&iavh<}L+$Jqz))<`IfKussVk%%Ah-Ti?Eo0hQH!rK%K=#EAw0 zwq@@~XNUXRnv8$;zv<6rCRJ6fPD^hfrh;0K?n z=p!u^3xOgWZ%f3+?+>H)9+w^$Tn1e;?UpVMJb!!;f)`6f&4|8mr+g)^@x>_rvnL0< zvD0Hu_N>$(Li7|Jgu0mRh&MV+<}`~Wi*+avM01E)Jtg=)-vViQKax!GeDc!xv$^mL z{#OVBA$U{(Zr8~Xm|cP@odkHC*1R8z6hcLY#N@3E-A8XEvpt066+3t9L_6Zg6j@9Q zj$$%~yO-OS6PUVrM2s)(T4#6=JpI_@Uz+!6=GdyVU?`!F=d;8#ZB@(5g7$A0(`eqY z8_i@3w$0*es5mrSjhW*qzrl!_LQWs4?VfLmo1Sd@Ztt53+etwzAT^8ow_*7Jp`Y|l z*UgSEwvxq+FYO!O*aLf-PinZYne7Ib6ny3u>MjQz=((r3NTEeU4=-i0LBq3H-VJH< z^>1RE3_JwrclUn9vb7HcGUaFRA0QHcnE;6)hnkp%lY1UII#WPAv?-;c?YH}LWB8Nl z{sx-@Z;QxWh9fX8SxLZk8;kMFlGD3Jc^QZVL4nO)1I$zQwvwM&_!kW+LMf&lApv#< zur|EyC|U@5OQuph$TC_ZU`{!vJp`13e9alaR0Dbn5ikLFH7>eIz4QbV|C=%7)F=qo z_>M&5N)d)7G(A%c>}UCrW!Ql_6_A{?R7&CL`;!KOb3 z8Z=$YkV-IF;c7zs{3-WDEFJzuakFbd*4LWd<_kBE8~BFcv}js_2OowRNzWCtCQ6&k z{&~Me92$m*@e0ANcWKuz)?YjB*VoSTx??-3Cc0l2U!X^;Bv@m87eKHukAljrD54R+ zE;@_w4NPe1>3`i5Qy*3^E9x#VB6?}v=~qIprrrd5|DFkg;v5ixo0IsBmik8=Y;zv2 z%Bcf%NE$a44bk^`i4VwDLTbX=q@j9;JWT9JncQ!+Y%2&HHk@1~*L8-{ZpY?(-a9J-1~<1ltr9i~D9`P{XTIFWA6IG8c4;6bFw*lzU-{+?b&%OcIoCiw00n>A1ra zFPE$y@>ebbZlf(sN_iWBzQKDV zmmaLX#zK!@ZdvCANfwV}9@2O&w)!5gSgQzHdk2Q`jG6KD7S+1R5&F)j6QTD^=hq&7 zHUW+r^da^%V(h(wonR(j?BOiC!;y=%nJvz?*aW&5E87qq;2z`EI(f zBJNNSMFF9U{sR-af5{IY&AtoGcoG)Iq-S^v{7+t0>7N(KRoPj;+2N5;9o_nxIGjJ@ z7bYQK)bX)vEhy~VL%N6g^NE@D5VtV+Q8U2%{ji_=6+i^G%xeskEhH>Sqr194PJ$fB zu1y^){?9Vkg(FY2h)3ZHrw0Z<@;(gd_dtF#6y_;Iwi{yX$?asr?0N0_B*CifEi7<6 zq`?OdQjCYbhVcg+7MSgIM|pJRu~`g?g3x?Tl+V}#$It`iD1j+!x+!;wS0+2e>#g?Z z*EA^k7W{jO1r^K~cD#5pamp+o@8&yw6;%b|uiT?{Wa=4+9<}aXWUuL#ZwN1a;lQod zW{pxWCYGXdEq9qAmvAB904}?97=re$>!I%wxPV#|f#@A*Y=qa%zHlDv^yWbR03%V0 zprLP+b(#fBqxI%FiF*-n8HtH6$8f(P6!H3V^ysgd8de-N(@|K!A< z^qP}jp(RaM9kQ(^K(U8O84?D)aU(g?1S8iWwe)gqpHCaFlJxb*ilr{KTnu4_@5{K- z)n=CCeCrPHO0WHz)dDtkbZfUfVBd?53}K>C5*-wC4hpDN8cGk3lu-ypq+EYpb_2H; z%vP4@&+c2p;thaTs$dc^1CDGlPG@A;yGR5@$UEqk6p58qpw#7lc<+W(WR;(vr(D>W z#(K$vE#uBkT=*q&uaZwzz=P5mjiee6>!lV?c}QIX%ZdkO1dHg>Fa#xcGT6~}1*2m9 zkc7l3ItD6Ie~o_aFjI$Ri=C!8uF4!Ky7iG9QTrxVbsQroi|r)SAon#*B*{}TB-?=@ z8~jJs;_R2iDd!$+n$%X6FO&PYS{YhDAS+U2o4su9x~1+U3z7YN5o0qUK&|g^klZ6X zj_vrM5SUTnz5`*}Hyts9ADwLu#x_L=nv$Z0`HqN`Zo=V>OQI)fh01n~*a%01%cx%0 z4LTFVjmW+ipVQv5rYcn3;d2o4qunWUY!p+?s~X~(ost@WR@r@EuDOSs8*MT4fiP>! zkfo^!PWJJ1MHgKS2D_hc?Bs?isSDO61>ebl$U*9*QY(b=i&rp3@3GV@z>KzcZOxip z^dzA~44;R~cnhWz7s$$v?_8y-k!DZys}Q?4IkSyR!)C0j$(Gm|t#e3|QAOFaV2}36 z?dPNY;@I=FaCwylc_;~kXlZsk$_eLkNb~TIl8QQ`mmH&$*zwwR8zHU*sId)rxHu*K z;yZWa8UmCwju%aSNLwD5fBl^b0Ux1%q8YR*uG`53Mi<`5uA^Dc6Ync)J3N7;zQ*75)hf%a@{$H+%S?SGT)ks60)?6j$ zspl|4Ad6@%-r1t*$tT(en!gIXTUDcsj?28ZEzz)dH)SV3bZ+pjMaW0oc~rOPZP@g! zb9E+ndeVO_Ib9c_>{)`01^`ZS198 z)(t=+{Azi11$eu%aU7jbwuQrO`vLOixuh~%4z@mKr_Oc;F%Uq01fA)^W&y+g16e?rkLhTxV!EqC%2}sx_1u7IBq|}Be&7WI z4I<;1-9tJsI&pQIhj>FPkQV9{(m!wYYV@i5h?A0#BN2wqlEwNDIq06|^2oYVa7<~h zI_OLan0Do*4R5P=a3H9`s5*>xU}_PSztg`+2mv)|3nIy=5#Z$%+@tZnr> zLcTI!Mxa`PY7%{;KW~!=;*t)R_sl<^b>eNO@w#fEt(tPMg_jpJpW$q_DoUlkY|uo> z0-1{ouA#;t%spf*7VjkK&$QrvwUERKt^Sdo)5@?qAP)>}Y!h4(JQ!7{wIdkA+|)bv z&8hBwoX4v|+fie}iTslaBX^i*TjwO}f{V)8*!dMmRPi%XAWc8<_IqK1jUsApk)+~R zNFTCD-h>M5Y{qTQ&0#j@I@tmXGj%rzhTW5%Bkh&sSc=$Fv;M@1y!zvYG5P2(2|(&W zlcbR1{--rJ&s!rB{G-sX5^PaM@3EqWVz_y9cwLR9xMig&9gq(voeI)W&{d6j1jh&< zARXi&APWE1FQWh7eoZjuP z;vdgX>zep^{{2%hem;e*gDJhK1Hj12nBLIJoL<=0+8SVEBx7!4Ea+hBY;A1gBwvY<)tj~T=H`^?3>zeWWm|LAwo*S4Z%bDVUe z6r)CH1H!(>OH#MXFJ2V(U(qxD{4Px2`8qfFLG+=a;B^~Te_Z!r3RO%Oc#ZAHKQxV5 zRYXxZ9T2A%NVJIu5Pu7!Mj>t%YDO$T@M=RR(~mi%sv(YXVl`yMLD;+WZ{vG9(@P#e zMo}ZiK^7^h6TV%cG+;jhJ0s>h&VERs=tuZz^Tlu~%d{ZHtq6hX$V9h)Bw|jVCMudd zwZ5l7In8NT)qEPGF$VSKg&fb0%R2RnUnqa){)V(X(s0U zkCdVZe6wy{+_WhZh3qLp245Y2RR$@g-!9PjJ&4~0cFSHMUn=>dapv)hy}|y91ZWTV zCh=z*!S3_?`$&-eZ6xIXUq8RGl9oK0BJw*TdU6A`LJqX9eS3X@F)g$jLkBWFscPhR zpCv8#KeAc^y>>Y$k^=r|K(DTC}T$0#jQBOwB#@`P6~*IuW_8JxCG}J4va{ zsZzt}tt+cv7=l&CEuVtjD6G2~_Meh%p4RGuY?hSt?(sreO_F}8r7Kp$qQdvCdZnDQ zxzc*qchE*E2=WK)^oRNa>Ttj`fpvF-JZ5tu5>X1xw)J@1!IqWjq)ESBG?J|ez`-Tc zi5a}GZx|w-h%5lNDE_3ho0hEXMoaofo#Z;$8|2;EDF&*L+e$u}K=u?pb;dv$SXeQM zD-~7P0i_`Wk$#YP$=hw3UVU+=^@Kuy$>6?~gIXx636jh{PHly_a2xNYe1l60`|y!7 z(u%;ILuW0DDJ)2%y`Zc~hOALnj1~txJtcdD#o4BCT68+8gZe`=^te6H_egxY#nZH&P*)hgYaoJ^qtmpeea`35Fw)cy!w@c#v6E29co8&D9CTCl%^GV|X;SpneSXzV~LXyRn-@K0Df z{tK-nDWA!q38M1~`xUIt_(MO^R(yNY#9@es9RQbY@Ia*xHhD&=k^T+ zJi@j2I|WcgW=PuAc>hs`(&CvgjL2a9Rx zCbZyUpi8NWUOi@S%t+Su4|r&UoU|ze9SVe7p@f1GBkrjkkq)T}X%Qo1g!SQ{O{P?m z-OfGyyWta+UCXH+-+(D^%kw#A1-U;?9129at7MeCCzC{DNgO zeSqsV>W^NIfTO~4({c}KUiuoH8A*J!Cb0*sp*w-Bg@YfBIPZFH!M}C=S=S7PLLcIG zs7K77g~W)~^|+mx9onzMm0qh(f~OsDTzVmRtz=aZTllgR zGUn~_5hw_k&rll<4G=G+`^Xlnw;jNYDJz@bE?|r866F2hA9v0-8=JO3g}IHB#b`hy zA42a0>{0L7CcabSD+F7?pGbS1KMvT{@1_@k!_+Ki|5~EMGt7T%u=79F)8xEiL5!EJ zzuxQ`NBliCoJMJdwu|);zRCD<5Sf?Y>U$trQ-;xj6!s5&w=9E7)%pZ+1Nh&8nCCwM zv5>Ket%I?cxr3vVva`YeR?dGxbG@pi{H#8@kFEf0Jq6~K4>kt26*bxv=P&jyE#e$| zDJB_~imk^-z|o!2njF2hL*|7sHCnzluhJjwLQGDmC)Y9 zr9ZN`s)uCd^XDvn)VirMgW~qfn1~SaN^7vcX#K1G`==UGaDVVx$0BQnubhX|{e z^i0}>k-;BP#Szk{cFjO{2x~LjK{^Upqd&<+03_iMLp0$!6_$@TbX>8U-f*-w-ew1?`CtD_0y_Lo|PfKi52p?`5$Jzx0E8`M0 zNIb?#!K$mM4X%`Ry_yhG5k@*+n4||2!~*+&pYLh~{`~o(W|o64^NrjP?-1Lgu?iK^ zTX6u3?#$?R?N!{599vg>G8RGHw)Hx&=|g4599y}mXNpM{EPKKXB&+m?==R3GsIq?G zL5fH={=zawB(sMlDBJ+{dgb)Vx3pu>L=mDV0{r1Qs{0Pn%TpopH{m(By4;{FBvi{I z$}x!Iw~MJOL~&)p93SDIfP3x%ROjg}X{Sme#hiJ&Yk&a;iR}V|n%PriZBY8SX2*;6 z4hdb^&h;Xz%)BDACY5AUsV!($lib4>11UmcgXKWpzRL8r2Srl*9Y(1uBQsY&hO&uv znDNff0tpHlLISam?o(lOp#CmFdH<6HmA0{UwfU#Y{8M+7od8b8|B|7ZYR9f<#+V|ZSaCQvI$~es~g(Pv{2&m_rKSB2QQ zMvT}$?Ll>V+!9Xh5^iy3?UG;dF-zh~RL#++roOCsW^cZ&({6q|?Jt6`?S8=16Y{oH zp50I7r1AC1(#{b`Aq5cw>ypNggHKM9vBx!W$eYIzD!4KbLsZGr2o8>g<@inmS3*>J zx8oG((8f!ei|M@JZB`p7+n<Q}?>h249<`7xJ?u}_n;Gq(&km#1ULN87CeTO~FY zS_Ty}0TgQhV zOh3T7{{x&LSYGQfKR1PDIkP!WnfC1$l+fs@Di+d4O=eVKeF~2fq#1<8hEvpwuqcaH z4A8u~r^gnY3u6}zj*RHjk{AHhrrDqaj?|6GaVJbV%o-nATw}ASFr!f`Oz|u_QPkR# z0mDudY1dZRlk@TyQ?%Eti=$_WNFtLpSx9=S^be{wXINp%MU?a`F66LNU<c;0&ngifmP9i;bj6&hdGMW^Kf8e6ZDXbQD&$QAAMo;OQ)G zW(qlHh;}!ZP)JKEjm$VZjTs@hk&4{?@+NADuYrr!R^cJzU{kGc1yB?;7mIyAWwhbeA_l_lw-iDVi7wcFurf5 z#Uw)A@a9fOf{D}AWE%<`s1L_AwpZ?F!Vac$LYkp<#A!!`XKaDC{A%)~K#5z6>Hv@V zBEqF(D5?@6r3Pwj$^krpPDCjB+UOszqUS;b2n>&iAFcw<*im2(b3|5u6SK!n9Sg4I z0KLcwA6{Mq?p%t>aW0W!PQ>iUeYvNjdKYqII!CE7SsS&Rj)eIw-K4jtI?II+0IdGq z2WT|L3RL?;GtGgt1LWfI4Ka`9dbZXc$TMJ~8#Juv@K^1RJN@yzdLS8$AJ(>g!U9`# zx}qr7JWlU+&m)VG*Se;rGisutS%!6yybi%B`bv|9rjS(xOUIvbNz5qtvC$_JYY+c& za*3*2$RUH8p%pSq>48xR)4qsp!Q7BEiJ*`^>^6INRbC@>+2q9?x(h0bpc>GaNFi$K zPH$6!#(~{8@0QZk=)QnM#I=bDx5vTvjm$f4K}%*s+((H2>tUTf==$wqyoI`oxI7>C z&>5fe)Yg)SmT)eA(|j@JYR1M%KixxC-Eceknf-;N=jJTwKvk#@|J^&5H0c+%KxHUI z6dQbwwVx3p?X<_VRVb2fStH?HH zFR@Mp=qX%#L3XL)+$PXKV|o|#DpHAoqvj6uQKe@M-mnhCSou7Dj4YuO6^*V`m)1lf z;)@e%1!Qg$10w8uEmz{ENb$^%u}B;J7sDd zump}onoD#!l=agcBR)iG!3AF0-63%@`K9G(CzKrm$VJ{v7^O9Ps7Zej|3m= zVXlR&yW6=Y%mD30G@|tf=yC7-#L!16Q=dq&@beWgaIL40k0n% z)QHrp2Jck#evLMM1RGt3WvQ936ZC9vEje0nFMfvmOHVI+&okB_K|l-;|4vW;qk>n~ z+|kk8#`K?x`q>`(f6A${wfw9Cx(^)~tX7<#TpxR#zYG2P+FY~mG{tnEkv~d6oUQA+ z&hNTL=~Y@rF`v-RZlts$nb$3(OL1&@Y11hhL9+zUb6)SP!;CD)^GUtUpCHBE`j1te zAGud@miCVFLk$fjsrcpjsadP__yj9iEZUW{Ll7PPi<$R;m1o!&Xdl~R_v0;oDX2z^!&8}zNGA}iYG|k zmehMd1%?R)u6R#<)B)1oe9TgYH5-CqUT8N7K-A-dm3hbm_W21p%8)H{O)xUlBVb+iUR}-v5dFaCyfSd zC6Bd7=N4A@+Bna=!-l|*_(nWGDpoyU>nH=}IOrLfS+-d40&(Wo*dDB9nQiA2Tse$R z;uq{`X7LLzP)%Y9aHa4YQ%H?htkWd3Owv&UYbr5NUDAH^<l@Z0Cx%`N+B*i!!1u>D8%;Qt1$ zE5O0{-`9gdDxZ!`0m}ywH!;c{oBfL-(BH<&SQ~smbcobU!j49O^f4&IIYh~f+hK*M zZwTp%{ZSAhMFj1qFaOA+3)p^gnXH^=)`NTYgTu!CLpEV2NF=~-`(}7p^Eof=@VUbd z_9U|8qF7Rueg&$qpSSkN%%%DpbV?8E8ivu@ensI0toJ7Eas^jyFReQ1JeY9plb^{m z&eQO)qPLZQ6O;FTr*aJq=$cMN)QlQO@G&%z?BKUs1&I^`lq>=QLODwa`(mFGC`0H< zOlc*|N?B5&!U6BuJvkL?s1&nsi$*5cCv7^j_*l&$-sBmRS85UIrE--7eD8Gr3^+o? zqG-Yl4S&E;>H>k^a0GdUI(|n1`ws@)1%sq2XBdK`mqrNq_b4N{#VpouCXLzNvjoFv zo9wMQ6l0+FT+?%N(ka*;%m~(?338bu32v26!{r)|w8J`EL|t$}TA4q_FJRX5 zCPa{hc_I(7TGE#@rO-(!$1H3N-C0{R$J=yPCXCtGk{4>=*B56JdXU9cQVwB`6~cQZ zf^qK21x_d>X%dT!!)CJQ3mlHA@ z{Prkgfs6=Tz%63$6Zr8CO0Ak3A)Cv#@BVKr&aiKG7RYxY$Yx>Bj#3gJk*~Ps-jc1l z;4nltQwwT4@Z)}Pb!3xM?+EW0qEKA)sqzw~!C6wd^{03-9aGf3Jmt=}w-*!yXupLf z;)>-7uvWN4Unn8b4kfIza-X=x*e4n5pU`HtgpFFd))s$C@#d>aUl3helLom+RYb&g zI7A9GXLRZPl}iQS*d$Azxg-VgcUr*lpLnbPKUV{QI|bsG{8bLG<%CF( zMoS4pRDtLVYOWG^@ox^h8xL~afW_9DcE#^1eEC1SVSb1BfDi^@g?#f6e%v~Aw>@w- zIY0k+2lGWNV|aA*e#`U3=+oBDmGeInfcL)>*!w|*;mWiKNG6wP6AW4-4imN!W)!hE zA02~S1*@Q`fD*+qX@f3!2yJX&6FsEfPditB%TWo3=HA;T3o2IrjS@9SSxv%{{7&4_ zdS#r4OU41~GYMiib#z#O;zohNbhJknrPPZS6sN$%HB=jUnlCO_w5Gw5EeE@KV>soy z2EZ?Y|4RQDDjt5y!WBlZ(8M)|HP<0YyG|D%RqD+K#e7-##o3IZxS^wQ5{Kbzb6h(i z#(wZ|^ei>8`%ta*!2tJzwMv+IFHLF`zTU8E^Mu!R*45_=ccqI};Zbyxw@U%a#2}%f zF>q?SrUa_a4H9l+uW8JHh2Oob>NyUwG=QH~-^ZebU*R@67DcXdz2{HVB4#@edz?B< z5!rQH3O0>A&ylROO%G^fimV*LX7>!%re{_Sm6N>S{+GW1LCnGImHRoF@csnFzn@P0 zM=jld0z%oz;j=>c7mMwzq$B^2mae7NiG}%>(wtmsDXkWk{?BeMpTrIt3Mizq?vRsf zi_WjNp+61uV(%gEU-Vf0;>~vcDhe(dzWdaf#4mH3o^v{0EWhj?E?$5v02sV@xL0l4 zX0_IMFtQ44PfWBbPYN#}qxa%=J%dlR{O!KyZvk^g5s?sTNycWYPJ^FK(nl3k?z-5t z39#hKrdO7V(@!TU)LAPY&ngnZ1MzLEeEiZznn7e-jLCy8LO zu^7_#z*%I-BjS#Pg-;zKWWqX-+Ly$T!4`vTe5ZOV0j?TJVA*2?*=82^GVlZIuH%9s zXiV&(T(QGHHah=s&7e|6y?g+XxZGmK55`wGV>@1U)Th&=JTgJq>4mI&Av2C z)w+kRoj_dA!;SfTfkgMPO>7Dw6&1*Hi1q?54Yng`JO&q->^CX21^PrU^JU#CJ_qhV zSG>afB%>2fx<~g8p=P8Yzxqc}s@>>{g7}F!;lCXvF#RV)^fyYb_)iKVCz1xEq=fJ| z0a7DMCK*FuP=NM*5h;*D`R4y$6cpW-E&-i{v`x=Jbk_xSn@2T3q!3HoAOB`@5Vg6) z{PW|@9o!e;v1jZ2{=Uw6S6o{g82x6g=k!)cFSC*oemHaVjg?VpEmtUuD2_J^A~$4* z3O7HsbA6wxw{TP5Kk)(Vm?gKo+_}11vbo{Tp_5x79P~#F)ahQXT)tSH5;;14?s)On zel1J>1x>+7;g1Iz2FRpnYz;sD0wG9Q!vuzE9yKi3@4a9Nh1!GGN?hA)!mZEnnHh&i zf?#ZEN2sFbf~kV;>K3UNj1&vFhc^sxgj8FCL4v>EOYL?2uuT`0eDH}R zmtUJMxVrV5H{L53hu3#qaWLUa#5zY?f5ozIn|PkMWNP%n zWB5!B0LZB0kLw$k39=!akkE9Q>F4j+q434jB4VmslQ;$ zKiO#FZ`p|dKS716jpcvR{QJkSNfDVhr2%~eHrW;fU45>>snr*S8Vik-5eN5k*c2Mp zyxvX&_cFbB6lODXznHHT|rsURe2!swomtrqc~w5 zymTM8!w`1{04CBprR!_F{5LB+2_SOuZN{b*!J~1ZiPpP-M;);!ce!rOPDLtgR@Ie1 zPreuqm4!H)hYePcW1WZ0Fyaqe%l}F~Orr)~+;mkS&pOhP5Ebb`cnUt!X_QhP4_4p( z8YKQCDKGIy>?WIFm3-}Br2-N`T&FOi?t)$hjphB9wOhBXU#Hb+zm&We_-O)s(wc`2 z8?VsvU;J>Ju7n}uUb3s1yPx_F*|FlAi=Ge=-kN?1;`~6szP%$3B0|8Sqp%ebM)F8v zADFrbeT0cgE>M0DMV@_Ze*GHM>q}wWMzt|GYC%}r{OXRG3Ij&<+nx9;4jE${Fj_r* z`{z1AW_6Myd)i6e0E-h&m{{CvzH=Xg!&(bLYgRMO_YVd8JU7W+7MuGWNE=4@OvP9+ zxi^vqS@5%+#gf*Z@RVyU9N1sO-(rY$24LGsg1>w>s6ST^@)|D9>cT50maXLUD{Fzf zt~tp{OSTEKg3ZSQyQQ5r51){%=?xlZ54*t1;Ow)zLe3i?8tD8YyY^k%M)e`V*r+vL zPqUf&m)U+zxps+NprxMHF{QSxv}>lE{JZETNk1&F+R~bp{_T$dbXL2UGnB|hgh*p4h$clt#6;NO~>zuyY@C-MD@)JCc5XrYOt`wW7! z_ti2hhZBMJNbn0O-uTxl_b6Hm313^fG@e;RrhIUK9@# z+DHGv_Ow$%S8D%RB}`doJjJy*aOa5mGHVHz0e0>>O_%+^56?IkA5eN+L1BVCp4~m=1eeL zb;#G!#^5G%6Mw}r1KnaKsLvJB%HZL)!3OxT{k$Yo-XrJ?|7{s4!H+S2o?N|^Z z)+?IE9H7h~Vxn5hTis^3wHYuOU84+bWd)cUKuHapq=&}WV#OxHpLab`NpwHm8LmOo zjri+!k;7j_?FP##CpM+pOVx*0wExEex z@`#)K<-ZrGyArK;a%Km`^+We|eT+#MygHOT6lXBmz`8|lyZOwL1+b+?Z$0OhMEp3R z&J=iRERpv~TC=p2-BYLC*?4 zxvPs9V@g=JT0>zky5Poj=fW_M!c)Xxz1<=&_ZcL=LMZJqlnO1P^xwGGW*Z+yTBvbV z-IFe6;(k1@$1;tS>{%pXZ_7w+i?N4A2=TXnGf=YhePg8bH8M|Lk-->+w8Y+FjZ;L=wSGwxfA`gqSn)f(XNuSm>6Y z@|#e-)I(PQ^G@N`%|_DZSb4_pkaEF0!-nqY+t#pyA>{9^*I-zw4SYA1_z2Bs$XGUZbGA;VeMo%CezHK0lO={L%G)dI-+8w?r9iexdoB{?l zbJ}C?huIhWXBVs7oo{!$lOTlvCLZ_KN1N+XJGuG$rh<^eUQIqcI7^pmqhBSaOKNRq zrx~w^?9C?*&rNwP_SPYmo;J-#!G|{`$JZK7DxsM3N^8iR4vvn>E4MU&Oe1DKJvLc~ zCT>KLZ1;t@My zRj_2hI^61T&LIz)S!+AQIV23n1>ng+LUvzv;xu!4;wpqb#EZz;F)BLUzT;8UA1x*6vJ zicB!3Mj03s*kGV{g`fpC?V^s(=JG-k1EMHbkdP4P*1^8p_TqO|;!Zr%GuP$8KLxuf z=pv*H;kzd;P|2`JmBt~h6|GxdU~@weK5O=X&5~w$HpfO}@l-T7@vTCxVOwCkoPQv8 z@aV_)I5HQtfs7^X=C03zYmH4m0S!V@JINm6#(JmZRHBD?T!m^DdiZJrhKpBcur2u1 zf9e4%k$$vcFopK5!CC`;ww(CKL~}mlxK_Pv!cOsFgVkNIghA2Au@)t6;Y3*2gK=5d z?|@1a)-(sQ%uFOmJ7v2iG&l&m^u&^6DJM#XzCrF%r>{2XKyxLD2rgWBD;i(!e4InDQBDg==^z;AzT2z~OmV0!?Z z0S9pX$+E;w3WN;v&NYT=+G8hf=6w0E1$0AOr61}eOvE8W1jX%>&Mjo7&!ulawgzLH zbcb+IF(s^3aj12WSi#pzIpijJJzkP?JzRawnxmNDSUR#7!29vHULCE<3Aa#be}ie~d|!V+ z%l~s9Odo$G&fH!t!+`rUT0T9DulF!Yq&BfQWFZV1L9D($r4H(}Gnf6k3^wa7g5|Ws zj7%d`!3(0bb55yhC6@Q{?H|2os{_F%o=;-h{@Yyyn*V7?{s%Grvpe!H^kl6tF4Zf5 z{Jv1~yZ*iIWL_9C*8pBMQArfJJ0d9Df6Kl#wa}7Xa#Ef_5B7=X}DzbQXVPfCwTO@9+@;A^Ti6il_C>g?A-GFwA0#U;t4;wOm-4oS})h z5&on>NAu67O?YCQr%7XIzY%LS4bha9*e*4bU4{lGCUmO2UQ2U)QOqClLo61Kx~3dI zmV3*(P6F_Tr-oP%x!0kTnnT?Ep5j;_IQ^pTRp=e8dmJtI4YgWd0}+b2=ATkOhgpXe z;jmw+FBLE}UIs4!&HflFr4)vMFOJ19W4f2^W(=2)F%TAL)+=F>IE$=e=@j-*bFLSg z)wf|uFQu+!=N-UzSef62u0-C8Zc7 zo6@F)c+nZA{H|+~7i$DCU0pL{0Ye|fKLuV^w!0Y^tT$isu%i1Iw&N|tX3kwFKJN(M zXS`k9js66o$r)x?TWL}Kxl`wUDUpwFx(w4Yk%49;$sgVvT~n8AgfG~HUcDt1TRo^s zdla@6heJB@JV z!vK;BUMznhzGK6PVtj0)GB=zTv6)Q9Yt@l#fv7>wKovLobMV-+(8)NJmyF8R zcB|_K7=FJGGn^X@JdFaat0uhKjp3>k#^&xE_}6NYNG?kgTp>2Iu?ElUjt4~E-?`Du z?mDCS9wbuS%fU?5BU@Ijx>1HG*N?gIP+<~xE4u=>H`8o((cS5M6@_OK%jSjFHirQK zN9@~NXFx*jS{<|bgSpC|SAnA@I)+GB=2W|JJChLI_mx+-J(mSJ!b)uUom6nH0#2^(L@JBlV#t zLl?j54s`Y3vE^c_3^Hl0TGu*tw_n?@HyO@ZrENxA+^!)OvUX28gDSF*xFtQzM$A+O zCG=n#6~r|3zt=8%GuG} z<#VCZ%2?3Q(Ad#Y7GMJ~{U3>E{5e@z6+rgZLX{Cxk^p-7dip^d29;2N1_mm4QkASo z-L`GWWPCq$uCo;X_BmGIpJFBlhl<8~EG{vOD1o|X$aB9KPhWO_cKiU*$HWEgtf=fn zsO%9bp~D2c@?*K9jVN@_vhR03>M_8h!_~%aN!Cnr?s-!;U3SVfmhRwk11A^8Ns`@KeE}+ zN$H}a1U6E;*j5&~Og!xHdfK5M<~xka)x-0N)K_&e7AjMz`toDzasH+^1bZlC!n()crk9kg@$(Y{wdKvbuUd04N^8}t1iOgsKF zGa%%XWx@WoVaNC1!|&{5ZbkopFre-Lu(LCE5HWZBoE#W@er9W<>R=^oYxBvypN#x3 zq#LC8&q)GFP=5^-bpHj?LW=)-g+3_)Ylps!3^YQ{9~O9&K)xgy zMkCWaApU-MI~e^cV{Je75Qr7eF%&_H)BvfyKL=gIA>;OSq(y z052BFz3E(Prg~09>|_Z@!qj}@;8yxnw+#Ej0?Rk<y}4ghbD569B{9hSFr*^ygZ zr6j7P#gtZh6tMk6?4V$*Jgz+#&ug;yOr>=qdI#9U&^am2qoh4Jy}H2%a|#Fs{E(5r z%!ijh;VuGA6)W)cJZx+;9Bp1LMUzN~x_8lQ#D3+sL{be-Jyeo@@dv7XguJ&S5vrH` z>QxOMWn7N-T!D@1(@4>ZlL^y5>m#0!HKovs12GRav4z!>p(1~xok8+_{| z#Ae4{9#NLh#Vj2&JuIn5$d6t@__`o}umFo(n0QxUtd2GKCyE+erwXY?`cm*h&^9*8 zJ+8x6fRZI-e$CRygofIQN^dWysCxgkyr{(_oBwwSRxZora1(%(aC!5BTtj^+YuevI zx?)H#(xlALUp6QJ!=l9N__$cxBZ5p&7;qD3PsXRFVd<({Kh+mShFWJNpy`N@ab7?9 zv5=klvCJ4bx|-pvOO2-+G)6O?$&)ncA#Urze2rlBfp#htudhx-NeRnJ@u%^_bfw4o z4|{b8SkPV3b>Wera1W(+N@p9H>dc6{cnkh-sgr?e%(YkWvK+0YXVwk0=d`)}*47*B z5JGkEdVix!w7-<%r0JF~`ZMMPe;f0EQHuYHxya`puazyph*ZSb1mJAt^k4549BfS; zK7~T&lRb=W{s&t`DJ$B}s-eH1&&-wEOH1KWsKn0a(ZI+G!v&W4A*cl>qAvUv6pbUR z#(f#EKV8~hk&8oayBz4vaswc(?qw1vn`yC zZQDl2PCB-&Uu@g9ZQHhO+v(W0bNig{-k0;;`+wM@#@J)8r?qOYs#&vUna8ILxN7S{ zp1s41KnR8miQJtJtOr|+qk}wrLt+N*z#5o`TmD1)E&QD(Vh&pjZJ_J*0!8dy_ z>^=@v=J)C`x&gjqAYu`}t^S=DFCtc0MkBU2zf|69?xW`Ck~(6zLD)gSE{7n~6w8j_ zoH&~$ED2k5-yRa0!r8fMRy z;QjBYUaUnpd}mf%iVFPR%Dg9!d>g`01m~>2s))`W|5!kc+_&Y>wD@@C9%>-lE`WB0 zOIf%FVD^cj#2hCkFgi-fgzIfOi+ya)MZK@IZhHT5FVEaSbv-oDDs0W)pA0&^nM0TW zmgJmd7b1R7b0a`UwWJYZXp4AJPteYLH>@M|xZFKwm!t3D3&q~av?i)WvAKHE{RqpD{{%OhYkK?47}+}` zrR2(Iv9bhVa;cDzJ%6ntcSbx7v7J@Y4x&+eWSKZ*eR7_=CVIUSB$^lfYe@g+p|LD{ zPSpQmxx@b$%d!05|H}WzBT4_cq?@~dvy<7s&QWtieJ9)hd4)$SZz}#H2UTi$CkFWW|I)v_-NjuH!VypONC=1`A=rm_jfzQ8Fu~1r8i{q-+S_j$ z#u^t&Xnfi5tZtl@^!fUJhx@~Cg0*vXMK}D{>|$#T*+mj(J_@c{jXBF|rm4-8%Z2o! z2z0o(4%8KljCm^>6HDK!{jI7p+RAPcty_~GZ~R_+=+UzZ0qzOwD=;YeZt*?3%UGdr z`c|BPE;yUbnyARUl&XWSNJ<+uRt%!xPF&K;(l$^JcA_CMH6)FZt{>6ah$|(9$2fc~ z=CD00uHM{qv;{Zk9FR0~u|3|Eiqv9?z2#^GqylT5>6JNZwKqKBzzQpKU2_pmtD;CT zi%Ktau!Y2Tldfu&b0UgmF(SSBID)15*r08eoUe#bT_K-G4VecJL2Pa=6D1K6({zj6 za(2Z{r!FY5W^y{qZ}08+h9f>EKd&PN90f}Sc0ejf%kB4+f#T8Q1=Pj=~#pi$U zp#5rMR%W25>k?<$;$x72pkLibu1N|jX4cWjD3q^Pk3js!uK6h7!dlvw24crL|MZs_ zb%Y%?Fyp0bY0HkG^XyS76Ts*|Giw{31LR~+WU5NejqfPr73Rp!xQ1mLgq@mdWncLy z%8}|nzS4P&`^;zAR-&nm5f;D-%yNQPwq4N7&yULM8bkttkD)hVU>h>t47`{8?n2&4 zjEfL}UEagLUYwdx0sB2QXGeRmL?sZ%J!XM`$@ODc2!y|2#7hys=b$LrGbvvjx`Iqi z&RDDm3YBrlKhl`O@%%&rhLWZ*ABFz2nHu7k~3@e4)kO3%$=?GEFUcCF=6-1n!x^vmu+Ai*amgXH+Rknl6U>#9w;A} zn2xanZSDu`4%%x}+~FG{Wbi1jo@wqBc5(5Xl~d0KW(^Iu(U3>WB@-(&vn_PJt9{1`e9Iic@+{VPc`vP776L*viP{wYB2Iff8hB%E3|o zGMOu)tJX!`qJ}ZPzq7>=`*9TmETN7xwU;^AmFZ-ckZjV5B2T09pYliaqGFY|X#E-8 z20b>y?(r-Fn5*WZ-GsK}4WM>@TTqsxvSYWL6>18q8Q`~JO1{vLND2wg@58OaU!EvT z1|o+f1mVXz2EKAbL!Q=QWQKDZpV|jznuJ}@-)1&cdo z^&~b4Mx{*1gurlH;Vhk5g_cM&6LOHS2 zRkLfO#HabR1JD4Vc2t828dCUG#DL}f5QDSBg?o)IYYi@_xVwR2w_ntlpAW0NWk$F1 z$If?*lP&Ka1oWfl!)1c3fl`g*lMW3JOn#)R1+tfwrs`aiFUgz3;XIJ>{QFxLCkK30 zNS-)#DON3yb!7LBHQJ$)4y%TN82DC2-9tOIqzhZ27@WY^<6}vXCWcR5iN{LN8{0u9 zNXayqD=G|e?O^*ms*4P?G%o@J1tN9_76e}E#66mr89%W_&w4n66~R;X_vWD(oArwj z4CpY`)_mH2FvDuxgT+akffhX0b_slJJ*?Jn3O3~moqu2Fs1oL*>7m=oVek2bnprnW zixkaIFU%+3XhNA@@9hyhFwqsH2bM|`P?G>i<-gy>NflhrN{$9?LZ1ynSE_Mj0rADF zhOz4FnK}wpLmQuV zgO4_Oz9GBu_NN>cPLA=`SP^$gxAnj;WjJnBi%Q1zg`*^cG;Q)#3Gv@c^j6L{arv>- zAW%8WrSAVY1sj$=umcAf#ZgC8UGZGoamK}hR7j6}i8#np8ruUlvgQ$j+AQglFsQQq zOjyHf22pxh9+h#n$21&$h?2uq0>C9P?P=Juw0|;oE~c$H{#RGfa>| zj)Iv&uOnaf@foiBJ}_;zyPHcZt1U~nOcNB{)og8Btv+;f@PIT*xz$x!G?u0Di$lo7 zOugtQ$Wx|C($fyJTZE1JvR~i7LP{ zbdIwqYghQAJi9p}V&$=*2Azev$6K@pyblphgpv8^9bN!?V}{BkC!o#bl&AP!3DAjM zmWFsvn2fKWCfjcAQmE+=c3Y7j@#7|{;;0f~PIodmq*;W9Fiak|gil6$w3%b_Pr6K_ zJEG@&!J%DgBZJDCMn^7mk`JV0&l07Bt`1ymM|;a)MOWz*bh2#d{i?SDe9IcHs7 zjCrnyQ*Y5GzIt}>`bD91o#~5H?4_nckAgotN{2%!?wsSl|LVmJht$uhGa+HiH>;av z8c?mcMYM7;mvWr6noUR{)gE!=i7cZUY7e;HXa221KkRoc2UB>s$Y(k%NzTSEr>W(u z<(4mcc)4rB_&bPzX*1?*ra%VF}P1nwiP5cykJ&W{!OTlz&Td0pOkVp+wc z@k=-Hg=()hNg=Q!Ub%`BONH{ z_=ZFgetj@)NvppAK2>8r!KAgi>#%*7;O-o9MOOfQjV-n@BX6;Xw;I`%HBkk20v`qoVd0)}L6_49y1IhR z_OS}+eto}OPVRn*?UHC{eGyFU7JkPz!+gX4P>?h3QOwGS63fv4D1*no^6PveUeE5% zlehjv_3_^j^C({a2&RSoVlOn71D8WwMu9@Nb@=E_>1R*ve3`#TF(NA0?d9IR_tm=P zOP-x;gS*vtyE1Cm zG0L?2nRUFj#aLr-R1fX*$sXhad)~xdA*=hF3zPZhha<2O$Ps+F07w*3#MTe?)T8|A!P!v+a|ot{|^$q(TX`35O{WI0RbU zCj?hgOv=Z)xV?F`@HKI11IKtT^ocP78cqHU!YS@cHI@{fPD?YXL)?sD~9thOAv4JM|K8OlQhPXgnevF=F7GKD2#sZW*d za}ma31wLm81IZxX(W#A9mBvLZr|PoLnP>S4BhpK8{YV_}C|p<)4#yO{#ISbco92^3 zv&kCE(q9Wi;9%7>>PQ!zSkM%qqqLZW7O`VXvcj;WcJ`2~v?ZTYB@$Q&^CTfvy?1r^ z;Cdi+PTtmQwHX_7Kz?r#1>D zS5lWU(Mw_$B&`ZPmqxpIvK<~fbXq?x20k1~9az-Q!uR78mCgRj*eQ>zh3c$W}>^+w^dIr-u{@s30J=)1zF8?Wn|H`GS<=>Om|DjzC{}Jt?{!fSJe*@$H zg>wFnlT)k#T?LslW zu$^7Uy~$SQ21cE?3Ijl+bLfuH^U5P^$@~*UY#|_`uvAIe(+wD2eF}z_y!pvomuVO; zS^9fbdv)pcm-B@CW|Upm<7s|0+$@@<&*>$a{aW+oJ%f+VMO<#wa)7n|JL5egEgoBv zl$BY(NQjE0#*nv=!kMnp&{2Le#30b)Ql2e!VkPLK*+{jv77H7)xG7&=aPHL7LK9ER z5lfHxBI5O{-3S?GU4X6$yVk>lFn;ApnwZybdC-GAvaznGW-lScIls-P?Km2mF>%B2 zkcrXTk+__hj-3f48U%|jX9*|Ps41U_cd>2QW81Lz9}%`mTDIhE)jYI$q$ma7Y-`>% z8=u+Oftgcj%~TU}3nP8&h7k+}$D-CCgS~wtWvM|UU77r^pUw3YCV80Ou*+bH0!mf0 zxzUq4ed6y>oYFz7+l18PGGzhB^pqSt)si=9M>~0(Bx9*5r~W7sa#w+_1TSj3Jn9mW zMuG9BxN=}4645Cpa#SVKjFst;9UUY@O<|wpnZk$kE+to^4!?0@?Cwr3(>!NjYbu?x z1!U-?0_O?k!NdM^-rIQ8p)%?M+2xkhltt*|l=%z2WFJhme7*2xD~@zk#`dQR$6Lmd zb3LOD4fdt$Cq>?1<%&Y^wTWX=eHQ49Xl_lFUA(YQYHGHhd}@!VpYHHm=(1-O=yfK#kKe|2Xc*9}?BDFN zD7FJM-AjVi)T~OG)hpSWqH>vlb41V#^G2B_EvYlWhDB{Z;Q9-0)ja(O+By`31=biA zG&Fs#5!%_mHi|E4Nm$;vVQ!*>=_F;ZC=1DTPB#CICS5fL2T3XmzyHu?bI;m7D4@#; ztr~;dGYwb?m^VebuULtS4lkC_7>KCS)F@)0OdxZIFZp@FM_pHnJes8YOvwB|++#G( z&dm*OP^cz95Wi15vh`Q+yB>R{8zqEhz5of>Po$9LNE{xS<)lg2*roP*sQ}3r3t<}; zPbDl{lk{pox~2(XY5=qg0z!W-x^PJ`VVtz$git7?)!h>`91&&hESZy1KCJ2nS^yMH z!=Q$eTyRi68rKxdDsdt+%J_&lapa{ds^HV9Ngp^YDvtq&-Xp}60B_w@Ma>_1TTC;^ zpbe!#gH}#fFLkNo#|`jcn?5LeUYto%==XBk6Ik0kc4$6Z+L3x^4=M6OI1=z5u#M%0 z0E`kevJEpJjvvN>+g`?gtnbo$@p4VumliZV3Z%CfXXB&wPS^5C+7of2tyVkMwNWBiTE2 z8CdPu3i{*vR-I(NY5syRR}I1TJOV@DJy-Xmvxn^IInF>Tx2e)eE9jVSz69$6T`M9-&om!T+I znia!ZWJRB28o_srWlAxtz4VVft8)cYloIoVF=pL zugnk@vFLXQ_^7;%hn9x;Vq?lzg7%CQR^c#S)Oc-8d=q_!2ZVH764V z!wDKSgP}BrVV6SfCLZnYe-7f;igDs9t+K*rbMAKsp9L$Kh<6Z;e7;xxced zn=FGY<}CUz31a2G}$Q(`_r~75PzM4l_({Hg&b@d8&jC}B?2<+ed`f#qMEWi z`gm!STV9E4sLaQX+sp5Nu9*;9g12naf5?=P9p@H@f}dxYprH+3ju)uDFt^V{G0APn zS;16Dk{*fm6&BCg#2vo?7cbkkI4R`S9SSEJ=#KBk3rl69SxnCnS#{*$!^T9UUmO#&XXKjHKBqLdt^3yVvu8yn|{ zZ#%1CP)8t-PAz(+_g?xyq;C2<9<5Yy<~C74Iw(y>uUL$+$mp(DRcCWbCKiGCZw@?_ zdomfp+C5xt;j5L@VfhF*xvZdXwA5pcdsG>G<8II-|1dhAgzS&KArcb0BD4ZZ#WfiEY{hkCq5%z9@f|!EwTm;UEjKJsUo696V>h zy##eXYX}GUu%t{Gql8vVZKkNhQeQ4C%n|RmxL4ee5$cgwlU+?V7a?(jI#&3wid+Kz5+x^G!bb#$q>QpR#BZ}Xo5UW^ zD&I`;?(a}Oys7-`I^|AkN?{XLZNa{@27Dv^s4pGowuyhHuXc zuctKG2x0{WCvg_sGN^n9myJ}&FXyGmUQnW7fR$=bj$AHR88-q$D!*8MNB{YvTTEyS zn22f@WMdvg5~o_2wkjItJN@?mDZ9UUlat2zCh(zVE=dGi$rjXF7&}*sxac^%HFD`Y zTM5D3u5x**{bW!68DL1A!s&$2XG@ytB~dX-?BF9U@XZABO`a|LM1X3HWCllgl0+uL z04S*PX$%|^WAq%jkzp~%9HyYIF{Ym?k)j3nMwPZ=hlCg9!G+t>tf0o|J2%t1 ztC+`((dUplgm3`+0JN~}&FRRJ3?l*>Y&TfjS>!ShS`*MwO{WIbAZR#<%M|4c4^dY8 z{Rh;-!qhY=dz5JthbWoovLY~jNaw>%tS4gHVlt5epV8ekXm#==Po$)}mh^u*cE>q7*kvX&gq)(AHoItMYH6^s6f(deNw%}1=7O~bTHSj1rm2|Cq+3M z93djjdomWCTCYu!3Slx2bZVy#CWDozNedIHbqa|otsUl+ut?>a;}OqPfQA05Yim_2 zs@^BjPoFHOYNc6VbNaR5QZfSMh2S*`BGwcHMM(1@w{-4jVqE8Eu0Bi%d!E*^Rj?cR z7qgxkINXZR)K^=fh{pc0DCKtrydVbVILI>@Y0!Jm>x-xM!gu%dehm?cC6ok_msDVA*J#{75%4IZt}X|tIVPReZS#aCvuHkZxc zHVMtUhT(wp09+w9j9eRqz~LtuSNi2rQx_QgQ(}jBt7NqyT&ma61ldD(s9x%@q~PQl zp6N*?=N$BtvjQ_xIT{+vhb1>{pM0Arde0!X-y))A4znDrVx8yrP3B1(7bKPE5jR@5 zwpzwT4cu~_qUG#zYMZ_!2Tkl9zP>M%cy>9Y(@&VoB84#%>amTAH{(hL4cDYt!^{8L z645F>BWO6QaFJ-{C-i|-d%j7#&7)$X7pv#%9J6da#9FB5KyDhkA+~)G0^87!^}AP>XaCSScr;kL;Z%RSPD2CgoJ;gpYT5&6NUK$86$T?jRH=w8nI9Z534O?5fk{kd z`(-t$8W|#$3>xoMfXvV^-A(Q~$8SKDE^!T;J+rQXP71XZ(kCCbP%bAQ1|%$%Ov9_a zyC`QP3uPvFoBqr_+$HenHklqyIr>PU_Fk5$2C+0eYy^~7U&(!B&&P2%7#mBUhM!z> z_B$Ko?{Pf6?)gpYs~N*y%-3!1>o-4;@1Zz9VQHh)j5U1aL-Hyu@1d?X;jtDBNk*vMXPn@ z+u@wxHN*{uHR!*g*4Xo&w;5A+=Pf9w#PeZ^x@UD?iQ&${K2c}UQgLRik-rKM#Y5rdDphdcNTF~cCX&9ViRP}`>L)QA4zNXeG)KXFzSDa6 zd^St;inY6J_i=5mcGTx4_^Ys`M3l%Q==f>{8S1LEHn{y(kbxn5g1ezt4CELqy)~TV6{;VW>O9?5^ ztcoxHRa0jQY7>wwHWcxA-BCwzsP>63Kt&3fy*n#Cha687CQurXaRQnf5wc9o8v7Rw zNwGr2fac;Wr-Ldehn7tF^(-gPJwPt@VR1f;AmKgxN&YPL;j=0^xKM{!wuU|^mh3NE zy35quf}MeL!PU;|{OW_x$TBothLylT-J>_x6p}B_jW1L>k)ps6n%7Rh z96mPkJIM0QFNYUM2H}YF5bs%@Chs6#pEnloQhEl?J-)es!(SoJpEPoMTdgA14-#mC zghayD-DJWtUu`TD8?4mR)w5E`^EHbsz2EjH5aQLYRcF{l7_Q5?CEEvzDo(zjh|BKg z3aJl_n#j&eFHsUw4~lxqnr!6NL*se)6H=A+T1e3xUJGQrd}oSPwSy5+$tt{2t5J5@(lFxl43amsARG74iyNC}uuS zd2$=(r6RdamdGx^eatX@F2D8?U23tDpR+Os?0Gq2&^dF+$9wiWf?=mDWfjo4LfRwL zI#SRV9iSz>XCSgEj!cW&9H-njJopYiYuq|2w<5R2!nZ27DyvU4UDrHpoNQZiGPkp@ z1$h4H46Zn~eqdj$pWrv;*t!rTYTfZ1_bdkZmVVIRC21YeU$iS-*XMNK`#p8Z_DJx| zk3Jssf^XP7v0X?MWFO{rACltn$^~q(M9rMYoVxG$15N;nP)A98k^m3CJx8>6}NrUd@wp-E#$Q0uUDQT5GoiK_R{ z<{`g;8s>UFLpbga#DAf%qbfi`WN1J@6IA~R!YBT}qp%V-j!ybkR{uY0X|x)gmzE0J z&)=eHPjBxJvrZSOmt|)hC+kIMI;qgOnuL3mbNR0g^<%|>9x7>{}>a2qYSZAGPt4it?8 zNcLc!Gy0>$jaU?}ZWxK78hbhzE+etM`67*-*x4DN>1_&{@5t7_c*n(qz>&K{Y?10s zXsw2&nQev#SUSd|D8w7ZD2>E<%g^; zV{yE_O}gq?Q|zL|jdqB^zcx7vo(^})QW?QKacx$yR zhG|XH|8$vDZNIfuxr-sYFR{^csEI*IM#_gd;9*C+SysUFejP0{{z7@P?1+&_o6=7V|EJLQun^XEMS)w(=@eMi5&bbH*a0f;iC~2J74V2DZIlLUHD&>mlug5+v z6xBN~8-ovZylyH&gG#ptYsNlT?-tzOh%V#Y33zlsJ{AIju`CjIgf$@gr8}JugRq^c zAVQ3;&uGaVlVw}SUSWnTkH_6DISN&k2QLMBe9YU=sA+WiX@z)FoSYX`^k@B!j;ZeC zf&**P?HQG6Rk98hZ*ozn6iS-dG}V>jQhb3?4NJB*2F?6N7Nd;EOOo;xR7acylLaLy z9)^lykX39d@8@I~iEVar4jmjjLWhR0d=EB@%I;FZM$rykBNN~jf>#WbH4U{MqhhF6 zU??@fSO~4EbU4MaeQ_UXQcFyO*Rae|VAPLYMJEU`Q_Q_%s2*>$#S^)&7er+&`9L=1 z4q4ao07Z2Vsa%(nP!kJ590YmvrWg+YrgXYs_lv&B5EcoD`%uL79WyYA$0>>qi6ov7 z%`ia~J^_l{p39EY zv>>b}Qs8vxsu&WcXEt8B#FD%L%ZpcVtY!rqVTHe;$p9rbb5O{^rFMB>auLn-^;s+-&P1#h~mf~YLg$8M9 zZ4#87;e-Y6x6QO<{McUzhy(%*6| z)`D~A(TJ$>+0H+mct(jfgL4x%^oC^T#u(bL)`E2tBI#V1kSikAWmOOYrO~#-cc_8! zCe|@1&mN2{*ceeiBldHCdrURk4>V}79_*TVP3aCyV*5n@jiNbOm+~EQ_}1#->_tI@ zqXv+jj2#8xJtW508rzFrYcJxoek@iW6SR@1%a%Bux&;>25%`j3UI`0DaUr7l79`B1 zqqUARhW1^h6=)6?;@v>xrZNM;t}{yY3P@|L}ey@gG( z9r{}WoYN(9TW&dE2dEJIXkyHA4&pU6ki=rx&l2{DLGbVmg4%3Dlfvn!GB>EVaY_%3+Df{fBiqJV>~Xf8A0aqUjgpa} zoF8YXO&^_x*Ej}nw-$-F@(ddB>%RWoPUj?p8U{t0=n>gAI83y<9Ce@Q#3&(soJ{64 z37@Vij1}5fmzAuIUnXX`EYe;!H-yTVTmhAy;y8VZeB#vD{vw9~P#DiFiKQ|kWwGFZ z=jK;JX*A;Jr{#x?n8XUOLS;C%f|zj-7vXtlf_DtP7bpurBeX%Hjwr z4lI-2TdFpzkjgiv!8Vfv`=SP+s=^i3+N~1ELNWUbH|ytVu>EyPN_3(4TM^QE1swRo zoV7Y_g)a>28+hZG0e7g%@2^s>pzR4^fzR-El}ARTmtu!zjZLuX%>#OoU3}|rFjJg} zQ2TmaygxJ#sbHVyiA5KE+yH0LREWr%^C*yR|@gM$nK2P zo}M}PV0v))uJh&33N>#aU376@ZH79u(Yw`EQ2hM3SJs9f99+cO6_pNW$j$L-CtAfe zYfM)ccwD!P%LiBk!eCD?fHCGvgMQ%Q2oT_gmf?OY=A>&PaZQOq4eT=lwbaf}33LCH zFD|)lu{K7$8n9gX#w4~URjZxWm@wlH%oL#G|I~Fb-v^0L0TWu+`B+ZG!yII)w05DU z>GO?n(TN+B=>HdxVDSlIH76pta$_LhbBg;eZ`M7OGcqt||qi zogS72W1IN%=)5JCyOHWoFP7pOFK0L*OAh=i%&VW&4^LF@R;+K)t^S!96?}^+5QBIs zjJNTCh)?)4k^H^g1&jc>gysM`y^8Rm3qsvkr$9AeWwYpa$b22=yAd1t<*{ zaowSEFP+{y?Ob}8&cwfqoy4Pb9IA~VnM3u!trIK$&&0Op#Ql4j>(EW?UNUv#*iH1$ z^j>+W{afcd`{e&`-A{g}{JnIzYib)!T56IT@YEs{4|`sMpW3c8@UCoIJv`XsAw!XC z34|Il$LpW}CIHFC5e*)}00I5{%OL*WZRGzC0?_}-9{#ue?-ug^ zLE|uv-~6xnSs_2_&CN9{9vyc!Xgtn36_g^wI0C4s0s^;8+p?|mm;Odt3`2ZjwtK;l zfd6j)*Fr#53>C6Y8(N5?$H0ma;BCF3HCjUs7rpb2Kf*x3Xcj#O8mvs#&33i+McX zQpBxD8!O{5Y8D&0*QjD=Yhl9%M0)&_vk}bmN_Ud^BPN;H=U^bn&(csl-pkA+GyY0Z zKV7sU_4n;}uR78ouo8O%g*V;79KY?3d>k6%gpcmQsKk&@Vkw9yna_3asGt`0Hmj59 z%0yiF*`jXhByBI9QsD=+>big5{)BGe&+U2gAARGe3ID)xrid~QN_{I>k}@tzL!Md_ z&=7>TWciblF@EMC3t4-WX{?!m!G6$M$1S?NzF*2KHMP3Go4=#ZHkeIv{eEd;s-yD# z_jU^Ba06TZqvV|Yd;Z_sN%$X=!T+&?#p+OQIHS%!LO`Hx0q_Y0MyGYFNoM{W;&@0@ zLM^!X4KhdtsET5G<0+|q0oqVXMW~-7LW9Bg}=E$YtNh1#1D^6Mz(V9?2g~I1( zoz9Cz=8Hw98zVLwC2AQvp@pBeKyidn6Xu0-1SY1((^Hu*-!HxFUPs)yJ+i`^BC>PC zjwd0mygOVK#d2pRC9LxqGc6;Ui>f{YW9Bvb>33bp^NcnZoH~w9(lM5@JiIlfa-6|k ziy31UoMN%fvQfhi8^T+=yrP{QEyb-jK~>$A4SZT-N56NYEbpvO&yUme&pWKs3^94D zH{oXnUTb3T@H+RgzML*lejx`WAyw*?K7B-I(VJx($2!NXYm%3`=F~TbLv3H<{>D?A zJo-FDYdSA-(Y%;4KUP2SpHKAIcv9-ld(UEJE7=TKp|Gryn;72?0LHqAN^fk6%8PCW z{g_-t)G5uCIf0I`*F0ZNl)Z>))MaLMpXgqWgj-y;R+@A+AzDjsTqw2Mo9ULKA3c70 z!7SOkMtZb+MStH>9MnvNV0G;pwSW9HgP+`tg}e{ij0H6Zt5zJ7iw`hEnvye!XbA@!~#%vIkzowCOvq5I5@$3wtc*w2R$7!$*?}vg4;eDyJ_1=ixJuEp3pUS27W?qq(P^8$_lU!mRChT}ctvZz4p!X^ zOSp|JOAi~f?UkwH#9k{0smZ7-#=lK6X3OFEMl7%)WIcHb=#ZN$L=aD`#DZKOG4p4r zwlQ~XDZ`R-RbF&hZZhu3(67kggsM-F4Y_tI^PH8PMJRcs7NS9ogF+?bZB*fcpJ z=LTM4W=N9yepVvTj&Hu~0?*vR1HgtEvf8w%Q;U0^`2@e8{SwgX5d(cQ|1(!|i$km! zvY03MK}j`sff;*-%mN~ST>xU$6Bu?*Hm%l@0dk;j@%>}jsgDcQ)Hn*UfuThz9(ww_ zasV`rSrp_^bp-0sx>i35FzJwA!d6cZ5#5#nr@GcPEjNnFHIrtUYm1^Z$;{d&{hQV9 z6EfFHaIS}46p^5I-D_EcwwzUUuO}mqRh&T7r9sfw`)G^Q%oHxEs~+XoM?8e*{-&!7 z7$m$lg9t9KP9282eke608^Q2E%H-xm|oJ8=*SyEo} z@&;TQ3K)jgspgKHyGiKVMCz>xmC=H5Fy3!=TP)-R3|&1S-B)!6q50wfLHKM@7Bq6E z44CY%G;GY>tC`~yh!qv~YdXw! zSkquvYNs6k1r7>Eza?Vkkxo6XRS$W7EzL&A`o>=$HXgBp{L(i^$}t`NcnAxzbH8Ht z2!;`bhKIh`f1hIFcI5bHI=ueKdzmB9)!z$s-BT4ItyY|NaA_+o=jO%MU5as9 zc2)aLP>N%u>wlaXTK!p)r?+~)L+0eCGb5{8WIk7K52$nufnQ+m8YF+GQc&{^(zh-$ z#wyWV*Zh@d!b(WwXqvfhQX)^aoHTBkc;4ossV3&Ut*k>AI|m+{#kh4B!`3*<)EJVj zwrxK>99v^k4&Y&`Awm>|exo}NvewV%E+@vOc>5>%H#BK9uaE2$vje zWYM5fKuOTtn96B_2~~!xJPIcXF>E_;yO8AwpJ4)V`Hht#wbO3Ung~@c%%=FX4)q+9 z99#>VC2!4l`~0WHs9FI$Nz+abUq# zz`Of97})Su=^rGp2S$)7N3rQCj#0%2YO<R&p>$<#lgXcUj=4H_{oAYiT3 z44*xDn-$wEzRw7#@6aD)EGO$0{!C5Z^7#yl1o;k0PhN=aVUQu~eTQ^Xy{z8Ow6tk83 z4{5xe%(hx)%nD&|e*6sTWH`4W&U!Jae#U4TnICheJmsw{l|CH?UA{a6?2GNgpZLyzU2UlFu1ZVwlALmh_DOs03J^Cjh1im`E3?9&zvNmg(MuMw&0^Lu$(#CJ*q6DjlKsY-RMJ^8yIY|{SQZ*9~CH|u9L z`R78^r=EbbR*_>5?-)I+$6i}G)%mN(`!X72KaV(MNUP7Nv3MS9S|Pe!%N2AeOt5zG zVJ;jI4HZ$W->Ai_4X+`9c(~m=@ek*m`ZQbv3ryI-AD#AH=`x$~WeW~M{Js57(K7(v ze5`};LG|%C_tmd>bkufMWmAo&B+DT9ZV~h(4jg0>^aeAqL`PEUzJJtI8W1M!bQWpv zvN(d}E1@nlYa!L!!A*RN!(Q3F%J?5PvQ0udu?q-T)j3JKV~NL>KRb~w-lWc685uS6 z=S#aR&B8Sc8>cGJ!!--?kwsJTUUm`Jk?7`H z7PrO~xgBrSW2_tTlCq1LH8*!o?pj?qxy8}(=r_;G18POrFh#;buWR0qU24+XUaVZ0 z?(sXcr@-YqvkCmHr{U2oPogHL{r#3r49TeR<{SJX1pcUqyWPrkYz^X8#QW~?F)R5i z>p^!i<;qM8Nf{-fd6!_&V*e_9qP6q(s<--&1Ttj01j0w>bXY7y1W*%Auu&p|XSOH=)V7Bd4fUKh&T1)@cvqhuD-d=?w}O zjI%i(f|thk0Go*!d7D%0^ztBfE*V=(ZIN84f5HU}T9?ulmEYzT5usi=DeuI*d|;M~ zp_=Cx^!4k#=m_qSPBr5EK~E?3J{dWWPH&oCcNepYVqL?nh4D5ynfWip$m*YlZ8r^Z zuFEUL-nW!3qjRCLIWPT0x)FDL7>Yt7@8dA?R2kF@WE>ysMY+)lTsgNM#3VbXVGL}F z1O(>q>2a+_`6r5Xv$NZAnp=Kgnr3)cL(^=8ypEeOf3q8(HGe@7Tt59;yFl||w|mnO zHDxg2G3z8=(6wjj9kbcEY@Z0iOd7Gq5GiPS5% z*sF1J<#daxDV2Z8H>wxOF<;yKzMeTaSOp_|XkS9Sfn6Mpe9UBi1cSTieGG5$O;ZLIIJ60Y>SN4vC?=yE_CWlo(EEE$e4j?z&^FM%kNmRtlbEL^dPPgvs9sbK5fGw*r@ z+!EU@u$T8!nZh?Fdf_qk$VuHk^yVw`h`_#KoS*N%epIIOfQUy_&V}VWDGp3tplMbf z5Se1sJUC$7N0F1-9jdV2mmGK{-}fu|Nv;12jDy0<-kf^AmkDnu6j~TPWOgy1MT68|D z=4=50jVbUKdKaQgD`eWGr3I&^<6uhkjz$YwItY8%Yp9{z4-{6g{73<_b*@XJ4Nm3-3z z?BW3{aY_ccRjb@W1)i5nLg|7BnWS!B`_Uo9CWaE`Ij327QH?i)9A}4Ug4wmxVVa^b z-4+m%-wwOl7cKH7+=x&nrCrbEC)Q$fpg&V83#uEH;C=GNMz`ps@^RxK%T*8%OPnC` z{WO~J%nxYJ`x|N%?&i7?;{_8t^jM&=50HlaOQj8fS}_`moH$c;vI<|cruPFnpT8yU zS%rPOCUSd5Zdb(zwk`hqwTQn)*&n)uYsP*F_(~xEWq}C= zv30kFmZFwJZ@ELVX3?$dXQh|icO7UrL*_5G=I^xXjImz`ZPp>?g#tf(ej~KaIU0algsG!IS09;>?MvqGg#c{i+}qY|{P8W~O%#>|gFd z<1dr$-oxyRGN17yZo1OwLnzwYs0|;IS_nymNB0IlSzPQ%-r`?T=;_XQ^~&#}b|AB} zkNbN5uB?-sUB-T5QLlg%Uk3)uHB;>VIzGe9_J9 zaeISkQm!v(9d(0ML^b9fR^sfHFlH?7Mvddt37OuR{|O0{uv)(&-6<87W4 zyO>s!=cPgP3O&7xxU5DlIPw_o3O>6o6Qb?JWs3qw#p3sBc3g$?Dx zi(6D+DYgV;GrUis-CL%Qe{nvZnwaVXmbhH(|GFh|Q)k=1uvA$I@1DXI7bKlQ@8D6P zS?(*?><>)G49q0wr;NajpxP4W2G)kHl6^=Z>hrNEI4Mwd_$O6$1dXF;Q#hE(-eeW6 zz03GJF%Wl?HO=_ztv5*zRlcU~{+{k%#N59mgm~eK>P!QZ6E?#Cu^2)+K8m@ySvZ*5 z|HDT}BkF@3!l(0%75G=1u2hETXEj!^1Z$!)!lyGXlWD!_vqGE$Z)#cUVBqlORW>0^ zDjyVTxwKHKG|0}j-`;!R-p>}qQfBl(?($7pP<+Y8QE#M8SCDq~k<+>Q^Zf@cT_WdX3~BSe z+|KK|7OL5Hm5(NFP~j>Ct3*$wi0n0!xl=(C61`q&cec@mFlH(sy%+RH<=s)8aAPN`SfJdkAQjdv82G5iRdv8 zh{9wHUZaniSEpslXl^_ODh}mypC?b*9FzLjb~H@3DFSe;D(A-K3t3eOTB(m~I6C;(-lKAvit(70k`%@+O*Ztdz;}|_TS~B?Tpmi=QKC^m_ z2YpEaT3iiz*;T~ap1yiA)a`dKMwu`^UhIUeltNQ1Yjo=q@bI@&3zH?rVUg=IxLy-ni zyxDu%-Fr{H6owTjZU2O5>nDb=q&Jz_TjeSq%!2m40x&U6w~GQ({quPL73IsJS;f`$ zsuhioqCBj(gJ>2hoo)Gou7(WP*pX)f=Y=!=k!&1K?EYY%jJ~X&DnK{^saPQK<1BJ z_A`_{%ZozcB(3w$z^To^6d|XuT@=X~wtW!+{4ID@N{AB~J6AL5vuY>JwvWCNFKsKh zd}@>q@_WV#QZ&UJ0#?X(pXR!oyXOEG3rqzHbCzGLONDb042i$})fM@XF)uSP(DHUc z^&{|$*xe{cs?Gp8=B%RY3L7#$ve$?TWh>MZdxF1zH1v}1z+$Ov#G7?%D)bBCyDe*% zSeKSpETC2V1){II>@UwJi>4uBN+iAx+82E~gb|Cr&8E^i&)A!uv-g?jzH99wU}8+# z$nh>yvb;TwZmS@7LrvuCu_d0-WxFNI&C7%sWuTL%YU!l|I1{|->=dlOeHOCtUO#zkS3ESO8LHV4hTdQL5EdV zuWD33fFPH}HPrW^s$Qn1Xgp&AT6<-He{{4%eIu3rN=iK|9mURdKXfB&Q?qGok%!cs ze53UP{Z!TO-Y@q2;;k2avA3`lm4OoN4@S*k=UA)7H;qZ`d8`XaYFCv?Ba+uGW@r5v z&&{nf(24WSBOhc7!qF^@0cz;XcUynNaj6w2349;s!K{KVqs5yS{ z7VubS`2OzT^5#1~6Tt^RTvt9-J|D2F>y~>2;jeF>g`hx5l%B3H=aLExQihuYngzlnBTYOTHJQMzl>kwqN5JYs)Ej zblA@ntkUS~xi+}y6|(81helS}Q~&VB37qyV|S3Y=><^1wh%msQM?fz z<58MX(=|PSUKCF#)dbhR%D&xgCD?$aR0qen+wpp6 zst}vX18!Be96TD??j1HsHTUx(a&@F?=gT`Q$oJFFyrh^;zgz!(NlAHGn0cJy@us=w zNhC#l5G;H}+>49Nsh12=ZPO2r*2OBQe5kpb&1?*PIBFitK8}FUfb~S-#hKfF0o#&d z#3aPkB$9scYku&kA6{0xHnBV#&Wei5J>5T-XX-gUXEPo+9b7WL=*XESc(3BshL`aj zXp}QIp*40}oWJt*l043e8_5;H5PI5c)U&IEw5dF(4zjX0y_lk9 zAp@!mK>WUqHo)-jop=DoK>&no>kAD=^qIE7qis&_*4~ z6q^EF$D@R~3_xseCG>Ikb6Gfofb$g|75PPyyZN&tiRxqovo_k zO|HA|sgy#B<32gyU9x^&)H$1jvw@qp+1b(eGAb)O%O!&pyX@^nQd^9BQ4{(F8<}|A zhF&)xusQhtoXOOhic=8#Xtt5&slLia3c*a?dIeczyTbC#>FTfiLST57nc3@Y#v_Eg#VUv zT8cKH#f3=1PNj!Oroz_MAR*pow%Y0*6YCYmUy^7`^r|j23Q~^*TW#cU7CHf0eAD_0 zEWEVddxFgQ7=!nEBQ|ibaScslvhuUk^*%b#QUNrEB{3PG@uTxNwW}Bs4$nS9wc(~O zG7Iq>aMsYkcr!9#A;HNsJrwTDYkK8ikdj{M;N$sN6BqJ<8~z>T20{J8Z2rRUuH7~3 z=tgS`AgxbBOMg87UT4Lwge`*Y=01Dvk>)^{Iu+n6fuVX4%}>?3czOGR$0 zpp*wp>bsFFSV`V;r_m+TZns$ZprIi`OUMhe^cLE$2O+pP3nP!YB$ry}2THx2QJs3< za1;>d-AggCarrQ>&Z!d@;mW+!q6eXhb&`GbzUDSxpl8AJ#Cm#tuc)_xh(2NV=5XMs zrf_ozRYO$NkC=pKFX5OH8v1>0i9Z$ec`~Mf+_jQ68spn(CJwclDhEEkH2Qw;${J$clv__nUjn5jA0wCLEnu1j;v!0vB>Ri6m9`;R{JMS%^)4FC zU0Z44+u$I$w=Bj|iu4DT5h~sS`C*zbmX?@-crY}E+hy>}2~C0Nn(EKk@5^qO4@l@! z6O0lr%tzGC`D^)8xU3FnMZVm0kX1sBWhaQyzVoXFWwr%Ny?=2M{5s#5i7fTu3gEkG zc{(Pr$v=;`Y#&`y*J}#M9ux>0?xu!`$9cUKm#Bdd_&S#LPTS?ZPV6zN6>W6JTS~-LfjL{mB=b(KMk3 z2HjBSlJeyUVqDd=Mt!=hpYsvby2GL&3~zm;0{^nZJq+4vb?5HH4wufvr}IX42sHeK zm@x?HN$8TsTavXs)tLDFJtY9b)y~Tl@7z4^I8oUQq4JckH@~CVQ;FoK(+e0XAM>1O z(ei}h?)JQp>)d=6ng-BZF1Z5hsAKW@mXq+hU?r8I(*%`tnIIOXw7V6ZK(T9RFJJe@ zZS!aC+p)Gf2Ujc=a6hx4!A1Th%YH!Lb^xpI!Eu` zmJO{9rw){B1Ql18d%F%da+Tbu1()?o(zT7StYqK6_w`e+fjXq5L^y(0 z09QA6H4oFj59c2wR~{~>jUoDzDdKz}5#onYPJRwa`SUO)Pd4)?(ENBaFVLJr6Kvz= zhTtXqbx09C1z~~iZt;g^9_2nCZ{};-b4dQJbv8HsWHXPVg^@(*!@xycp#R?a|L!+` zY5w))JWV`Gls(=}shH0#r*;~>_+-P5Qc978+QUd>J%`fyn{*TsiG-dWMiJXNgwBaT zJ=wgYFt+1ACW)XwtNx)Q9tA2LPoB&DkL16P)ERWQlY4%Y`-5aM9mZ{eKPUgI!~J3Z zkMd5A_p&v?V-o-6TUa8BndiX?ooviev(DKw=*bBVOW|=zps9=Yl|-R5@yJe*BPzN}a0mUsLn{4LfjB_oxpv(mwq# zSY*%E{iB)sNvWfzg-B!R!|+x(Q|b@>{-~cFvdDHA{F2sFGA5QGiIWy#3?P2JIpPKg6ncI^)dvqe`_|N=8 '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH="\\\"\\\"" + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/lib/biokbase/catalog/Impl.py b/lib/biokbase/catalog/Impl.py index 2e55ed75..255e5705 100644 --- a/lib/biokbase/catalog/Impl.py +++ b/lib/biokbase/catalog/Impl.py @@ -21,9 +21,9 @@ class Catalog: # state. A method could easily clobber the state set by another while # the latter method is running. ######################################### noqa - VERSION = "0.0.1" - GIT_URL = "https://github.com/kbase/catalog" - GIT_COMMIT_HASH = "fda05a2962373163e4983dc5187b1c51cd1455b1" + VERSION = "0.1.0" + GIT_URL = "https://github.com/kbase/catalog.git" + GIT_COMMIT_HASH = "bfd28df246cd39404293ea4cf4cd4200950fc900" #BEGIN_CLASS_HEADER #END_CLASS_HEADER diff --git a/lib/biokbase/catalog/version.py b/lib/biokbase/catalog/version.py index 006d3cff..33363d57 100644 --- a/lib/biokbase/catalog/version.py +++ b/lib/biokbase/catalog/version.py @@ -1,2 +1,2 @@ # File that simply defines version information -CATALOG_VERSION = '2.3.0' +CATALOG_VERSION = '2.3.1' diff --git a/lib/java/us/kbase/catalog/CurrentRepoParams.java b/lib/java/us/kbase/catalog/CurrentRepoParams.java deleted file mode 100644 index f433d68e..00000000 --- a/lib/java/us/kbase/catalog/CurrentRepoParams.java +++ /dev/null @@ -1,82 +0,0 @@ - -package us.kbase.catalog; - -import java.util.HashMap; -import java.util.Map; -import javax.annotation.Generated; -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; - - -/** - *

Original spec-file type: CurrentRepoParams

- * 
- * Describes how to find repository details.
- * module_name - name of module defined in kbase.yaml file;
- * with_disabled - optional flag adding disabled repos (default value is false).
- *
- * - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -@Generated("com.googlecode.jsonschema2pojo") -@JsonPropertyOrder({ - "module_name", - "with_disabled" -}) -public class CurrentRepoParams { - - @JsonProperty("module_name") - private String moduleName; - @JsonProperty("with_disabled") - private Long withDisabled; - private Map additionalProperties = new HashMap(); - - @JsonProperty("module_name") - public String getModuleName() { - return moduleName; - } - - @JsonProperty("module_name") - public void setModuleName(String moduleName) { - this.moduleName = moduleName; - } - - public CurrentRepoParams withModuleName(String moduleName) { - this.moduleName = moduleName; - return this; - } - - @JsonProperty("with_disabled") - public Long getWithDisabled() { - return withDisabled; - } - - @JsonProperty("with_disabled") - public void setWithDisabled(Long withDisabled) { - this.withDisabled = withDisabled; - } - - public CurrentRepoParams withWithDisabled(Long withDisabled) { - this.withDisabled = withDisabled; - return this; - } - - @JsonAnyGetter - public Map getAdditionalProperties() { - return this.additionalProperties; - } - - @JsonAnySetter - public void setAdditionalProperties(String name, Object value) { - this.additionalProperties.put(name, value); - } - - @Override - public String toString() { - return ((((((("CurrentRepoParams"+" [moduleName=")+ moduleName)+", withDisabled=")+ withDisabled)+", additionalProperties=")+ additionalProperties)+"]"); - } - -} diff --git a/lib/java/us/kbase/catalog/GetRawStatsParams.java b/lib/java/us/kbase/catalog/GetRawStatsParams.java deleted file mode 100644 index 9fd851d1..00000000 --- a/lib/java/us/kbase/catalog/GetRawStatsParams.java +++ /dev/null @@ -1,80 +0,0 @@ - -package us.kbase.catalog; - -import java.util.HashMap; -import java.util.Map; -import javax.annotation.Generated; -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; - - -/** - *

Original spec-file type: GetRawStatsParams

- * 
- * Get raw usage metrics; available only to Admins.
- *
- * - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -@Generated("com.googlecode.jsonschema2pojo") -@JsonPropertyOrder({ - "begin", - "end" -}) -public class GetRawStatsParams { - - @JsonProperty("begin") - private Long begin; - @JsonProperty("end") - private Long end; - private Map additionalProperties = new HashMap(); - - @JsonProperty("begin") - public Long getBegin() { - return begin; - } - - @JsonProperty("begin") - public void setBegin(Long begin) { - this.begin = begin; - } - - public GetRawStatsParams withBegin(Long begin) { - this.begin = begin; - return this; - } - - @JsonProperty("end") - public Long getEnd() { - return end; - } - - @JsonProperty("end") - public void setEnd(Long end) { - this.end = end; - } - - public GetRawStatsParams withEnd(Long end) { - this.end = end; - return this; - } - - @JsonAnyGetter - public Map getAdditionalProperties() { - return this.additionalProperties; - } - - @JsonAnySetter - public void setAdditionalProperties(String name, Object value) { - this.additionalProperties.put(name, value); - } - - @Override - public String toString() { - return ((((((("GetRawStatsParams"+" [begin=")+ begin)+", end=")+ end)+", additionalProperties=")+ additionalProperties)+"]"); - } - -} diff --git a/lib/java/us/kbase/catalog/HistoryRepoParams.java b/lib/java/us/kbase/catalog/HistoryRepoParams.java deleted file mode 100644 index 8e63f6fd..00000000 --- a/lib/java/us/kbase/catalog/HistoryRepoParams.java +++ /dev/null @@ -1,121 +0,0 @@ - -package us.kbase.catalog; - -import java.util.HashMap; -import java.util.Map; -import javax.annotation.Generated; -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; - - -/** - *

Original spec-file type: HistoryRepoParams

- * 
- * Describes how to find repository details (including old versions). In case neither of
- *     version and git_commit_hash is specified last version is returned.
- * module_name - name of module defined in kbase.yaml file;
- * timestamp - optional parameter limiting search by certain version timestamp;
- * git_commit_hash - optional parameter limiting search by certain git commit hash;
- * with_disabled - optional flag adding disabled repos (default value is false).
- *
- * - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -@Generated("com.googlecode.jsonschema2pojo") -@JsonPropertyOrder({ - "module_name", - "timestamp", - "git_commit_hash", - "include_disabled" -}) -public class HistoryRepoParams { - - @JsonProperty("module_name") - private String moduleName; - @JsonProperty("timestamp") - private Long timestamp; - @JsonProperty("git_commit_hash") - private String gitCommitHash; - @JsonProperty("include_disabled") - private Long includeDisabled; - private Map additionalProperties = new HashMap(); - - @JsonProperty("module_name") - public String getModuleName() { - return moduleName; - } - - @JsonProperty("module_name") - public void setModuleName(String moduleName) { - this.moduleName = moduleName; - } - - public HistoryRepoParams withModuleName(String moduleName) { - this.moduleName = moduleName; - return this; - } - - @JsonProperty("timestamp") - public Long getTimestamp() { - return timestamp; - } - - @JsonProperty("timestamp") - public void setTimestamp(Long timestamp) { - this.timestamp = timestamp; - } - - public HistoryRepoParams withTimestamp(Long timestamp) { - this.timestamp = timestamp; - return this; - } - - @JsonProperty("git_commit_hash") - public String getGitCommitHash() { - return gitCommitHash; - } - - @JsonProperty("git_commit_hash") - public void setGitCommitHash(String gitCommitHash) { - this.gitCommitHash = gitCommitHash; - } - - public HistoryRepoParams withGitCommitHash(String gitCommitHash) { - this.gitCommitHash = gitCommitHash; - return this; - } - - @JsonProperty("include_disabled") - public Long getIncludeDisabled() { - return includeDisabled; - } - - @JsonProperty("include_disabled") - public void setIncludeDisabled(Long includeDisabled) { - this.includeDisabled = includeDisabled; - } - - public HistoryRepoParams withIncludeDisabled(Long includeDisabled) { - this.includeDisabled = includeDisabled; - return this; - } - - @JsonAnyGetter - public Map getAdditionalProperties() { - return this.additionalProperties; - } - - @JsonAnySetter - public void setAdditionalProperties(String name, Object value) { - this.additionalProperties.put(name, value); - } - - @Override - public String toString() { - return ((((((((((("HistoryRepoParams"+" [moduleName=")+ moduleName)+", timestamp=")+ timestamp)+", gitCommitHash=")+ gitCommitHash)+", includeDisabled=")+ includeDisabled)+", additionalProperties=")+ additionalProperties)+"]"); - } - -} diff --git a/lib/java/us/kbase/catalog/Icon.java b/lib/java/us/kbase/catalog/Icon.java deleted file mode 100644 index b08419ce..00000000 --- a/lib/java/us/kbase/catalog/Icon.java +++ /dev/null @@ -1,42 +0,0 @@ - -package us.kbase.catalog; - -import java.util.HashMap; -import java.util.Map; -import javax.annotation.Generated; -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; - - -/** - *

Original spec-file type: Icon

- * - * - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -@Generated("com.googlecode.jsonschema2pojo") -@JsonPropertyOrder({ - -}) -public class Icon { - - private Map additionalProperties = new HashMap(); - - @JsonAnyGetter - public Map getAdditionalProperties() { - return this.additionalProperties; - } - - @JsonAnySetter - public void setAdditionalProperties(String name, Object value) { - this.additionalProperties.put(name, value); - } - - @Override - public String toString() { - return ((("Icon"+" [additionalProperties=")+ additionalProperties)+"]"); - } - -} diff --git a/lib/java/us/kbase/catalog/ListReposParams.java b/lib/java/us/kbase/catalog/ListReposParams.java deleted file mode 100644 index c492e89c..00000000 --- a/lib/java/us/kbase/catalog/ListReposParams.java +++ /dev/null @@ -1,63 +0,0 @@ - -package us.kbase.catalog; - -import java.util.HashMap; -import java.util.Map; -import javax.annotation.Generated; -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; - - -/** - *

Original spec-file type: ListReposParams

- * 
- * Describes how to filter repositories.
- * with_disabled - optional flag adding disabled repos (default value is false).
- *
- * - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -@Generated("com.googlecode.jsonschema2pojo") -@JsonPropertyOrder({ - "with_disabled" -}) -public class ListReposParams { - - @JsonProperty("with_disabled") - private Long withDisabled; - private Map additionalProperties = new HashMap(); - - @JsonProperty("with_disabled") - public Long getWithDisabled() { - return withDisabled; - } - - @JsonProperty("with_disabled") - public void setWithDisabled(Long withDisabled) { - this.withDisabled = withDisabled; - } - - public ListReposParams withWithDisabled(Long withDisabled) { - this.withDisabled = withDisabled; - return this; - } - - @JsonAnyGetter - public Map getAdditionalProperties() { - return this.additionalProperties; - } - - @JsonAnySetter - public void setAdditionalProperties(String name, Object value) { - this.additionalProperties.put(name, value); - } - - @Override - public String toString() { - return ((((("ListReposParams"+" [withDisabled=")+ withDisabled)+", additionalProperties=")+ additionalProperties)+"]"); - } - -} diff --git a/lib/java/us/kbase/catalog/RepoDetails.java b/lib/java/us/kbase/catalog/RepoDetails.java deleted file mode 100644 index 76bb9032..00000000 --- a/lib/java/us/kbase/catalog/RepoDetails.java +++ /dev/null @@ -1,228 +0,0 @@ - -package us.kbase.catalog; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import javax.annotation.Generated; -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; - - -/** - *

Original spec-file type: RepoDetails

- * 
- * method_ids - list of method ids (each id is fully qualified, i.e. contains module
- *     name prefix followed by slash);
- * widget_ids - list of widget ids (each id is name of JavaScript file stored in
- *     repo's 'ui/widgets' folder).
- *
- * - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -@Generated("com.googlecode.jsonschema2pojo") -@JsonPropertyOrder({ - "module_name", - "git_url", - "git_commit_hash", - "version", - "module_description", - "service_language", - "owners", - "readme", - "method_ids", - "widget_ids" -}) -public class RepoDetails { - - @JsonProperty("module_name") - private java.lang.String moduleName; - @JsonProperty("git_url") - private java.lang.String gitUrl; - @JsonProperty("git_commit_hash") - private java.lang.String gitCommitHash; - @JsonProperty("version") - private java.lang.String version; - @JsonProperty("module_description") - private java.lang.String moduleDescription; - @JsonProperty("service_language") - private java.lang.String serviceLanguage; - @JsonProperty("owners") - private List owners; - @JsonProperty("readme") - private java.lang.String readme; - @JsonProperty("method_ids") - private List methodIds; - @JsonProperty("widget_ids") - private List widgetIds; - private Map additionalProperties = new HashMap(); - - @JsonProperty("module_name") - public java.lang.String getModuleName() { - return moduleName; - } - - @JsonProperty("module_name") - public void setModuleName(java.lang.String moduleName) { - this.moduleName = moduleName; - } - - public RepoDetails withModuleName(java.lang.String moduleName) { - this.moduleName = moduleName; - return this; - } - - @JsonProperty("git_url") - public java.lang.String getGitUrl() { - return gitUrl; - } - - @JsonProperty("git_url") - public void setGitUrl(java.lang.String gitUrl) { - this.gitUrl = gitUrl; - } - - public RepoDetails withGitUrl(java.lang.String gitUrl) { - this.gitUrl = gitUrl; - return this; - } - - @JsonProperty("git_commit_hash") - public java.lang.String getGitCommitHash() { - return gitCommitHash; - } - - @JsonProperty("git_commit_hash") - public void setGitCommitHash(java.lang.String gitCommitHash) { - this.gitCommitHash = gitCommitHash; - } - - public RepoDetails withGitCommitHash(java.lang.String gitCommitHash) { - this.gitCommitHash = gitCommitHash; - return this; - } - - @JsonProperty("version") - public java.lang.String getVersion() { - return version; - } - - @JsonProperty("version") - public void setVersion(java.lang.String version) { - this.version = version; - } - - public RepoDetails withVersion(java.lang.String version) { - this.version = version; - return this; - } - - @JsonProperty("module_description") - public java.lang.String getModuleDescription() { - return moduleDescription; - } - - @JsonProperty("module_description") - public void setModuleDescription(java.lang.String moduleDescription) { - this.moduleDescription = moduleDescription; - } - - public RepoDetails withModuleDescription(java.lang.String moduleDescription) { - this.moduleDescription = moduleDescription; - return this; - } - - @JsonProperty("service_language") - public java.lang.String getServiceLanguage() { - return serviceLanguage; - } - - @JsonProperty("service_language") - public void setServiceLanguage(java.lang.String serviceLanguage) { - this.serviceLanguage = serviceLanguage; - } - - public RepoDetails withServiceLanguage(java.lang.String serviceLanguage) { - this.serviceLanguage = serviceLanguage; - return this; - } - - @JsonProperty("owners") - public List getOwners() { - return owners; - } - - @JsonProperty("owners") - public void setOwners(List owners) { - this.owners = owners; - } - - public RepoDetails withOwners(List owners) { - this.owners = owners; - return this; - } - - @JsonProperty("readme") - public java.lang.String getReadme() { - return readme; - } - - @JsonProperty("readme") - public void setReadme(java.lang.String readme) { - this.readme = readme; - } - - public RepoDetails withReadme(java.lang.String readme) { - this.readme = readme; - return this; - } - - @JsonProperty("method_ids") - public List getMethodIds() { - return methodIds; - } - - @JsonProperty("method_ids") - public void setMethodIds(List methodIds) { - this.methodIds = methodIds; - } - - public RepoDetails withMethodIds(List methodIds) { - this.methodIds = methodIds; - return this; - } - - @JsonProperty("widget_ids") - public List getWidgetIds() { - return widgetIds; - } - - @JsonProperty("widget_ids") - public void setWidgetIds(List widgetIds) { - this.widgetIds = widgetIds; - } - - public RepoDetails withWidgetIds(List widgetIds) { - this.widgetIds = widgetIds; - return this; - } - - @JsonAnyGetter - public Map getAdditionalProperties() { - return this.additionalProperties; - } - - @JsonAnySetter - public void setAdditionalProperties(java.lang.String name, Object value) { - this.additionalProperties.put(name, value); - } - - @Override - public java.lang.String toString() { - return ((((((((((((((((((((((("RepoDetails"+" [moduleName=")+ moduleName)+", gitUrl=")+ gitUrl)+", gitCommitHash=")+ gitCommitHash)+", version=")+ version)+", moduleDescription=")+ moduleDescription)+", serviceLanguage=")+ serviceLanguage)+", owners=")+ owners)+", readme=")+ readme)+", methodIds=")+ methodIds)+", widgetIds=")+ widgetIds)+", additionalProperties=")+ additionalProperties)+"]"); - } - -} diff --git a/lib/java/us/kbase/catalog/RepoVersion.java b/lib/java/us/kbase/catalog/RepoVersion.java deleted file mode 100644 index 9f1ddd14..00000000 --- a/lib/java/us/kbase/catalog/RepoVersion.java +++ /dev/null @@ -1,98 +0,0 @@ - -package us.kbase.catalog; - -import java.util.HashMap; -import java.util.Map; -import javax.annotation.Generated; -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; - - -/** - *

Original spec-file type: RepoVersion

- * 
- * timestamp will be epoch time
- *
- * - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -@Generated("com.googlecode.jsonschema2pojo") -@JsonPropertyOrder({ - "timestamp", - "git_commit_hash", - "include_disabled" -}) -public class RepoVersion { - - @JsonProperty("timestamp") - private Long timestamp; - @JsonProperty("git_commit_hash") - private String gitCommitHash; - @JsonProperty("include_disabled") - private Long includeDisabled; - private Map additionalProperties = new HashMap(); - - @JsonProperty("timestamp") - public Long getTimestamp() { - return timestamp; - } - - @JsonProperty("timestamp") - public void setTimestamp(Long timestamp) { - this.timestamp = timestamp; - } - - public RepoVersion withTimestamp(Long timestamp) { - this.timestamp = timestamp; - return this; - } - - @JsonProperty("git_commit_hash") - public String getGitCommitHash() { - return gitCommitHash; - } - - @JsonProperty("git_commit_hash") - public void setGitCommitHash(String gitCommitHash) { - this.gitCommitHash = gitCommitHash; - } - - public RepoVersion withGitCommitHash(String gitCommitHash) { - this.gitCommitHash = gitCommitHash; - return this; - } - - @JsonProperty("include_disabled") - public Long getIncludeDisabled() { - return includeDisabled; - } - - @JsonProperty("include_disabled") - public void setIncludeDisabled(Long includeDisabled) { - this.includeDisabled = includeDisabled; - } - - public RepoVersion withIncludeDisabled(Long includeDisabled) { - this.includeDisabled = includeDisabled; - return this; - } - - @JsonAnyGetter - public Map getAdditionalProperties() { - return this.additionalProperties; - } - - @JsonAnySetter - public void setAdditionalProperties(String name, Object value) { - this.additionalProperties.put(name, value); - } - - @Override - public String toString() { - return ((((((((("RepoVersion"+" [timestamp=")+ timestamp)+", gitCommitHash=")+ gitCommitHash)+", includeDisabled=")+ includeDisabled)+", additionalProperties=")+ additionalProperties)+"]"); - } - -} diff --git a/lib/java/us/kbase/catalog/SelectModuleParams.java b/lib/java/us/kbase/catalog/SelectModuleParams.java deleted file mode 100644 index f7f59a67..00000000 --- a/lib/java/us/kbase/catalog/SelectModuleParams.java +++ /dev/null @@ -1,101 +0,0 @@ - -package us.kbase.catalog; - -import java.util.HashMap; -import java.util.Map; -import javax.annotation.Generated; -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; - - -/** - *

Original spec-file type: SelectModuleParams

- * 
- * Describes how to find module/repository details.
- * module_name - name of module defined in kbase.yaml file;
- * git_url - the url used to register the module
- * include_disabled - optional flag, set to true to include disabled repos
- *
- * - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -@Generated("com.googlecode.jsonschema2pojo") -@JsonPropertyOrder({ - "module_name", - "git_url", - "include_disabled" -}) -public class SelectModuleParams { - - @JsonProperty("module_name") - private String moduleName; - @JsonProperty("git_url") - private String gitUrl; - @JsonProperty("include_disabled") - private Long includeDisabled; - private Map additionalProperties = new HashMap(); - - @JsonProperty("module_name") - public String getModuleName() { - return moduleName; - } - - @JsonProperty("module_name") - public void setModuleName(String moduleName) { - this.moduleName = moduleName; - } - - public SelectModuleParams withModuleName(String moduleName) { - this.moduleName = moduleName; - return this; - } - - @JsonProperty("git_url") - public String getGitUrl() { - return gitUrl; - } - - @JsonProperty("git_url") - public void setGitUrl(String gitUrl) { - this.gitUrl = gitUrl; - } - - public SelectModuleParams withGitUrl(String gitUrl) { - this.gitUrl = gitUrl; - return this; - } - - @JsonProperty("include_disabled") - public Long getIncludeDisabled() { - return includeDisabled; - } - - @JsonProperty("include_disabled") - public void setIncludeDisabled(Long includeDisabled) { - this.includeDisabled = includeDisabled; - } - - public SelectModuleParams withIncludeDisabled(Long includeDisabled) { - this.includeDisabled = includeDisabled; - return this; - } - - @JsonAnyGetter - public Map getAdditionalProperties() { - return this.additionalProperties; - } - - @JsonAnySetter - public void setAdditionalProperties(String name, Object value) { - this.additionalProperties.put(name, value); - } - - @Override - public String toString() { - return ((((((((("SelectModuleParams"+" [moduleName=")+ moduleName)+", gitUrl=")+ gitUrl)+", includeDisabled=")+ includeDisabled)+", additionalProperties=")+ additionalProperties)+"]"); - } - -} diff --git a/lib/java/us/kbase/catalog/SetRepoStateParams.java b/lib/java/us/kbase/catalog/SetRepoStateParams.java deleted file mode 100644 index 38a0113a..00000000 --- a/lib/java/us/kbase/catalog/SetRepoStateParams.java +++ /dev/null @@ -1,119 +0,0 @@ - -package us.kbase.catalog; - -import java.util.HashMap; -import java.util.Map; -import javax.annotation.Generated; -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; - - -/** - *

Original spec-file type: SetRepoStateParams

- * 
- * Describes how to find repository details.
- * module_name - name of module defined in kbase.yaml file;
- * multiple state fields? (approvalState, buildState, versionState)
- * state - one of 'pending', 'ready', 'building', 'testing', 'disabled'.
- *
- * - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -@Generated("com.googlecode.jsonschema2pojo") -@JsonPropertyOrder({ - "module_name", - "github_repo", - "registration_state", - "error_message" -}) -public class SetRepoStateParams { - - @JsonProperty("module_name") - private String moduleName; - @JsonProperty("github_repo") - private String githubRepo; - @JsonProperty("registration_state") - private String registrationState; - @JsonProperty("error_message") - private String errorMessage; - private Map additionalProperties = new HashMap(); - - @JsonProperty("module_name") - public String getModuleName() { - return moduleName; - } - - @JsonProperty("module_name") - public void setModuleName(String moduleName) { - this.moduleName = moduleName; - } - - public SetRepoStateParams withModuleName(String moduleName) { - this.moduleName = moduleName; - return this; - } - - @JsonProperty("github_repo") - public String getGithubRepo() { - return githubRepo; - } - - @JsonProperty("github_repo") - public void setGithubRepo(String githubRepo) { - this.githubRepo = githubRepo; - } - - public SetRepoStateParams withGithubRepo(String githubRepo) { - this.githubRepo = githubRepo; - return this; - } - - @JsonProperty("registration_state") - public String getRegistrationState() { - return registrationState; - } - - @JsonProperty("registration_state") - public void setRegistrationState(String registrationState) { - this.registrationState = registrationState; - } - - public SetRepoStateParams withRegistrationState(String registrationState) { - this.registrationState = registrationState; - return this; - } - - @JsonProperty("error_message") - public String getErrorMessage() { - return errorMessage; - } - - @JsonProperty("error_message") - public void setErrorMessage(String errorMessage) { - this.errorMessage = errorMessage; - } - - public SetRepoStateParams withErrorMessage(String errorMessage) { - this.errorMessage = errorMessage; - return this; - } - - @JsonAnyGetter - public Map getAdditionalProperties() { - return this.additionalProperties; - } - - @JsonAnySetter - public void setAdditionalProperties(String name, Object value) { - this.additionalProperties.put(name, value); - } - - @Override - public String toString() { - return ((((((((((("SetRepoStateParams"+" [moduleName=")+ moduleName)+", githubRepo=")+ githubRepo)+", registrationState=")+ registrationState)+", errorMessage=")+ errorMessage)+", additionalProperties=")+ additionalProperties)+"]"); - } - -} diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 00000000..1d01a90a --- /dev/null +++ b/settings.gradle @@ -0,0 +1,8 @@ +/* + * This file was generated by the Gradle 'init' task. + * + * The settings file is used to specify which projects to include in your build. + * For more detailed information on multi-project builds, please refer to https://docs.gradle.org/8.14.3/userguide/multi_project_builds.html in the Gradle documentation. + */ + +rootProject.name = 'catalog' diff --git a/lib/java/us/kbase/catalog/AppClientGroup.java b/src/main/java/us/kbase/catalog/AppClientGroup.java similarity index 100% rename from lib/java/us/kbase/catalog/AppClientGroup.java rename to src/main/java/us/kbase/catalog/AppClientGroup.java diff --git a/lib/java/us/kbase/catalog/BasicModuleInfo.java b/src/main/java/us/kbase/catalog/BasicModuleInfo.java similarity index 100% rename from lib/java/us/kbase/catalog/BasicModuleInfo.java rename to src/main/java/us/kbase/catalog/BasicModuleInfo.java diff --git a/lib/java/us/kbase/catalog/BasicModuleVersionInfo.java b/src/main/java/us/kbase/catalog/BasicModuleVersionInfo.java similarity index 100% rename from lib/java/us/kbase/catalog/BasicModuleVersionInfo.java rename to src/main/java/us/kbase/catalog/BasicModuleVersionInfo.java diff --git a/lib/java/us/kbase/catalog/BuildInfo.java b/src/main/java/us/kbase/catalog/BuildInfo.java similarity index 100% rename from lib/java/us/kbase/catalog/BuildInfo.java rename to src/main/java/us/kbase/catalog/BuildInfo.java diff --git a/lib/java/us/kbase/catalog/BuildLog.java b/src/main/java/us/kbase/catalog/BuildLog.java similarity index 100% rename from lib/java/us/kbase/catalog/BuildLog.java rename to src/main/java/us/kbase/catalog/BuildLog.java diff --git a/lib/java/us/kbase/catalog/BuildLogLine.java b/src/main/java/us/kbase/catalog/BuildLogLine.java similarity index 100% rename from lib/java/us/kbase/catalog/BuildLogLine.java rename to src/main/java/us/kbase/catalog/BuildLogLine.java diff --git a/lib/java/us/kbase/catalog/CatalogClient.java b/src/main/java/us/kbase/catalog/CatalogClient.java similarity index 99% rename from lib/java/us/kbase/catalog/CatalogClient.java rename to src/main/java/us/kbase/catalog/CatalogClient.java index a5e6505e..8b44ec37 100644 --- a/lib/java/us/kbase/catalog/CatalogClient.java +++ b/src/main/java/us/kbase/catalog/CatalogClient.java @@ -101,6 +101,7 @@ public boolean isInsecureHttpConnectionAllowed() { /** Deprecated. Use isInsecureHttpConnectionAllowed(). * @deprecated */ + @Deprecated public boolean isAuthAllowedForHttp() { return caller.isAuthAllowedForHttp(); } @@ -116,6 +117,7 @@ public void setIsInsecureHttpConnectionAllowed(boolean allowed) { /** Deprecated. Use setIsInsecureHttpConnectionAllowed(). * @deprecated */ + @Deprecated public void setAuthAllowedForHttp(boolean isAuthAllowedForHttp) { caller.setAuthAllowedForHttp(isAuthAllowedForHttp); } diff --git a/lib/java/us/kbase/catalog/ClientGroupConfig.java b/src/main/java/us/kbase/catalog/ClientGroupConfig.java similarity index 100% rename from lib/java/us/kbase/catalog/ClientGroupConfig.java rename to src/main/java/us/kbase/catalog/ClientGroupConfig.java diff --git a/lib/java/us/kbase/catalog/ClientGroupFilter.java b/src/main/java/us/kbase/catalog/ClientGroupFilter.java similarity index 100% rename from lib/java/us/kbase/catalog/ClientGroupFilter.java rename to src/main/java/us/kbase/catalog/ClientGroupFilter.java diff --git a/lib/java/us/kbase/catalog/CompilationReport.java b/src/main/java/us/kbase/catalog/CompilationReport.java similarity index 100% rename from lib/java/us/kbase/catalog/CompilationReport.java rename to src/main/java/us/kbase/catalog/CompilationReport.java diff --git a/lib/java/us/kbase/catalog/ExecAggrStats.java b/src/main/java/us/kbase/catalog/ExecAggrStats.java similarity index 100% rename from lib/java/us/kbase/catalog/ExecAggrStats.java rename to src/main/java/us/kbase/catalog/ExecAggrStats.java diff --git a/lib/java/us/kbase/catalog/ExecAggrTableParams.java b/src/main/java/us/kbase/catalog/ExecAggrTableParams.java similarity index 100% rename from lib/java/us/kbase/catalog/ExecAggrTableParams.java rename to src/main/java/us/kbase/catalog/ExecAggrTableParams.java diff --git a/lib/java/us/kbase/catalog/FavoriteCount.java b/src/main/java/us/kbase/catalog/FavoriteCount.java similarity index 100% rename from lib/java/us/kbase/catalog/FavoriteCount.java rename to src/main/java/us/kbase/catalog/FavoriteCount.java diff --git a/lib/java/us/kbase/catalog/FavoriteItem.java b/src/main/java/us/kbase/catalog/FavoriteItem.java similarity index 100% rename from lib/java/us/kbase/catalog/FavoriteItem.java rename to src/main/java/us/kbase/catalog/FavoriteItem.java diff --git a/lib/java/us/kbase/catalog/FavoriteUser.java b/src/main/java/us/kbase/catalog/FavoriteUser.java similarity index 100% rename from lib/java/us/kbase/catalog/FavoriteUser.java rename to src/main/java/us/kbase/catalog/FavoriteUser.java diff --git a/lib/java/us/kbase/catalog/Function.java b/src/main/java/us/kbase/catalog/Function.java similarity index 100% rename from lib/java/us/kbase/catalog/Function.java rename to src/main/java/us/kbase/catalog/Function.java diff --git a/lib/java/us/kbase/catalog/FunctionPlace.java b/src/main/java/us/kbase/catalog/FunctionPlace.java similarity index 100% rename from lib/java/us/kbase/catalog/FunctionPlace.java rename to src/main/java/us/kbase/catalog/FunctionPlace.java diff --git a/lib/java/us/kbase/catalog/GetBuildLogParams.java b/src/main/java/us/kbase/catalog/GetBuildLogParams.java similarity index 100% rename from lib/java/us/kbase/catalog/GetBuildLogParams.java rename to src/main/java/us/kbase/catalog/GetBuildLogParams.java diff --git a/lib/java/us/kbase/catalog/GetClientGroupParams.java b/src/main/java/us/kbase/catalog/GetClientGroupParams.java similarity index 100% rename from lib/java/us/kbase/catalog/GetClientGroupParams.java rename to src/main/java/us/kbase/catalog/GetClientGroupParams.java diff --git a/lib/java/us/kbase/catalog/GetExecAggrStatsParams.java b/src/main/java/us/kbase/catalog/GetExecAggrStatsParams.java similarity index 100% rename from lib/java/us/kbase/catalog/GetExecAggrStatsParams.java rename to src/main/java/us/kbase/catalog/GetExecAggrStatsParams.java diff --git a/lib/java/us/kbase/catalog/GetExecRawStatsParams.java b/src/main/java/us/kbase/catalog/GetExecRawStatsParams.java similarity index 100% rename from lib/java/us/kbase/catalog/GetExecRawStatsParams.java rename to src/main/java/us/kbase/catalog/GetExecRawStatsParams.java diff --git a/lib/java/us/kbase/catalog/GetLocalFunctionDetails.java b/src/main/java/us/kbase/catalog/GetLocalFunctionDetails.java similarity index 100% rename from lib/java/us/kbase/catalog/GetLocalFunctionDetails.java rename to src/main/java/us/kbase/catalog/GetLocalFunctionDetails.java diff --git a/lib/java/us/kbase/catalog/GetSecureConfigParamsInput.java b/src/main/java/us/kbase/catalog/GetSecureConfigParamsInput.java similarity index 100% rename from lib/java/us/kbase/catalog/GetSecureConfigParamsInput.java rename to src/main/java/us/kbase/catalog/GetSecureConfigParamsInput.java diff --git a/lib/java/us/kbase/catalog/IOTags.java b/src/main/java/us/kbase/catalog/IOTags.java similarity index 100% rename from lib/java/us/kbase/catalog/IOTags.java rename to src/main/java/us/kbase/catalog/IOTags.java diff --git a/lib/java/us/kbase/catalog/ListBuildParams.java b/src/main/java/us/kbase/catalog/ListBuildParams.java similarity index 100% rename from lib/java/us/kbase/catalog/ListBuildParams.java rename to src/main/java/us/kbase/catalog/ListBuildParams.java diff --git a/lib/java/us/kbase/catalog/ListFavoriteCounts.java b/src/main/java/us/kbase/catalog/ListFavoriteCounts.java similarity index 100% rename from lib/java/us/kbase/catalog/ListFavoriteCounts.java rename to src/main/java/us/kbase/catalog/ListFavoriteCounts.java diff --git a/lib/java/us/kbase/catalog/ListLocalFunctionParams.java b/src/main/java/us/kbase/catalog/ListLocalFunctionParams.java similarity index 100% rename from lib/java/us/kbase/catalog/ListLocalFunctionParams.java rename to src/main/java/us/kbase/catalog/ListLocalFunctionParams.java diff --git a/lib/java/us/kbase/catalog/ListModuleParams.java b/src/main/java/us/kbase/catalog/ListModuleParams.java similarity index 100% rename from lib/java/us/kbase/catalog/ListModuleParams.java rename to src/main/java/us/kbase/catalog/ListModuleParams.java diff --git a/lib/java/us/kbase/catalog/ListServiceModuleParams.java b/src/main/java/us/kbase/catalog/ListServiceModuleParams.java similarity index 100% rename from lib/java/us/kbase/catalog/ListServiceModuleParams.java rename to src/main/java/us/kbase/catalog/ListServiceModuleParams.java diff --git a/lib/java/us/kbase/catalog/LocalFunctionDetails.java b/src/main/java/us/kbase/catalog/LocalFunctionDetails.java similarity index 100% rename from lib/java/us/kbase/catalog/LocalFunctionDetails.java rename to src/main/java/us/kbase/catalog/LocalFunctionDetails.java diff --git a/lib/java/us/kbase/catalog/LocalFunctionInfo.java b/src/main/java/us/kbase/catalog/LocalFunctionInfo.java similarity index 100% rename from lib/java/us/kbase/catalog/LocalFunctionInfo.java rename to src/main/java/us/kbase/catalog/LocalFunctionInfo.java diff --git a/lib/java/us/kbase/catalog/LocalFunctionTags.java b/src/main/java/us/kbase/catalog/LocalFunctionTags.java similarity index 100% rename from lib/java/us/kbase/catalog/LocalFunctionTags.java rename to src/main/java/us/kbase/catalog/LocalFunctionTags.java diff --git a/lib/java/us/kbase/catalog/LogExecStatsParams.java b/src/main/java/us/kbase/catalog/LogExecStatsParams.java similarity index 100% rename from lib/java/us/kbase/catalog/LogExecStatsParams.java rename to src/main/java/us/kbase/catalog/LogExecStatsParams.java diff --git a/lib/java/us/kbase/catalog/ModifySecureConfigParamsInput.java b/src/main/java/us/kbase/catalog/ModifySecureConfigParamsInput.java similarity index 100% rename from lib/java/us/kbase/catalog/ModifySecureConfigParamsInput.java rename to src/main/java/us/kbase/catalog/ModifySecureConfigParamsInput.java diff --git a/lib/java/us/kbase/catalog/ModuleInfo.java b/src/main/java/us/kbase/catalog/ModuleInfo.java similarity index 100% rename from lib/java/us/kbase/catalog/ModuleInfo.java rename to src/main/java/us/kbase/catalog/ModuleInfo.java diff --git a/lib/java/us/kbase/catalog/ModuleState.java b/src/main/java/us/kbase/catalog/ModuleState.java similarity index 100% rename from lib/java/us/kbase/catalog/ModuleState.java rename to src/main/java/us/kbase/catalog/ModuleState.java diff --git a/lib/java/us/kbase/catalog/ModuleVersion.java b/src/main/java/us/kbase/catalog/ModuleVersion.java similarity index 100% rename from lib/java/us/kbase/catalog/ModuleVersion.java rename to src/main/java/us/kbase/catalog/ModuleVersion.java diff --git a/lib/java/us/kbase/catalog/ModuleVersionInfo.java b/src/main/java/us/kbase/catalog/ModuleVersionInfo.java similarity index 100% rename from lib/java/us/kbase/catalog/ModuleVersionInfo.java rename to src/main/java/us/kbase/catalog/ModuleVersionInfo.java diff --git a/lib/java/us/kbase/catalog/ModuleVersionLookupParams.java b/src/main/java/us/kbase/catalog/ModuleVersionLookupParams.java similarity index 100% rename from lib/java/us/kbase/catalog/ModuleVersionLookupParams.java rename to src/main/java/us/kbase/catalog/ModuleVersionLookupParams.java diff --git a/lib/java/us/kbase/catalog/Parameter.java b/src/main/java/us/kbase/catalog/Parameter.java similarity index 100% rename from lib/java/us/kbase/catalog/Parameter.java rename to src/main/java/us/kbase/catalog/Parameter.java diff --git a/lib/java/us/kbase/catalog/RegisterRepoParams.java b/src/main/java/us/kbase/catalog/RegisterRepoParams.java similarity index 100% rename from lib/java/us/kbase/catalog/RegisterRepoParams.java rename to src/main/java/us/kbase/catalog/RegisterRepoParams.java diff --git a/lib/java/us/kbase/catalog/ReleaseReview.java b/src/main/java/us/kbase/catalog/ReleaseReview.java similarity index 100% rename from lib/java/us/kbase/catalog/ReleaseReview.java rename to src/main/java/us/kbase/catalog/ReleaseReview.java diff --git a/lib/java/us/kbase/catalog/RequestedReleaseInfo.java b/src/main/java/us/kbase/catalog/RequestedReleaseInfo.java similarity index 100% rename from lib/java/us/kbase/catalog/RequestedReleaseInfo.java rename to src/main/java/us/kbase/catalog/RequestedReleaseInfo.java diff --git a/lib/java/us/kbase/catalog/SecureConfigParameter.java b/src/main/java/us/kbase/catalog/SecureConfigParameter.java similarity index 100% rename from lib/java/us/kbase/catalog/SecureConfigParameter.java rename to src/main/java/us/kbase/catalog/SecureConfigParameter.java diff --git a/lib/java/us/kbase/catalog/SelectModuleVersion.java b/src/main/java/us/kbase/catalog/SelectModuleVersion.java similarity index 100% rename from lib/java/us/kbase/catalog/SelectModuleVersion.java rename to src/main/java/us/kbase/catalog/SelectModuleVersion.java diff --git a/lib/java/us/kbase/catalog/SelectModuleVersionParams.java b/src/main/java/us/kbase/catalog/SelectModuleVersionParams.java similarity index 100% rename from lib/java/us/kbase/catalog/SelectModuleVersionParams.java rename to src/main/java/us/kbase/catalog/SelectModuleVersionParams.java diff --git a/lib/java/us/kbase/catalog/SelectOneLocalFunction.java b/src/main/java/us/kbase/catalog/SelectOneLocalFunction.java similarity index 100% rename from lib/java/us/kbase/catalog/SelectOneLocalFunction.java rename to src/main/java/us/kbase/catalog/SelectOneLocalFunction.java diff --git a/lib/java/us/kbase/catalog/SelectOneModuleParams.java b/src/main/java/us/kbase/catalog/SelectOneModuleParams.java similarity index 100% rename from lib/java/us/kbase/catalog/SelectOneModuleParams.java rename to src/main/java/us/kbase/catalog/SelectOneModuleParams.java diff --git a/lib/java/us/kbase/catalog/SetRegistrationStateParams.java b/src/main/java/us/kbase/catalog/SetRegistrationStateParams.java similarity index 100% rename from lib/java/us/kbase/catalog/SetRegistrationStateParams.java rename to src/main/java/us/kbase/catalog/SetRegistrationStateParams.java diff --git a/lib/java/us/kbase/catalog/SpecFile.java b/src/main/java/us/kbase/catalog/SpecFile.java similarity index 100% rename from lib/java/us/kbase/catalog/SpecFile.java rename to src/main/java/us/kbase/catalog/SpecFile.java diff --git a/lib/java/us/kbase/catalog/UpdateGitUrlParams.java b/src/main/java/us/kbase/catalog/UpdateGitUrlParams.java similarity index 100% rename from lib/java/us/kbase/catalog/UpdateGitUrlParams.java rename to src/main/java/us/kbase/catalog/UpdateGitUrlParams.java diff --git a/lib/java/us/kbase/catalog/VersionCommitInfo.java b/src/main/java/us/kbase/catalog/VersionCommitInfo.java similarity index 100% rename from lib/java/us/kbase/catalog/VersionCommitInfo.java rename to src/main/java/us/kbase/catalog/VersionCommitInfo.java diff --git a/lib/java/us/kbase/catalog/VolumeMount.java b/src/main/java/us/kbase/catalog/VolumeMount.java similarity index 100% rename from lib/java/us/kbase/catalog/VolumeMount.java rename to src/main/java/us/kbase/catalog/VolumeMount.java diff --git a/lib/java/us/kbase/catalog/VolumeMountConfig.java b/src/main/java/us/kbase/catalog/VolumeMountConfig.java similarity index 100% rename from lib/java/us/kbase/catalog/VolumeMountConfig.java rename to src/main/java/us/kbase/catalog/VolumeMountConfig.java diff --git a/lib/java/us/kbase/catalog/VolumeMountFilter.java b/src/main/java/us/kbase/catalog/VolumeMountFilter.java similarity index 100% rename from lib/java/us/kbase/catalog/VolumeMountFilter.java rename to src/main/java/us/kbase/catalog/VolumeMountFilter.java From aad1a6d24a34eab9bf2b62f2fc0ad3ccb48ac74e Mon Sep 17 00:00:00 2001 From: MrCreosote Date: Wed, 3 Sep 2025 17:17:52 -0700 Subject: [PATCH 110/110] Fix JITPack build builds by default on Java 8. Need to look into how to bump that --- build.gradle | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 81035700..dcd377d6 100644 --- a/build.gradle +++ b/build.gradle @@ -21,7 +21,9 @@ repositories { } compileJava { - options.release = 11 + // TODO BUILD remove when we no longer support java 8, use `options.release = 11` if needed + java.sourceCompatibility = JavaVersion.VERSION_1_8 + java.targetCompatibility = JavaVersion.VERSION_1_8 } java {