From 7c255e6a96ad4ee414837ab98149dc17d1f8fe51 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Fri, 4 Apr 2025 13:07:36 -0400 Subject: [PATCH 1/5] Remove timeline panel and add query times --- .../debug_toolbar/panels/mql/panel.py | 2 - .../debug_toolbar/panels/mql/tracking.py | 1 + .../templates/debug_toolbar/panels/mql.html | 223 +++++++++--------- 3 files changed, 106 insertions(+), 120 deletions(-) diff --git a/django_mongodb_extensions/debug_toolbar/panels/mql/panel.py b/django_mongodb_extensions/debug_toolbar/panels/mql/panel.py index 37ad6ca..a1d8647 100644 --- a/django_mongodb_extensions/debug_toolbar/panels/mql/panel.py +++ b/django_mongodb_extensions/debug_toolbar/panels/mql/panel.py @@ -58,8 +58,6 @@ def disable_instrumentation(self): def generate_stats(self, request, response): self.record_stats( { - "databases": sorted(self._databases.items()), "queries": self._queries, - "sql_time": self._sql_time, } ) diff --git a/django_mongodb_extensions/debug_toolbar/panels/mql/tracking.py b/django_mongodb_extensions/debug_toolbar/panels/mql/tracking.py index 535ce84..31870e9 100644 --- a/django_mongodb_extensions/debug_toolbar/panels/mql/tracking.py +++ b/django_mongodb_extensions/debug_toolbar/panels/mql/tracking.py @@ -34,6 +34,7 @@ def log(self, op, duration, args, kwargs=None): if self.logger: self.logger._queries.append( { + "alias": self.db.alias, "sql": operation, "time": "%.3f" % duration, } diff --git a/django_mongodb_extensions/templates/debug_toolbar/panels/mql.html b/django_mongodb_extensions/templates/debug_toolbar/panels/mql.html index cb896ba..858103c 100644 --- a/django_mongodb_extensions/templates/debug_toolbar/panels/mql.html +++ b/django_mongodb_extensions/templates/debug_toolbar/panels/mql.html @@ -1,124 +1,111 @@ {% load i18n l10n %} - {% if queries %} - - - - - - - - - - - - - - - - - - - - {% for query in queries %} - - - - - - - - - - - - - {% endfor %} - -
{% trans "Query" %}{% trans "Timeline" %}{% trans "Time (ms)" %}{% trans "Action" %}
- - -
{{ query.sql|safe }}
- {% if query.similar_count %} - - - {% blocktrans with count=query.similar_count %}{{ count }} similar queries.{% endblocktrans %} - - {% endif %} - {% if query.duplicate_count %} - - - {% blocktrans with dupes=query.duplicate_count %}Duplicated {{ dupes }} times.{% endblocktrans %} - - {% endif %} - 		- - - {% if query.starts_trans %} - - {% endif %} - {% if query.ends_trans %} - - {% endif %} - - - {{ query.duration|floatformat:"2" }} - - {% if query.params %} - {% if query.is_select %} -
- {{ query.form.as_div }} - - - {% if query.vendor == 'mysql' %} - - {% endif %} -
- {% endif %} - {% endif %} -
-
-

{% trans "Connection:" %} {{ query.alias }}

- {% if query.iso_level %} -

{% trans "Isolation level:" %} {{ query.iso_level }}

- {% endif %} - {% if query.trans_status %} -

{% trans "Transaction status:" %} {{ query.trans_status }}

- {% endif %} - {% if query.stacktrace %} - 
{{ query.stacktrace }}
- {% endif %} - {% if query.template_info %} - - {% for line in query.template_info.context %} - - - - - {% endfor %} -
{{ line.num }}{{ line.content }}
-

{{ query.template_info.name|default:_("(unknown)") }}

- {% endif %} -
-
+ + + + + + + + + + + + + + + {% for query in queries %} + + + + + + + + + + + {% endfor %} + +
{% trans "Query" %}{% trans "Time (ms)" %}
+ + + + +
{{ query.sql|safe }}
+ {% if query.similar_count %} + + + {% blocktrans with count=query.similar_count %}{{ count }} similar queries.{% endblocktrans %} + + {% endif %} + {% if query.duplicate_count %} + + + {% blocktrans with dupes=query.duplicate_count %}Duplicated {{ dupes }} times.{% endblocktrans %} + + {% endif %} + 		{{ query.time|floatformat:"2" }}
+
+

+ {% trans "Connection:" %} {{ query.alias }} +

+ {% if query.iso_level %} +

+ {% trans "Isolation level:" %} {{ query.iso_level }} +

+ {% endif %} + {% if query.trans_status %} +

+ {% trans "Transaction status:" %} {{ query.trans_status }} +

+ {% endif %} + {% if query.stacktrace %}
{{ query.stacktrace }}
{% endif %} + {% if query.template_info %} + + {% for line in query.template_info.context %} + + + + + {% endfor %} +
{{ line.num }} + {{ line.content }} +
+

+ {{ query.template_info.name|default:_("(unknown)") }} +

+ {% endif %} +
+
{% else %} -

{% trans "No MQL queries were recorded during this request." %}

+

{% trans "No MQL queries were recorded during this request." %}

{% endif %} From 2a9d280b948e980698598507ff0321aa0ce8208f Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Tue, 8 Apr 2025 08:32:47 -0400 Subject: [PATCH 2/5] Start over with sql template; remove djlint --- .pre-commit-config.yaml | 6 - .../templates/debug_toolbar/panels/mql.html | 223 +++++++++--------- 2 files changed, 118 insertions(+), 111 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5c722e9..b79d6dd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,9 +14,3 @@ repos: - id: ruff args: [ --fix ] - id: ruff-format - -- repo: https://github.com/djlint/djLint - rev: v1.36.3 - hooks: - - id: djlint-reformat-django - - id: djlint-django diff --git a/django_mongodb_extensions/templates/debug_toolbar/panels/mql.html b/django_mongodb_extensions/templates/debug_toolbar/panels/mql.html index 858103c..63cf293 100644 --- a/django_mongodb_extensions/templates/debug_toolbar/panels/mql.html +++ b/django_mongodb_extensions/templates/debug_toolbar/panels/mql.html @@ -1,111 +1,124 @@ {% load i18n l10n %} + {% if queries %} - - - - - - - - - - - - - - - {% for query in queries %} - - - - - - - - - - - {% endfor %} - -
{% trans "Query" %}{% trans "Time (ms)" %}
- - - - -
{{ query.sql|safe }}
- {% if query.similar_count %} - - - {% blocktrans with count=query.similar_count %}{{ count }} similar queries.{% endblocktrans %} - - {% endif %} - {% if query.duplicate_count %} - - - {% blocktrans with dupes=query.duplicate_count %}Duplicated {{ dupes }} times.{% endblocktrans %} - - {% endif %} - 		{{ query.time|floatformat:"2" }}
-
-

- {% trans "Connection:" %} {{ query.alias }} -

- {% if query.iso_level %} -

- {% trans "Isolation level:" %} {{ query.iso_level }} -

- {% endif %} - {% if query.trans_status %} -

- {% trans "Transaction status:" %} {{ query.trans_status }} -

- {% endif %} - {% if query.stacktrace %}
{{ query.stacktrace }}
{% endif %} - {% if query.template_info %} - - {% for line in query.template_info.context %} - - - - - {% endfor %} -
{{ line.num }} - {{ line.content }} -
-

- {{ query.template_info.name|default:_("(unknown)") }} -

- {% endif %} -
-
+ + + + + + + + + + + + + + + + + + + + {% for query in queries %} + + + + + + + + + + + + + {% endfor %} + +
{% translate "Query" %}{% translate "Timeline" %}{% translate "Time (ms)" %}{% translate "Action" %}
+ + +
{{ query.sql|safe }}
+ {% if query.similar_count %} + + + {% blocktranslate with count=query.similar_count %}{{ count }} similar queries.{% endblocktranslate %} + + {% endif %} + {% if query.duplicate_count %} + + + {% blocktranslate with dupes=query.duplicate_count %}Duplicated {{ dupes }} times.{% endblocktranslate %} + + {% endif %} + 		+ + + {% if query.starts_trans %} + + {% endif %} + {% if query.ends_trans %} + + {% endif %} + + + {{ query.duration|floatformat:"2" }} + + {% if query.params %} + {% if query.is_select %} +
+ {{ query.form.as_div }} + + + {% if query.vendor == 'mysql' %} + + {% endif %} +
+ {% endif %} + {% endif %} +
+
+

{% translate "Connection:" %} {{ query.alias }}

+ {% if query.iso_level %} +

{% translate "Isolation level:" %} {{ query.iso_level }}

+ {% endif %} + {% if query.trans_status %} +

{% translate "Transaction status:" %} {{ query.trans_status }}

+ {% endif %} + {% if query.stacktrace %} + 
{{ query.stacktrace }}
+ {% endif %} + {% if query.template_info %} + + {% for line in query.template_info.context %} + + + + + {% endfor %} +
{{ line.num }}{{ line.content }}
+

{{ query.template_info.name|default:_("(unknown)") }}

+ {% endif %} +
+
{% else %} -

{% trans "No MQL queries were recorded during this request." %}

+

{% translate "No SQL queries were recorded during this request." %}

{% endif %} From 89ba23561087738d86e44b4e4444cf03ce716696 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Tue, 8 Apr 2025 12:07:25 -0400 Subject: [PATCH 3/5] Fix query times; comment unused template features --- .../debug_toolbar/panels/mql/tracking.py | 2 +- .../templates/debug_toolbar/panels/mql.html | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/django_mongodb_extensions/debug_toolbar/panels/mql/tracking.py b/django_mongodb_extensions/debug_toolbar/panels/mql/tracking.py index 31870e9..e0c3219 100644 --- a/django_mongodb_extensions/debug_toolbar/panels/mql/tracking.py +++ b/django_mongodb_extensions/debug_toolbar/panels/mql/tracking.py @@ -36,7 +36,7 @@ def log(self, op, duration, args, kwargs=None): { "alias": self.db.alias, "sql": operation, - "time": "%.3f" % duration, + "duration": "%.3f" % duration, } ) self.logger._databases[self.db.alias] = { diff --git a/django_mongodb_extensions/templates/debug_toolbar/panels/mql.html b/django_mongodb_extensions/templates/debug_toolbar/panels/mql.html index 63cf293..d12c187 100644 --- a/django_mongodb_extensions/templates/debug_toolbar/panels/mql.html +++ b/django_mongodb_extensions/templates/debug_toolbar/panels/mql.html @@ -23,18 +23,18 @@ - - + {% comment %}{% endcomment %} + {% comment %}{% endcomment %} {% translate "Query" %} - {% translate "Timeline" %} + {% comment %}{% translate "Timeline" %}{% endcomment %} {% translate "Time (ms)" %} - {% translate "Action" %} + {% comment %}{% translate "Action" %}{% endcomment %} @@ -59,6 +59,7 @@ {% endif %} + {% comment %} @@ -70,9 +71,11 @@ {% endif %} + {% endcomment %} {{ query.duration|floatformat:"2" }} + {% comment %} {% if query.params %} {% if query.is_select %} @@ -87,6 +90,7 @@ {% endif %} {% endif %} + {% endcomment %} From 77b1a7966bbedf70cc11e1719a5ee809ffc8f303 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Tue, 8 Apr 2025 17:15:21 -0400 Subject: [PATCH 4/5] Update _sql_time to calculate total duration --- django_mongodb_extensions/debug_toolbar/panels/mql/tracking.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django_mongodb_extensions/debug_toolbar/panels/mql/tracking.py b/django_mongodb_extensions/debug_toolbar/panels/mql/tracking.py index e0c3219..9c9c757 100644 --- a/django_mongodb_extensions/debug_toolbar/panels/mql/tracking.py +++ b/django_mongodb_extensions/debug_toolbar/panels/mql/tracking.py @@ -32,6 +32,7 @@ def log(self, op, duration, args, kwargs=None): args = ", ".join(repr(arg) for arg in args) operation = f"db.{self.collection_name}{op}({args})" if self.logger: + self.logger._sql_time += duration self.logger._queries.append( { "alias": self.db.alias, From 84ac172f6a9b8ddd90226639660ea3a3f3941a57 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Thu, 10 Apr 2025 13:42:24 -0400 Subject: [PATCH 5/5] WIP trusted publishing --- .github/dependabot.yml | 16 ++++ .github/workflows/release-python.yml | 99 +++++++++++++++++++++++++ .github/workflows/test-python.yml | 107 +++++++++++++++++++++++++++ .github/workflows/zizmor.yml | 32 ++++++++ .pre-commit-config.yaml | 81 ++++++++++++++++---- justfile | 28 ++++++- 6 files changed, 347 insertions(+), 16 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/release-python.yml create mode 100644 .github/workflows/test-python.yml create mode 100644 .github/workflows/zizmor.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..5bf500b --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,16 @@ +version: 2 +updates: + # GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + groups: + actions: + patterns: + - "*" + # Python + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml new file mode 100644 index 0000000..519ef9d --- /dev/null +++ b/.github/workflows/release-python.yml @@ -0,0 +1,99 @@ +name: Python Wheels + +on: + push: + branches: ["main"] + tags: + - "**" + pull_request: + workflow_dispatch: + +concurrency: + group: wheels-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash -eux {0} + +jobs: + + build_dist: + name: Build Distribution Files + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: actions/setup-python@v5 + with: + # Build sdist on lowest supported Python + python-version: '3.9' + + - name: Install build + run: | + python -m pip install build + + - name: build the dist files + run: | + python -m build . + + - name: Upload the dist files + uses: actions/upload-artifact@v4 + with: + name: dist-${{ github.run_id }} + path: ./dist/*.* + + test_dist: + needs: [build_dist] + name: Test Distribution Files + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: actions/setup-python@v5 + with: + # Build sdist on lowest supported Python + python-version: '3.9' + + - name: Download the dists + uses: actions/download-artifact@v4 + with: + name: dist-${{ github.run_id }} + path: dist/ + + - name: Test the sdist + run: | + cd dist + pip install *.tar.gz + python -c "import flask_pymongo" + pip uninstall -y flask_pymongo + + - name: Test the wheel + run: | + cd dist + pip install *.whl + python -c "import flask_pymongo" + pip uninstall -y flask_pymongo + + publish: + # https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/#publishing-the-distribution-to-pypi + needs: [test_dist] + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + environment: release + permissions: + id-token: write + steps: + - name: Download the dists + uses: actions/download-artifact@v4 + with: + name: dist-${{ github.run_id }} + path: dist/ + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml new file mode 100644 index 0000000..583dbca --- /dev/null +++ b/.github/workflows/test-python.yml @@ -0,0 +1,107 @@ +name: Python Tests + +on: + push: + branches: ["main"] + pull_request: + +concurrency: + group: tests-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash -eux {0} + +env: + MIN_PYTHON: "3.9" + MIN_MONGODB: "4.0" + MAX_MONGODB: "8.0" + +jobs: + static: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 0 + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + python-version: ${{ matrix.python-version }} + - uses: extractions/setup-just@v3 + - run: just install + - run: just lint + - run: just docs + - run: just doctest + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ["ubuntu-latest", "macos-latest", "windows-latest"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + fail-fast: false + name: CPython ${{ matrix.python-version }}-${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 0 + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + python-version: ${{ matrix.python-version }} + - uses: extractions/setup-just@v3 + - name: Start MongoDB on Linux + if: ${{ startsWith(runner.os, 'Linux') }} + uses: supercharge/mongodb-github-action@1.12.0 + with: + mongodb-version: ${{ env.MAX_MONGODB }} + mongodb-replica-set: test-rs + - name: Start MongoDB on MacOS + if: ${{ startsWith(runner.os, 'macOS') }} + run: | + brew tap mongodb/brew + brew install mongodb/brew/mongodb-community@${MAX_MONGODB} + brew services start mongodb-community@${MAX_MONGODB} + - name: Start MongoDB on Windows + if: ${{ startsWith(runner.os, 'Windows') }} + shell: powershell + run: | + mkdir data + mongod --remove + mongod --install --dbpath=$(pwd)/data --logpath=$PWD/mongo.log + net start MongoDB + - run: just install + - run: just test + + build-min: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 0 + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + python-version: ${{ env.MIN_PYTHON }} + - uses: extractions/setup-just@v3 + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + python-version: ${{ env.MIN_PYTHON }} + - uses: extractions/setup-just@v3 + - uses: supercharge/mongodb-github-action@1.12.0 + with: + mongodb-version: ${{ env.MIN_MONGODB }} + mongodb-replica-set: test-rs + - name: Run unit tests with minimum dependency versions + run: | + uv sync --python=${MIN_PYTHON} --resolution=lowest-direct + just test diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml new file mode 100644 index 0000000..0fbdbd6 --- /dev/null +++ b/.github/workflows/zizmor.yml @@ -0,0 +1,32 @@ +name: GitHub Actions Security Analysis with zizmor + +on: + push: + branches: ["main"] + pull_request: + branches: ["**"] + +jobs: + zizmor: + name: zizmor latest via Cargo + runs-on: ubuntu-latest + permissions: + security-events: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Setup Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 + - name: Get zizmor + run: cargo install zizmor + - name: Run zizmor + run: zizmor --format sarif . > results.sarif + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Upload SARIF file + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: results.sarif + category: zizmor diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b79d6dd..1808b53 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,16 +1,67 @@ -# See https://pre-commit.com for more information -# See https://pre-commit.com/hooks.html for more hooks + repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 - hooks: - - id: trailing-whitespace - - id: end-of-file-fixer - - id: check-yaml - - id: check-added-large-files -- repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.3 - hooks: - - id: ruff - args: [ --fix ] - - id: ruff-format +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-toml + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: forbid-new-submodules + - id: trailing-whitespace + +# We use the Python version instead of the original version which seems to require Docker +# https://github.com/koalaman/shellcheck-precommit +- repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.10.0.1 + hooks: + - id: shellcheck + name: shellcheck + args: ["--severity=warning"] + stages: [manual] + +- repo: https://github.com/sirosen/check-jsonschema + rev: 0.31.0 + hooks: + - id: check-github-workflows + args: ["--verbose"] + +- repo: https://github.com/codespell-project/codespell + rev: "v2.3.0" + hooks: + - id: codespell + args: ["-L", "nd"] + stages: [manual] + +- repo: https://github.com/adamchainz/blacken-docs + rev: "1.19.1" + hooks: + - id: blacken-docs + additional_dependencies: [black==24.*] + +- repo: https://github.com/pre-commit/pygrep-hooks + rev: "v1.10.0" + hooks: + - id: rst-backticks + - id: rst-directive-colons + - id: rst-inline-touching-normal + +- repo: https://github.com/hukkin/mdformat + rev: 0.7.21 + hooks: + - id: mdformat + # Optionally add plugins + additional_dependencies: + - mdformat-gfm + +- repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.9.1 + hooks: + # Run the linter. + - id: ruff + args: [ --fix, --show-fixes ] + # Run the formatter. + - id: ruff-format diff --git a/justfile b/justfile index 789b1ea..1270741 100644 --- a/justfile +++ b/justfile @@ -1,2 +1,28 @@ +docs_build := "docs/_build" +sphinx_opts:= "-d " + docs_build + "/doctrees docs" + +# Default target executed when no arguments are given. +[private] default: - echo 'Hello, world!' + @just --list + +install: + uv sync + uv run pre-commit install + +test *args: + uv run pytest {{args}} + +lint: + uv run pre-commit run --hook-stage manual --all-files + +docs: + uv run sphinx-build -T -b html {{sphinx_opts}} {{docs_build}} + +doctest: + uv run python -m doctest -v examples/wiki/wiki.py + uv run sphinx-build -E -b doctest {{sphinx_opts}} {{docs_build}}/doctest + uv run sphinx-build -b linkcheck {{sphinx_opts}} {{docs_build}}/linkcheck + +typing: + uv run mypy --install-types --non-interactive .