From bc2d929b68f5580ce60df9a64805d571d52dd7e3 Mon Sep 17 00:00:00 2001 From: matthewhegarty Date: Sat, 20 Sep 2025 17:15:03 +0100 Subject: [PATCH 1/8] Modernize dependencies, tooling, and development environment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Update Python support to 3.9-3.13 with modern dependency constraints - Update Django support to 4.2-5.2+ with minimum version requirements - Modernize Docker setup with PostgreSQL 14 and optimized Python 3.11 image - Update Docker Compose to v2 syntax and fix container networking - Add pre-commit hooks with latest tool versions (black, isort, flake8, bandit) - Fix Django model warnings by adding DEFAULT_AUTO_FIELD setting - Update example app with corrected admin URLs and database configuration - Apply consistent code formatting and resolve all linting issues - Update development documentation and scripts for current setup - Add pyproject.toml for modern Python project configuration 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- .dockerignore | 68 +++++++++++++++++++ .pre-commit-config.yaml | 28 ++++++-- Dockerfile | 44 ++++++++++-- Makefile | 23 +++++-- develop.sh | 9 ++- docker-compose.yaml | 10 +-- example/README.rst | 67 +++++------------- example/project/settings.py | 18 ++++- example/winners/admin.py | 2 +- example/winners/models.py | 3 +- example/winners/tests/test_admin.py | 1 - example/winners/tests/test_fields.py | 26 ++++--- example/winners/tests/test_models.py | 11 ++- example/winners/tests/test_utils.py | 15 ++-- example/winners/urls.py | 3 +- import_export_celery/admin_actions.py | 9 ++- import_export_celery/fields.py | 17 ++++- .../migrations/0001_initial.py | 2 +- .../migrations/0003_exportjob.py | 2 +- .../migrations/0010_auto_20231013_0904.py | 1 + ...011_alter_exportjob_email_on_completion.py | 1 + import_export_celery/model_config.py | 2 +- import_export_celery/models/exportjob.py | 15 ++-- import_export_celery/models/importjob.py | 12 ++-- import_export_celery/tasks.py | 29 +++----- import_export_celery/utils.py | 10 +-- pyproject.toml | 51 ++++++++++++++ setup-dev-env.sh | 27 +++++++- setup.py | 18 ++++- tox.ini | 16 ++--- 30 files changed, 364 insertions(+), 176 deletions(-) create mode 100644 .dockerignore create mode 100644 pyproject.toml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..44a7c08 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,68 @@ +# Git +.git +.gitignore + +# Python +__pycache__ +*.pyc +*.pyo +*.pyd +.Python +build +develop-eggs +dist +downloads +eggs +.eggs +lib +lib64 +parts +sdist +var +wheels +*.egg-info +.installed.cfg +*.egg + +# Virtual environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo + +# Testing +.tox/ +.coverage +htmlcov/ +.pytest_cache/ +.mypy_cache/ +coverage.xml +*.cover +.hypothesis/ + +# Documentation +docs/_build/ + +# OS +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +# Project specific +db/ +pyenv/ +CLAUDE.md +screenshots/ \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0e90cc9..e76c301 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,29 +1,38 @@ default_language_version: - python: python3.7 + python: python3.11 repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v5.0.0 hooks: - id: check-added-large-files - id: check-case-conflict - id: check-merge-conflict - id: end-of-file-fixer - id: trailing-whitespace + - id: check-yaml + - id: check-toml + - id: check-json - repo: https://github.com/asottile/pyupgrade - rev: v2.31.1 + rev: v3.17.0 hooks: - id: pyupgrade - args: [--py37-plus] + args: [--py39-plus] - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 24.10.0 hooks: - id: black +- repo: https://github.com/PyCQA/isort + rev: 5.13.2 + hooks: + - id: isort + args: ["--profile", "black"] + - repo: https://github.com/PyCQA/flake8 - rev: 4.0.1 + rev: 7.1.1 hooks: - id: flake8 additional_dependencies: @@ -31,3 +40,10 @@ repos: - flake8-comprehensions - flake8-tidy-imports - flake8-typing-imports + +- repo: https://github.com/PyCQA/bandit + rev: 1.7.10 + hooks: + - id: bandit + args: ["-c", "pyproject.toml"] + additional_dependencies: ["bandit[toml]"] diff --git a/Dockerfile b/Dockerfile index 5cd24d7..9b718f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,38 @@ -FROM python:3.7 -RUN pip3 install poetry celery -RUN apt-get update ; apt-get install -yq python3-psycopg2 gdal-bin -ARG UID -RUN useradd test --uid $UID -RUN chsh test -s /bin/bash +# Multi-stage build for smaller images +FROM python:3.11-slim AS base + +# Set environment variables +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/proj \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +# Install system dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + libpq-dev \ + git \ + && rm -rf /var/lib/apt/lists/* + +# Create user +ARG UID=1000 +RUN useradd test --uid $UID --create-home --shell /bin/bash + +# Install poetry +RUN pip install poetry==1.7.1 + +# Set working directory +WORKDIR /proj + +# Copy dependency files +COPY requirements_test.txt ./ + +# Install dependencies +RUN pip install -r requirements_test.txt + +# Install redis for celery +RUN pip install redis + +# Switch to non-root user +USER test diff --git a/Makefile b/Makefile index 01ff748..4c6673b 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,21 @@ -docker-compose: Dockerfile +docker compose: Dockerfile mkdir -p pyenv mkdir -p db - sudo docker-compose build --build-arg UID=$(shell id -u) - sudo docker-compose up -d web postgres - sudo docker exec -it django-import-export-celery_web_1 /proj/setup-dev-env.sh - sudo docker-compose down + docker compose build --build-arg UID=$(shell id -u) + docker compose up -d postgres redis + @echo "Waiting for PostgreSQL to be ready..." + @sleep 15 + @echo "Starting web container..." + docker compose up -d web + @sleep 10 + @echo "Running setup script..." + docker exec django-import-export-celery-web-1 /proj/setup-dev-env.sh + @echo "" + @echo "✅ Setup complete! You can now:" + @echo " ./develop.sh # Enter development environment" + @echo " docker exec -it django-import-export-celery-web-1 bash # Manual container access" + @echo " docker compose up -d celery # Start celery worker" + @echo "" + @echo "🌐 Django admin will be available at: http://localhost:8001/admin/" + @echo "👤 Login: admin / admin" diff --git a/develop.sh b/develop.sh index 86897e3..4c6b7c0 100755 --- a/develop.sh +++ b/develop.sh @@ -1,4 +1,7 @@ #!/bin/bash -docker-compose down -docker-compose up -d -exec docker exec -u test -it django-import-export-celery_web_1 bash --init-file "/proj/dev-entrypoint.sh" +docker compose down +docker compose up -d +echo "Waiting for services to start..." +sleep 10 +echo "Entering development container..." +exec docker exec -u test -it django-import-export-celery-web-1 bash --init-file "/proj/dev-entrypoint.sh" diff --git a/docker-compose.yaml b/docker-compose.yaml index 51fe731..229edc1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,4 +1,3 @@ -version: '2' services: web: build: . @@ -17,8 +16,8 @@ services: - ./pyenv:/home/test celery: build: . - entrypoint: poetry run celery -A project.celery worker -l info - links: + entrypoint: sh -c "pip install -e /proj && pip install redis && celery -A project.celery worker -l info" + depends_on: - postgres - redis tty: true @@ -31,10 +30,13 @@ services: redis: image: redis postgres: - image: mdillon/postgis:9.6-alpine + image: postgres:14-alpine + ports: + - "5432:5432" volumes: - ./db:/var/lib/postgresql/data environment: POSTGRES_PASSWORD: foobar POSTGRES_USER: pguser PGDATA: /var/lib/postgresql/data + POSTGRES_HOST_AUTH_METHOD: md5 diff --git a/example/README.rst b/example/README.rst index 36d6c97..4efee05 100644 --- a/example/README.rst +++ b/example/README.rst @@ -1,74 +1,41 @@ Install ======= -Launch docker-compose +Quick setup with make: .. code-block:: bash - docker-compose up + make -Attach to docker-compose +Or manual setup: .. code-block:: bash - docker attach djangoimportexportcelery_web - -Install Django dependencies: - -.. code-block:: bash - - cd example - pipenv install - pipenv shell - -Initialize database tables: - -.. code-block:: bash - - python manage.py migrate - -Create a super-user for the admin: - -.. code-block:: bash - - python manage.py createsuperuser - -Restart docker-compose - -.. code-block:: bash - - docker-compose down - + docker compose up -d postgres redis + docker compose up -d web + docker exec django-import-export-celery-web-1 /proj/setup-dev-env.sh Run === -Launch docker-compose - -.. code-block:: bash - - docker-compose up - -Attach to docker-compose +Enter the development container: .. code-block:: bash - docker attach djangoimportexportcelery_web + docker exec -it django-import-export-celery-web-1 bash -Enter pipenv shell: +Run the Django server: .. code-block:: bash - cd example - pipenv shell - - -Actually run the server - -.. code-block:: bash + cd example + export DATABASE_HOST=postgres + python manage.py runserver 0.0.0.0:8000 - python manage.py runserver 0.0.0.0:8000 +The example app will be available from http://localhost:8001/admin -The example app will be available from http://127.0.0.1:8001/admin +**Login credentials:** +- Username: admin +- Password: admin -Note: parts of this example app were taken from the [djano-leaflet](https://github.com/makinacorpus/django-leaflet/tree/master/example) example app. +Note: parts of this example app were taken from the [django-leaflet](https://github.com/makinacorpus/django-leaflet/tree/master/example) example app. diff --git a/example/project/settings.py b/example/project/settings.py index c4ea7fe..475ed94 100644 --- a/example/project/settings.py +++ b/example/project/settings.py @@ -76,6 +76,10 @@ BROKER_URL = os.environ.get("REDIS_URL", "redis://redis") REDIS_URL = os.environ.get("REDIS_URL", "redis://redis") + +# Modern Celery configuration +CELERY_BROKER_URL = os.environ.get("REDIS_URL", "redis://redis") +CELERY_RESULT_BACKEND = os.environ.get("REDIS_URL", "redis://redis") # Database # https://docs.djangoproject.com/en/1.10/ref/settings/#databases @@ -83,7 +87,9 @@ DATABASES = { "default": { "ENGINE": "django.db.backends.sqlite3", - "NAME": os.environ.get("DATABASE_NAME", os.path.join(BASE_DIR, "db.sqlite3")), + "NAME": os.environ.get( + "DATABASE_NAME", os.path.join(BASE_DIR, "db.sqlite3") + ), } } else: @@ -93,7 +99,7 @@ "NAME": os.environ.get("DATABASE_NAME", "pguser"), "USER": os.environ.get("DATABASE_USER", "pguser"), "PASSWORD": os.environ.get("DATABASE_PASSWORD", "foobar"), - "HOST": os.environ.get("DATABASE_HOST", "postgres"), + "HOST": os.environ.get("DATABASE_HOST", "localhost"), "PORT": os.environ.get("DATABASE_PORT", ""), }, } @@ -103,7 +109,10 @@ AUTH_PASSWORD_VALIDATORS = [ { - "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + "NAME": ( + "django.contrib.auth.password_validation" + ".UserAttributeSimilarityValidator" + ), }, { "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", @@ -130,6 +139,9 @@ USE_TZ = True +# Fix auto-created primary key warnings +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" + # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.10/howto/static-files/ diff --git a/example/winners/admin.py b/example/winners/admin.py index 9e7a117..e2a2299 100644 --- a/example/winners/admin.py +++ b/example/winners/admin.py @@ -1,7 +1,7 @@ # Copyright (C) 2016 o.s. Auto*Mat from django.contrib import admin - from import_export.admin import ImportExportMixin + from import_export_celery.admin_actions import create_export_job_action from . import models diff --git a/example/winners/models.py b/example/winners/models.py index 0609d60..77110c3 100644 --- a/example/winners/models.py +++ b/example/winners/models.py @@ -1,7 +1,6 @@ from django.db import models - -from import_export.resources import ModelResource from import_export.fields import Field +from import_export.resources import ModelResource class Winner(models.Model): diff --git a/example/winners/tests/test_admin.py b/example/winners/tests/test_admin.py index 939a652..d6c1887 100644 --- a/example/winners/tests/test_admin.py +++ b/example/winners/tests/test_admin.py @@ -1,5 +1,4 @@ from django.contrib.messages.storage.fallback import FallbackStorage - from django_admin_smoke_tests import tests diff --git a/example/winners/tests/test_fields.py b/example/winners/tests/test_fields.py index 80ff76a..696435e 100644 --- a/example/winners/tests/test_fields.py +++ b/example/winners/tests/test_fields.py @@ -1,8 +1,9 @@ +import unittest + import django -from django.test import TestCase, override_settings from django.conf import settings from django.core.files.storage import FileSystemStorage -import unittest +from django.test import TestCase, override_settings from import_export_celery.fields import lazy_initialize_storage_class @@ -16,7 +17,9 @@ class InitializeStorageClassTests(TestCase): def test_default(self): self.assertIsInstance(lazy_initialize_storage_class(), FileSystemStorage) - @unittest.skipUnless(django.VERSION < (5, 1), "Test only applicable for Django versions < 5.1") + @unittest.skipUnless( + django.VERSION < (5, 1), "Test only applicable for Django versions < 5.1" + ) @override_settings( IMPORT_EXPORT_CELERY_STORAGE="winners.tests.test_fields.FooTestingStorage" ) @@ -25,7 +28,9 @@ def test_old_style(self): del settings.STORAGES self.assertIsInstance(lazy_initialize_storage_class(), FooTestingStorage) - @unittest.skipUnless((4, 2) <= django.VERSION, "Test only applicable for Django 4.2 and later") + @unittest.skipUnless( + (4, 2) <= django.VERSION, "Test only applicable for Django 4.2 and later" + ) @override_settings( IMPORT_EXPORT_CELERY_STORAGE_ALIAS="test_import_export_celery", STORAGES={ @@ -37,14 +42,15 @@ def test_old_style(self): }, "default": { "BACKEND": "django.core.files.storage.FileSystemStorage", - } - } - + }, + }, ) def test_new_style(self): self.assertIsInstance(lazy_initialize_storage_class(), FooTestingStorage) - @unittest.skipUnless((4, 2) <= django.VERSION, "Test only applicable for Django 4.2 and later") + @unittest.skipUnless( + (4, 2) <= django.VERSION, "Test only applicable for Django 4.2 and later" + ) @override_settings( STORAGES={ "staticfiles": { @@ -52,9 +58,9 @@ def test_new_style(self): }, "default": { "BACKEND": "winners.tests.test_fields.FooTestingStorage", - } + }, } ) def test_default_storage(self): - """ Test that "default" storage is used when no alias is provided """ + """Test that "default" storage is used when no alias is provided""" self.assertIsInstance(lazy_initialize_storage_class(), FooTestingStorage) diff --git a/example/winners/tests/test_models.py b/example/winners/tests/test_models.py index cfc2916..5be4ea7 100644 --- a/example/winners/tests/test_models.py +++ b/example/winners/tests/test_models.py @@ -1,6 +1,7 @@ import os -from django.test import TestCase, override_settings + from django.core.files.base import ContentFile +from django.test import TestCase, override_settings from import_export_celery.models.exportjob import ExportJob from import_export_celery.models.importjob import ImportJob @@ -21,16 +22,12 @@ def test_delete_file_on_job_delete(self): class ExportJobTestCases(TestCase): def test_create_export_job_default_email_on_completion(self): - job = ExportJob.objects.create( - app_label="winners", model="Winner" - ) + job = ExportJob.objects.create(app_label="winners", model="Winner") job.refresh_from_db() self.assertTrue(job.email_on_completion) @override_settings(EXPORT_JOB_EMAIL_ON_COMPLETION=False) def test_create_export_job_false_email_on_completion(self): - job = ExportJob.objects.create( - app_label="winners", model="Winner" - ) + job = ExportJob.objects.create(app_label="winners", model="Winner") job.refresh_from_db() self.assertFalse(job.email_on_completion) diff --git a/example/winners/tests/test_utils.py b/example/winners/tests/test_utils.py index cefc09f..fcf671c 100644 --- a/example/winners/tests/test_utils.py +++ b/example/winners/tests/test_utils.py @@ -1,15 +1,15 @@ from django.test import TestCase, override_settings +from import_export_celery.models import ExportJob from import_export_celery.utils import ( - get_export_job_mail_subject, - get_export_job_mail_template, - get_export_job_mail_context, - get_export_job_email_on_completion, DEFAULT_EXPORT_JOB_COMPLETION_MAIL_SUBJECT, DEFAULT_EXPORT_JOB_COMPLETION_MAIL_TEMPLATE, DEFAULT_EXPORT_JOB_EMAIL_ON_COMPLETION, + get_export_job_email_on_completion, + get_export_job_mail_context, + get_export_job_mail_subject, + get_export_job_mail_template, ) -from import_export_celery.models import ExportJob class UtilsTestCases(TestCase): @@ -48,6 +48,9 @@ def test_get_export_job_mail_context(self): expected_context = { "app_label": "winners", "model": "Winner", - "link": f"http://127.0.0.1:8000/adminimport_export_celery/exportjob/{export_job.id}/change/", + "link": ( + f"http://127.0.0.1:8000/admin/import_export_celery/exportjob/" + f"{export_job.id}/change/" + ), } self.assertEqual(context, expected_context) diff --git a/example/winners/urls.py b/example/winners/urls.py index 620353e..a47e154 100644 --- a/example/winners/urls.py +++ b/example/winners/urls.py @@ -13,11 +13,12 @@ 1. Import the include() function: from django.conf.urls import url, include 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) """ + from django.conf import settings from django.conf.urls.static import static from django.contrib import admin from django.urls import path urlpatterns = [ - path("admin", admin.site.urls), + path("admin/", admin.site.urls), ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) diff --git a/import_export_celery/admin_actions.py b/import_export_celery/admin_actions.py index 2c06434..59891a3 100644 --- a/import_export_celery/admin_actions.py +++ b/import_export_celery/admin_actions.py @@ -1,14 +1,13 @@ -from django.utils import timezone import json from uuid import UUID -from django.utils.translation import gettext_lazy as _ -from django.urls import reverse from django.shortcuts import redirect - -from .models import ExportJob +from django.urls import reverse +from django.utils import timezone +from django.utils.translation import gettext_lazy as _ from . import tasks +from .models import ExportJob def run_import_job_action(modeladmin, request, queryset): diff --git a/import_export_celery/fields.py b/import_export_celery/fields.py index 74f8097..efd6138 100644 --- a/import_export_celery/fields.py +++ b/import_export_celery/fields.py @@ -3,20 +3,31 @@ def lazy_initialize_storage_class(): from django.conf import settings + try: from django.core.files.storage import storages + storages_defined = True except ImportError: storages_defined = False - if not hasattr(settings, 'IMPORT_EXPORT_CELERY_STORAGE') and storages_defined: + if not hasattr(settings, "IMPORT_EXPORT_CELERY_STORAGE") and storages_defined: # Use new style storages if defined - storage_alias = getattr(settings, "IMPORT_EXPORT_CELERY_STORAGE_ALIAS", "default") + storage_alias = getattr( + settings, "IMPORT_EXPORT_CELERY_STORAGE_ALIAS", "default" + ) storage_class = storages[storage_alias] else: # Use old style storages if defined from django.core.files.storage import get_storage_class - storage_class = get_storage_class(getattr(settings, "IMPORT_EXPORT_CELERY_STORAGE", "django.core.files.storage.FileSystemStorage")) + + storage_class = get_storage_class( + getattr( + settings, + "IMPORT_EXPORT_CELERY_STORAGE", + "django.core.files.storage.FileSystemStorage", + ) + ) return storage_class() return storage_class diff --git a/import_export_celery/migrations/0001_initial.py b/import_export_celery/migrations/0001_initial.py index 5ed9170..0ce2553 100644 --- a/import_export_celery/migrations/0001_initial.py +++ b/import_export_celery/migrations/0001_initial.py @@ -1,8 +1,8 @@ # Generated by Django 2.0.12 on 2019-06-28 13:55 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/import_export_celery/migrations/0003_exportjob.py b/import_export_celery/migrations/0003_exportjob.py index 13417a0..de3bd13 100644 --- a/import_export_celery/migrations/0003_exportjob.py +++ b/import_export_celery/migrations/0003_exportjob.py @@ -1,8 +1,8 @@ # Generated by Django 2.2.4 on 2019-11-13 11:27 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/import_export_celery/migrations/0010_auto_20231013_0904.py b/import_export_celery/migrations/0010_auto_20231013_0904.py index c4ea04d..700d24b 100644 --- a/import_export_celery/migrations/0010_auto_20231013_0904.py +++ b/import_export_celery/migrations/0010_auto_20231013_0904.py @@ -1,6 +1,7 @@ # Generated by Django 3.2.18 on 2023-10-13 09:04 from django.db import migrations + import import_export_celery.fields diff --git a/import_export_celery/migrations/0011_alter_exportjob_email_on_completion.py b/import_export_celery/migrations/0011_alter_exportjob_email_on_completion.py index 809e71d..67a413c 100644 --- a/import_export_celery/migrations/0011_alter_exportjob_email_on_completion.py +++ b/import_export_celery/migrations/0011_alter_exportjob_email_on_completion.py @@ -1,6 +1,7 @@ # Generated by Django 3.2.25 on 2024-11-22 12:19 from django.db import migrations, models + import import_export_celery.utils diff --git a/import_export_celery/model_config.py b/import_export_celery/model_config.py index b68d336..b06273e 100644 --- a/import_export_celery/model_config.py +++ b/import_export_celery/model_config.py @@ -1,6 +1,6 @@ +from celery.utils.log import get_task_logger from django.apps import apps from import_export.resources import modelresource_factory -from celery.utils.log import get_task_logger log = get_task_logger(__name__) diff --git a/import_export_celery/models/exportjob.py b/import_export_celery/models/exportjob.py index 9876d19..df7b08d 100644 --- a/import_export_celery/models/exportjob.py +++ b/import_export_celery/models/exportjob.py @@ -1,20 +1,17 @@ # Copyright (C) 2019 o.s. Auto*Mat -from django.utils import timezone import json from author.decorators import with_author - from django.contrib.contenttypes.models import ContentType -from django.db import models -from django.db import transaction -from django.dispatch import receiver - +from django.db import models, transaction from django.db.models.signals import post_save +from django.dispatch import receiver +from django.utils import timezone from django.utils.translation import gettext_lazy as _ from ..fields import ImportExportFileField from ..tasks import run_export_job -from ..utils import get_formats, get_export_job_email_on_completion +from ..utils import get_export_job_email_on_completion, get_formats @with_author @@ -126,9 +123,7 @@ def get_resource_choices(self): def get_format_choices(): """returns choices of available export formats""" return [ - (f.CONTENT_TYPE, f().get_title()) - for f in get_formats() - if f().can_export() + (f.CONTENT_TYPE, f().get_title()) for f in get_formats() if f().can_export() ] diff --git a/import_export_celery/models/importjob.py b/import_export_celery/models/importjob.py index 74e545e..b819821 100644 --- a/import_export_celery/models/importjob.py +++ b/import_export_celery/models/importjob.py @@ -1,23 +1,19 @@ # Copyright (C) 2019 o.s. Auto*Mat -from django.conf import settings -from django.utils import timezone +import logging from author.decorators import with_author - +from django.conf import settings from django.db import models, transaction +from django.db.models.signals import post_delete, post_save from django.dispatch import receiver - -from django.db.models.signals import post_save, post_delete +from django.utils import timezone from django.utils.translation import gettext_lazy as _ - from import_export.formats.base_formats import DEFAULT_FORMATS from ..fields import ImportExportFileField from ..tasks import run_import_job -import logging - logger = logging.getLogger(__name__) diff --git a/import_export_celery/tasks.py b/import_export_celery/tasks.py index 2b87759..4c5dff0 100644 --- a/import_export_celery/tasks.py +++ b/import_export_celery/tasks.py @@ -1,22 +1,19 @@ # Author: Timothy Hobbs hobbs.cz> -from django.utils import timezone +import logging import os from celery import shared_task - +from celery.utils.log import get_task_logger from django.conf import settings -from django.core.files.base import ContentFile from django.core.cache import cache - +from django.core.files.base import ContentFile +from django.utils import timezone from django.utils.encoding import force_str from django.utils.translation import gettext_lazy as _ from . import models from .model_config import ModelConfig -from .utils import send_export_job_completion_mail, get_formats - -from celery.utils.log import get_task_logger -import logging +from .utils import get_formats, send_export_job_completion_mail logger = logging.getLogger(__name__) @@ -56,9 +53,7 @@ def _run_import_job(import_job, dry_run=True): data = force_str(data, "utf8") dataset = import_format.create_dataset(data) except UnicodeDecodeError as e: - import_job.errors += ( - _("Imported file has a wrong encoding: %s" % e) + "\n" - ) + import_job.errors += _("Imported file has a wrong encoding: %s" % e) + "\n" change_job_status( import_job, "import", "Imported file has a wrong encoding", dry_run ) @@ -69,9 +64,7 @@ def _run_import_job(import_job, dry_run=True): change_job_status(import_job, "import", "Error reading file", dry_run) import_job.save() return - change_job_status( - import_job, "import", "2/5 Processing import data", dry_run - ) + change_job_status(import_job, "import", "2/5 Processing import data", dry_run) class Resource(model_config.resource): def __init__(self, import_job, *args, **kwargs): @@ -95,9 +88,7 @@ def before_import_row(self, row, **kwargs): skip_diff = resource._meta.skip_diff or resource._meta.skip_html_diff result = resource.import_data(dataset, dry_run=dry_run) - change_job_status( - import_job, "import", "4/5 Generating import summary", dry_run - ) + change_job_status(import_job, "import", "4/5 Generating import summary", dry_run) for error in result.base_errors: import_job.errors += f"\n{error.error}\n{error.traceback}\n" for line, errors in result.row_errors(): @@ -139,9 +130,7 @@ def before_import_row(self, row, **kwargs): + "" ) else: - cols = lambda row: "".join( - [str(field) for field in row.values] - ) + cols = lambda row: "".join([str(field) for field in row.values]) def cols_error(row): if hasattr(row.error, "message_dict"): diff --git a/import_export_celery/utils.py b/import_export_celery/utils.py index cbe05a7..3312722 100644 --- a/import_export_celery/utils.py +++ b/import_export_celery/utils.py @@ -1,15 +1,13 @@ import html2text +from django.conf import settings from django.core.mail import send_mail from django.template.loader import get_template -from django.conf import settings from django.urls import reverse from import_export.formats.base_formats import DEFAULT_FORMATS DEFAULT_EXPORT_JOB_EMAIL_ON_COMPLETION = True DEFAULT_EXPORT_JOB_COMPLETION_MAIL_SUBJECT = "Django: Export job completed" -DEFAULT_EXPORT_JOB_COMPLETION_MAIL_TEMPLATE = ( - "email/export_job_completion.html" -) +DEFAULT_EXPORT_JOB_COMPLETION_MAIL_TEMPLATE = "email/export_job_completion.html" IMPORT_EXPORT_CELERY_EXCLUDED_FORMATS = getattr( settings, "IMPORT_EXPORT_CELERY_EXCLUDED_FORMATS", @@ -86,9 +84,7 @@ def send_export_job_completion_mail(export_job): template_name = get_export_job_mail_template() context = get_export_job_mail_context(export_job) context.update({"export_job": export_job}) - html_message, text_message = build_html_and_text_message( - template_name, context - ) + html_message, text_message = build_html_and_text_message(template_name, context) send_mail( subject=subject, message=text_message, diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d8508a8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,51 @@ +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 88 +target-version = ['py39'] +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | build + | dist +)/ +''' + +[tool.isort] +profile = "black" +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true +line_length = 88 + +[tool.bandit] +exclude_dirs = ["tests", "example/winners/tests"] +skips = ["B101", "B601"] + +[tool.coverage.run] +source = ["import_export_celery"] +omit = [ + "*/migrations/*", + "*/tests/*", + "*/venv/*", + "*/.tox/*", +] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "raise AssertionError", + "raise NotImplementedError", +] \ No newline at end of file diff --git a/setup-dev-env.sh b/setup-dev-env.sh index 58bd72f..9d38490 100755 --- a/setup-dev-env.sh +++ b/setup-dev-env.sh @@ -1,4 +1,27 @@ #!/bin/bash +set -e + +echo "Setting up development environment..." + +# Install the package in development mode +pip install -e . + +# Install additional dependencies +pip install -r requirements_test.txt + +# Install redis client +pip install redis + cd example -poetry install -poetry run python3 manage.py migrate + +# Set database host to postgres when running in Docker +export DATABASE_HOST=postgres + +echo "Running Django migrations..." +python manage.py migrate + +echo "Creating superuser..." +echo "from django.contrib.auth.models import User; User.objects.filter(username='admin').exists() or User.objects.create_superuser('admin', 'admin@example.com', 'admin')" | python manage.py shell + +echo "Development environment setup complete!" +echo "You can now run: python manage.py runserver 0.0.0.0:8000" diff --git a/setup.py b/setup.py index 415e322..863407b 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,13 @@ here = os.path.abspath(os.path.dirname(__file__)) -requires = ["Django", "django-import-export", "django-author", "html2text"] +requires = [ + "Django>=4.2", + "django-import-export>=4.0", + "django-author>=1.2.0", + "html2text>=2020.1.16", + "celery>=5.3.0" +] try: version = ( @@ -44,8 +50,14 @@ "Intended Audience :: Developers", "Environment :: Web Environment", "Framework :: Django", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", + "Framework :: Django :: 4.2", + "Framework :: Django :: 5.0", + "Framework :: Django :: 5.1", + "Framework :: Django :: 5.2", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ], ) diff --git a/tox.ini b/tox.ini index 53ee723..306192a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,23 +1,19 @@ [tox] envlist = - py{36,37,38,39,310}-django32 - py{38,39,310}-django40 - py{38,39,310,311}-django41 - py{38,39,310,311,312}-django42 - py{310,311,312}-django50 - py{310,311,312}-django51 + py{39,310,311,312,313}-django42 + py{310,311,312,313}-django50 + py{310,311,312,313}-django51 + py{311,312,313}-django52 [testenv] deps = -rrequirements_test.txt coverage django-coverage-plugin - django32: django>=3.2,<3.3 - django40: django>=4.0,<4.1 - django41: django>=4.1,<4.2 django42: django>=4.2,<4.3 django50: django>=5.0,<5.1 - django51: django>=5.1a1,<5.2 + django51: django>=5.1,<5.2 + django52: django>=5.2,<5.3 setenv = DATABASE_TYPE=sqlite From 0f16573dd05d897a6e367b09a27432171f59af05 Mon Sep 17 00:00:00 2001 From: matthewhegarty Date: Sat, 20 Sep 2025 17:47:18 +0100 Subject: [PATCH 2/8] Update Docker Compose setup and documentation for improved usability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Update docker-compose.yaml to auto-start Django server and Celery worker - Add automatic migrations on web container startup - Fix port mapping consistency (8000:8000) across all documentation - Update Makefile to start both web and celery services automatically - Improve develop.sh to work alongside auto-started services - Update all README files with current setup instructions - Fix Docker Compose v2 syntax and modernize commands - Add environment variables for proper database connectivity - Include unique Celery node names to avoid duplicate warnings The setup now provides a fully automated development environment where both Django and Celery start automatically after running 'make'. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- Makefile | 16 ++++++----- README.rst | 2 +- develop.sh | 17 +++++++---- docker-compose.yaml | 14 ++++++---- example/README.rst | 28 ++++++++++++++++--- .../migrations/0002_alter_winner_id.py | 18 ++++++++++++ ...2_alter_exportjob_id_alter_importjob_id.py | 23 +++++++++++++++ 7 files changed, 96 insertions(+), 22 deletions(-) create mode 100644 example/winners/migrations/0002_alter_winner_id.py create mode 100644 import_export_celery/migrations/0012_alter_exportjob_id_alter_importjob_id.py diff --git a/Makefile b/Makefile index 4c6673b..fe14693 100644 --- a/Makefile +++ b/Makefile @@ -5,17 +5,19 @@ docker compose: Dockerfile docker compose up -d postgres redis @echo "Waiting for PostgreSQL to be ready..." @sleep 15 - @echo "Starting web container..." - docker compose up -d web + @echo "Starting web and celery containers..." + docker compose up -d web celery @sleep 10 @echo "Running setup script..." docker exec django-import-export-celery-web-1 /proj/setup-dev-env.sh @echo "" - @echo "✅ Setup complete! You can now:" - @echo " ./develop.sh # Enter development environment" - @echo " docker exec -it django-import-export-celery-web-1 bash # Manual container access" - @echo " docker compose up -d celery # Start celery worker" + @echo "✅ Setup complete! Django server and Celery worker are running automatically." + @echo "" + @echo "🌐 Django admin is available at: http://localhost:8000/admin/" @echo "" - @echo "🌐 Django admin will be available at: http://localhost:8001/admin/" + @echo "Optional commands:" + @echo " ./develop.sh # Enter development environment" + @echo " docker exec -it django-import-export-celery-web-1 bash # Manual container access" + @echo " docker compose logs celery # View celery logs" @echo "👤 Login: admin / admin" diff --git a/README.rst b/README.rst index 24a6a8b..5882202 100644 --- a/README.rst +++ b/README.rst @@ -257,7 +257,7 @@ You can enter a preconfigured dev environment by first running `make` and then l Before submitting a PR please run `flake8` and (in the examples directory) `python3 manange.py test`. -Please note, that you need to restart celery for changes to propogate to the workers. Do this with `docker-compose down celery`, `docker-compose up celery`. +Please note, that you need to restart celery for changes to propogate to the workers. Do this with `docker compose restart celery`. Commercial support ------------------ diff --git a/develop.sh b/develop.sh index 4c6b7c0..950ed17 100755 --- a/develop.sh +++ b/develop.sh @@ -1,7 +1,14 @@ #!/bin/bash -docker compose down -docker compose up -d +echo "Starting database and redis services..." +docker compose up -d postgres redis echo "Waiting for services to start..." -sleep 10 -echo "Entering development container..." -exec docker exec -u test -it django-import-export-celery-web-1 bash --init-file "/proj/dev-entrypoint.sh" +sleep 5 +echo "Entering development container (without auto-starting web server)..." +exec docker run --rm -it \ + --network django-import-export-celery_default \ + -v ./:/proj/ \ + -v ./pyenv:/home/test \ + -w /proj/ \ + -u test \ + -e DATABASE_HOST=postgres \ + django-import-export-celery-web bash --init-file "/proj/dev-entrypoint.sh" diff --git a/docker-compose.yaml b/docker-compose.yaml index 229edc1..d955d30 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,12 +1,14 @@ services: web: build: . - entrypoint: /bin/bash + entrypoint: sh -c "pip install -e /proj && cd example && python manage.py migrate && python manage.py runserver 0.0.0.0:8000" ports: - - "8001:8000" - links: + - "8000:8000" + depends_on: - postgres - redis + environment: + DATABASE_HOST: postgres tty: true stdin_open: true working_dir: /proj/ @@ -16,13 +18,15 @@ services: - ./pyenv:/home/test celery: build: . - entrypoint: sh -c "pip install -e /proj && pip install redis && celery -A project.celery worker -l info" + entrypoint: sh -c "pip install -e /proj && pip install redis && cd example && celery -A project worker --loglevel=info -n worker1" depends_on: - postgres - redis + environment: + DATABASE_HOST: postgres tty: true stdin_open: true - working_dir: /proj/example + working_dir: /proj/ user: test volumes: - ./:/proj/ diff --git a/example/README.rst b/example/README.rst index 4efee05..118b0ad 100644 --- a/example/README.rst +++ b/example/README.rst @@ -18,21 +18,41 @@ Or manual setup: Run === -Enter the development container: +The Django server starts automatically after setup. If you need to restart it: + +.. code-block:: bash + + docker compose restart web + +For debugging or manual control, enter the development container: .. code-block:: bash docker exec -it django-import-export-celery-web-1 bash -Run the Django server: +Start Celery worker: + +**Option 1: Using docker compose (recommended):** + +.. code-block:: bash + + docker compose up -d celery + +**Option 2: Manual startup (for debugging):** + +.. code-block:: bash + + docker exec -it django-import-export-celery-web-1 bash .. code-block:: bash cd example export DATABASE_HOST=postgres - python manage.py runserver 0.0.0.0:8000 + celery -A project worker --loglevel=info -n worker1 + +The example app will be available from http://localhost:8000/admin/ -The example app will be available from http://localhost:8001/admin +**Note:** Both Django and Celery need to be running for import/export jobs to work properly. **Login credentials:** - Username: admin diff --git a/example/winners/migrations/0002_alter_winner_id.py b/example/winners/migrations/0002_alter_winner_id.py new file mode 100644 index 0000000..7aeac56 --- /dev/null +++ b/example/winners/migrations/0002_alter_winner_id.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.6 on 2025-09-20 16:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('winners', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='winner', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ] diff --git a/import_export_celery/migrations/0012_alter_exportjob_id_alter_importjob_id.py b/import_export_celery/migrations/0012_alter_exportjob_id_alter_importjob_id.py new file mode 100644 index 0000000..9eb3e9d --- /dev/null +++ b/import_export_celery/migrations/0012_alter_exportjob_id_alter_importjob_id.py @@ -0,0 +1,23 @@ +# Generated by Django 5.2.6 on 2025-09-20 16:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('import_export_celery', '0011_alter_exportjob_email_on_completion'), + ] + + operations = [ + migrations.AlterField( + model_name='exportjob', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='importjob', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ] From a1e51d72d8655e70198a3ce62efafe3781a72691 Mon Sep 17 00:00:00 2001 From: matthewhegarty Date: Sat, 20 Sep 2025 18:21:28 +0100 Subject: [PATCH 3/8] removed out of date configuration for django STORAGES --- README.rst | 11 ++-------- example/project/settings.py | 7 +------ example/winners/tests/test_fields.py | 20 +++++++++--------- import_export_celery/fields.py | 31 ++++++---------------------- 4 files changed, 19 insertions(+), 50 deletions(-) diff --git a/README.rst b/README.rst index 5882202..717aaa7 100644 --- a/README.rst +++ b/README.rst @@ -173,7 +173,7 @@ To exclude or disable file formats from the admin site, configure `IMPORT_EXPORT Customizing File Storage Backend -------------------------------- -**If you are using the new Django 4.2 STORAGES**: +**If you are using the Django 4.2+ STORAGES**: By default, `import_export_celery` uses Django `default` storage. To use your own storage, use the the `IMPORT_EXPORT_CELERY_STORAGE_ALIAS` variable in your Django settings and adding the STORAGES definition. @@ -188,20 +188,13 @@ For instance: } IMPORT_EXPORT_CELERY_STORAGE_ALIAS = 'import_export_celery' -**DEPRECATED: If you are using old style storages**: - -Define a custom storage backend by adding the `IMPORT_EXPORT_CELERY_STORAGE` to your Django settings. For instance: - - :: - - IMPORT_EXPORT_CELERY_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" - Customizing Task Time Limits ---------------------------- By default, there is no time limit on celery import/export tasks. This can be customized by setting the following variables in your Django settings file. + :: # set import time limits (in seconds) diff --git a/example/project/settings.py b/example/project/settings.py index 475ed94..2527c30 100644 --- a/example/project/settings.py +++ b/example/project/settings.py @@ -74,10 +74,7 @@ WSGI_APPLICATION = "winners.wsgi.application" -BROKER_URL = os.environ.get("REDIS_URL", "redis://redis") -REDIS_URL = os.environ.get("REDIS_URL", "redis://redis") - -# Modern Celery configuration +# Celery configuration CELERY_BROKER_URL = os.environ.get("REDIS_URL", "redis://redis") CELERY_RESULT_BACKEND = os.environ.get("REDIS_URL", "redis://redis") # Database @@ -135,8 +132,6 @@ USE_I18N = True -USE_L10N = True - USE_TZ = True # Fix auto-created primary key warnings diff --git a/example/winners/tests/test_fields.py b/example/winners/tests/test_fields.py index 696435e..7702a4b 100644 --- a/example/winners/tests/test_fields.py +++ b/example/winners/tests/test_fields.py @@ -17,16 +17,16 @@ class InitializeStorageClassTests(TestCase): def test_default(self): self.assertIsInstance(lazy_initialize_storage_class(), FileSystemStorage) - @unittest.skipUnless( - django.VERSION < (5, 1), "Test only applicable for Django versions < 5.1" - ) - @override_settings( - IMPORT_EXPORT_CELERY_STORAGE="winners.tests.test_fields.FooTestingStorage" - ) - def test_old_style(self): - del settings.IMPORT_EXPORT_CELERY_STORAGE_ALIAS - del settings.STORAGES - self.assertIsInstance(lazy_initialize_storage_class(), FooTestingStorage) + # @unittest.skipUnless( + # django.VERSION < (5, 1), "Test only applicable for Django versions < 5.1" + # ) + # @override_settings( + # IMPORT_EXPORT_CELERY_STORAGE="winners.tests.test_fields.FooTestingStorage" + # ) + # def test_old_style(self): + # del settings.IMPORT_EXPORT_CELERY_STORAGE_ALIAS + # del settings.STORAGES + # self.assertIsInstance(lazy_initialize_storage_class(), FooTestingStorage) @unittest.skipUnless( (4, 2) <= django.VERSION, "Test only applicable for Django 4.2 and later" diff --git a/import_export_celery/fields.py b/import_export_celery/fields.py index efd6138..9dc840c 100644 --- a/import_export_celery/fields.py +++ b/import_export_celery/fields.py @@ -3,34 +3,15 @@ def lazy_initialize_storage_class(): from django.conf import settings + from django.core.files.storage import storages - try: - from django.core.files.storage import storages + if hasattr(settings, "IMPORT_EXPORT_CELERY_STORAGE_ALIAS"): + # Use the alias configured in STORAGES + return storages[settings.IMPORT_EXPORT_CELERY_STORAGE_ALIAS] - storages_defined = True - except ImportError: - storages_defined = False + # Otherwise, just use the default storage + return storages["default"] - if not hasattr(settings, "IMPORT_EXPORT_CELERY_STORAGE") and storages_defined: - # Use new style storages if defined - storage_alias = getattr( - settings, "IMPORT_EXPORT_CELERY_STORAGE_ALIAS", "default" - ) - storage_class = storages[storage_alias] - else: - # Use old style storages if defined - from django.core.files.storage import get_storage_class - - storage_class = get_storage_class( - getattr( - settings, - "IMPORT_EXPORT_CELERY_STORAGE", - "django.core.files.storage.FileSystemStorage", - ) - ) - return storage_class() - - return storage_class class ImportExportFileField(models.FileField): From 36bc37c5a807c9f2fcef0d8f92277d7770d6a6fb Mon Sep 17 00:00:00 2001 From: matthewhegarty Date: Sat, 20 Sep 2025 20:06:58 +0100 Subject: [PATCH 4/8] added further changes to resolve tox issue --- Makefile | 2 +- docker-compose.yaml | 19 +++++++++++++------ example/project/celery.py | 2 +- example/project/settings.py | 6 +++--- setup-dev-env.sh | 1 - tox.ini | 1 + 6 files changed, 19 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index fe14693..d3f0e41 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ docker compose: Dockerfile docker compose build --build-arg UID=$(shell id -u) docker compose up -d postgres redis @echo "Waiting for PostgreSQL to be ready..." - @sleep 15 + @sleep 10 @echo "Starting web and celery containers..." docker compose up -d web celery @sleep 10 diff --git a/docker-compose.yaml b/docker-compose.yaml index d955d30..89f0722 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -9,6 +9,7 @@ services: - redis environment: DATABASE_HOST: postgres + REDIS_URL: redis://redis:6379/0 tty: true stdin_open: true working_dir: /proj/ @@ -18,19 +19,25 @@ services: - ./pyenv:/home/test celery: build: . - entrypoint: sh -c "pip install -e /proj && pip install redis && cd example && celery -A project worker --loglevel=info -n worker1" + entrypoint: > + sh -c "pip install -e /proj && + pip install redis && + cd example && + celery -A project worker --loglevel=info -n worker1" depends_on: - - postgres - - redis + - postgres + - redis environment: - DATABASE_HOST: postgres + DATABASE_HOST: postgres + REDIS_URL: redis://redis:6379/0 tty: true stdin_open: true working_dir: /proj/ user: test volumes: - - ./:/proj/ - - ./pyenv:/home/test + - ./:/proj/ + - ./pyenv:/home/test + redis: image: redis postgres: diff --git a/example/project/celery.py b/example/project/celery.py index 41e393c..d47c356 100644 --- a/example/project/celery.py +++ b/example/project/celery.py @@ -15,5 +15,5 @@ # Using a string here means the worker will not have to # pickle the object when using Windows. -app.config_from_object("django.conf:settings") +app.config_from_object("django.conf:settings", namespace="CELERY") app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) diff --git a/example/project/settings.py b/example/project/settings.py index 2527c30..4aae1d2 100644 --- a/example/project/settings.py +++ b/example/project/settings.py @@ -15,7 +15,6 @@ # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ @@ -75,8 +74,9 @@ WSGI_APPLICATION = "winners.wsgi.application" # Celery configuration -CELERY_BROKER_URL = os.environ.get("REDIS_URL", "redis://redis") -CELERY_RESULT_BACKEND = os.environ.get("REDIS_URL", "redis://redis") +CELERY_BROKER_URL = os.environ.get("REDIS_URL", "redis://redis:6379/0") +CELERY_RESULT_BACKEND = os.environ.get("REDIS_URL", "redis://redis:6379/0") + # Database # https://docs.djangoproject.com/en/1.10/ref/settings/#databases diff --git a/setup-dev-env.sh b/setup-dev-env.sh index 9d38490..d4b7b94 100755 --- a/setup-dev-env.sh +++ b/setup-dev-env.sh @@ -24,4 +24,3 @@ echo "Creating superuser..." echo "from django.contrib.auth.models import User; User.objects.filter(username='admin').exists() or User.objects.create_superuser('admin', 'admin@example.com', 'admin')" | python manage.py shell echo "Development environment setup complete!" -echo "You can now run: python manage.py runserver 0.0.0.0:8000" diff --git a/tox.ini b/tox.ini index 306192a..4f6e980 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,7 @@ envlist = py{311,312,313}-django52 [testenv] +package = sdist deps = -rrequirements_test.txt coverage From c884aa59ac83d524b862469d96a7572ef143df07 Mon Sep 17 00:00:00 2001 From: matthewhegarty Date: Sat, 20 Sep 2025 20:14:29 +0100 Subject: [PATCH 5/8] fix dynamic version to resolve tox issue --- setup.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 863407b..579e214 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,13 @@ .strip() ) except subprocess.CalledProcessError: - version = "0.dev" + datetime.datetime.now().strftime("%Y%m%d%H%M%S") + # Use git commit hash for stable dev versions to avoid timestamp issues + try: + commit_hash = subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).decode("utf-8").strip() + version = f"0.dev+{commit_hash}" + except subprocess.CalledProcessError: + # Static fallback version for tox compatibility + version = "0.dev" setup( name="django-import-export-celery", From 24efc990bf640f1794ced773c67075ebd11e060c Mon Sep 17 00:00:00 2001 From: matthewhegarty Date: Sat, 20 Sep 2025 20:19:33 +0100 Subject: [PATCH 6/8] removed test related to pre django 4.2 --- example/winners/tests/test_fields.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/example/winners/tests/test_fields.py b/example/winners/tests/test_fields.py index 7702a4b..8c790ed 100644 --- a/example/winners/tests/test_fields.py +++ b/example/winners/tests/test_fields.py @@ -17,17 +17,6 @@ class InitializeStorageClassTests(TestCase): def test_default(self): self.assertIsInstance(lazy_initialize_storage_class(), FileSystemStorage) - # @unittest.skipUnless( - # django.VERSION < (5, 1), "Test only applicable for Django versions < 5.1" - # ) - # @override_settings( - # IMPORT_EXPORT_CELERY_STORAGE="winners.tests.test_fields.FooTestingStorage" - # ) - # def test_old_style(self): - # del settings.IMPORT_EXPORT_CELERY_STORAGE_ALIAS - # del settings.STORAGES - # self.assertIsInstance(lazy_initialize_storage_class(), FooTestingStorage) - @unittest.skipUnless( (4, 2) <= django.VERSION, "Test only applicable for Django 4.2 and later" ) From 512a0bcc087ec40a1c7fff068c6c473087be0900 Mon Sep 17 00:00:00 2001 From: matthewhegarty Date: Sun, 19 Oct 2025 18:01:40 +0100 Subject: [PATCH 7/8] reformatting and README update --- .dockerignore | 2 +- .pre-commit-config.yaml | 7 --- Makefile | 4 ++ README.rst | 34 +++++++++++++- .../migrations/0002_alter_winner_id.py | 10 ++-- import_export_celery/fields.py | 1 - ...8_alter_exportjob_id_alter_importjob_id.py | 18 +++++--- ...ptions_alter_importjob_options_and_more.py | 46 ++++++++++++------- .../migrations/0010_auto_20231013_0904.py | 39 ++++++++++++---- ...011_alter_exportjob_email_on_completion.py | 11 +++-- ...2_alter_exportjob_id_alter_importjob_id.py | 18 +++++--- import_export_celery/models/importjob.py | 4 +- pyproject.toml | 2 +- setup.py | 16 ++++--- 14 files changed, 141 insertions(+), 71 deletions(-) diff --git a/.dockerignore b/.dockerignore index 44a7c08..efa67b5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -65,4 +65,4 @@ Thumbs.db db/ pyenv/ CLAUDE.md -screenshots/ \ No newline at end of file +screenshots/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e76c301..975a5ec 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,10 +40,3 @@ repos: - flake8-comprehensions - flake8-tidy-imports - flake8-typing-imports - -- repo: https://github.com/PyCQA/bandit - rev: 1.7.10 - hooks: - - id: bandit - args: ["-c", "pyproject.toml"] - additional_dependencies: ["bandit[toml]"] diff --git a/Makefile b/Makefile index d3f0e41..b4f1592 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +RUN_TEST_COMMAND=PYTHONPATH=".:example:${PYTHONPATH}" python -W error -m django test example --settings=project.settings + docker compose: Dockerfile mkdir -p pyenv mkdir -p db @@ -21,3 +23,5 @@ docker compose: Dockerfile @echo " docker compose logs celery # View celery logs" @echo "👤 Login: admin / admin" +test: ## run tests with the default Python + $(RUN_TEST_COMMAND) diff --git a/README.rst b/README.rst index 717aaa7..f562b22 100644 --- a/README.rst +++ b/README.rst @@ -248,9 +248,39 @@ For developers of this library You can enter a preconfigured dev environment by first running `make` and then launching `./develop.sh` to get into a docker compose environment packed with **redis**, **celery**, **postgres** and everything you need to run and test django-import-export-celery. -Before submitting a PR please run `flake8` and (in the examples directory) `python3 manange.py test`. +Before submitting a PR please run `precommit` and ensure tests pass (see below). -Please note, that you need to restart celery for changes to propogate to the workers. Do this with `docker compose restart celery`. +Please note, that you need to restart celery for changes to propagate to the workers. Do this with `docker compose restart celery`. + +.. _create_venv: + +Create virtual environment +-------------------------- + +Once you have cloned and checked out the repository, you can install a new development environment as follows:: + + python -m venv .venv + source .venv/bin/activate + python -m pip install '.[dev]' + pip install psycopg2-binary django-admin-smoke-tests + +Run tests +--------- + +You can run the test suite with:: + + make test + +Formatting +---------- + +To install pre-commit:: + + python -m pip install pre-commit + +Then run:: + + pre-commit install Commercial support ------------------ diff --git a/example/winners/migrations/0002_alter_winner_id.py b/example/winners/migrations/0002_alter_winner_id.py index 7aeac56..2316c47 100644 --- a/example/winners/migrations/0002_alter_winner_id.py +++ b/example/winners/migrations/0002_alter_winner_id.py @@ -6,13 +6,15 @@ class Migration(migrations.Migration): dependencies = [ - ('winners', '0001_initial'), + ("winners", "0001_initial"), ] operations = [ migrations.AlterField( - model_name='winner', - name='id', - field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + model_name="winner", + name="id", + field=models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), ), ] diff --git a/import_export_celery/fields.py b/import_export_celery/fields.py index 9dc840c..6ef16c6 100644 --- a/import_export_celery/fields.py +++ b/import_export_celery/fields.py @@ -13,7 +13,6 @@ def lazy_initialize_storage_class(): return storages["default"] - class ImportExportFileField(models.FileField): def __init__(self, *args, **kwargs): kwargs["storage"] = lazy_initialize_storage_class diff --git a/import_export_celery/migrations/0008_alter_exportjob_id_alter_importjob_id.py b/import_export_celery/migrations/0008_alter_exportjob_id_alter_importjob_id.py index d8eab09..7cecf8a 100644 --- a/import_export_celery/migrations/0008_alter_exportjob_id_alter_importjob_id.py +++ b/import_export_celery/migrations/0008_alter_exportjob_id_alter_importjob_id.py @@ -6,18 +6,22 @@ class Migration(migrations.Migration): dependencies = [ - ('import_export_celery', '0007_auto_20210210_1831'), + ("import_export_celery", "0007_auto_20210210_1831"), ] operations = [ migrations.AlterField( - model_name='exportjob', - name='id', - field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + model_name="exportjob", + name="id", + field=models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), ), migrations.AlterField( - model_name='importjob', - name='id', - field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + model_name="importjob", + name="id", + field=models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), ), ] diff --git a/import_export_celery/migrations/0009_alter_exportjob_options_alter_importjob_options_and_more.py b/import_export_celery/migrations/0009_alter_exportjob_options_alter_importjob_options_and_more.py index 317cb44..150f656 100644 --- a/import_export_celery/migrations/0009_alter_exportjob_options_alter_importjob_options_and_more.py +++ b/import_export_celery/migrations/0009_alter_exportjob_options_alter_importjob_options_and_more.py @@ -6,36 +6,48 @@ class Migration(migrations.Migration): dependencies = [ - ('import_export_celery', '0008_alter_exportjob_id_alter_importjob_id'), + ("import_export_celery", "0008_alter_exportjob_id_alter_importjob_id"), ] operations = [ migrations.AlterModelOptions( - name='exportjob', - options={'verbose_name': 'Export job', 'verbose_name_plural': 'Export jobs'}, + name="exportjob", + options={ + "verbose_name": "Export job", + "verbose_name_plural": "Export jobs", + }, ), migrations.AlterModelOptions( - name='importjob', - options={'verbose_name': 'Import job', 'verbose_name_plural': 'Import jobs'}, + name="importjob", + options={ + "verbose_name": "Import job", + "verbose_name_plural": "Import jobs", + }, ), migrations.AlterField( - model_name='exportjob', - name='id', - field=models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + model_name="exportjob", + name="id", + field=models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), ), migrations.AlterField( - model_name='exportjob', - name='site_of_origin', - field=models.TextField(default='', max_length=255, verbose_name='Site of origin'), + model_name="exportjob", + name="site_of_origin", + field=models.TextField( + default="", max_length=255, verbose_name="Site of origin" + ), ), migrations.AlterField( - model_name='importjob', - name='errors', - field=models.TextField(blank=True, default='', verbose_name='Errors'), + model_name="importjob", + name="errors", + field=models.TextField(blank=True, default="", verbose_name="Errors"), ), migrations.AlterField( - model_name='importjob', - name='id', - field=models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + model_name="importjob", + name="id", + field=models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), ), ] diff --git a/import_export_celery/migrations/0010_auto_20231013_0904.py b/import_export_celery/migrations/0010_auto_20231013_0904.py index 700d24b..88dd68b 100644 --- a/import_export_celery/migrations/0010_auto_20231013_0904.py +++ b/import_export_celery/migrations/0010_auto_20231013_0904.py @@ -8,23 +8,42 @@ class Migration(migrations.Migration): dependencies = [ - ('import_export_celery', '0009_alter_exportjob_options_alter_importjob_options_and_more'), + ( + "import_export_celery", + "0009_alter_exportjob_options_alter_importjob_options_and_more", + ), ] operations = [ migrations.AlterField( - model_name='exportjob', - name='file', - field=import_export_celery.fields.ImportExportFileField(max_length=255, storage=import_export_celery.fields.lazy_initialize_storage_class, upload_to='django-import-export-celery-export-jobs', verbose_name='exported file'), + model_name="exportjob", + name="file", + field=import_export_celery.fields.ImportExportFileField( + max_length=255, + storage=import_export_celery.fields.lazy_initialize_storage_class, + upload_to="django-import-export-celery-export-jobs", + verbose_name="exported file", + ), ), migrations.AlterField( - model_name='importjob', - name='change_summary', - field=import_export_celery.fields.ImportExportFileField(blank=True, null=True, storage=import_export_celery.fields.lazy_initialize_storage_class, upload_to='django-import-export-celery-import-change-summaries', verbose_name='Summary of changes made by this import'), + model_name="importjob", + name="change_summary", + field=import_export_celery.fields.ImportExportFileField( + blank=True, + null=True, + storage=import_export_celery.fields.lazy_initialize_storage_class, + upload_to="django-import-export-celery-import-change-summaries", + verbose_name="Summary of changes made by this import", + ), ), migrations.AlterField( - model_name='importjob', - name='file', - field=import_export_celery.fields.ImportExportFileField(max_length=255, storage=import_export_celery.fields.lazy_initialize_storage_class, upload_to='django-import-export-celery-import-jobs', verbose_name='File to be imported'), + model_name="importjob", + name="file", + field=import_export_celery.fields.ImportExportFileField( + max_length=255, + storage=import_export_celery.fields.lazy_initialize_storage_class, + upload_to="django-import-export-celery-import-jobs", + verbose_name="File to be imported", + ), ), ] diff --git a/import_export_celery/migrations/0011_alter_exportjob_email_on_completion.py b/import_export_celery/migrations/0011_alter_exportjob_email_on_completion.py index 67a413c..b01b235 100644 --- a/import_export_celery/migrations/0011_alter_exportjob_email_on_completion.py +++ b/import_export_celery/migrations/0011_alter_exportjob_email_on_completion.py @@ -8,13 +8,16 @@ class Migration(migrations.Migration): dependencies = [ - ('import_export_celery', '0010_auto_20231013_0904'), + ("import_export_celery", "0010_auto_20231013_0904"), ] operations = [ migrations.AlterField( - model_name='exportjob', - name='email_on_completion', - field=models.BooleanField(default=import_export_celery.utils.get_export_job_email_on_completion, verbose_name='Send me an email when this export job is complete'), + model_name="exportjob", + name="email_on_completion", + field=models.BooleanField( + default=import_export_celery.utils.get_export_job_email_on_completion, + verbose_name="Send me an email when this export job is complete", + ), ), ] diff --git a/import_export_celery/migrations/0012_alter_exportjob_id_alter_importjob_id.py b/import_export_celery/migrations/0012_alter_exportjob_id_alter_importjob_id.py index 48d143c..ea00b88 100644 --- a/import_export_celery/migrations/0012_alter_exportjob_id_alter_importjob_id.py +++ b/import_export_celery/migrations/0012_alter_exportjob_id_alter_importjob_id.py @@ -6,18 +6,22 @@ class Migration(migrations.Migration): dependencies = [ - ('import_export_celery', '0011_alter_exportjob_email_on_completion'), + ("import_export_celery", "0011_alter_exportjob_email_on_completion"), ] operations = [ migrations.AlterField( - model_name='exportjob', - name='id', - field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + model_name="exportjob", + name="id", + field=models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), ), migrations.AlterField( - model_name='importjob', - name='id', - field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + model_name="importjob", + name="id", + field=models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), ), ] diff --git a/import_export_celery/models/importjob.py b/import_export_celery/models/importjob.py index b819821..edfc5e5 100644 --- a/import_export_celery/models/importjob.py +++ b/import_export_celery/models/importjob.py @@ -106,7 +106,5 @@ def auto_delete_file_on_delete(sender, instance, **kwargs): try: instance.file.delete() except Exception as e: - logger.error( - "Some error occurred while deleting ImportJob file: {0}".format(e) - ) + logger.error(f"Some error occurred while deleting ImportJob file: {e}") ImportJob.objects.filter(id=instance.id).delete() diff --git a/pyproject.toml b/pyproject.toml index d8508a8..fc65f87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,4 +48,4 @@ exclude_lines = [ "def __repr__", "raise AssertionError", "raise NotImplementedError", -] \ No newline at end of file +] diff --git a/setup.py b/setup.py index 579e214..15f135b 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ import codecs import os -from setuptools import setup, find_packages import subprocess -import datetime + +from setuptools import find_packages, setup here = os.path.abspath(os.path.dirname(__file__)) @@ -11,7 +11,7 @@ "django-import-export>=4.0", "django-author>=1.2.0", "html2text>=2020.1.16", - "celery>=5.3.0" + "celery>=5.3.0", ] try: @@ -23,7 +23,11 @@ except subprocess.CalledProcessError: # Use git commit hash for stable dev versions to avoid timestamp issues try: - commit_hash = subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).decode("utf-8").strip() + commit_hash = ( + subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]) + .decode("utf-8") + .strip() + ) version = f"0.dev+{commit_hash}" except subprocess.CalledProcessError: # Static fallback version for tox compatibility @@ -37,9 +41,7 @@ url="https://github.com/auto-mat/django-import-export-celery", download_url="http://pypi.python.org/pypi/django-import-export-celery/", description="Process long running django imports and exports in celery", - long_description=codecs.open( - os.path.join(here, "README.rst"), "r", "utf-8" - ).read(), + long_description=codecs.open(os.path.join(here, "README.rst"), "r", "utf-8").read(), long_description_content_type="text/x-rst", license=( "License :: OSI Approved :: GNU Lesser General Public License v3.0 or" From b0074d2f6972ced6f2b9c2e89d4f78f9560c19e5 Mon Sep 17 00:00:00 2001 From: matthewhegarty Date: Sun, 19 Oct 2025 18:10:01 +0100 Subject: [PATCH 8/8] updated readme --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index f562b22..58a8b99 100644 --- a/README.rst +++ b/README.rst @@ -269,6 +269,7 @@ Run tests You can run the test suite with:: + make # wait for docker to start make test Formatting