Modernize dependencies, tooling, and development environment

.dockerignore

@@ -0,0 +1,68 @@
+# Git
+.git
+.gitignore
+
+# Python
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+.Python
+build
+develop-eggs
+dist
+downloads
+eggs
+.eggs
+lib
+lib64
+parts
+sdist
+var
+wheels
+*.egg-info
+.installed.cfg
+*.egg
+
+# Virtual environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# Testing
+.tox/
+.coverage
+htmlcov/
+.pytest_cache/
+.mypy_cache/
+coverage.xml
+*.cover
+.hypothesis/
+
+# Documentation
+docs/_build/
+
+# OS
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# Project specific
+db/
+pyenv/
+CLAUDE.md
+screenshots/

.pre-commit-config.yaml

@@ -1,29 +1,38 @@
 default_language_version:
-  python: python3.7
+  python: python3.11
 
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.1.0
+  rev: v5.0.0
   hooks:
   - id: check-added-large-files
   - id: check-case-conflict
   - id: check-merge-conflict
   - id: end-of-file-fixer
   - id: trailing-whitespace
+  - id: check-yaml
+  - id: check-toml
+  - id: check-json
 
 - repo: https://github.com/asottile/pyupgrade
-  rev: v2.31.1
+  rev: v3.17.0
   hooks:
   - id: pyupgrade
-    args: [--py37-plus]
+    args: [--py39-plus]
 
 - repo: https://github.com/psf/black
-  rev: 22.3.0
+  rev: 24.10.0
   hooks:
   - id: black
 
+- repo: https://github.com/PyCQA/isort
+  rev: 5.13.2
+  hooks:
+  - id: isort
+    args: ["--profile", "black"]
+
 - repo: https://github.com/PyCQA/flake8
-  rev: 4.0.1
+  rev: 7.1.1
   hooks:
   - id: flake8
     additional_dependencies:

Dockerfile

@@ -1,6 +1,38 @@
-FROM python:3.7
-RUN pip3 install poetry celery
-RUN apt-get update ; apt-get install -yq python3-psycopg2 gdal-bin
-ARG UID
-RUN useradd test --uid $UID
-RUN chsh test -s /bin/bash
+# Multi-stage build for smaller images
+FROM python:3.11-slim AS base
+
+# Set environment variables
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONPATH=/proj \
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    libpq-dev \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create user
+ARG UID=1000
+RUN useradd test --uid $UID --create-home --shell /bin/bash
+
+# Install poetry
+RUN pip install poetry==1.7.1
+
+# Set working directory
+WORKDIR /proj
+
+# Copy dependency files
+COPY requirements_test.txt ./
+
+# Install dependencies
+RUN pip install -r requirements_test.txt
+
+# Install redis for celery
+RUN pip install redis
+
+# Switch to non-root user
+USER test

Makefile

@@ -1,8 +1,27 @@
-docker-compose: Dockerfile
+RUN_TEST_COMMAND=PYTHONPATH=".:example:${PYTHONPATH}" python -W error -m django test example --settings=project.settings
+
+docker compose: Dockerfile
 	mkdir -p pyenv
 	mkdir -p db
-	sudo docker-compose build --build-arg UID=$(shell id -u)
-	sudo docker-compose up -d web postgres
-	sudo docker exec -it django-import-export-celery_web_1 /proj/setup-dev-env.sh
-	sudo docker-compose down
+	docker compose build --build-arg UID=$(shell id -u)
+	docker compose up -d postgres redis
+	@echo "Waiting for PostgreSQL to be ready..."
+	@sleep 10
+	@echo "Starting web and celery containers..."
+	docker compose up -d web celery
+	@sleep 10
+	@echo "Running setup script..."
+	docker exec django-import-export-celery-web-1 /proj/setup-dev-env.sh
+	@echo ""
+	@echo "✅ Setup complete! Django server and Celery worker are running automatically."
+	@echo ""
+	@echo "🌐 Django admin is available at: http://localhost:8000/admin/"
+	@echo ""
+	@echo "Optional commands:"
+	@echo "  ./develop.sh                                          # Enter development environment"
+	@echo "  docker exec -it django-import-export-celery-web-1 bash # Manual container access"
+	@echo "  docker compose logs celery                            # View celery logs"
+	@echo "👤 Login: admin / admin"
 
+test: ## run tests with the default Python
+	$(RUN_TEST_COMMAND)

README.rst

@@ -173,7 +173,7 @@ To exclude or disable file formats from the admin site, configure `IMPORT_EXPORT
 Customizing File Storage Backend
 --------------------------------
 
-**If you are using the new Django 4.2 STORAGES**:
+**If you are using the Django 4.2+ STORAGES**:
 
 By default, `import_export_celery` uses Django `default` storage.
 To use your own storage, use the the `IMPORT_EXPORT_CELERY_STORAGE_ALIAS` variable in your Django settings and adding the STORAGES definition.
@@ -188,20 +188,13 @@ For instance:
         }
         IMPORT_EXPORT_CELERY_STORAGE_ALIAS = 'import_export_celery'
 
-**DEPRECATED: If you are using old style storages**:
-
-Define a custom storage backend by adding the `IMPORT_EXPORT_CELERY_STORAGE` to your Django settings. For instance:
-
-    ::
-
-        IMPORT_EXPORT_CELERY_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
-
 
 Customizing Task Time Limits
 ----------------------------
 
 By default, there is no time limit on celery import/export tasks. This can be customized by setting the following variables in your Django settings file.
 
+
     ::
 
         # set import time limits (in seconds)
@@ -255,9 +248,40 @@ For developers of this library
 
 You can enter a preconfigured dev environment by first running `make` and then launching `./develop.sh` to get into a docker compose environment packed with **redis**, **celery**, **postgres** and everything you need to run and test django-import-export-celery.
 
-Before submitting a PR please run `flake8` and (in the examples directory) `python3 manange.py test`.
+Before submitting a PR please run `precommit` and ensure tests pass (see below).
+
+Please note, that you need to restart celery for changes to propagate to the workers. Do this with `docker compose restart celery`.
+
+.. _create_venv:
+
+Create virtual environment
+--------------------------
+
+Once you have cloned and checked out the repository, you can install a new development environment as follows::
+
+  python -m venv .venv
+  source .venv/bin/activate
+  python -m pip install '.[dev]'
+  pip install psycopg2-binary django-admin-smoke-tests
+
+Run tests
+---------
+
+You can run the test suite with::
+
+  make # wait for docker to start
+  make test
+
+Formatting
+----------
+
+To install pre-commit::
+
+  python -m pip install pre-commit
+
+Then run::
 
-Please note, that you need to restart celery for changes to propogate to the workers. Do this with `docker-compose down celery`, `docker-compose up celery`.
+  pre-commit install
 
 Commercial support
 ------------------

develop.sh

@@ -1,4 +1,14 @@
 #!/bin/bash
-docker-compose down
-docker-compose up -d
-exec docker exec -u test -it django-import-export-celery_web_1 bash --init-file "/proj/dev-entrypoint.sh"
+echo "Starting database and redis services..."
+docker compose up -d postgres redis
+echo "Waiting for services to start..."
+sleep 5
+echo "Entering development container (without auto-starting web server)..."
+exec docker run --rm -it \
+  --network django-import-export-celery_default \
+  -v ./:/proj/ \
+  -v ./pyenv:/home/test \
+  -w /proj/ \
+  -u test \
+  -e DATABASE_HOST=postgres \
+  django-import-export-celery-web bash --init-file "/proj/dev-entrypoint.sh"

docker-compose.yaml

@@ -1,13 +1,15 @@
-version: '2'
 services:
  web:
   build: .
-  entrypoint: /bin/bash
+  entrypoint: sh -c "pip install -e /proj && cd example && python manage.py migrate && python manage.py runserver 0.0.0.0:8000"
   ports:
-   - "8001:8000"
-  links:
+   - "8000:8000"
+  depends_on:
    - postgres
    - redis
+  environment:
+   DATABASE_HOST: postgres
+   REDIS_URL: redis://redis:6379/0
   tty: true
   stdin_open: true
   working_dir: /proj/
@@ -17,24 +19,35 @@ services:
    - ./pyenv:/home/test
  celery:
   build: .
-  entrypoint: poetry run celery -A project.celery worker -l info
-  links:
-   - postgres
-   - redis
+  entrypoint: >
+    sh -c "pip install -e /proj &&
+           pip install redis &&
+           cd example &&
+           celery -A project worker --loglevel=info -n worker1"
+  depends_on:
+    - postgres
+    - redis
+  environment:
+    DATABASE_HOST: postgres
+    REDIS_URL: redis://redis:6379/0
   tty: true
   stdin_open: true
-  working_dir: /proj/example
+  working_dir: /proj/
   user: test
   volumes:
-   - ./:/proj/
-   - ./pyenv:/home/test
+    - ./:/proj/
+    - ./pyenv:/home/test
+
  redis:
   image: redis
  postgres:
-  image: mdillon/postgis:9.6-alpine
+  image: postgres:14-alpine
+  ports:
+   - "5432:5432"
   volumes:
    - ./db:/var/lib/postgresql/data
   environment:
    POSTGRES_PASSWORD: foobar
    POSTGRES_USER: pguser
    PGDATA: /var/lib/postgresql/data
+   POSTGRES_HOST_AUTH_METHOD: md5