diff --git a/.github/workflows/release_actions.yml b/.github/workflows/release_actions.yml index 2dab2dc9..df087179 100644 --- a/.github/workflows/release_actions.yml +++ b/.github/workflows/release_actions.yml @@ -10,17 +10,21 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + fetch-depth: 0 # setuptools-scm needs full git history + + - name: Install uv + uses: astral-sh/setup-uv@v4 - - name: Install pyinstaller + - name: Install package and dependencies shell: bash - run: python -m pip install pyinstaller + run: | + uv pip install --system pyinstaller + uv pip install --system "commcare-export[executable]" - name: Generate exe shell: bash - run: | - pip install commcare-export - pip install -r build_exe/requirements.txt - pyinstaller --dist ./dist/linux commcare-export.spec + run: pyinstaller --dist ./dist/linux commcare-export.spec - name: Upload release assets uses: AButler/upload-release-assets@v3.0 @@ -34,17 +38,21 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + fetch-depth: 0 # setuptools-scm needs full git history + + - name: Install uv + uses: astral-sh/setup-uv@v4 - - name: Install pyinstaller + - name: Install package and dependencies shell: pwsh - run: python -m pip install pyinstaller + run: | + uv pip install --system pyinstaller + uv pip install --system "commcare-export[executable]" - name: Generate exe shell: pwsh - run: | - pip install commcare-export - pip install -r build_exe/requirements.txt - pyinstaller --dist ./dist/windows commcare-export.spec + run: pyinstaller --dist ./dist/windows commcare-export.spec - name: Upload release assets uses: AButler/upload-release-assets@v3.0 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 867a1420..8520051f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,28 +35,39 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 50 - - run: git fetch --tags origin # So we can use git describe. actions/checkout@v4 does not pull tags. + fetch-depth: 0 # setuptools-scm needs full git history # MySQL set up - - run: sudo service mysql start # Ubuntu already includes mysql no need to use service + - run: sudo service mysql start - run: mysql -uroot -proot -e "CREATE USER '${{ env.DB_USER }}'@'%';" - run: mysql -uroot -proot -e "GRANT ALL PRIVILEGES ON *.* TO '${{ env.DB_USER }}'@'%';" - - uses: actions/setup-python@v5 + # Install uv + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + cache-dependency-glob: "pyproject.toml" + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - cache: 'pip' + - run: sudo apt-get install pandoc - - run: pip install --upgrade pip - - run: pip install setuptools - - run: python setup.py sdist - - run: pip install dist/* - - run: pip install pymysql psycopg2 pyodbc - - run: pip install coverage coveralls - - run: pip install mypy - - run: pip install pytest - - run: pip install -e ".[test]" + + # Build and install package + - name: Build distribution + run: uv build + + - name: Install package with extras + run: | + uv pip install --system dist/*.whl + uv pip install --system pymysql psycopg2 pyodbc + uv pip install --system coverage coveralls + uv pip install --system mypy + uv pip install --system pytest + uv pip install --system -e ".[test]" # Run tests and save test coverage - run: coverage run -m pytest @@ -66,8 +77,10 @@ jobs: MSSQL_URL: mssql+pyodbc://sa:${{ env.DB_PASSWORD }}@localhost/ HQ_USERNAME: ${{ secrets.HQ_USERNAME }} HQ_API_KEY: ${{ secrets.HQ_API_KEY }} + # Convert coverage data for Coveralls - run: coverage lcov -o coverage/lcov.info + # Use Coveralls to track coverage - name: Coveralls uses: coverallsapp/github-action@v2 @@ -76,8 +89,34 @@ jobs: flag-name: run-${{ join(matrix.*, '-') }} github-token: ${{ secrets.GITHUB_TOKEN }} - # Check typing - - run: mypy --install-types --non-interactive commcare_export/ tests/ migrations/ setup.py + typing: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + cache-dependency-glob: "pyproject.toml" + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Build distribution + run: uv build + + - name: Install package with mypy + run: | + uv pip install --system dist/*.whl + uv pip install --system mypy + uv pip install --system -e ".[test]" + + - run: mypy --install-types --non-interactive commcare_export/ tests/ migrations/ finish: needs: test diff --git a/.gitignore b/.gitignore index 69607db9..529d350f 100644 --- a/.gitignore +++ b/.gitignore @@ -49,4 +49,11 @@ nosetests.xml # Excel ~*.xlsx -commcare_export.log \ No newline at end of file +commcare_export.log + +# uv +.uv/ +uv.lock + +# Build artifacts +*.whl diff --git a/README.md b/README.md index 1421ed23..d031d5b3 100644 --- a/README.md +++ b/README.md @@ -69,10 +69,10 @@ $ venv ## Install CommCare Export -Install CommCare Export via `pip` +[uv](https://docs.astral.sh/uv/) is a fast Python package installer and resolver. ```shell -$ pip install commcare-export +$ uv pip install commcare-export ``` ## CommCare HQ @@ -563,28 +563,27 @@ via the `--output-format ` option, and it can be directed to a file with Dependencies ------------ -Required dependencies will be automatically installed via pip. But since -you may not care about all export formats, the various dependencies there -are optional. Here is how you might install them: +Required dependencies will be automatically installed. Optional dependencies +for specific export formats can be installed as extras: ```shell # To export "xlsx" -$ pip install "commcare-export[xlsx]" +$ uv pip install "commcare-export[xlsx]" # To export "xls" -$ pip install "commcare-export[xls]" +$ uv pip install "commcare-export[xls]" # To sync with a Postgres database -$ pip install "commcare-export[postgres]" +$ uv pip install "commcare-export[postgres]" # To sync with a mysql database -$ pip install "commcare-export[mysql]" +$ uv pip install "commcare-export[mysql]" # To sync with a database which uses odbc (e.g. mssql) -$ pip install "commcare-export[odbc]" +$ uv pip install "commcare-export[odbc]" # To sync with another SQL database supported by SQLAlchemy -$ pip install "commcare-export[base_sql]" +$ uv pip install "commcare-export[base_sql]" # Then install the Python package for your database ``` @@ -598,11 +597,11 @@ Contributing 2\. Clone your fork, install into a virtualenv, and start a feature branch ```shell -$ git clone git@github.com:dimagi/commcare-export.git +$ git clone git@github.com:your-username/commcare-export.git $ cd commcare-export -$ python3 -m venv venv -$ source venv/bin/activate -$ pip install -e ".[test]" +$ uv venv +$ source .venv/bin/activate # On Windows: .venv\Scripts\activate +$ uv pip install -e ".[test]" $ git checkout -b my-super-duper-feature ``` @@ -651,18 +650,18 @@ $ git tag -a "X.YY.0" -m "Release X.YY.0" $ git push --tags ``` -2\. Create the source distribution +2\. Create the distribution ```shell -$ python setup.py sdist +$ uv build ``` -Ensure that the archive (`dist/commcare-export-X.YY.0.tar.gz`) has the correct version number (matching the tag name). + +Ensure that the archives in `dist/` have the correct version number (matching the tag name). 3\. Upload to pypi ```shell -$ pip install twine -$ twine upload -u dimagi dist/commcare-export-X.YY.0.tar.gz +$ uv publish ``` 4\. Verify upload diff --git a/commcare_export/version.py b/commcare_export/version.py index a061cb64..3a82a5a2 100644 --- a/commcare_export/version.py +++ b/commcare_export/version.py @@ -1,52 +1,36 @@ -import io -import re -import os.path -import subprocess +"""Version information for commcare-export. -__all__ = ['__version__', 'stored_version', 'git_version'] +The version is managed by setuptools-scm and stored in the VERSION file. +""" +from pathlib import Path -VERSION_PATH = os.path.join(os.path.dirname(__file__), 'VERSION') +__all__ = ['__version__'] +VERSION_PATH = Path(__file__).parent / 'VERSION' -def stored_version(): - if os.path.exists(VERSION_PATH): - with io.open(VERSION_PATH, encoding='ascii') as fh: - return fh.read().strip() - else: - return None +def get_version(): + """Read version from VERSION file written by setuptools-scm during build. -def git_version(): - if os.environ.get('DET_EXECUTABLE'): - return None - - described_version_bytes = subprocess.Popen( - ['git', 'describe'], - stdout=subprocess.PIPE - ).communicate()[0].strip() - version_raw = described_version_bytes.decode('ascii') - return parse_version(version_raw) - - -def parse_version(version_raw): - """Attempt to convert a git version to a version - compatible with PEP440: https://peps.python.org/pep-0440/ + For development installs, setuptools-scm handles version detection automatically. + For built distributions, the version is in the VERSION file. """ - match = re.match('(\d+\.\d+\.\d+)(?:-(\d+).*)?', version_raw) - if match: - tag_version, lead_count = match.groups() - if lead_count: - tag_version += f".dev{lead_count}" - return tag_version + if VERSION_PATH.exists(): + return VERSION_PATH.read_text(encoding='ascii').strip() - return version_raw + # During development with editable install, try to get version from setuptools-scm + try: + from setuptools_scm import get_version as scm_get_version + return scm_get_version(root='..', relative_to=__file__) + except Exception: + pass + # Final fallback for edge cases (e.g., PyInstaller executable) + return "unknown" -def version(): - return stored_version() or git_version() +__version__ = get_version() -__version__ = version() if __name__ == '__main__': print(__version__) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..4d83f937 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,112 @@ +[build-system] +requires = ["setuptools>=64", "setuptools-scm>=8"] +build-backend = "setuptools.build_meta" + +[project] +name = "commcare-export" +dynamic = ["version"] +description = "A command-line tool (and Python library) to extract data from CommCare HQ into a SQL database or Excel workbook" +readme = "README.md" +license = {text = "MIT"} +authors = [ + {name = "Dimagi", email = "information@dimagi.com"} +] +requires-python = ">=3.9" +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Healthcare Industry", + "Intended Audience :: Science/Research", + "Intended Audience :: System Administrators", + "Intended Audience :: End Users/Desktop", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Database", + "Topic :: Software Development :: Interpreters", + "Topic :: System :: Archiving", + "Topic :: System :: Distributed Computing", +] + +dependencies = [ + "alembic", + "backoff>=2.0", + "jsonpath-ng~=1.6.0", + "ndg-httpsclient", + "openpyxl==3.1.5", + "python-dateutil", + "pytz", + "requests", + "simplejson", + "sqlalchemy~=1.4", + "sqlalchemy-migrate", +] + +[project.optional-dependencies] +test = [ + "pytest", + "psycopg2-binary", + "mock", +] +base_sql = [ + "SQLAlchemy", + "alembic", +] +postgres = [ + "SQLAlchemy", + "alembic", + "psycopg2-binary", +] +mysql = [ + "SQLAlchemy", + "alembic", + "pymysql", +] +odbc = [ + "SQLAlchemy", + "alembic", + "pyodbc", +] +xlsx = [ + "openpyxl", +] +xls = [ + "xlwt", +] +executable = [ + "chardet", + "psycopg2-binary", + "pymysql", + "pyodbc", + "xlwt", + "openpyxl", +] + +[project.urls] +Homepage = "https://github.com/dimagi/commcare-export" + +[project.scripts] +commcare-export = "commcare_export.cli:entry_point" +commcare-export-utils = "commcare_export.utils_cli:entry_point" + +[tool.setuptools] +packages = ["commcare_export"] +include-package-data = true + +[tool.setuptools.package-data] +commcare_export = ["VERSION"] + +[tool.setuptools-scm] +version_file = "commcare_export/VERSION" +version_scheme = "guess-next-dev" +local_scheme = "no-local-version" + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +addopts = ["-vv", "--tb=short"] diff --git a/setup.py b/setup.py deleted file mode 100644 index f4cd04fa..00000000 --- a/setup.py +++ /dev/null @@ -1,121 +0,0 @@ -import glob -import io -import os.path -import re -import sys - -import setuptools -from setuptools.command.test import test as TestCommand - -VERSION_PATH = 'commcare_export/VERSION' - -# Overwrite VERSION if we are actually building for a distribution to pypi -# This code path requires dependencies, etc, to be available -if 'sdist' in sys.argv: - import commcare_export.version - with io.open(VERSION_PATH, 'w', encoding='ascii') as fh: - fh.write(commcare_export.version.git_version()) - -# This import requires either commcare_export/VERSION or to be in a git clone (as does the package in general) -import commcare_export - -version = commcare_export.version.version() - -# Crash if the VERSION is not a simple version and it is going to register or upload -if 'register' in sys.argv or 'upload' in sys.argv: - version = commcare_export.version.stored_version() - if not version or not re.match(r'\d+\.\d+\.\d+', version): - print('Version %s is not an appropriate version for publicizing!' % - version) - sys.exit(1) - -readme = 'README.md' - - -class PyTest(TestCommand): # type: ignore - def finalize_options(self): - TestCommand.finalize_options(self) - self.test_args = ['-vv', '--tb=short'] - self.test_suite = True - - def run_tests(self): - #import here, cause outside the eggs aren't loaded - import pytest - errno = pytest.main(['tests/'] + self.test_args) - sys.exit(errno) - - -test_deps = ['pytest', 'psycopg2-binary', 'mock'] -base_sql_deps = ["SQLAlchemy", "alembic"] -postgres = ["psycopg2-binary"] -mysql = ["pymysql"] -odbc = ["pyodbc"] - -setuptools.setup( - name="commcare-export", - version=version, - description='A command-line tool (and Python library) to extract data from ' - 'CommCare HQ into a SQL database or Excel workbook', - long_description=io.open(readme, encoding='utf-8').read(), - long_description_content_type='text/markdown', - author='Dimagi', - author_email='information@dimagi.com', - url="https://github.com/dimagi/commcare-export", - entry_points={ - 'console_scripts': [ - 'commcare-export = commcare_export.cli:entry_point', - 'commcare-export-utils = commcare_export.utils_cli:entry_point' - ] - }, - packages=setuptools.find_packages(exclude=['tests*']), - data_files=[ - (os.path.join('share', 'commcare-export', 'examples'), - glob.glob('examples/*.json') + glob.glob('examples/*.xlsx')), - ], - include_package_data=True, - license='MIT', - python_requires=">=3.9", - install_requires=[ - 'alembic', - 'backoff>=2.0', - 'jsonpath-ng~=1.6.0', - 'ndg-httpsclient', - 'openpyxl==3.1.5', - 'python-dateutil', - 'pytz', - 'requests', - 'simplejson', - 'sqlalchemy~=1.4', - 'sqlalchemy-migrate' - ], - extras_require={ - 'test': test_deps, - 'base_sql': base_sql_deps, - 'postgres': base_sql_deps + postgres, - 'mysql': base_sql_deps + mysql, - 'odbc': base_sql_deps + odbc, - 'xlsx': ["openpyxl"], - 'xls': ["xlwt"], - }, - cmdclass={'test': PyTest}, - classifiers=[ - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'Intended Audience :: Healthcare Industry', - 'Intended Audience :: Science/Research', - 'Intended Audience :: System Administrators', - 'Intended Audience :: End Users/Desktop', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - 'Topic :: Database', - 'Topic :: Software Development :: Interpreters', - 'Topic :: System :: Archiving', - 'Topic :: System :: Distributed Computing', - ] -) diff --git a/tests/test_version.py b/tests/test_version.py deleted file mode 100644 index c6559fe2..00000000 --- a/tests/test_version.py +++ /dev/null @@ -1,19 +0,0 @@ -import pytest - -from commcare_export.version import parse_version - - -@pytest.mark.parametrize( - "input,output", - [ - ("1.2.3", "1.2.3"), - ("1.2", "1.2"), - ("0.1.5-3", "0.1.5.dev3"), - ("0.1.5-3-g1234567", "0.1.5.dev3"), - ("0.1.5-4-g1234567-dirty", "0.1.5.dev4"), - ("0.1.5-15-g1234567-dirty-123", "0.1.5.dev15"), - ("a.b.c", "a.b.c"), - ] -) -def test_parse_version(input, output): - assert parse_version(input) == output