From a668f0dc303c8b105e1bf5f9729b2d69ecb91898 Mon Sep 17 00:00:00 2001 From: Ian Lee Date: Thu, 17 Apr 2025 11:25:32 -0700 Subject: [PATCH 1/7] Upgraded packaging infrastructure Ref: - https://build.pypa.io/en/stable/installation.html - https://packaging.python.org/en/latest/guides/writing-pyproject-toml/ - https://packaging.python.org/en/latest/guides/modernize-setup-py-project/ --- Makefile | 2 +- pyproject.toml | 41 +++++++++++++++++++++++++++++++++++++ requirements/dev.txt | 1 + setup.py | 48 -------------------------------------------- 4 files changed, 43 insertions(+), 49 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/Makefile b/Makefile index 5d7abcf..d0824f7 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ test: pyflakes scraper release: test - python3 setup.py sdist bdist_wheel + python3 -m build . upload: twine upload --skip-existing dist/* diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c5f2d4c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,41 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "llnl-scraper" +version = "0.16.0" +description = "Package for extracting software repository metadata" +readme = "README.md" +requires-python = ">=3.8" +authors = [{ name = "Ian Lee", email = "lee1001@llnl.gov" }] +license = { text = "MIT" } +keywords = [] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] + +[project.urls] +Homepage = "https://github.com/llnl/scraper" + +[project.scripts] +scraper = "scraper.gen_code_gov_json:main" + +[tool.setuptools] +packages = ["scraper"] + +[tool.setuptools.dynamic] +dependencies = { file = ["requirements/production.txt"] } diff --git a/requirements/dev.txt b/requirements/dev.txt index 5aff939..f34ad3e 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -3,6 +3,7 @@ # Development tools ipython twine +build # Testing tools bandit diff --git a/setup.py b/setup.py deleted file mode 100644 index 593d1b1..0000000 --- a/setup.py +++ /dev/null @@ -1,48 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -from setuptools import find_packages, setup - -with open("README.md") as fh: - long_description = fh.read() - -with open("requirements/production.txt") as fp: - lines = [x.strip() for x in fp.readlines() if x] - install_reqs = [x for x in lines if not x.startswith("#")] - -setup( - name="llnl-scraper", - version="0.15.0", - description="Package for extracting software repository metadata", - long_description=long_description, - long_description_content_type="text/markdown", - author="Ian Lee", - author_email="lee1001@llnl.gov", - url="https://github.com/llnl/scraper", - packages=find_packages(), - install_requires=install_reqs, - python_requires=">=3.6", - entry_points={ - "console_scripts": [ - "scraper = scraper.gen_code_gov_json:main", - ] - }, - scripts=[ - "scripts/codegov_compute_hours.py", - ], - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - ], -) From 7e892c1a2e26024a20b574a689382723c6d32261 Mon Sep 17 00:00:00 2001 From: Ian Lee Date: Thu, 17 Apr 2025 11:55:18 -0700 Subject: [PATCH 2/7] Fixed configuration about dependencies --- pyproject.toml | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c5f2d4c..a9803ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,17 @@ classifiers = [ "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] +dependencies = [ + "github3.py>=2.0.0", + "msrest>=0.6.4", + "python-dateutil>=2.7.3", + "python-gitlab>=1.6.0", + "pytz>=2017.3", + "requests>=2.16", + "setuptools>=24.2.0", + "stashy>=0.3", + "vsts>=0.1.25", +] [project.urls] Homepage = "https://github.com/llnl/scraper" @@ -34,8 +45,15 @@ Homepage = "https://github.com/llnl/scraper" [project.scripts] scraper = "scraper.gen_code_gov_json:main" -[tool.setuptools] -packages = ["scraper"] - -[tool.setuptools.dynamic] -dependencies = { file = ["requirements/production.txt"] } +[project.optional-dependencies] +dev = [ + "ipython", + "twine", + "build", + "bandit", + "black", + "flake8", + "isort", + "pyflakes", + "safety", +] From 9593f4a843fde58063e6ff2b008471ef79e340ce Mon Sep 17 00:00:00 2001 From: Ian Lee Date: Thu, 17 Apr 2025 11:55:45 -0700 Subject: [PATCH 3/7] Removed requirements/ files in favor of specifying in pyproject.toml --- requirements/dev.txt | 14 -------------- requirements/production.txt | 9 --------- 2 files changed, 23 deletions(-) delete mode 100644 requirements/dev.txt delete mode 100644 requirements/production.txt diff --git a/requirements/dev.txt b/requirements/dev.txt deleted file mode 100644 index f34ad3e..0000000 --- a/requirements/dev.txt +++ /dev/null @@ -1,14 +0,0 @@ --r production.txt - -# Development tools -ipython -twine -build - -# Testing tools -bandit -black -flake8 -isort -pyflakes -safety diff --git a/requirements/production.txt b/requirements/production.txt deleted file mode 100644 index f462d59..0000000 --- a/requirements/production.txt +++ /dev/null @@ -1,9 +0,0 @@ -github3.py>=2.0.0 -msrest>=0.6.4 -python-dateutil>=2.7.3 -python-gitlab>=1.6.0 -pytz>=2017.3 -requests>=2.16 -setuptools>=24.2.0 -stashy>=0.3 -vsts>=0.1.25 From d628961d4553671153d6a080817825e48338caaf Mon Sep 17 00:00:00 2001 From: Ian Lee Date: Fri, 18 Apr 2025 06:21:52 -0700 Subject: [PATCH 4/7] Removed top level requirements file --- requirements.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index b0c3540..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --r requirements/dev.txt From d87b51c16135c41a5d70e1b4fe6db1b9f7a3968a Mon Sep 17 00:00:00 2001 From: Ian Lee Date: Fri, 18 Apr 2025 06:52:30 -0700 Subject: [PATCH 5/7] Updated github actions to use pyproject.toml --- .github/workflows/main.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fbf1483..37dd484 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,18 +21,21 @@ jobs: name: Python ${{ matrix.python-version }} Tests steps: - uses: actions/checkout@v3 + - name: Set up Python uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} architecture: x64 - - name: Python Tests + + - name: Install project with dev dependencies + run: | + python -m pip install --upgrade pip + pip install .[dev] + + - name: Run tests run: | - conda create --quiet --name test pytest - export PATH="/usr/share/miniconda/bin:$PATH" - source activate test - pip install bandit black isort flake8 - pip install . - npm install -g markdownlint-cli@0.33.0 make test - scraper -h + + - name: Show scraper help + run: scraper -h From f412191ec4e539060e3e3e1aa08fe83b81dc144f Mon Sep 17 00:00:00 2001 From: Ian Lee Date: Fri, 18 Apr 2025 06:53:59 -0700 Subject: [PATCH 6/7] Removed manifest file since it only referenced requirements files which are gone --- MANIFEST.in | 1 - 1 file changed, 1 deletion(-) delete mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index e534d45..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include requirements/*.txt From 31ba6bf71c4ee31780f241b8724088fec0305404 Mon Sep 17 00:00:00 2001 From: Ian Lee Date: Fri, 18 Apr 2025 07:00:01 -0700 Subject: [PATCH 7/7] Re-add markdownlint installation to github actions --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 37dd484..1da0bbf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -32,6 +32,7 @@ jobs: run: | python -m pip install --upgrade pip pip install .[dev] + npm install -g markdownlint-cli@0.33.0 - name: Run tests run: |