Architecture coverage test

.codecov.yml

@@ -1,23 +1,62 @@
 coverage:
   status:
     project:
-      default:
-        target: 88%
+      default: false
+
+    # core package
+    core:
+      target: 88%
+      paths:
+        - "src/metatrain/utils/.*"
+        - "src/metatrain/cli/.*"
+
     patch:
       default:
         informational: true
+
+    component:
+      default: true
+
+component_management:
+  individual_components:
+
+    # stable architectures
+    - component_id: gap
+      name: GAP
+      paths:
+        - "src/metatrain/gap/.*"
+
+    - component_id: soap_bpnn
+      name: SOAP-BPNN
+      paths:
+        - "src/metatrain/soap_bpnn/.*"
+
+    - component_id: pet
+      name: PET
+      paths:
+        - "src/metatrain/pet/.*"
+
+    - component_id: llpr
+      name: LLPR
+      paths:
+        - "src/metatrain/llpr/.*"
+
+    # deprecated architectures
+    - component_id: nanopet
+      name: NanoPET
+      paths:
+        - "src/metatrain/deprecated/nanopet/.*"
+
+    # experimental architectures
+    - component_id: flashmd
+      name: FlashMD
+      paths:
+        - "src/metatrain/experimental/flashmd/.*"
+
   ignore:
     # no need to report coverage of the test code themself
-    - "tests/.*"
-    - "examples/.*"
     - "src/metatrain/utils/testing/.*"
-    # we don't check coverage for the architectures
-    - "src/metatrain/deprecated/.*"
-    - "src/metatrain/experimental/.*"
-    - "src/metatrain/gap/.*"
-    - "src/metatrain/pet/.*"
-    - "src/metatrain/soap_bpnn/.*"
-    - "src/metatrain/llpr/.*"
+    - "src/metatrain/**/tests/.*"
     # we don't currently run distributed tests on CI
     - "src/metatrain/utils/distributed/.*"
 

.github/workflows/architecture-tests.yml

@@ -36,9 +36,18 @@ jobs:
     - name: Setup dependencies
       run: |
         uv tool install tox --with tox-uv
+        pip install coverage[toml]
 
-    - name: run architecture tests
-      run: tox -e ${{ matrix.architecture-name }}-tests
+    - name: run architecture tests and generate coverage
+      run: |
+        tox -e ${{ matrix.architecture-name }}-tests
+        coverage xml -o coverage.xml
       env:
         # Use the CPU only version of torch when building/running the code
         PIP_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu
+
+    - name: upload to codecov.io
+      uses: codecov/codecov-action@v5
+      with:
+        fail_ci_if_error: false
+        files: coverage.xml

.github/workflows/tests.yml

@@ -51,10 +51,8 @@ jobs:
         uv pip install coverage[toml]
         uv tool install tox --with tox-uv
 
-    - name: run Python tests and collect coverage
-      run: |
-        tox -e tests
-        coverage xml --data-file tests/.coverage
+    - name: run Python tests
+      run: tox -e tests
       env:
         # Use the CPU only version of torch when building/running the code
         PIP_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu

docs/src/architectures/generate.py

@@ -216,10 +216,27 @@ def generate_rst(
             **template_variables
         )
 
+    codecov_arch_path = architecture_name.replace(".", "/")
+    badge_string = (
+        f".. image:: https://codecov.io/gh/metatensor/metatrain/branch/architecture-coverage-test/graph/badge.svg?component={architecture_real_name}"
+        f"\n   :target: https://codecov.io/gh/metatensor/metatrain/tree/architecture-coverage-test/src/metatrain/{codecov_arch_path}"
+    )
+
     # Prepend docstring with reference and append missing sections
+    lines = docstring.split("\n")
+    title_id = 0
+    underline_id = 1
     docstring = (
-        f".. _architecture-{template_variables['architecture']}:" + "\n\n" + docstring
+        f".. _architecture-{template_variables['architecture']}:\n\n"
+        + lines[title_id]
+        + "\n"
+        + lines[underline_id]
+        + "\n\n"
+        + badge_string
+        + "\n"
+        + "\n".join(lines[2:])
     )
+
     # Check for missing sections and add them to the end of the docstring
     for section in SECTIONS:
         section_var = "{{SECTION_" + section.upper() + "}}"

pyproject.toml

@@ -94,8 +94,11 @@ exclude_lines = [
 ]
 omit = [
     "*/site-packages/*",
-    "*/metatrain/experimental/*",
-    "*/metatrain/utils/testing/*"
+    "*/tests/*",
+    "src/metatrain/*/tests/*",
+    "src/metatrain/*/*/tests/*",
+    "src/metatrain/utils/testing/*",
+    "examples/*"
 ]
 
 [tool.coverage.paths]
@@ -106,6 +109,7 @@ source = [
 
 [tool.coverage.run]
 patch = ["subprocess"]
+relative_files = true
 
 [tool.ruff]
 exclude = ["docs/src/examples/**", "src/metatrain/_version.py"]

tox.ini

@@ -23,12 +23,14 @@ passenv = *
 setenv =
     UV_EXTRA_INDEX_URL = {env:PIP_EXTRA_INDEX_URL:}
     UV_INDEX_STRATEGY = unsafe-best-match
+    COVERAGE_FILE = {toxinidir}/.coverage
 lint_folders =
     "{toxinidir}/developer/" \
     "{toxinidir}/examples/" \
     "{toxinidir}/src/" \
     "{toxinidir}/tests/" \
     "{toxinidir}/docs/src/"
+
 # We apply stricter checks on the main code, but not on tests, examples, or
 # external architectures
 lint_strict_folders =
@@ -40,6 +42,7 @@ lint_strict_folders =
     "{toxinidir}/src/metatrain/soap_bpnn"
 exclude_tests_regex = "(^|/)[^/]*test[^/]*$"
 exclude_tests_glob = "**/*test*"
+
 [testenv:lint]
 description = Run linters and type checks
 package = skip
@@ -108,7 +111,7 @@ commands =
         --numprocesses=auto \
         --cov=metatrain \
         --cov-append \
-        --cov-report= \
+        --cov-report=term \
         --import-mode=append \
         {posargs}
 
@@ -132,68 +135,73 @@ commands =
     check-manifest {toxinidir}
 
 [testenv:soap-bpnn-tests]
-description = Run SOAP-BPNN tests with pytest
+description = Run SOAP-BPNN tests with pytest and collect coverage
 passenv = *
 deps =
     pytest
+    pytest-cov
     spherical  # tensor target tests
     cmake
 extras = soap-bpnn
 changedir = src/metatrain/soap_bpnn/tests/
 commands =
-    pytest {posargs}
-
+    pytest --cov=metatrain.soap_bpnn --cov-append --cov-report=term {posargs}
 
 [testenv:pet-tests]
-description = Run PET tests with pytest
+description = Run PET tests with pytest and collect coverage
 passenv = *
 deps =
     pytest
+    pytest-cov
 extras = pet
 changedir = src/metatrain/pet/tests/
 commands =
-    pytest {posargs}
+    pytest --cov=metatrain.pet --cov-append --cov-report=term {posargs}
 
 [testenv:gap-tests]
-description = Run GAP tests with pytest
+description = Run GAP tests with pytest and collect coverage
 passenv = *
 deps =
     pytest
+    pytest-cov
 extras = gap
 changedir = src/metatrain/gap/tests/
 commands =
-    pytest {posargs}
+    pytest --cov=metatrain.gap --cov-append --cov-report=term {posargs}
 
 [testenv:nanopet-tests]
-description = Run NanoPET tests with pytest
+description = Run NanoPET tests with pytest and collect coverage
 passenv = *
 deps =
     pytest
+    pytest-cov
     spherical  # for nanoPET spherical target
 extras = nanopet
 changedir = src/metatrain/deprecated/nanopet/tests/
 commands =
-    pytest {posargs}
+    pytest --cov=metatrain.deprecated.nanopet --cov-append --cov-report=term {posargs}
 
 [testenv:llpr-tests]
-description = Run LLPR tests with pytest
+description = Run LLPR tests with pytest and collect coverage
 passenv = *
 deps =
     pytest
+    pytest-cov
 extras = llpr
 changedir = src/metatrain/llpr/tests/
 commands =
-    pytest {posargs}
+    pytest --cov=metatrain.llpr --cov-append --cov-report=term {posargs}
 
 [testenv:flashmd-tests]
-description = Run FlashMD tests with pytest
+description = Run FlashMD tests with pytest and collect coverage
 passenv = *
 deps =
     pytest
+    pytest-cov
 extras = flashmd
 changedir = src/metatrain/experimental/flashmd/tests/
 commands =
-    pytest {posargs}
+    pytest --cov=metatrain.experimental.flashmd --cov-append --cov-report=term {posargs}
 
 [testenv:docs]
 description = builds the documentation with sphinx