From 94cf2b513661622ea916f3658869d7bd53aa7a36 Mon Sep 17 00:00:00 2001 From: lanluo-nvidia Date: Fri, 30 May 2025 09:23:56 -0700 Subject: [PATCH 1/9] add fail fast actions --- .github/workflows/build-test-linux-x86_64.yml | 55 ++++++++++++++++--- tests/py/core/test_classes.py | 5 +- .../automatic_plugin/test_automatic_plugin.py | 2 + .../dynamo/backend/test_backend_compiler.py | 2 + tests/py/dynamo/distributed/test_nccl_ops.py | 2 + .../lowering/test_aten_lowering_passes.py | 3 + .../py/dynamo/lowering/test_decompositions.py | 1 + tests/py/dynamo/models/test_dyn_models.py | 2 + tests/py/dynamo/models/test_engine_cache.py | 1 + .../dynamo/models/test_export_kwargs_serde.py | 2 + tests/py/dynamo/models/test_export_serde.py | 1 + tests/py/dynamo/models/test_models.py | 1 + tests/py/dynamo/models/test_models_export.py | 1 + .../partitioning/test_fast_partitioning.py | 2 + .../partitioning/test_global_partitioning.py | 1 + .../test_000_convert_module_to_trt_engine.py | 5 +- .../dynamo/runtime/test_output_allocator.py | 2 + .../runtime/test_pre_allocated_outputs.py | 2 + 18 files changed, 79 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-test-linux-x86_64.yml b/.github/workflows/build-test-linux-x86_64.yml index 5f24a0c7ad..8d8ff734db 100644 --- a/.github/workflows/build-test-linux-x86_64.yml +++ b/.github/workflows/build-test-linux-x86_64.yml @@ -75,9 +75,50 @@ jobs: smoke-test-script: ${{ matrix.smoke-test-script }} trigger-event: ${{ github.event_name }} + tests-py-fail-fast: + name: Test fail fast [Python] + needs: [filter-matrix, build] + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/tensorrt + package-name: torch_tensorrt + pre-script: packaging/pre_build_script.sh + post-script: packaging/post_build_script.sh + smoke-test-script: packaging/smoke_test_script.sh + uses: ./.github/workflows/linux-test.yml + with: + job-name: tests-py-critical-fail-fast + repository: "pytorch/tensorrt" + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.filter-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + script: | + export USE_HOST_DEPS=1 + export CI_BUILD=1 + export LD_LIBRARY_PATH=/usr/lib64:$LD_LIBRARY_PATH + pushd . + cd tests/py + python -m pip install -r requirements.txt + + # test dynamo + python -m pytest -m critical --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_backend_test_results.xml -n 4 dynamo/backend/ + python -m pytest -m critical -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_models_result.xml --ir dynamo dynamo/models/ + python -m pytest -m critical --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_automatic_plugin_results.xml dynamo/automatic_plugin/ + python -m pytest -m critical --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_partitioning_results.xml dynamo/partitioning/ + python -m pytest -m critical --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_lowering_results.xml dynamo/lowering/ + python -m pytest -m critical --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_runtime_results.xml dynamo/runtime/ + + # test core + python -m pytest -m critical --junitxml=${RUNNER_TEST_RESULTS_DIR}/core_test_results.xml core/ + popd + tests-py-torchscript-fe: name: Test torchscript frontend [Python] - needs: [filter-matrix, build] + needs: [filter-matrix, build, tests-py-fail-fast] strategy: fail-fast: false matrix: @@ -113,7 +154,7 @@ jobs: tests-py-dynamo-converters: name: Test dynamo converters [Python] - needs: [filter-matrix, build] + needs: [filter-matrix, build, tests-py-fail-fast] strategy: fail-fast: false matrix: @@ -148,7 +189,7 @@ jobs: tests-py-dynamo-fe: name: Test dynamo frontend [Python] - needs: [filter-matrix, build] + needs: [filter-matrix, build, tests-py-fail-fast] strategy: fail-fast: false matrix: @@ -180,7 +221,7 @@ jobs: tests-py-dynamo-serde: name: Test dynamo export serde [Python] - needs: [filter-matrix, build] + needs: [filter-matrix, build, tests-py-fail-fast] strategy: fail-fast: false matrix: @@ -245,7 +286,7 @@ jobs: tests-py-dynamo-core: name: Test dynamo core [Python] - needs: [filter-matrix, build] + needs: [filter-matrix, build, tests-py-fail-fast] strategy: fail-fast: false matrix: @@ -278,7 +319,7 @@ jobs: tests-py-dynamo-cudagraphs: name: Test dynamo cudagraphs [Python] - needs: [filter-matrix, build] + needs: [filter-matrix, build, tests-py-fail-fast] strategy: fail-fast: false matrix: @@ -311,7 +352,7 @@ jobs: tests-py-core: name: Test core [Python] - needs: [filter-matrix, build] + needs: [filter-matrix, build, tests-py-fail-fast] strategy: fail-fast: false matrix: diff --git a/tests/py/core/test_classes.py b/tests/py/core/test_classes.py index 62abeb6b1a..3bf7fc8ae8 100644 --- a/tests/py/core/test_classes.py +++ b/tests/py/core/test_classes.py @@ -2,16 +2,17 @@ import unittest from typing import Dict +import pytest +import tensorrt as trt import torch import torch_tensorrt import torch_tensorrt as torchtrt import torchvision.models as models from torch_tensorrt.dynamo.runtime._TorchTensorRTModule import TorchTensorRTModule -import tensorrt as trt - class TestDevice(unittest.TestCase): + @pytest.mark.critical def test_from_string_constructor(self): device = torchtrt.Device("cuda:0") self.assertEqual(device.device_type, torchtrt.DeviceType.GPU) diff --git a/tests/py/dynamo/automatic_plugin/test_automatic_plugin.py b/tests/py/dynamo/automatic_plugin/test_automatic_plugin.py index 8ab47def08..024e28d3fd 100644 --- a/tests/py/dynamo/automatic_plugin/test_automatic_plugin.py +++ b/tests/py/dynamo/automatic_plugin/test_automatic_plugin.py @@ -1,5 +1,6 @@ from typing import Tuple +import pytest import torch import torch.nn as nn import torch_tensorrt @@ -66,6 +67,7 @@ class TestAutomaticPlugin(DispatchTestCase): ((256, 256), torch.int), ] ) + @pytest.mark.critical def test_mul_plugin_float(self, input_shape, dtype): class elementwise_mul(nn.Module): def forward(self, lhs, rhs): diff --git a/tests/py/dynamo/backend/test_backend_compiler.py b/tests/py/dynamo/backend/test_backend_compiler.py index 6369d3805c..39bd3a9c17 100644 --- a/tests/py/dynamo/backend/test_backend_compiler.py +++ b/tests/py/dynamo/backend/test_backend_compiler.py @@ -1,6 +1,7 @@ # type: ignore from copy import deepcopy +import pytest import torch import torch_tensorrt from torch.testing._internal.common_utils import TestCase, run_tests @@ -10,6 +11,7 @@ class TestTRTModuleNextCompilation(TestCase): + @pytest.mark.critical def test_trt_module_next_full_support(self): class FullySupportedMultiOp(torch.nn.Module): def forward(self, x, y): diff --git a/tests/py/dynamo/distributed/test_nccl_ops.py b/tests/py/dynamo/distributed/test_nccl_ops.py index 89c94300b7..67bd09596b 100644 --- a/tests/py/dynamo/distributed/test_nccl_ops.py +++ b/tests/py/dynamo/distributed/test_nccl_ops.py @@ -1,5 +1,6 @@ import os +import pytest import torch import torch.distributed as dist import torch.nn as nn @@ -17,6 +18,7 @@ class TestGatherNcclOpsConverter(DispatchTestCase): + @pytest.mark.critical @parameterized.expand([8]) def test_nccl_ops(self, linear_layer_dim): class DistributedGatherModel(nn.Module): diff --git a/tests/py/dynamo/lowering/test_aten_lowering_passes.py b/tests/py/dynamo/lowering/test_aten_lowering_passes.py index 69c91db475..2571ce5443 100644 --- a/tests/py/dynamo/lowering/test_aten_lowering_passes.py +++ b/tests/py/dynamo/lowering/test_aten_lowering_passes.py @@ -1,3 +1,4 @@ +import pytest import torch import torch_tensorrt from torch.testing._internal.common_utils import TestCase, run_tests @@ -6,6 +7,7 @@ class TestInputAsOutput(TestCase): + @pytest.mark.critical def test_input_as_output(self): class InputAsOutput(torch.nn.Module): def forward(self, x, y): @@ -56,6 +58,7 @@ def forward(self, x, y): class TestLoweringPassMembership(TestCase): + @pytest.mark.critical def insert_at_end(self): from torch_tensorrt.dynamo.lowering.passes import ( ATEN_LOWERING_PASSES, diff --git a/tests/py/dynamo/lowering/test_decompositions.py b/tests/py/dynamo/lowering/test_decompositions.py index b63e0f3bf7..903f9e0eee 100644 --- a/tests/py/dynamo/lowering/test_decompositions.py +++ b/tests/py/dynamo/lowering/test_decompositions.py @@ -14,6 +14,7 @@ class TestLowering(TestCase): + @pytest.mark.critical def test_lowering_inplace_op(self): class InPlace(torch.nn.Module): def __init__(self, *args, **kwargs) -> None: diff --git a/tests/py/dynamo/models/test_dyn_models.py b/tests/py/dynamo/models/test_dyn_models.py index d5627499f5..f8b9b59aea 100644 --- a/tests/py/dynamo/models/test_dyn_models.py +++ b/tests/py/dynamo/models/test_dyn_models.py @@ -11,6 +11,7 @@ assertions = unittest.TestCase() +@pytest.mark.critical @pytest.mark.unit def test_base_dynamic(ir): """ @@ -175,6 +176,7 @@ def forward(self, x): ) +@pytest.mark.critical @pytest.mark.unit def test_resnet_dynamic(ir): """ diff --git a/tests/py/dynamo/models/test_engine_cache.py b/tests/py/dynamo/models/test_engine_cache.py index 0bc7c665b3..158b9edb43 100644 --- a/tests/py/dynamo/models/test_engine_cache.py +++ b/tests/py/dynamo/models/test_engine_cache.py @@ -57,6 +57,7 @@ def load(self, hash: str, prefix: str = "blob") -> Optional[bytes]: class TestHashFunction(TestCase): + @pytest.mark.critical def test_reexport_is_equal(self): pyt_model = models.resnet18(pretrained=True).eval().to("cuda") example_inputs = (torch.randn((100, 3, 224, 224)).to("cuda"),) diff --git a/tests/py/dynamo/models/test_export_kwargs_serde.py b/tests/py/dynamo/models/test_export_kwargs_serde.py index 928d62e7ba..242dbb54fd 100644 --- a/tests/py/dynamo/models/test_export_kwargs_serde.py +++ b/tests/py/dynamo/models/test_export_kwargs_serde.py @@ -23,6 +23,7 @@ @pytest.mark.unit +@pytest.mark.critical def test_custom_model(): class net(nn.Module): def __init__(self): @@ -83,6 +84,7 @@ def forward(self, x, b=5, c=None, d=None): @pytest.mark.unit +@pytest.mark.critical def test_custom_model_with_dynamo_trace(): class net(nn.Module): def __init__(self): diff --git a/tests/py/dynamo/models/test_export_serde.py b/tests/py/dynamo/models/test_export_serde.py index 52e5eefb63..017214b25f 100644 --- a/tests/py/dynamo/models/test_export_serde.py +++ b/tests/py/dynamo/models/test_export_serde.py @@ -17,6 +17,7 @@ trt_ep_path = os.path.join(tempfile.gettempdir(), "trt.ep") +@pytest.mark.critical @pytest.mark.unit def test_base_full_compile(ir): """ diff --git a/tests/py/dynamo/models/test_models.py b/tests/py/dynamo/models/test_models.py index b0ebbf5fa4..86251a38c9 100644 --- a/tests/py/dynamo/models/test_models.py +++ b/tests/py/dynamo/models/test_models.py @@ -85,6 +85,7 @@ def test_resnet18_cpu_offload(ir): @pytest.mark.unit +@pytest.mark.critical def test_mobilenet_v2(ir): model = models.mobilenet_v2(pretrained=True).eval().to("cuda") input = torch.randn((1, 3, 224, 224)).to("cuda") diff --git a/tests/py/dynamo/models/test_models_export.py b/tests/py/dynamo/models/test_models_export.py index 0c28b23bba..b30bd1b2cf 100644 --- a/tests/py/dynamo/models/test_models_export.py +++ b/tests/py/dynamo/models/test_models_export.py @@ -81,6 +81,7 @@ def test_mobilenet_v2(ir): @pytest.mark.unit +@pytest.mark.critical def test_efficientnet_b0(ir): model = timm.create_model("efficientnet_b0", pretrained=True).eval().to("cuda") input = torch.randn((1, 3, 224, 224)).to("cuda") diff --git a/tests/py/dynamo/partitioning/test_fast_partitioning.py b/tests/py/dynamo/partitioning/test_fast_partitioning.py index 30f691d9e9..b53d34d73c 100644 --- a/tests/py/dynamo/partitioning/test_fast_partitioning.py +++ b/tests/py/dynamo/partitioning/test_fast_partitioning.py @@ -1,6 +1,7 @@ from copy import deepcopy import numpy as np +import pytest import torch from torch.testing._internal.common_utils import TestCase, run_tests from torch_tensorrt.dynamo import partitioning @@ -55,6 +56,7 @@ def forward(self, x, y): "Single operators can be segmented if full compilation is required", ) + @pytest.mark.critical def test_partition_fully_supported_multi_op(self): class FullySupportedMultiOp(torch.nn.Module): def __init__(self, *args, **kwargs) -> None: diff --git a/tests/py/dynamo/partitioning/test_global_partitioning.py b/tests/py/dynamo/partitioning/test_global_partitioning.py index 887fa35659..aaf5add515 100644 --- a/tests/py/dynamo/partitioning/test_global_partitioning.py +++ b/tests/py/dynamo/partitioning/test_global_partitioning.py @@ -95,6 +95,7 @@ def forward(self, x, y): "Single operators can be segmented if full compilation is required", ) + @pytest.mark.critical def test_partition_fully_supported_multi_op(self): class FullySupportedMultiOp(torch.nn.Module): def __init__(self, *args, **kwargs) -> None: diff --git a/tests/py/dynamo/runtime/test_000_convert_module_to_trt_engine.py b/tests/py/dynamo/runtime/test_000_convert_module_to_trt_engine.py index b513ff46c8..9f0a7e5949 100644 --- a/tests/py/dynamo/runtime/test_000_convert_module_to_trt_engine.py +++ b/tests/py/dynamo/runtime/test_000_convert_module_to_trt_engine.py @@ -1,14 +1,15 @@ import unittest +import pytest +import tensorrt as trt import torch import torch_tensorrt from torch_tensorrt.dynamo.runtime import PythonTorchTensorRTModule from torch_tensorrt.dynamo.utils import COSINE_THRESHOLD, cosine_similarity -import tensorrt as trt - class TestConvertModuleToTrtEngine(unittest.TestCase): + @pytest.mark.critical def test_convert_module(self): class Test(torch.nn.Module): def forward(self, a, b): diff --git a/tests/py/dynamo/runtime/test_output_allocator.py b/tests/py/dynamo/runtime/test_output_allocator.py index c915f42173..2ec2e229ed 100644 --- a/tests/py/dynamo/runtime/test_output_allocator.py +++ b/tests/py/dynamo/runtime/test_output_allocator.py @@ -48,6 +48,7 @@ class TestOutputAllocatorStaticModel(TestCase): ("cpp_runtime", False), ] ) + @pytest.mark.critical def test_cudagraphs_and_output_allocator(self, _, use_python_runtime): model = StaticModel().eval().cuda() inputs = [torch.randn((2, 3), dtype=torch.float).cuda()] @@ -157,6 +158,7 @@ class TestOutputAllocatorDDSModel(TestCase): ("cpp_runtime", False), ] ) + @pytest.mark.critical def test_cudagraphs_and_output_allocator(self, _, use_python_runtime): model = DDSModel().eval().cuda() inputs = (torch.randint(low=0, high=3, size=(10,), dtype=torch.int).to("cuda"),) diff --git a/tests/py/dynamo/runtime/test_pre_allocated_outputs.py b/tests/py/dynamo/runtime/test_pre_allocated_outputs.py index b8c7b61fb3..2bffc1b35f 100644 --- a/tests/py/dynamo/runtime/test_pre_allocated_outputs.py +++ b/tests/py/dynamo/runtime/test_pre_allocated_outputs.py @@ -1,3 +1,4 @@ +import pytest import torch import torch_tensorrt as torchtrt from parameterized import parameterized @@ -14,6 +15,7 @@ class TestPreAllocatedOutputs(TestCase): ("cpp_runtime", False), ] ) + @pytest.mark.critical def test_pre_allocated_outputs_default(self, _, use_python_runtime): class SampleModel(torch.nn.Module): def forward(self, x): From e012fbdab92c80558cfe779cbc58ca823887ce58 Mon Sep 17 00:00:00 2001 From: lanluo-nvidia Date: Fri, 30 May 2025 09:56:00 -0700 Subject: [PATCH 2/9] test --- .github/workflows/build-test-linux-x86_64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test-linux-x86_64.yml b/.github/workflows/build-test-linux-x86_64.yml index 8d8ff734db..da7cc7a497 100644 --- a/.github/workflows/build-test-linux-x86_64.yml +++ b/.github/workflows/build-test-linux-x86_64.yml @@ -253,7 +253,7 @@ jobs: tests-py-torch-compile-be: name: Test torch compile backend [Python] - needs: [filter-matrix, build] + needs: [filter-matrix, build, tests-py-fail-fast] strategy: fail-fast: false matrix: From 8259280485bc77437c714b20d925ffcc533a3268 Mon Sep 17 00:00:00 2001 From: lanluo-nvidia Date: Mon, 2 Jun 2025 15:33:08 -0700 Subject: [PATCH 3/9] add coverage report --- .coveragerc | 27 ++++++++++++++++++++ .gitignore | 4 ++- pyproject.toml | 7 +++++ tests/py/dynamo/models/test_models_export.py | 1 - 4 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000000..971d784707 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,27 @@ +[run] +source = + py/torch_tensorrt/ + . +omit = + tests/* +relative_files = true +branch = true + +[paths] +source = + py/torch_tensorrt/ + */site-packages/torch_tensorrt/ + +[report] +# Configure minimum coverage threshold +fail_under = 20 +exclude_lines = + # Don't count debug-only code + pragma: no cover + # Don't count empty functions + def __repr__ + # Don't count error handling + raise NotImplementedError + +[html] +directory = coverage_html diff --git a/.gitignore b/.gitignore index 496f53ca2b..e66f053237 100644 --- a/.gitignore +++ b/.gitignore @@ -75,4 +75,6 @@ tests/py/dynamo/models/*.ep *.deb *.tar.xz MODULE.bazel.lock -*.whl \ No newline at end of file +*.whl +.coverage +*.log \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 3bb857e3e0..2e0ff930ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,6 +102,13 @@ Changelog = "https://github.com/pytorch/tensorrt/releases" package-dir = { "" = "py" } include-package-data = false +[tool.pytest.ini_options] +testpaths = ["tests/py"] +addopts = "--cov=py/torch_tensorrt --cov-report=html" + +[tool.coverage.run] +branch = true + [tool.uv] package = true environments = ["sys_platform == 'linux'", "sys_platform == 'windows'"] diff --git a/tests/py/dynamo/models/test_models_export.py b/tests/py/dynamo/models/test_models_export.py index b30bd1b2cf..0c28b23bba 100644 --- a/tests/py/dynamo/models/test_models_export.py +++ b/tests/py/dynamo/models/test_models_export.py @@ -81,7 +81,6 @@ def test_mobilenet_v2(ir): @pytest.mark.unit -@pytest.mark.critical def test_efficientnet_b0(ir): model = timm.create_model("efficientnet_b0", pretrained=True).eval().to("cuda") input = torch.randn((1, 3, 224, 224)).to("cuda") From d084891e36e1ef8d76c80ad7312781b219908c17 Mon Sep 17 00:00:00 2001 From: lanluo-nvidia Date: Mon, 2 Jun 2025 16:49:17 -0700 Subject: [PATCH 4/9] test --- .coveragerc | 27 --------------------------- .gitignore | 1 + pyproject.toml | 39 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 39 insertions(+), 28 deletions(-) delete mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 971d784707..0000000000 --- a/.coveragerc +++ /dev/null @@ -1,27 +0,0 @@ -[run] -source = - py/torch_tensorrt/ - . -omit = - tests/* -relative_files = true -branch = true - -[paths] -source = - py/torch_tensorrt/ - */site-packages/torch_tensorrt/ - -[report] -# Configure minimum coverage threshold -fail_under = 20 -exclude_lines = - # Don't count debug-only code - pragma: no cover - # Don't count empty functions - def __repr__ - # Don't count error handling - raise NotImplementedError - -[html] -directory = coverage_html diff --git a/.gitignore b/.gitignore index e66f053237..f7d554ebe9 100644 --- a/.gitignore +++ b/.gitignore @@ -77,4 +77,5 @@ tests/py/dynamo/models/*.ep MODULE.bazel.lock *.whl .coverage +coverage.xml *.log \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 2e0ff930ea..f561129c78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -104,11 +104,48 @@ include-package-data = false [tool.pytest.ini_options] testpaths = ["tests/py"] -addopts = "--cov=py/torch_tensorrt --cov-report=html" +addopts = "-ra --cov=py/torch_tensorrt --cov-report=html --cov-report=xml:coverage.xml --cov-config=pyproject.toml" + +[tool.coverage.paths] +source = [ + "py/torch_tensorrt/", + "*/site-packages/torch_tensorrt/" +] +omit = [ + "tests/*" +] [tool.coverage.run] +relative_files=true branch = true +[tool.coverage.report] +fail_under = 20 +skip_covered = true +ignore_errors = true +exclude_lines = [ + "pragma: no cover", + # Don't complain about missing debug or verbose code + "def __repr__", + "if verbose", + # Don't complain if tests don't hit defensive exception handling code + "raise AssertionError", + "raise NotImplementedError", + "raise RuntimeError", + "raise ValueError", + "raise KeyError", + "raise AttributeError", + "except ImportError", + # Don't complain if non-runnable code isn't run + "if __name__ == \"__main__\":", + "if TYPE_CHECKING:", + # Don't complain about abstract methods, they aren't run + "@(abc\\.)?abstractmethod", +] + +[tool.coverage.html] +directory = "coverage_html" + [tool.uv] package = true environments = ["sys_platform == 'linux'", "sys_platform == 'windows'"] From 4a5a1e231a1af6894babd4414a187bd534ca66c4 Mon Sep 17 00:00:00 2001 From: lanluo-nvidia Date: Wed, 4 Jun 2025 10:56:58 -0700 Subject: [PATCH 5/9] resolve comments --- .github/workflows/build-test-linux-x86_64.yml | 34 +++++++++++++------ pyproject.toml | 19 ++++------- .../py/dynamo/lowering/test_decompositions.py | 2 +- tests/py/dynamo/models/test_reexport.py | 1 + .../partitioning/test_fast_partitioning.py | 3 +- .../test_000_convert_module_to_trt_engine.py | 5 ++- tests/py/requirements.txt | 1 + 7 files changed, 37 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build-test-linux-x86_64.yml b/.github/workflows/build-test-linux-x86_64.yml index da7cc7a497..607dbc3721 100644 --- a/.github/workflows/build-test-linux-x86_64.yml +++ b/.github/workflows/build-test-linux-x86_64.yml @@ -101,20 +101,32 @@ jobs: export CI_BUILD=1 export LD_LIBRARY_PATH=/usr/lib64:$LD_LIBRARY_PATH pushd . - cd tests/py - python -m pip install -r requirements.txt + python -m pip install -r tests/py/requirements.txt + cov_param="--cov=torch_tensorrt --cov-report= --cov-config=pyproject.toml" + + # test dynamo backend + python -m pytest -m critical ${cov_param} -n 4 tests/py/dynamo/backend/ + + # test dynamo models + python -m pytest -m critical ${cov_param} -ra --cov-append --ir dynamo tests/py/dynamo/models/ - # test dynamo - python -m pytest -m critical --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_backend_test_results.xml -n 4 dynamo/backend/ - python -m pytest -m critical -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_models_result.xml --ir dynamo dynamo/models/ - python -m pytest -m critical --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_automatic_plugin_results.xml dynamo/automatic_plugin/ - python -m pytest -m critical --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_partitioning_results.xml dynamo/partitioning/ - python -m pytest -m critical --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_lowering_results.xml dynamo/lowering/ - python -m pytest -m critical --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_runtime_results.xml dynamo/runtime/ + # test dynamo automatic plugin + python -m pytest -m critical ${cov_param} --cov-append tests/py/dynamo/automatic_plugin/ + # test dynamo partitioning + python -m pytest -m critical ${cov_param} --cov-append tests/py/dynamo/partitioning/ + + # test dynamo lowering + python -m pytest -m critical ${cov_param} --cov-append tests/py/dynamo/lowering/ + + # test dynamo runtime + python -m pytest ${cov_param} --cov-append tests/py/dynamo/runtime/test_000_* + python -m pytest -m critical ${cov_param} --cov-append --ignore tests/py/dynamo/runtime/test_000_* tests/py/dynamo/runtime/ + # test core - python -m pytest -m critical --junitxml=${RUNNER_TEST_RESULTS_DIR}/core_test_results.xml core/ - popd + python -m pytest -m critical ${cov_param} --cov-append tests/py/core/ + + coverage report --fail-under=20 tests-py-torchscript-fe: name: Test torchscript frontend [Python] diff --git a/pyproject.toml b/pyproject.toml index f561129c78..b6145db18b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,6 +77,7 @@ dev = [ "isort", "ruff", "pytest", + "pytest-cov", "pytest-xdist", "parameterized>=0.2.0", "expecttest==0.1.6", @@ -102,25 +103,22 @@ Changelog = "https://github.com/pytorch/tensorrt/releases" package-dir = { "" = "py" } include-package-data = false -[tool.pytest.ini_options] -testpaths = ["tests/py"] -addopts = "-ra --cov=py/torch_tensorrt --cov-report=html --cov-report=xml:coverage.xml --cov-config=pyproject.toml" - [tool.coverage.paths] source = [ - "py/torch_tensorrt/", - "*/site-packages/torch_tensorrt/" + "py/torch_tensorrt", + "*/site-packages/torch_tensorrt" ] + +[tool.coverage.run] +source = ["torch_tensorrt"] omit = [ "tests/*" ] - -[tool.coverage.run] +data_file=".coverage" relative_files=true branch = true [tool.coverage.report] -fail_under = 20 skip_covered = true ignore_errors = true exclude_lines = [ @@ -143,9 +141,6 @@ exclude_lines = [ "@(abc\\.)?abstractmethod", ] -[tool.coverage.html] -directory = "coverage_html" - [tool.uv] package = true environments = ["sys_platform == 'linux'", "sys_platform == 'windows'"] diff --git a/tests/py/dynamo/lowering/test_decompositions.py b/tests/py/dynamo/lowering/test_decompositions.py index 903f9e0eee..1eae282a48 100644 --- a/tests/py/dynamo/lowering/test_decompositions.py +++ b/tests/py/dynamo/lowering/test_decompositions.py @@ -9,7 +9,7 @@ ) from torch.testing._internal.common_utils import TestCase, run_tests from torch_tensorrt.dynamo.utils import ATOL, RTOL - +import pytest from ..testing_utilities import DECIMALS_OF_AGREEMENT, lower_graph_testing diff --git a/tests/py/dynamo/models/test_reexport.py b/tests/py/dynamo/models/test_reexport.py index 297410ae55..4a341dc9ac 100644 --- a/tests/py/dynamo/models/test_reexport.py +++ b/tests/py/dynamo/models/test_reexport.py @@ -71,6 +71,7 @@ def forward(self, x): @pytest.mark.unit +@pytest.mark.critical def test_base_full_compile_multiple_outputs(ir): """ This tests export serde functionality on a base model diff --git a/tests/py/dynamo/partitioning/test_fast_partitioning.py b/tests/py/dynamo/partitioning/test_fast_partitioning.py index b53d34d73c..c143b4d442 100644 --- a/tests/py/dynamo/partitioning/test_fast_partitioning.py +++ b/tests/py/dynamo/partitioning/test_fast_partitioning.py @@ -84,7 +84,7 @@ def forward(self, x, y): 1, "All operators are supported, there should be one segment", ) - + @pytest.mark.critical def test_partition_partially_supported_multi_op(self): class PartiallySupportedMultiOp(torch.nn.Module): def __init__(self, *args, **kwargs) -> None: @@ -114,6 +114,7 @@ def forward(self, x, y): "Unsupported operators interleave supported ones, expected 2 segments", ) + @pytest.mark.critical def test_partition_partially_supported_with_torch_executed_ops(self): class PartiallySupportedMultiOp(torch.nn.Module): def __init__(self, *args, **kwargs) -> None: diff --git a/tests/py/dynamo/runtime/test_000_convert_module_to_trt_engine.py b/tests/py/dynamo/runtime/test_000_convert_module_to_trt_engine.py index 9f0a7e5949..b513ff46c8 100644 --- a/tests/py/dynamo/runtime/test_000_convert_module_to_trt_engine.py +++ b/tests/py/dynamo/runtime/test_000_convert_module_to_trt_engine.py @@ -1,15 +1,14 @@ import unittest -import pytest -import tensorrt as trt import torch import torch_tensorrt from torch_tensorrt.dynamo.runtime import PythonTorchTensorRTModule from torch_tensorrt.dynamo.utils import COSINE_THRESHOLD, cosine_similarity +import tensorrt as trt + class TestConvertModuleToTrtEngine(unittest.TestCase): - @pytest.mark.critical def test_convert_module(self): class Test(torch.nn.Module): def forward(self, a, b): diff --git a/tests/py/requirements.txt b/tests/py/requirements.txt index 13aa40fe44..4a372f75c6 100644 --- a/tests/py/requirements.txt +++ b/tests/py/requirements.txt @@ -6,6 +6,7 @@ numpy parameterized>=0.2.0 pytest>=8.2.1 pytest-xdist>=3.6.1 +pytest-cov pyyaml timm>=1.0.3 flashinfer-python; python_version < "3.13" From abeeecc366d6893b960c24ae2ab64933f6826849 Mon Sep 17 00:00:00 2001 From: lanluo-nvidia Date: Wed, 4 Jun 2025 13:50:32 -0700 Subject: [PATCH 6/9] generate coverage report --- .github/workflows/build-test-linux-x86_64.yml | 8 ++++---- tests/py/dynamo/partitioning/test_fast_partitioning.py | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-test-linux-x86_64.yml b/.github/workflows/build-test-linux-x86_64.yml index 607dbc3721..bc12f51a60 100644 --- a/.github/workflows/build-test-linux-x86_64.yml +++ b/.github/workflows/build-test-linux-x86_64.yml @@ -102,7 +102,7 @@ jobs: export LD_LIBRARY_PATH=/usr/lib64:$LD_LIBRARY_PATH pushd . python -m pip install -r tests/py/requirements.txt - cov_param="--cov=torch_tensorrt --cov-report= --cov-config=pyproject.toml" + cov_param="--cov=torch_tensorrt --cov-report=xml:${RUNNER_TEST_RESULTS_DIR}/coverage.xml --cov-config=pyproject.toml" # test dynamo backend python -m pytest -m critical ${cov_param} -n 4 tests/py/dynamo/backend/ @@ -121,11 +121,11 @@ jobs: # test dynamo runtime python -m pytest ${cov_param} --cov-append tests/py/dynamo/runtime/test_000_* - python -m pytest -m critical ${cov_param} --cov-append --ignore tests/py/dynamo/runtime/test_000_* tests/py/dynamo/runtime/ - + python -m pytest -m critical ${cov_param} --cov-append --ignore tests/py/dynamo/runtime/test_000_* tests/py/dynamo/runtime/ + # test core python -m pytest -m critical ${cov_param} --cov-append tests/py/core/ - + coverage report --fail-under=20 tests-py-torchscript-fe: diff --git a/tests/py/dynamo/partitioning/test_fast_partitioning.py b/tests/py/dynamo/partitioning/test_fast_partitioning.py index c143b4d442..728b3b9f84 100644 --- a/tests/py/dynamo/partitioning/test_fast_partitioning.py +++ b/tests/py/dynamo/partitioning/test_fast_partitioning.py @@ -84,6 +84,7 @@ def forward(self, x, y): 1, "All operators are supported, there should be one segment", ) + @pytest.mark.critical def test_partition_partially_supported_multi_op(self): class PartiallySupportedMultiOp(torch.nn.Module): From 63c8d442e858ef50f4db1102a83b013d70b1a0f1 Mon Sep 17 00:00:00 2001 From: lanluo-nvidia Date: Wed, 4 Jun 2025 17:09:19 -0700 Subject: [PATCH 7/9] install sqlite --- .github/scripts/install-torch-tensorrt.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/scripts/install-torch-tensorrt.sh b/.github/scripts/install-torch-tensorrt.sh index fd88207eb4..59f9fa697b 100644 --- a/.github/scripts/install-torch-tensorrt.sh +++ b/.github/scripts/install-torch-tensorrt.sh @@ -1,6 +1,12 @@ #set -exou pipefail set -x +dnf install -y sqlite-devel + +ldconfig -p | grep sqlite + +find /usr -print | grep libsqlite3.so + TORCH_TORCHVISION=$(grep "^torch" ${PWD}/py/requirements.txt) INDEX_URL=https://download.pytorch.org/whl/${CHANNEL}/${CU_VERSION} PLATFORM=$(python -c "import sys; print(sys.platform)") From 6d37407eaa9a443cd11299b367d82ca482677605 Mon Sep 17 00:00:00 2001 From: lanluo-nvidia Date: Thu, 5 Jun 2025 11:06:08 -0700 Subject: [PATCH 8/9] test --- .github/workflows/build-test-linux-x86_64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test-linux-x86_64.yml b/.github/workflows/build-test-linux-x86_64.yml index bc12f51a60..6522b49b90 100644 --- a/.github/workflows/build-test-linux-x86_64.yml +++ b/.github/workflows/build-test-linux-x86_64.yml @@ -102,7 +102,7 @@ jobs: export LD_LIBRARY_PATH=/usr/lib64:$LD_LIBRARY_PATH pushd . python -m pip install -r tests/py/requirements.txt - cov_param="--cov=torch_tensorrt --cov-report=xml:${RUNNER_TEST_RESULTS_DIR}/coverage.xml --cov-config=pyproject.toml" + cov_param="--cov=torch_tensorrt --cov-report= --cov-config=pyproject.toml" # test dynamo backend python -m pytest -m critical ${cov_param} -n 4 tests/py/dynamo/backend/ From aa99f2670f81ef2d4e7e1c927a9acf914893b00e Mon Sep 17 00:00:00 2001 From: lanluo-nvidia Date: Fri, 6 Jun 2025 08:34:16 -0700 Subject: [PATCH 9/9] add workflow run trigger. --- .github/workflows/build-test-linux-aarch64-jetpack.yml | 7 ++++++- .github/workflows/build-test-linux-aarch64.yml | 7 ++++++- .github/workflows/build-test-windows.yml | 7 ++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-test-linux-aarch64-jetpack.yml b/.github/workflows/build-test-linux-aarch64-jetpack.yml index 744a802bfa..29b6fea881 100644 --- a/.github/workflows/build-test-linux-aarch64-jetpack.yml +++ b/.github/workflows/build-test-linux-aarch64-jetpack.yml @@ -1,7 +1,11 @@ name: Build and test Linux aarch64 wheels for Jetpack on: - pull_request: + #pull_request: + workflow_run: + workflows: ["Build and test Linux x86_64 wheels"] + types: + - completed push: branches: - main @@ -15,6 +19,7 @@ on: jobs: generate-matrix: + if: ${{ github.event.workflow_run.conclusion == 'success' }} uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main with: package-type: wheel diff --git a/.github/workflows/build-test-linux-aarch64.yml b/.github/workflows/build-test-linux-aarch64.yml index 5358cf4b26..4ffbe31b66 100644 --- a/.github/workflows/build-test-linux-aarch64.yml +++ b/.github/workflows/build-test-linux-aarch64.yml @@ -1,7 +1,11 @@ name: Build and test Linux aarch64 wheels on: - pull_request: + #pull_request: + workflow_run: + workflows: ["Build and test Linux x86_64 wheels"] + types: + - completed push: branches: - main @@ -15,6 +19,7 @@ on: jobs: generate-matrix: + if: ${{ github.event.workflow_run.conclusion == 'success' }} uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main with: package-type: wheel diff --git a/.github/workflows/build-test-windows.yml b/.github/workflows/build-test-windows.yml index 7a974411c8..b7cb143a50 100644 --- a/.github/workflows/build-test-windows.yml +++ b/.github/workflows/build-test-windows.yml @@ -1,7 +1,11 @@ name: Build and test Windows wheels on: - pull_request: + #pull_request: + workflow_run: + workflows: ["Build and test Linux x86_64 wheels"] + types: + - completed push: branches: - main @@ -15,6 +19,7 @@ on: jobs: generate-matrix: + if: ${{ github.event.workflow_run.conclusion == 'success' }} uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main with: package-type: wheel