Ensure that E2E tests do not run for packages that won't contain any E2E (DataDog#20967)

* Ensrue that E2E tests do not run for packages that won't contain any E2E

* Add changelog

* Fix typos and remove debug print

* Modify existing test to ensure it is not skipped

Author: AAraKKe

ddev/changelog.d/20967.fixed

@@ -0,0 +1 @@
+Skip E2E test execution for packages that do not define them.

ddev/src/ddev/cli/env/test.py

@@ -78,12 +78,18 @@ def test_command(
     from ddev.config.constants import AppEnvVars
     from ddev.e2e.config import EnvDataStorage
     from ddev.e2e.constants import E2EMetadata
+    from ddev.repo.constants import NOT_E2E_TESTABLE
     from ddev.utils.ci import running_in_ci
     from ddev.utils.structures import EnvVars
 
     app: Application = ctx.obj
-    storage = EnvDataStorage(app.data_dir)
     integration = app.repo.integrations.get(intg_name)
+
+    if integration.name in NOT_E2E_TESTABLE:
+        app.display_info(f"Selected target {integration.name!r} does not have E2E tests to run. Skipping.")
+        return
+
+    storage = EnvDataStorage(app.data_dir)
     active_envs = storage.get_environments(integration.name)
 
     if environment is None:
@@ -109,6 +115,7 @@ def test_command(
             and (not data.get('platforms') or app.platform.name in data['platforms'])
             and (python_filter is None or data.get('python') == python_filter)
         ]
+
     elif environment == 'active':
         env_names = active_envs
     else:

ddev/src/ddev/repo/constants.py

@@ -3,6 +3,7 @@
 # Licensed under a 3-clause BSD style license (see LICENSE)
 CONFIG_DIRECTORY = '.ddev'
 NOT_SHIPPABLE = frozenset(['datadog_checks_dev', 'datadog_checks_tests_helper', 'ddev'])
+NOT_E2E_TESTABLE = frozenset(['datadog_checks_dev', 'datadog_checks_base', 'datadog_checks_tests_helper', 'ddev'])
 FULL_NAMES = {
     'core': 'integrations-core',
     'extras': 'integrations-extras',

ddev/tests/cli/env/test_test.py

@@ -1,7 +1,10 @@
 # (C) Datadog, Inc. 2024-present
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
+from contextlib import nullcontext
+
 import mock
+import pytest
 
 from tests.helpers.mocks import MockPopen
 
@@ -23,3 +26,24 @@ def test_env_vars_repo(ddev, helpers, data_dir, write_result_file, mocker):
     with mock.patch('ddev.utils.structures.EnvVars', side_effect=MockEnvVars):
         result = ddev('env', 'test', 'postgres', 'py3.12')
         assert result.exit_code == 0, result.output
+        # Ensure test was not skipped
+        assert "does not have E2E tests to run" not in result.output
+
+
+@pytest.mark.parametrize(
+    'target, expectation',
+    [
+        ('datadog_checks_dev', nullcontext()),
+        ('datadog_checks_base', nullcontext()),
+        # This will raise an OSError because the package is not a valid integration
+        ('datadog_checks_tests_helper', pytest.raises(OSError)),
+        ('ddev', nullcontext()),
+    ],
+    ids=['datadog_checks_dev', 'datadog_checks_base', 'datadog_checks_tests_helper', 'ddev'],
+)
+@pytest.mark.parametrize('env', ['py3.12', 'all', ''], ids=['py3.12', 'all', 'no-env'])
+def test_env_test_not_e2e_testable(ddev, target: str, env: str, expectation):
+    with expectation:
+        result = ddev('env', 'test', target, env)
+        assert result.exit_code == 0, result.output
+        assert "does not have E2E tests to run" in result.output