diff --git a/.coderabbit.yaml b/.coderabbit.yaml deleted file mode 100644 index 3a1c9de283..0000000000 --- a/.coderabbit.yaml +++ /dev/null @@ -1,72 +0,0 @@ -# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json -# https://docs.coderabbit.ai/getting-started/configure-coderabbit/ -# https://github.com/coderabbitai/awesome-coderabbit/blob/main/README.md#configuration-examples -language: en-US -tone_instructions: '' # this is where we can tell it to not compliment the authors as happz doesn't like it :) -early_access: true -enable_free_tier: true -reviews: - profile: chill # less nitpicky - request_changes_workflow: false - high_level_summary: false - high_level_summary_placeholder: '@coderabbitai summary' - high_level_summary_in_walkthrough: false - auto_title_placeholder: '@coderabbitai' - auto_title_instructions: '' - review_status: false - commit_status: false - fail_commit_status: false - collapse_walkthrough: true - changed_files_summary: true - sequence_diagrams: true - assess_linked_issues: true - related_issues: true - related_prs: true - suggested_labels: true - auto_apply_labels: false - suggested_reviewers: true - auto_assign_reviewers: false - poem: false - labeling_instructions: [] - path_filters: [] - path_instructions: [] - abort_on_close: true - disable_cache: false - auto_review: - enabled: false # Only do reviews when tagged - auto_incremental_review: false # Do not update reviews on each new commit - ignore_title_keywords: [] - labels: [] - drafts: false - base_branches: [] - finishing_touches: - docstrings: - enabled: false -chat: - auto_reply: false # Needs to be tagged to start a chat - create_issues: false - integrations: - jira: - usage: auto - linear: - usage: disabled -knowledge_base: - opt_out: false - web_search: - enabled: true - learnings: - scope: auto - issues: - scope: auto - jira: - usage: auto - project_keys: [] - linear: - usage: disabled - team_keys: [] - pull_requests: - scope: auto -code_generation: - docstrings: # disabled above - language: en-US - path_instructions: [] diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index dae03eaaf7..e59fc6ceac 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,14 +1,21 @@ +# This is a comment. +# Each line is a file pattern followed by one or more owners. + +# These owners will be the default owners for everything in +# the repo. Unless a later match takes precedence +* @psss @lukaszachy @happz @thrix @janhavlin + +# Packaging +/pyproject.toml @psss @lukaszachy @happz @thrix @janhavlin @martinhoyer +/tmt.spec @psss @lukaszachy @happz @thrix @janhavlin @martinhoyer + # Provision plugins /tmt/steps/provision/artemis.py @happz +/tmt/steps/provision/testcloud.py @frantisekz /tmt/steps/provision/mrack* @dav-pascual -/tmt/steps/provision/testcloud.py @lbrabec # Report plugins /tmt/steps/report/polarion.py @KwisatzHaderach -/tmt/steps/report/reportportal.py @4N0body5 # Export plugins /tmt/export/polarion.py @KwisatzHaderach - -# Specific areas -/tmt/trying.py @falconizmi diff --git a/.github/actions/setup-docs-env/action.yml b/.github/actions/setup-docs-env/action.yml deleted file mode 100644 index 963f4ef6ef..0000000000 --- a/.github/actions/setup-docs-env/action.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: 'Setup Documentation Environment' -description: 'Sets up Python and Hatch environment for documentation builds' - -inputs: - python-version: - description: 'Python version to use' - default: '3.x' - required: false - -runs: - using: "composite" - steps: - - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python-version }} - cache: 'pip' - - - name: Cache Hatch environments - uses: actions/cache@v4 - with: - path: ${{ github.workspace }}/.hatch_envs - key: hatch-envs-${{ runner.os }}-${{ hashFiles('pyproject.toml') }} - restore-keys: | - hatch-envs-${{ runner.os }}- - - - name: Install hatch - run: pip install hatch - shell: bash diff --git a/.github/renovate.json b/.github/renovate.json deleted file mode 100644 index ef69a1d232..0000000000 --- a/.github/renovate.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "packageRules": [ - { - "groupName": "CI and devDependencies", - "matchManagers": ["github-actions", "pre-commit"] - }, - { - "groupName": "Runtime and test", - "matchManagers": ["pep621"] - } - ], - "separateMajorMinor": false, - "extends": [ - "config:recommended", - "schedule:weekly", - ":enablePreCommit", - ":semanticCommitTypeAll(chore)" - ] -} diff --git a/.github/workflows/doc-tests.yml b/.github/workflows/doc-tests.yml index 30a4f9465e..1c70979c1b 100644 --- a/.github/workflows/doc-tests.yml +++ b/.github/workflows/doc-tests.yml @@ -22,39 +22,24 @@ jobs: with: persist-credentials: false - - uses: ./.github/actions/setup-docs-env + - uses: actions/setup-python@v5 + with: + python-version: '3.x' + cache: 'pip' + + - name: Cache Hatch environments + uses: actions/cache@v4 + with: + path: | + ${{ github.workspace }}/.hatch_envs + key: hatch-envs-${{ hashFiles('pyproject.toml') }} + + - name: Install hatch + run: pip install hatch - name: Run sphinx html builder + # -W = warnings as error run: hatch run docs:html -W - - name: Check GitHub API Rate Limit - id: rate_limit_check - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - API_URL="https://api.github.com/rate_limit" - RESPONSE=$(curl -s -H "Authorization: token $GH_TOKEN" -H "Accept: application/vnd.github.v3+json" $API_URL) - REMAINING_CORE=$(echo "$RESPONSE" | jq -r '.resources.core.remaining') - echo "Remaining core API calls: $REMAINING_CORE" - THRESHOLD=100 - echo "remaining_core=$REMAINING_CORE" >> $GITHUB_OUTPUT - echo "threshold=$THRESHOLD" >> $GITHUB_OUTPUT - if [ "$REMAINING_CORE" -lt "$THRESHOLD" ]; then - echo "skip_linkcheck=true" >> $GITHUB_OUTPUT - echo "Low GitHub API core rate limit ($REMAINING_CORE remaining). Skipping linkcheck." - else - echo "skip_linkcheck=false" >> $GITHUB_OUTPUT - echo "Sufficient GitHub API core rate limit ($REMAINING_CORE remaining)." - fi - shell: bash - - name: Run sphinx linkcheck - if: steps.rate_limit_check.outputs.skip_linkcheck == 'false' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: hatch run docs:linkcheck - - - name: Report Skipped Linkcheck - if: steps.rate_limit_check.outputs.skip_linkcheck == 'true' - run: | - echo "::warning title=Linkcheck Skipped Due to Rate Limit::The Sphinx documentation link check was skipped because the remaining GitHub API core rate limit (${{ steps.rate_limit_check.outputs.remaining_core }} remaining) was below the threshold (${{ steps.rate_limit_check.outputs.threshold }}). The link check will be performed when the rate limit resets." diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 0b15de5393..75d958d5d4 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -11,10 +11,6 @@ jobs: env: SKIP: no-commit-to-branch steps: - # Required because of https://github.com/hadolint/hadolint/issues/886 - - run: | - wget -O /usr/local/bin/hadolint https://github.com/hadolint/hadolint/releases/latest/download/hadolint-Linux-x86_64 - chmod +x /usr/local/bin/hadolint - uses: actions/checkout@v4 with: persist-credentials: false diff --git a/.github/workflows/publish-images.yml b/.github/workflows/publish-images.yml deleted file mode 100644 index 0e6519d428..0000000000 --- a/.github/workflows/publish-images.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: publish_images -on: workflow_dispatch - -jobs: - publish-images: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Build - tmt - uses: redhat-actions/buildah-build@v2 - id: build-image-tmt - with: - image: tmt - containerfiles: ./containers/Containerfile.mini - - name: Build - tmt-all - uses: redhat-actions/buildah-build@v2 - id: build-image-tmt-all - with: - image: tmt-all - containerfiles: ./containers/Containerfile.full - - name: Push To quay.io - tmt - id: push-to-quay-tmt - uses: redhat-actions/push-to-registry@v2 - with: - image: ${{ steps.build-image-tmt.outputs.image }} - tags: ${{ steps.build-image-tmt.outputs.tags }} - registry: quay.io/teemtee - username: teemtee+github_action - password: ${{ secrets.QUAY_TEEMTEE_SECRET }} - - name: Push To quay.io - tmt-all - id: push-to-quay-tmt-all - uses: redhat-actions/push-to-registry@v2 - with: - image: ${{ steps.build-image-tmt-all.outputs.image }} - tags: ${{ steps.build-image-tmt-all.outputs.tags }} - registry: quay.io/teemtee - username: teemtee+github_action - password: ${{ secrets.QUAY_TEEMTEE_SECRET }} diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index a0ce91fc58..33d3600b69 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -26,17 +26,16 @@ jobs: name: Differential ShellCheck uses: redhat-plumbers-in-action/differential-shellcheck@v4 with: - # Keep exclusions in sync with shellcheck-py hook in .pre-commit-config.yaml exclude-path: | tests/** - examples/** + examples/**/test.sh tmt/steps/execute/scripts/*.sh.j2 tmt/templates/** token: ${{ secrets.GITHUB_TOKEN }} - if: ${{ always() }} name: Upload artifact with ShellCheck defects in SARIF format - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: Differential ShellCheck SARIF path: ${{ steps.ShellCheck.outputs.sarif }} diff --git a/.gitignore b/.gitignore index fdebac38ff..a488e5b83d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,17 +2,15 @@ /tmp/ docs/code/autodocs/*.rst -docs/code/template-extensions.rst +docs/code/template-filters.rst docs/plugins/discover.rst docs/plugins/execute.rst docs/plugins/finish.rst docs/plugins/prepare.rst docs/plugins/provision.rst docs/plugins/report.rst -docs/plugins/prepare-feature.rst docs/plugins/test-checks.rst docs/plugins/hardware-matrix.rst -docs/guide/test-runner-guest-compatibility-matrix.inc.rst docs/_build docs/spec docs/stories diff --git a/.hadolint.yaml b/.hadolint.yaml deleted file mode 100644 index 4f6158a7b2..0000000000 --- a/.hadolint.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# See https://github.com/hadolint/hadolint?tab=readme-ov-file#configure -failure-threshold: style -format: tty -ignored: - # Using latest is prone to errors if the image will ever update. Pin the version explicitly to a release tag. - - DL3007 - # Pin versions in apk add - - DL3018 - # Specify version with dnf install -y - - - DL3041 -strict-labels: true -disable-ignore-pragma: false -trustedRegistries: [] diff --git a/.packit.yaml b/.packit.yaml index e892d74f99..049c405426 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -25,145 +25,144 @@ targets: &all-targets - fedora-all - epel-9 -# Uncomment below line if OpenScanHub scans are failing -# osh_diff_scan_after_copr_build: false +osh_diff_scan_after_copr_build: false - -# Common definitions -_: - # Copr jobs under the packit project - - &copr-under-packit +jobs: + # Build pull requests + - &copr_build job: copr_build - additional_repos: - - copr://@teemtee/stable - - # Copr jobs under the teemtee project - - &copr-under-teemtee - <<: *copr-under-packit - list_on_homepage: True - preserve_project: True - owner: "@teemtee" + trigger: pull_request + actions: + <<: *base-actions + get-current-version: + # Use `dev888` instead of the last version digit to make + # builds older than the final release and newer than + # copr builds from main. Remove the remaining suffix + # generated by `hatch version` as well so that build + # with the latest timestamp always wins. + - bash -c "hatch version | sed -E 's/\\.[0-9]+\\.dev.*/.dev888/'" - # Test jobs base setup - - &test-base + # Test core + - &tests job: tests trigger: pull_request - tf_extra_params: - environments: - - &copr-teemtee-stable - artifacts: - - type: repository-file - id: https://copr.fedorainfracloud.org/coprs/g/teemtee/stable/repo/fedora-rawhide/group_teemtee-stable-fedora-rawhide.repo - - type: repository-file - id: https://copr.fedorainfracloud.org/coprs/g/teemtee/stable/repo/epel-9/group_teemtee-stable-epel-9.repo - - # Latest fedora & epel targets - - &latest-targets - - fedora-latest-stable - - epel-9 - - # Internal jobs - - &internal - use_internal_tf: True - targets: + targets: &latest-targets - fedora-latest-stable + - epel-9 + identifier: core tf_extra_params: - environments: - - &tmt-cloud-resources - <<: *copr-teemtee-stable - settings: - provisioning: - tags: - BusinessUnit: tmt + test: + tmt: + name: /plans/features/(core|basic) - # Full test requires - - &require-full-tests - require: + # Test full + # Do not run extended unit tests, that plan gets its own job because + # of podman vs systemd-resolved flakiness. + - <<: *tests + targets: *all-targets + require: &require-full-tests label: present: - ci | full test absent: - status | discuss - - -jobs: - # Build released bits to stable - - <<: *copr-under-teemtee - trigger: release - project: stable - - # Build commits merged to main to latest - - <<: *copr-under-teemtee - trigger: commit - branch: main - project: latest - release_suffix: "{PACKIT_PROJECT_BRANCH}" - - # Build pull requests - - <<: *copr-under-packit - trigger: pull_request - - # Test core - - <<: *test-base - targets: *latest-targets - identifier: core - tmt_plan: '/plans/features/(core|basic)' - - # Test full - - <<: *test-base - <<: *require-full-tests identifier: full - # Do not run extended unit tests. That plan gets its own job - # because of podman vs systemd-resolved flakiness. - tmt_plan: '^(?!/plans/features/extended-unit-tests).*$' + tf_extra_params: + test: + tmt: + name: '^(?!/plans/features/extended-unit-tests).*$' # Extended unit tests - - <<: *test-base - <<: *require-full-tests + - <<: *tests targets: [ fedora-latest-stable ] + require: *require-full-tests identifier: extended-unit-tests - tmt_plan: '/plans/features/extended-unit-tests$' + tf_extra_params: + test: + tmt: + name: '/plans/features/extended-unit-tests$' - # Test virtual & bootc provision plugins - - <<: *test-base - <<: *internal - <<: *require-full-tests + # Test virtual provision + - <<: *tests + use_internal_tf: True + targets: &internal-targets + - fedora-latest-stable + require: *require-full-tests identifier: provision - tmt_plan: '/plans/provision/(bootc|virtual)' tf_extra_params: + test: + tmt: + name: /plans/provision/virtual environments: - tmt: context: how: provision - <<: *tmt-cloud-resources + - &tmt-cloud-resources + settings: + provisioning: + tags: + BusinessUnit: tmt # Test internal plugins - - <<: *test-base - <<: *internal - <<: *require-full-tests + - <<: *tests + use_internal_tf: True + targets: *internal-targets + require: *require-full-tests identifier: internal-plugins fmf_url: "https://gitlab.cee.redhat.com/baseos-qe/tmt.git" + # Tag cloud resources for tmt + tf_extra_params: + environments: + - *tmt-cloud-resources # Test internal wow - - <<: *test-base - <<: *internal - <<: *require-full-tests + - <<: *tests + use_internal_tf: True + targets: *internal-targets + require: *require-full-tests identifier: internal-wow fmf_url: "https://gitlab.cee.redhat.com/baseos-qe/integration_scripts.git" - tmt_plan: '/tmt/integration/plan' + tmt_plan: "/tmt/integration/plan" + tf_extra_params: + environments: + - *tmt-cloud-resources + + # Build commits to main + - <<: *copr_build + trigger: commit + branch: main + list_on_homepage: True + preserve_project: True + owner: "@teemtee" + project: tmt + release_suffix: "{PACKIT_PROJECT_BRANCH}" + actions: + <<: *base-actions + get-current-version: + # Get rid of the the final version digit to make copr + # builds older than the final release + - bash -c "hatch version | sed -E 's/\\.[0-9]+\\.dev/.dev/'" + + # Release to copr + - <<: *copr_build + trigger: release + list_on_homepage: True + preserve_project: True + owner: "@teemtee" + project: tmt + actions: *base-actions - # Propose downstream pull requests + # Fedora releases - job: propose_downstream trigger: release dist_git_branches: *all-targets - # Create koji builds - job: koji_build trigger: commit - allowed_pr_authors: ["packit", "all_committers"] + allowed_pr_authors: [ "packit", "psss", "lzachar" ] + allowed_committers: [ "packit", "psss", "lzachar" ] dist_git_branches: *all-targets - # Create bodhi updates - job: bodhi_update trigger: commit dist_git_branches: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9de4daa502..b21c98add2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,11 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks repos: + - repo: https://github.com/hhatto/autopep8 + rev: 'v2.3.1' + hooks: + - id: autopep8 + - repo: https://github.com/pre-commit/pre-commit-hooks rev: "v5.0.0" hooks: @@ -16,7 +21,7 @@ repos: - id: trailing-whitespace - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.15.0" + rev: "v1.12.0" hooks: - id: mypy language_version: "3.9" @@ -27,23 +32,22 @@ repos: # # For now, we simply copy & paste from pyproject.toml :( additional_dependencies: - - "click>=8.0.3" - - "docutils>=0.16" - - "fmf>=1.7.0" - - "jinja2>=2.11.3" - - "packaging>=20" - - "pint>=0.16.1" - - "pydantic>=1.10.14" - - "pygments>=2.7.4" - - "requests>=2.25.1" - - "ruamel.yaml>=0.16.6" - - "urllib3>=1.26.5, <3.0" + - "click>=8.0.3,!=8.1.4" # 8.1.3 / 8.1.6 TODO type annotations tmt.cli.Context -> click.core.Context click/issues/2558 + - "docutils>=0.16" # 0.16 is the current one available for RHEL9 + - "fmf>=1.3.0" + - "jinja2>=2.11.3" # 3.1.2 / 3.1.2 + - "packaging>=20" # 20 seems to be available with RHEL8 + - "pint>=0.16.1" # 0.16.1 + - "pygments>=2.7.4" # 2.7.4 is the current one available for RHEL9 + - "requests>=2.25.1" # 2.28.2 / 2.31.0 + - "ruamel.yaml>=0.16.6" # 0.17.32 / 0.17.32 + - "urllib3>=1.26.5, <2.0" # 1.26.16 / 2.0.4 - "jira>=3.5.0, <4" # report-junit - "lxml>=4.6.5" - - "typing-extensions>=4.12.2; python_version < '3.13'" + - "typing-extensions>=4.9.0; python_version < '3.13'" - "pytest" - "requre" # Do not install *types-click* - it's not recommended with Click 8 & newer, @@ -62,9 +66,10 @@ repos: - "types-lxml" pass_filenames: false + args: [--config-file=pyproject.toml] - repo: https://github.com/RobertCraigie/pyright-python - rev: v1.1.399 + rev: v1.1.385 hooks: - id: pyright language_version: "3.9" @@ -75,17 +80,16 @@ repos: # # For now, we simply copy & paste from pyproject.toml :( additional_dependencies: - - "click>=8.0.3" - - "docutils>=0.16" - - "fmf>=1.7.0" - - "jinja2>=2.11.3" - - "packaging>=20" - - "pint>=0.16.1" - - "pydantic>=1.10.14" - - "pygments>=2.7.4" - - "requests>=2.25.1" - - "ruamel.yaml>=0.16.6" - - "urllib3>=1.26.5, <3.0" + - "click>=8.0.3,!=8.1.4" # 8.1.3 / 8.1.6 TODO type annotations tmt.cli.Context -> click.core.Context click/issues/2558 + - "docutils>=0.16" # 0.16 is the current one available for RHEL9 + - "fmf>=1.3.0" + - "jinja2>=2.11.3" # 3.1.2 / 3.1.2 + - "packaging>=20" # 20 seems to be available with RHEL8 + - "pint>=0.16.1" # 0.16.1 / 0.19.x TODO: Pint 0.20 requires larger changes to tmt.hardware + - "pygments>=2.7.4" # 2.7.4 is the current one available for RHEL9 + - "requests>=2.25.1" # 2.28.2 / 2.31.0 + - "ruamel.yaml>=0.16.6" # 0.17.32 / 0.17.32 + - "urllib3>=1.26.5, <2.0" # 1.26.16 / 2.0.4 - "jira>=3.5.0, <4" # report-junit @@ -110,24 +114,20 @@ repos: - "types-lxml" - repo: https://github.com/python-jsonschema/check-jsonschema - rev: "0.33.0" + rev: "0.29.4" hooks: - id: check-metaschema name: "Check JSON schemas validity" files: ^tmt/schemas/.*\.yaml - repo: https://github.com/adrienverge/yamllint - rev: v1.37.0 + rev: v1.35.1 hooks: - id: yamllint - files: > - (?x)^( - tmt/(?:schemas|steps/prepare/feature)/.+\.yaml| - tests/policy/policies/.+/.+\.yaml - )$ + files: ^tmt/schemas/.*\.yaml - repo: https://github.com/ansible-community/ansible-lint.git - rev: v25.2.1 + rev: v24.9.2 hooks: - id: ansible-lint args: @@ -143,45 +143,42 @@ repos: # in order to be parsed by ansible-lint - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.11.5 + rev: v0.7.0 hooks: - id: ruff args: - '--fix' - '--show-fixes' - - id: ruff-format - repo: https://github.com/teemtee/tmt.git - rev: 1.46.0 + rev: 1.37.0 hooks: - id: tmt-lint additional_dependencies: - "docutils>=0.16" # 0.16 is the current one available for RHEL9 - "packaging>=20" # 20 seems to be available with RHEL8 - "pint<0.20" - - "pydantic>=1.10.14" - "pygments>=2.7.4" # 2.7.4 is the current one available for RHEL9 # Help installation by reducing the set of inspected botocore release. # There is *a lot* of them, and hatch might fetch many of them. - "botocore>=1.25.10" # 1.25.10 is the current one available for RHEL9 - "typing-extensions>=4.9.0; python_version < '3.13'" - - "fmf>=1.6.0" - repo: https://github.com/codespell-project/codespell - rev: v2.4.1 + rev: v2.3.0 hooks: - id: codespell additional_dependencies: - tomli # Required for python < 3.11 - repo: https://github.com/djlint/djLint - rev: v1.36.4 + rev: v1.35.2 hooks: - id: djlint files: "\\.j2" types_or: ['jinja'] - repo: https://github.com/aristanetworks/j2lint.git - rev: v1.2.0 + rev: v1.1.0 hooks: - id: j2lint args: @@ -191,23 +188,3 @@ repos: - jinja-variable-lower-case - single-statement-per-line - "--" - - - repo: https://github.com/AleksaC/hadolint-py - rev: v2.12.1b3 - hooks: - - - id: hadolint - files: ^containers/.* - - - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.10.0.1 - hooks: - - id: shellcheck - # Local shellcheck for developers - keep exclusions in sync with .github/workflows/shellcheck.yml - exclude: | - (?x)^( - tests/.*| - examples/.*| - tmt/steps/execute/scripts/.*\.sh\.j2| - tmt/templates/.* - )$ diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 58a0c5cd92..57bf139739 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,9 +1,5 @@ # Config for building https://tmt.readthedocs.io/ version: 2 - -sphinx: - configuration: docs/conf.py - python: install: - method: pip diff --git a/.yamllint b/.yamllint deleted file mode 100644 index bf8a6f079c..0000000000 --- a/.yamllint +++ /dev/null @@ -1,35 +0,0 @@ ---- - -# The following settings are recommended by ansible-lint project. They -# may conflict with our other YAML files, but as of now, we test only -# a very small subset of all YAML files in the repository. Once we start -# extending the coverage, it may become a problem, and we will need to -# resolve it, but the settings below seem reasonable enough that any -# YAML file could comply. - -extends: default - -rules: - comments: - min-spaces-from-content: 1 - - comments-indentation: false - - braces: - max-spaces-inside: 1 - - line-length: - max: 100 - - octal-values: - forbid-implicit-octal: true - forbid-explicit-octal: true - - truthy: - level: error - - allowed-values: - - "true" - - "false" - - check-keys: true diff --git a/Makefile b/Makefile index 68d8f94bb8..6a57137e0f 100644 --- a/Makefile +++ b/Makefile @@ -7,9 +7,9 @@ # Prepare variables TMP = $(CURDIR)/tmp -ccred=$(shell env TERM="$${TERM:-linux}" tput setaf 1) -ccgreen=$(shell env TERM="$${TERM:-linux}" tput setaf 2) -ccend=$(shell env TERM="$${TERM:-linux}" tput sgr0) +ccred=$(shell tput setaf 1) +ccgreen=$(shell tput setaf 2) +ccend=$(shell tput sgr0) # Define special targets .DEFAULT_GOAL := help @@ -82,46 +82,22 @@ ver2spec: ## ## Containers ## +images: ## Build tmt images for podman/docker + podman build -t tmt --squash -f ./containers/Containerfile.mini . + podman build -t tmt-all --squash -f ./containers/Containerfile.full . -# Base images of our tmt container images, collected from `FROM ...` directives in Containerfiles. -TMT_DISTRO_IMAGE_BASES = $(shell grep -h 'FROM ' containers/Containerfile.* | cut -d' ' -f2 | sort | uniq) - -# All tmt image targets will begin with this string. -TMT_DISTRO_IMAGE_TARGET_PREFIX = images - -# All tmt container images will begin with this string. -TMT_DISTRO_CONTAINER_IMAGE_NAME_PREFIX = tmt/container - -# The list of tmt container images. -TMT_DISTRO_CONTAINER_IMAGES := $(TMT_DISTRO_CONTAINER_IMAGE_NAME_PREFIX)/tmt:latest \ - $(TMT_DISTRO_CONTAINER_IMAGE_NAME_PREFIX)/tmt-all:latest - -# The list of targets building individual tmt images. -TMT_DISTRO_IMAGES_TARGETS := $(foreach image,$(TMT_DISTRO_CONTAINER_IMAGES),images/$(subst :,\:,$(image))) - -# Base images of our test images, collected from `FROM ...` directives in Containerfiles TMT_TEST_IMAGE_BASES = $(shell grep -rh 'FROM ' containers/ | cut -d' ' -f2 | sort | uniq) +TMT_TEST_IMAGE_TARGET_PREFIX = images-tests +TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX = tmt/tests/container -# All tmt test image targets will begin with this string. -TMT_TEST_IMAGE_TARGET_PREFIX = images/test - -# All tmt test container images will begin with this string. -TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX = tmt/container/test - -# The list of tmt test container images. TMT_TEST_CONTAINER_IMAGES := $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/alpine:latest \ $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/alpine/upstream:latest \ $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/centos/7:latest \ $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/centos/7/upstream:latest \ $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/centos/stream9:latest \ $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/centos/stream9/upstream:latest \ - $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/centos/stream10:latest \ - $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/centos/stream10/upstream:latest \ $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/coreos:stable \ $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/coreos/ostree:stable \ - $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/latest:latest \ - $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/latest/upstream:latest \ - $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/latest/unprivileged:latest \ $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/rawhide:latest \ $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/rawhide/upstream:latest \ $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/rawhide/unprivileged:latest \ @@ -131,56 +107,31 @@ TMT_TEST_CONTAINER_IMAGES := $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/alpine:late $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/40:latest \ $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/40/upstream:latest \ $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/40/unprivileged:latest \ + $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/39:latest \ + $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/39/upstream:latest \ + $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/39/unprivileged:latest \ $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/ubi/8/upstream:latest \ $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/ubuntu/22.04/upstream:latest \ - $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/debian/12.7/upstream:latest \ - $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/latest/bootc:latest + $(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/debian/12.7/upstream:latest -# The list of targets building individual tmt test images. -TMT_TEST_IMAGES_TARGETS := $(foreach image,$(TMT_TEST_CONTAINER_IMAGES),images/test/$(subst :,\:,$(image))) +TMT_TEST_IMAGES_TARGETS := $(foreach image,$(TMT_TEST_CONTAINER_IMAGES),images-tests/$(subst :,\:,$(image))) -images: $(TMT_DISTRO_IMAGES_TARGETS) ## Build tmt images for podman/docker - podman images | grep 'localhost/$(TMT_DISTRO_CONTAINER_IMAGE_NAME_PREFIX)/' | sort - -images/test: $(TMT_TEST_IMAGES_TARGETS) ## Build customized images for tests +images-tests: $(TMT_TEST_IMAGES_TARGETS) ## Build customized images for tests podman images | grep 'localhost/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/' | sort -images/test/bases: ## Download base images for custom test images +images-tests-bases: ## Download base images for custom test images podman pull $(TMT_TEST_IMAGE_BASES) -# Build a single container: -define do-build-container-image = -@ echo "$(ccred)$$(date '+%Y-%m-%d %H:%M:%S')$(ccend) $(ccgreen)Building$(ccend) $(ccred)${1}$(ccend) $(ccgreen)image...$(ccend)" -podman build ${3} -t ${1} -f ./containers/${2} . -@ echo "$(ccred)$$(date '+%Y-%m-%d %H:%M:%S')$(ccend) $(ccgreen)Building$(ccend) $(ccred)${1}$(ccend) $(ccgreen)image done$(ccend)" -endef - -# Return an image name from the given target: -define container-image-target-to-name = -$(subst $(TMT_DISTRO_IMAGE_TARGET_PREFIX)/,,${1}) -endef - -# Return a test image name from the given target: define test-container-image-target-to-name = $(subst $(TMT_TEST_IMAGE_TARGET_PREFIX)/,,${1}) endef -# Build tmt image: -define build-container-image = -$(call do-build-container-image,$(call container-image-target-to-name,${1}),${2},--squash) -endef - -# Build tmt test image: define build-test-container-image = -$(call do-build-container-image,$(call test-container-image-target-to-name,${1}),${2},) +@ echo "$(ccgreen)Building $(ccred)$(call test-container-image-target-to-name,$@)$(ccend) $(ccgreen)image...$(ccend)" +podman build -t $(call test-container-image-target-to-name,${1}) -f ./containers/${2} . +@ echo endef -$(TMT_DISTRO_IMAGE_TARGET_PREFIX)/$(TMT_DISTRO_CONTAINER_IMAGE_NAME_PREFIX)/tmt\:latest: - $(call build-container-image,$@,Containerfile.mini) - -$(TMT_DISTRO_IMAGE_TARGET_PREFIX)/$(TMT_DISTRO_CONTAINER_IMAGE_NAME_PREFIX)/tmt-all\:latest: - $(call build-container-image,$@,Containerfile.full) - $(TMT_TEST_IMAGE_TARGET_PREFIX)/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/alpine\:latest: $(call build-test-container-image,$@,alpine/Containerfile) @@ -193,12 +144,6 @@ $(TMT_TEST_IMAGE_TARGET_PREFIX)/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/centos/7 $(TMT_TEST_IMAGE_TARGET_PREFIX)/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/centos/7/upstream\:latest: $(call build-test-container-image,$@,centos/7/Containerfile.upstream) -$(TMT_TEST_IMAGE_TARGET_PREFIX)/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/centos/stream10\:latest: - $(call build-test-container-image,$@,centos/stream10/Containerfile) - -$(TMT_TEST_IMAGE_TARGET_PREFIX)/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/centos/stream10/upstream\:latest: - $(call build-test-container-image,$@,centos/stream10/Containerfile.upstream) - $(TMT_TEST_IMAGE_TARGET_PREFIX)/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/centos/stream9\:latest: $(call build-test-container-image,$@,centos/stream9/Containerfile) @@ -220,15 +165,6 @@ $(TMT_TEST_IMAGE_TARGET_PREFIX)/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/r $(TMT_TEST_IMAGE_TARGET_PREFIX)/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/rawhide/unprivileged\:latest: $(call build-test-container-image,$@,fedora/rawhide/Containerfile.unprivileged) -$(TMT_TEST_IMAGE_TARGET_PREFIX)/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/latest\:latest: - $(call build-test-container-image,$@,fedora/latest/Containerfile) - -$(TMT_TEST_IMAGE_TARGET_PREFIX)/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/latest/upstream\:latest: - $(call build-test-container-image,$@,fedora/latest/Containerfile.upstream) - -$(TMT_TEST_IMAGE_TARGET_PREFIX)/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/latest/unprivileged\:latest: - $(call build-test-container-image,$@,fedora/latest/Containerfile.unprivileged) - $(TMT_TEST_IMAGE_TARGET_PREFIX)/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/41\:latest: $(call build-test-container-image,$@,fedora/41/Containerfile) @@ -247,6 +183,15 @@ $(TMT_TEST_IMAGE_TARGET_PREFIX)/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/4 $(TMT_TEST_IMAGE_TARGET_PREFIX)/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/40/unprivileged\:latest: $(call build-test-container-image,$@,fedora/40/Containerfile.unprivileged) +$(TMT_TEST_IMAGE_TARGET_PREFIX)/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/39\:latest: + $(call build-test-container-image,$@,fedora/39/Containerfile) + +$(TMT_TEST_IMAGE_TARGET_PREFIX)/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/39/upstream\:latest: + $(call build-test-container-image,$@,fedora/39/Containerfile.upstream) + +$(TMT_TEST_IMAGE_TARGET_PREFIX)/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/39/unprivileged\:latest: + $(call build-test-container-image,$@,fedora/39/Containerfile.unprivileged) + $(TMT_TEST_IMAGE_TARGET_PREFIX)/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/ubi/8/upstream\:latest: $(call build-test-container-image,$@,ubi/8/Containerfile.upstream) @@ -256,13 +201,11 @@ $(TMT_TEST_IMAGE_TARGET_PREFIX)/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/ubuntu/2 $(TMT_TEST_IMAGE_TARGET_PREFIX)/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/debian/12.7/upstream\:latest: $(call build-test-container-image,$@,debian/12.7/Containerfile.upstream) -$(TMT_TEST_IMAGE_TARGET_PREFIX)/$(TMT_TEST_CONTAINER_IMAGE_NAME_PREFIX)/fedora/latest/bootc\:latest: - $(call build-test-container-image,$@,fedora/latest/bootc/Containerfile) ## ## Development ## develop: _deps ## Install development requirements - sudo dnf install -y expect gcc git python3-nitrate {libvirt,krb5,libpq,python3}-devel jq podman buildah /usr/bin/python3.9 + sudo dnf install -y gcc git python3-nitrate {libvirt,krb5,libpq,python3}-devel jq podman buildah /usr/bin/python3.9 # Git vim tags and cleanup tags: @@ -280,12 +223,7 @@ clean: ## Remove all temporary files, packaging artifacts and docs rm -rf examples/convert/{main.fmf,test.md,Manual} Manual rm -f tests/full/repo_copy.tgz -clean/images: ## Remove tmt images - for image in $(TMT_DISTRO_CONTAINER_IMAGES); do \ - podman rmi -i "$$image"; \ - done - -clean/images/test: ## Remove all custom images built for tests +clean-test-images: ## Remove all custom images built for tests for image in $(TMT_TEST_CONTAINER_IMAGES); do \ podman rmi -i "$$image"; \ done diff --git a/README.rst b/README.rst index 52962d1807..9406437ce4 100644 --- a/README.rst +++ b/README.rst @@ -3,8 +3,6 @@ tmt - Test Management Tool ================================================================== -|docs| |matrix| |fedora-pkg| |copr-build| |pypi-version| |license| - The ``tmt`` tool provides a user-friendly way to work with tests. You can comfortably create new tests, safely and easily run tests across different environments, review test results, debug test @@ -22,43 +20,4 @@ human and machine readable way close to the source code. Thanks to inheritance and elasticity metadata are organized in the structure efficiently, preventing unnecessary duplication. - -| Check the `overview`__ section for `install`__ instructions and - a couple of inspirative `examples`__. -| See the `guide`__ for a more detailed introduction to available - features. -| Find detailed descriptions of the options in the - `specification`__ and `plugins`__ section. - -__ https://tmt.readthedocs.io/en/stable/overview.html -__ https://tmt.readthedocs.io/en/stable/overview.html#install -__ https://tmt.readthedocs.io/en/stable/overview.html#examples -__ https://tmt.readthedocs.io/en/stable/guide.html -__ https://tmt.readthedocs.io/en/stable/spec.html -__ https://tmt.readthedocs.io/en/stable/plugins/index.html - -Use the ``tldr tmt`` command to quickly get started. - -.. |docs| image:: https://img.shields.io/badge/Read%20the%20Docs-8CA1AF?logo=readthedocs&logoColor=fff - :target: https://tmt.readthedocs.io/ - :alt: Documentation - -.. |matrix| image:: https://img.shields.io/badge/Matrix-000?logo=matrix&logoColor=fff - :target: https://matrix.to/#/#tmt:fedoraproject.org - :alt: Matrix chat - -.. |fedora-pkg| image:: https://img.shields.io/fedora/v/tmt?logo=fedora&logoColor=fff&color=fff&labelColor=51a2da - :target: https://src.fedoraproject.org/rpms/tmt - :alt: Fedora package - -.. |copr-build| image:: https://img.shields.io/badge/dynamic/json?logo=fedora&color=blue&label=dev-build&query=builds.latest.state&url=https%3A%2F%2Fcopr.fedorainfracloud.org%2Fapi_3%2Fpackage%3Fownername%3D%40teemtee%26projectname%3Dlatest%26packagename%3Dtmt%26with_latest_build%3DTrue - :target: https://copr.fedorainfracloud.org/coprs/g/teemtee/latest/ - :alt: Copr build status - -.. |pypi-version| image:: https://img.shields.io/badge/PyPI-3775A9?logo=pypi&logoColor=fff - :target: https://pypi.org/project/tmt/ - :alt: PyPI - -.. |license| image:: https://img.shields.io/badge/license-MIT-blue.svg - :target: https://opensource.org/licenses/MIT - :alt: MIT License +See the https://tmt.readthedocs.io web for detailed documentation. diff --git a/ansible/packages.yml b/ansible/packages.yml index 48f3c3deca..6edb5120fe 100644 --- a/ansible/packages.yml +++ b/ansible/packages.yml @@ -1,5 +1,3 @@ ---- - - name: Install packages hosts: all tasks: diff --git a/containers/Containerfile.full b/containers/Containerfile.full index 381e0703eb..6d8d3a2c18 100644 --- a/containers/Containerfile.full +++ b/containers/Containerfile.full @@ -1,24 +1,20 @@ -FROM quay.io/fedora/fedora:latest - -RUN <> /etc/sudo.conf +RUN set -x && \ + sed -i '/tsflags=nodocs/d' /etc/dnf/dnf.conf && \ + echo "Set disable_coredump false" >> /etc/sudo.conf # Install necessary packages -dnf install -y dnf-plugins-core -dnf copr enable -y @teemtee/stable -dnf install -y tmt+all beakerlib -dnf autoremove -y -dnf clean all - -# Prepare a directory for experimenting -mkdir /tmt -EOF +RUN set -x && \ + dnf install -y dnf-plugins-core && \ + dnf copr enable -y @teemtee/tmt && \ + dnf install -y tmt+all-[0-9].[0-9][0-9].[0-9]* beakerlib && \ + dnf autoremove -y && \ + dnf clean all # Prepare files for experimenting +RUN mkdir /tmt COPY .fmf /tmt/.fmf COPY tests /tmt/tests COPY plans /tmt/plans @@ -26,4 +22,4 @@ COPY stories /tmt/stories # Run all plans under the regular user by default WORKDIR /tmt -CMD ["tmt", "run", "-av", "provision", "-h", "local"] +CMD tmt run -av provision -h local diff --git a/containers/Containerfile.mini b/containers/Containerfile.mini index 067b9672a6..38a30b227c 100644 --- a/containers/Containerfile.mini +++ b/containers/Containerfile.mini @@ -1,24 +1,20 @@ -FROM quay.io/fedora/fedora:latest - -RUN <> /etc/sudo.conf +RUN set -x && \ + sed -i '/tsflags=nodocs/d' /etc/dnf/dnf.conf && \ + echo "Set disable_coredump false" >> /etc/sudo.conf # Install necessary packages -dnf install -y dnf-plugins-core -dnf copr enable -y @teemtee/stable -dnf install -y tmt -dnf autoremove -y -dnf clean all - -# Prepare a directory for experimenting -mkdir /tmt -EOF +RUN set -x && \ + dnf install -y dnf-plugins-core && \ + dnf copr enable -y @teemtee/tmt && \ + dnf install -y tmt-[0-9].[0-9][0-9].[0-9]* && \ + dnf autoremove -y && \ + dnf clean all # Prepare files for experimenting +RUN mkdir /tmt COPY .fmf /tmt/.fmf COPY tests /tmt/tests COPY plans/main.fmf /tmt/plans/main.fmf @@ -26,4 +22,4 @@ COPY plans/features/core.fmf /tmt/plans/core.fmf # Run all plans under the regular user by default WORKDIR /tmt -CMD ["tmt", "run", "-av", "provision", "-h", "local"] +CMD tmt run -av provision -h local diff --git a/containers/alpine/Containerfile b/containers/alpine/Containerfile index 1316df3879..e277e0a649 100644 --- a/containers/alpine/Containerfile +++ b/containers/alpine/Containerfile @@ -6,15 +6,7 @@ FROM docker.io/library/alpine:3.19 -RUN <> /etc/sudoers + +USER fedora diff --git a/containers/fedora/39/Containerfile.upstream b/containers/fedora/39/Containerfile.upstream new file mode 100644 index 0000000000..3012c667d2 --- /dev/null +++ b/containers/fedora/39/Containerfile.upstream @@ -0,0 +1,10 @@ +# +# A Fedora 39 image tailored for tmt test suite +# +# tmt/tests/fedora/39/upstream:latest +# + +FROM quay.io/fedora/fedora:39 + + # Populate dnf cache +RUN dnf makecache diff --git a/containers/fedora/40/Containerfile b/containers/fedora/40/Containerfile index 25e7af14c5..bad431615d 100644 --- a/containers/fedora/40/Containerfile +++ b/containers/fedora/40/Containerfile @@ -6,17 +6,8 @@ FROM quay.io/fedora/fedora:40 -# hadolint ignore=DL3040 -RUN <> /etc/sudoers -EOF + # Populate dnf cache +RUN dnf makecache \ + # Create unprivileged user and setup sudo for it + && dnf install -y /usr/sbin/useradd \ + && useradd fedora \ + && usermod -aG wheel fedora \ + && echo -e 'fedora\tALL=(ALL)\tNOPASSWD: ALL' >> /etc/sudoers USER fedora diff --git a/containers/fedora/40/Containerfile.upstream b/containers/fedora/40/Containerfile.upstream index 8660931030..71459667a5 100644 --- a/containers/fedora/40/Containerfile.upstream +++ b/containers/fedora/40/Containerfile.upstream @@ -6,12 +6,5 @@ FROM quay.io/fedora/fedora:40 -RUN <> /etc/sudoers -EOF + # Populate dnf cache +RUN dnf makecache \ + # Create unprivileged user and setup sudo for it + && dnf install -y /usr/sbin/useradd \ + && useradd fedora \ + && usermod -aG wheel fedora \ + && echo -e 'fedora\tALL=(ALL)\tNOPASSWD: ALL' >> /etc/sudoers USER fedora diff --git a/containers/fedora/41/Containerfile.upstream b/containers/fedora/41/Containerfile.upstream index 833f653228..97b44a046a 100644 --- a/containers/fedora/41/Containerfile.upstream +++ b/containers/fedora/41/Containerfile.upstream @@ -6,12 +6,5 @@ FROM quay.io/fedora/fedora:41 -RUN <> /etc/sudoers -EOF - -USER fedora diff --git a/containers/fedora/latest/Containerfile.upstream b/containers/fedora/latest/Containerfile.upstream deleted file mode 100644 index 527cc8d29f..0000000000 --- a/containers/fedora/latest/Containerfile.upstream +++ /dev/null @@ -1,17 +0,0 @@ -# -# A latest Fedora image tailored for tmt test suite -# -# tmt/tests/fedora/latest/upstream:latest -# - -FROM quay.io/fedora/fedora:latest - -RUN <> /etc/sudoers -EOF + # Populate dnf cache +RUN dnf makecache \ + # Create unprivileged user and setup sudo for it + && dnf install -y /usr/sbin/useradd \ + && useradd fedora \ + && usermod -aG wheel fedora \ + && echo -e 'fedora\tALL=(ALL)\tNOPASSWD: ALL' >> /etc/sudoers USER fedora diff --git a/containers/fedora/rawhide/Containerfile.upstream b/containers/fedora/rawhide/Containerfile.upstream index c50d2d84fd..4f838fdaba 100644 --- a/containers/fedora/rawhide/Containerfile.upstream +++ b/containers/fedora/rawhide/Containerfile.upstream @@ -6,12 +6,5 @@ FROM quay.io/fedora/fedora:rawhide -RUN < /etc/apt/apt.conf.d/99force-ipv4 - -# Try to use us mirror, we are getting random stability issues on the main archive -sed -i 's|http://archive.ubuntu.com/ubuntu|http://us.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list - -# Populate apt cache -apt-get update - -# Make sure the image is built with the latest packages -apt-get upgrade -y -EOF + # Populate apt cache +RUN apt update diff --git a/docs/Makefile b/docs/Makefile index c38acd77bf..2c8349125f 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -8,9 +8,6 @@ REPODIR = .. # Path to tmt source directory TMTDIR = $(REPODIR)/tmt -# Path to docs source directory -DOCSDIR = $(REPODIR)/docs - # Path to directory with scripts generating documentation sources SCRIPTSDIR = scripts @@ -27,7 +24,13 @@ GENERATED_DIRECTORIES = spec stories STEPS = discover provision prepare execute finish report # A list of `plugins/*.rst` files to generate -PLUGIN_TARGETS = $(addsuffix .rst,$(addprefix plugins/,$(STEPS))) plugins/prepare-feature.rst plugins/test-checks.rst +PLUGIN_TARGETS = $(addsuffix .rst,$(addprefix plugins/,$(STEPS))) plugins/test-checks.rst + +# A source of logo picture for Sphinx docs favicon +LOGO_SRC = https://raw.githubusercontent.com/teemtee/docs/main/logo/tmt-small.png + +# Local filepath to fetched logo +LOGO_DST = _static/tmt-small.png .DEFAULT_GOAL := help @@ -36,13 +39,13 @@ PLUGIN_TARGETS = $(addsuffix .rst,$(addprefix plugins/,$(STEPS))) plugins/prepar generate-plugins \ $(PLUGIN_TARGETS) \ generate-stories \ - generate-template-extensions \ + generate-template-filters \ generate-autodocs clean ## ## Generate documentation sources from inputs ## -generate: $(LOGO_DST) generate-runner-guest-matrix generate-lint-checks generate-template-extensions generate-plugins generate-stories generate-autodocs ## Refresh all generated documentation sources +generate: $(LOGO_DST) generate-lint-checks generate-template-filters generate-plugins generate-stories generate-autodocs ## Refresh all generated documentation sources # # Targets creating directories for generated documentation sources @@ -64,25 +67,25 @@ define plugins-dependencies = $(SCRIPTSDIR)/generate-plugins.py $(PLUGINS_TEMPLATE) $(TMTDIR)/steps/__init__.py $(TMTDIR)/steps/$(call plugins-to-step,$@)/*.py endef -# Render a list of dependencies of a `plugins/feature.rst` target -define plugins-feature-dependencies = -$(SCRIPTSDIR)/generate-plugins.py $(PLUGINS_TEMPLATE) $(TMTDIR)/steps/__init__.py $(TMTDIR)/steps/prepare/__init__.py $(TMTDIR)/steps/prepare/feature/*.py -endef - # Render a list of dependencies of a `plugins/test-checks.rst` target define plugins-checks-dependencies = $(SCRIPTSDIR)/generate-plugins.py $(PLUGINS_TEMPLATE) $(TMTDIR)/steps/__init__.py $(TMTDIR)/checks/*.py endef +# We can ignore the error: later, during the build, if the logo is +# missing, Sphinx will complain. +$(LOGO_DST): + -curl -f $(LOGO_SRC) -o $(LOGO_DST) + # Generate plugin documentation sources for a given step define build-plugins = $(SCRIPTSDIR)/generate-plugins.py $(call plugins-to-step,$@) $(PLUGINS_TEMPLATE) $@ endef -code/template-extensions.rst: $(SCRIPTSDIR)/generate-template-extensions.py \ - $(TEMPLATESDIR)/template-extensions.rst.j2 \ - $(TMTDIR)/utils/templates.py - $(SCRIPTSDIR)/generate-template-extensions.py $(TEMPLATESDIR)/template-extensions.rst.j2 $@ +code/template-filters.rst: $(SCRIPTSDIR)/generate-template-filters.py \ + $(TEMPLATESDIR)/template-filters.rst.j2 \ + $(TMTDIR)/utils/templates.py + $(SCRIPTSDIR)/generate-template-filters.py $(TEMPLATESDIR)/template-filters.rst.j2 $@ plugins/discover.rst: $(call plugins-dependencies) $(call build-plugins) @@ -102,9 +105,6 @@ plugins/provision.rst: $(call plugins-dependencies) plugins/report.rst: $(call plugins-dependencies) $(call build-plugins) -plugins/prepare-feature.rst: $(call plugins-feature-dependencies) - $(call build-plugins) - plugins/test-checks.rst: $(call plugins-checks-dependencies) $(call build-plugins) @@ -113,11 +113,6 @@ plugins/hardware-matrix.rst: $(SCRIPTSDIR)/generate-hardware-matrix.py \ $(shell find $(REPODIR)/spec/hardware -name '*.fmf') $(SCRIPTSDIR)/generate-hardware-matrix.py $(TEMPLATESDIR)/hardware-matrix.rst.j2 $@ -guide/test-runner-guest-compatibility-matrix.inc.rst: $(SCRIPTSDIR)/generate-test-runner-guest-matrix.py \ - $(TEMPLATESDIR)/test-runner-guest-compatibility-matrix.inc.rst.j2 \ - $(DOCSDIR)/test-runner-guest-compatibility.yaml - $(SCRIPTSDIR)/generate-test-runner-guest-matrix.py $(TEMPLATESDIR)/test-runner-guest-compatibility-matrix.inc.rst.j2 $(DOCSDIR)/test-runner-guest-compatibility.yaml $@ - spec/lint.rst: $(SCRIPTSDIR)/generate-lint-checks.py \ $(TEMPLATESDIR)/lint-checks.rst.j2 \ $(TMTDIR)/base.py @@ -135,15 +130,13 @@ generate-plugins: $(PLUGIN_TARGETS) generate-hardware-matrix ## Generate docume generate-hardware-matrix: plugins/hardware-matrix.rst ## Generate HW requirement support matrix -generate-runner-guest-matrix: guide/test-runner-guest-compatibility-matrix.inc.rst ## Generate runner vs. guest compatibility matrix - generate-stories: stories $(TEMPLATESDIR)/story.rst.j2 ## Generate documentation sources for stories $(SCRIPTSDIR)/generate-stories.py $(TEMPLATESDIR)/story.rst.j2 -generate-template-extensions: code/template-extensions.rst ## Generate documentation sources for Jinja2 template extensions +generate-template-filters: code/template-filters.rst ## Generate documentation sources for Jinja2 template filters clean: ## Remove all generated content - rm -rf _build $(GENERATED_DIRECTORIES) code/autodocs/*.rst code/template-extensions.rst plugins/hardware-matrix.rst guide/test-runner-guest-compatibility-matrix.inc.rst $(PLUGIN_TARGETS) + rm -rf _build $(GENERATED_DIRECTORIES) code/autodocs/*.rst code/template-filters.rst plugins/hardware-matrix.rst $(PLUGIN_TARGETS) $(LOGO_DST) ## ## Help! diff --git a/docs/_static/tmt-custom.css b/docs/_static/tmt-custom.css index f9f758f9d2..0c34d77531 100644 --- a/docs/_static/tmt-custom.css +++ b/docs/_static/tmt-custom.css @@ -4,14 +4,3 @@ /* Used for HW requirement support matrix */ .supported { color: green; } .unsupported { color: red; } -.logo { - padding: 10px 50px !important; -} - -.rst-content .note .admonition-title { - display: block !important; -} - -.rst-content .warning .admonition-title { - display: block !important; -} diff --git a/docs/code/index.rst b/docs/code/index.rst index 0e0bbe729c..54fb4a8307 100644 --- a/docs/code/index.rst +++ b/docs/code/index.rst @@ -14,7 +14,7 @@ the documentation generated from sources linked below. :maxdepth: 2 Class Overview - Template Extensions + Template Filters Plugin Introduction tmt diff --git a/docs/conf.py b/docs/conf.py index 8f7cc52feb..adfa235251 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -29,8 +29,8 @@ # Fall back to sphinx_rtd_theme if available ('sphinx_rtd_theme', 'sphinx_rtd_theme'), # The default theme - (None, 'default'), -] + (None, 'default') + ] # NOTE: this one is defined somewhere below, among original Sphinx config fields, # but we need it as early as possible to be set when loading themes. @@ -38,7 +38,9 @@ html_theme_path = [] -def _load_theme(theme_package_name: str, theme_name: str) -> bool: +def _load_theme( + theme_package_name: str, + theme_name: str) -> bool: try: theme_package = importlib.import_module(theme_package_name) @@ -72,8 +74,7 @@ def _load_theme(theme_package_name: str, theme_name: str) -> bool: except ValueError: raise tmt.utils.GeneralError( - f"Cannot split TMT_DOCS_THEME '{theme_specs}' into theme package and theme name." - ) + f"Cannot split TMT_DOCS_THEME '{theme_specs}' into theme package and theme name.") if not _load_theme(theme_package_name, theme_name): raise tmt.utils.GeneralError(f"Cannot load theme from TMT_DOCS_THEME, '{theme_specs}'.") @@ -108,7 +109,7 @@ def _load_theme(theme_package_name: str, theme_name: str) -> bool: 'sphinx.ext.autodoc', 'sphinx.ext.autodoc.typehints', 'sphinx_rtd_theme', -] + ] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -188,7 +189,7 @@ def _load_theme(theme_package_name: str, theme_name: str) -> bool: autodoc_default_options = { # Enable to "ignore" re-imported names in `tmt.__all__` 'ignore-module-all': True -} + } autoclass_content = "both" autodoc_typehints_format = 'short' @@ -216,13 +217,12 @@ def _load_theme(theme_package_name: str, theme_name: str) -> bool: # The name of an image file (relative to this directory) to place at the top # of the sidebar. -html_logo = 'https://raw.githubusercontent.com/teemtee/docs/main/logo/tmt-logo-dark-background.png' +html_logo = 'https://raw.githubusercontent.com/teemtee/docs/main/logo/tmt-transparent-175x175.png' # The name of an image file (within the static path) to use as favicon of the -# docs. -html_favicon = ( - 'https://raw.githubusercontent.com/teemtee/docs/main/logo/tmt-logo-dark-background.svg' -) +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +html_favicon = 'https://raw.githubusercontent.com/teemtee/docs/main/logo/tmt-favicon.ico' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -299,45 +299,26 @@ def _load_theme(theme_package_name: str, theme_name: str) -> bool: # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [(master_man, '', 'tmt Documentation', [author], 1)] +man_pages = [ + (master_man, '', 'tmt Documentation', + [author], 1) + ] # If true, show URL addresses after external links. # man_show_urls = False # -- Options for linkcheck builder ---------------------------------------- -linkcheck_request_headers = { - r"https://github\.com/.*": { - "User-Agent": "tmt-docs-linkcheck/1.0 (GitHub Actions)", - }, -} - -github_token = os.environ.get('GITHUB_TOKEN') - -if github_token: - linkcheck_request_headers[r"https://github\.com/.*"]["Authorization"] = ( - f"Bearer {github_token}" - ) - print("INFO: Using GITHUB_TOKEN for linkcheck requests to github.com") -else: - print("INFO: GITHUB_TOKEN not found. linkcheck requests to github will be unauthenticated.") - linkcheck_retries = 3 linkcheck_ignore = [ # Github "source code line" anchors are apparently too dynamic for linkcheck # to detect correctly. The link exists, a browser can open it, but linkcheck # reports a broken link. - r'https://github.com/packit/packit/blob/main/packit/utils/logging.py#L10', - # The site repeatedly refuses to serve pages to github - r'https://www.cpu-world.com.*', - # Stack Overflow uses captcha and these links are not essential - r'https://stackoverflow.com.*', -] + r'https://github.com/packit/packit/blob/main/packit/utils/logging.py#L10' + ] def generate_tmt_docs(app: Sphinx, config: Any) -> None: - """ - Run `make generate` to populate the auto-generated sources - """ + """ Run `make generate` to populate the auto-generated sources """ conf_dir = Path(app.confdir) subprocess.run(["make", "generate"], cwd=conf_dir, check=True) diff --git a/docs/contribute.rst b/docs/contribute.rst index 3dbe794b1d..e7b0478ef8 100644 --- a/docs/contribute.rst +++ b/docs/contribute.rst @@ -58,7 +58,7 @@ as something that will be pasted into release notes: Why should I care? * It helps others (and yourself) find relevant commits quickly. -* The summary line will be reused later (e.g. for rpm changelog). +* The summary line will be re-used later (e.g. for rpm changelog). * Some tools do not handle wrapping, so it is then hard to read. * You will make the maintainers happy to read beautiful commits :) @@ -200,23 +200,23 @@ The following tags can be used to enable given test under the respective provision method plan: provision-artemis - For tests checking the :ref:`/plugins/provision/artemis` + For tests checking the :ref:`/spec/plans/provision/artemis` plugin functionality. provision-beaker - For tests checking the :ref:`/plugins/provision/beaker` + For tests checking the :ref:`/spec/plans/provision/beaker` plugin functionality using the ``mrack`` plugin. provision-connect - For tests checking the :ref:`/plugins/provision/connect` + For tests checking the :ref:`/spec/plans/provision/connect` plugin functionality. provision-container - For tests checking the :ref:`/plugins/provision/container` + For tests checking the :ref:`/spec/plans/provision/container` provision method using the ``podman`` plugin. provision-virtual - For tests checking the :ref:`/plugins/provision/virtual.testcloud` + For tests checking the :ref:`/spec/plans/provision/virtual` provision method using the ``testcloud`` plugin. provision-ssh @@ -253,7 +253,7 @@ __ https://requre.readthedocs.io/en/latest/ Images ------------------------------------------------------------------ -Tests which exercise the :ref:`/plugins/provision/container` +Tests which exercise the :ref:`/spec/plans/provision/container` provisioning plugin with various guest environments should use the custom-built set of container images rather than using the upstream ones directly. We built custom images to have better control over the initial @@ -312,17 +312,17 @@ For example, the following images can be found: .. code-block:: # Latest Alpine, with added Bash to simulate proper essential setup: - localhost/tmt/container/test/alpine + localhost/tmt/tests/container/alpine # Various CentOS releases: - localhost/tmt/container/test/centos/7 - localhost/tmt/container/test/centos/stream9 + localhost/tmt/tests/container/centos/7 + localhost/tmt/tests/container/centos/stream9 # Fedora rawhide, with dnf5 pre-installed: - localhost/tmt/container/test/fedora/rawhide + localhost/tmt/tests/container/fedora/rawhide # Same, but with password-less sudo set up: - localhost/tmt/container/test/fedora/rawhide/unprivileged + localhost/tmt/tests/container/fedora/rawhide/unprivileged __ https://ostreedev.github.io/ostree/ @@ -331,17 +331,17 @@ To build these images, run the following: .. code-block:: shell # Build all images... - make images/test + make images-tests # ... or just a single one: - make images/test/tmt/container/test/fedora/rawhide:latest + make images-tests/tmt/tests/container/fedora/rawhide:latest Tests that need to use various container images should trigger this command before running the actual test cases: .. code-block:: bash - rlRun "make -C images/test" + rlRun "make -C images-tests" To list built container images, run the following: @@ -353,7 +353,7 @@ To remove these images from your local system, run the following: .. code-block:: shell - make clean/images/test + make clean-test-images .. _docs: @@ -499,27 +499,6 @@ variable to include additional file: override theme CSS, it is recommended to add ``!important`` flag. -tldr pages ------------------------------------------------------------------- - -The ``tldr`` pages are maintained in the central `tldr-pages`__ -repository. To modify existing pages or add new ones, submit your -changes directly there by following their `contribution -guidelines`__. - -Translations of existing pages into other languages are welcomed. -If you'd like to help translate pages, please follow the same -contribution process described above. - -__ https://github.com/tldr-pages/tldr -__ https://github.com/tldr-pages/tldr/blob/main/CONTRIBUTING.md - -.. note:: - - Changes made directly to documentation in this repository will - not be reflected in the tldr pages collection. - - Pull Requests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -533,12 +512,6 @@ commits with your changes on the top of the branch instead of amending the original commit and doing a force push. This will make it easier for the reviewers to see what has recently changed. -It's good to keep the pull request up-to-date with the ``main`` -branch. Rebase regularly or use ``/packit build`` command in the -pull request comment if there were significant changes on the -default branch otherwise newly added tests might cause unexpected -and irrelevant failures in your test jobs. - Once the pull request has been successfully reviewed and all tests passed, please rebase on the latest ``main`` branch content and squash the changes into a single commit. Use multiple commits to @@ -609,9 +582,6 @@ Each completed pull request review helps you, little by little, to get familiar with larger part of the project code and empowers you to contribute more easily in the future. -Pull requests ready for review can be easily filtered and found at -the `review tab`__. - For instructions how to locally try a change on your laptop see the :ref:`develop` section. Basically just enable the development environment and check out the pull request branch or use the @@ -647,7 +617,6 @@ Even partial review which happens sooner is beneficial, saves time. Every single comment helps to improve and move the project forward. No question is a dumb question. Every feedback counts! -__ https://github.com/orgs/teemtee/projects/1/views/6 __ https://cli.github.com @@ -716,14 +685,7 @@ Regular Follow the steps below to create a new major or minor release: -* Update ``overview.rst`` with new contributors since the last - release. To identify contributors whose first ever commit to - the repository occurred *after* the last release tag, run: - - .. code-block:: bash - - ./scripts/list-new-contributors - +* Update ``overview.rst`` with new contributors since the last release * Review the release notes in ``releases.rst``, update as needed * Add a ``Release x.y.z`` commit, empty if needed: ``git commit --allow-empty -m "Release x.y.z"`` * Create a pull request with the commit, ensure tests pass, merge it @@ -741,8 +703,9 @@ Create a new `github release`__ based on the tag above Finally, if everything went well: * Close the corresponding release milestone -* Once the non development `copr build`__ is completed, run the - `publish-images`__ workflow to build fresh container image. +* Once the non development `copr build`__ is completed, move the + ``quay`` branch to point to the release commit as well to build + fresh `container images`__. Handle manually what did not went well: @@ -756,8 +719,8 @@ Handle manually what did not went well: __ https://github.com/teemtee/tmt/releases/ __ https://tmt.readthedocs.io/en/stable/releases.html __ https://src.fedoraproject.org/rpms/tmt/pull-requests -__ https://copr.fedorainfracloud.org/coprs/g/teemtee/stable/builds/ -__ https://github.com/teemtee/tmt/actions/workflows/publish-images.yml +__ https://copr.fedorainfracloud.org/coprs/g/teemtee/tmt/builds/ +__ https://quay.io/repository/teemtee/tmt __ https://pypi.org/project/tmt/ @@ -776,31 +739,3 @@ fix needs to be released before the regular schedule: * Tag the commit and publish the release in the same way as for regular release * Create a pull request with the hotfix release notes changes - - -Releaser ------------------------------------------------------------------- - -Taking care of a new ``tmt`` release is not just about performing -the final steps described above. In this role you should shepherd -the issues and pull requests like sheep so that they make it to -the ``main`` branch by the proposed deadline. Here's a couple of -recommendations which could help you to make the release process -smooth and timely: - -* continually watch the issues & pull requests and gently push - them forward if any of them seems to get stuck -* bring attention especially to those with the high priority, the - ``priority | must`` issues and pull requests should be finished - ideally one week before the release deadline -* regularly check the pull request progress and highlight those - which are waiting for feedback on the review sessions -* if there is anything not clear and needs discussion bring it to - the chat or raise the topic on the weekly sessions -* do not hesitate to contact assignees directly, e.g. on the chat, - if there is no update for a longer time, consider also - reassigning the issue to another contributor if necessary -* if there are pull requests ready for merging but not included in - the release, it might make sense to squeeze them in, to make the - development more fluent, just make sure they do not slow down - important issues diff --git a/docs/examples.rst b/docs/examples.rst index 2d896490a0..666a4c19e0 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -635,7 +635,7 @@ Dynamic ``ref`` Evaluation ------------------------------------------------------------------ When using test branching for test maintenance it becomes handy to -be able to set :ref:`ref` dynamically +be able to set :ref:`ref` dynamically depending on the provided :ref:`/spec/context`. This is possible using a special file in tmt format stored in a default branch of a tests repository. That special file should contain rules assigning diff --git a/docs/guide.rst b/docs/guide.rst index 2a9432543d..e09066544f 100644 --- a/docs/guide.rst +++ b/docs/guide.rst @@ -19,8 +19,7 @@ The First Steps ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Installing the main package with the core functionality is quite -straightforward. No worry, the :ref:`/stories/install/minimal` -package has just a few dependencies: +straightforward. No worry, there are just a few dependencies: .. code-block:: shell @@ -86,8 +85,8 @@ dependencies here: tmt run -a provision -h virtual Don't care about the disk space? Simply install ``tmt+all`` and -you'll get :ref:`/stories/install/all` available functionality at -hand. Check the help to list all supported provision methods: +you'll get all available functionality at hand. Check the help to +list all supported provision methods: .. code-block:: shell @@ -193,9 +192,9 @@ Note that each of the steps above uses the ``how`` keyword to choose the desired method which should be applied. Steps can provide multiple implementations which enables you to choose the best one for your use case. For example to prepare the guest it's -possible to use the :ref:`/plugins/prepare/install` method for -simple package installations, :ref:`/plugins/prepare/ansible` -for more complex system setup or :ref:`/plugins/prepare/shell` +possible to use the :ref:`/spec/plans/prepare/install` method for +simple package installations, :ref:`/spec/plans/prepare/ansible` +for more complex system setup or :ref:`/spec/plans/prepare/shell` for arbitrary shell commands. @@ -777,6 +776,47 @@ This gives you a nice flexibility to extend the metadata when and where needed as your project organically grows. +Efficient Metadata Handling +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``tmt`` offers several features to help you manage your test metadata efficiently, reducing duplication and making it easier to maintain large test suites. This section highlights some of these key features. + +Minimize Duplication +-------------------- + +Leverage the following features to avoid repetition in your metadata: + +* **Inheritance**: The Flexible Metadata Format (fmf) allows metadata to be inherited from parent directories. This is a powerful way to define common attributes at a higher level, which are then automatically picked up by tests and plans in subdirectories. For a detailed explanation, see the :ref:`inheritance` section. + +* **YAML Anchors and Aliases**: For repetition within a single YAML file, you can use `YAML anchors and aliases `_. This allows you to define a chunk of YAML once and reuse it multiple times within the same file. Learn more about this in the :ref:`anchors-aliases` section. + +Adapt to Context +---------------- + +* **Adjust**: The :ref:`/spec/core/adjust` attribute provides a flexible way to modify metadata based on the current context (e.g., different distributions, architectures, or environments). This allows you to tailor tests and plans without duplicating the entire metadata structure. For example, you can adjust test requirements or enable/disable tests based on specific conditions. + +Share Tests Across Repositories +------------------------------- + +``tmt`` allows you to discover and execute tests that reside in different Git repositories. This is particularly useful for sharing common test libraries or when tests are maintained by different teams. + +To include tests from an external repository, specify the ``url`` and optionally a ``ref`` (branch, tag, or commit) in the ``discover`` step of your plan: + +.. code-block:: yaml + + discover: + how: fmf + url: https://github.com/teemtee/tmt.git + ref: main + # You can also specify a path within the repository + # path: /tests/core + +This will clone the specified repository and discover tests according to the fmf metadata found there. This enables modular test organization and promotes reusability of test code across projects. You can also use local paths to other repositories if they are available on the same filesystem. +For general information on configuring test discovery, see :ref:`/spec/plans/discover`. + +By utilizing these features, you can create a more maintainable, scalable, and efficient test metadata structure. + + .. _link-issues: Link Issues @@ -874,40 +914,6 @@ copied to the guest. ``finish`` steps yet. -.. _when-config: - -Conditional step configuration ------------------------------- - -.. versionadded:: 1.40 - -Sometimes, the plan is expected to cover a broad set of environments; -however, some step configurations may not be applicable everywhere. -While :ref:`/spec/core/adjust` can be used to construct the plan -in this way, it soon becomes difficult to read. - -Using the ``when`` key makes it easier to restrict a step configuration -to run only if any of the specified rules matches. -The syntax is the same as in ``adjust`` and :ref:`/spec/context`. - -.. code-block:: yaml - - prepare: - - name: Prepare config to run only on Fedora - when: distro == fedora - how: shell - script: ./fedora_specific.sh - - name: Runs always - how: shell - script: ./setup.sh - - name: More rules in 'when' key - how: shell - script: ./something.sh - when: - - arch != x86_64 - - initiator == human && distro == fedora - - .. _multihost-testing: Multihost Testing @@ -972,7 +978,7 @@ to their services, synchronization, etc. tmt fully supports one test being executed multiple times. This is especially visible in the format of results, see -:ref:`/spec/results`. Every test is assigned a "serial +:ref:`/spec/plans/results`. Every test is assigned a "serial number", if the same test appears in multiple discover phases, each instance would be given a different serial number. The serial number and the guest from which a result comes from are then saved for each diff --git a/docs/guide/test-runner.inc.rst b/docs/guide/test-runner.inc.rst index 9566dc7901..708def4da5 100644 --- a/docs/guide/test-runner.inc.rst +++ b/docs/guide/test-runner.inc.rst @@ -62,5 +62,3 @@ __ https://docs.testing-farm.io/Testing%20Farm/0.1/cli.html __ https://docs.testing-farm.io/Testing%20Farm/0.1/onboarding.html __ https://docs.testing-farm.io/Testing%20Farm/0.1/test-environment.html#_composes __ https://docs.testing-farm.io/Testing%20Farm/0.1/test-runner.html#_supported_ansible_collections - -.. include:: guide/test-runner-guest-compatibility-matrix.inc.rst diff --git a/docs/overview.rst b/docs/overview.rst index 3b4fdbd79e..5917d09188 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -259,15 +259,11 @@ Then you can install either everything or only those you need:: sudo dnf install tmt-redhat-all sudo dnf install tmt-redhat-* -Do you like to check the released bits as soon as they are out? -Enable the ``stable`` copr repository and install from there:: +Impatient to try the fresh features as soon as possible? Install +the latest greatest version from the ``copr`` repository:: - sudo dnf copr enable @teemtee/stable - -Impatient to try the fresh features as soon as they are merged -into the ``main`` branch? Enable the ``latest`` copr repository:: - - sudo dnf copr enable @teemtee/latest + sudo dnf copr enable @teemtee/tmt + sudo dnf install tmt Not sure, just want to try out how it works? Experiment safely and easily inside a container:: @@ -407,23 +403,10 @@ TMT_FORCE_COLOR TMT_SHOW_TRACEBACK By default, when tmt reports an error, the corresponding - traceback is not printed out. By setting this variable, the - traceback and details would be shown: - - TMT_SHOW_TRACEBACK=0 (or unset) - Render only exception and its causes. - - TMT_SHOW_TRACEBACK=1 - Render also call stack for exception and each of its causes. - - TMT_SHOW_TRACEBACK=2 - Render also call stack for exception and each of its causes, - plus all local variables in each frame, trimmed to first 1024 - characters of their values. - - TMT_SHOW_TRACEBACK=full - Render everything that can be show: all causes, their call - stacks, all frames and all locals in their completeness. + traceback is not printed out. When ``TMT_SHOW_TRACEBACK`` is + set to any string except ``0``, traceback would be printed out. + When set to ``full``, traceback would list also local variables + in each stack frame. TMT_OUTPUT_WIDTH By default, the output width of commands like ``tmt * show`` is constrained @@ -492,7 +475,7 @@ TMT_SCRIPTS_DIR .. versionadded:: 1.38 -__ https://tmt.readthedocs.io/en/stable/plugins/execute.html#tmt +__ https://tmt.readthedocs.io/en/stable/spec/plans.html#tmt TMT_SSH_* Every environment variable in this format would be treated as an SSH @@ -524,9 +507,8 @@ during ``prepare``, ``execute`` and ``finish`` steps: TMT_TREE The full path of the working directory where the metadata tree - is copied. This usually contains the whole git repository where - tmt plans are located in. Notice that it might not contain tmt - tests if tmt plans and tests are in different git repositories. + is copied. This usually contains the whole git repository from + which tests have been executed. TMT_PLAN_DATA Path to the common directory used for storing logs and other @@ -630,8 +612,6 @@ TMT_TEST_PIDFILE_ROOT of temporary directory permissions, e.g. ``chmod 1777``, to allow access to users with all privilege levels. -.. _plugin-variables: - Plugin Variables ---------------- @@ -688,9 +668,6 @@ example, an interactive mode would be enabled in this run:: # Here the environment variable will take effect: $ TMT_PLUGIN_DISCOVER_FMF_VERBOSE=2 tmt run -a discover -h fmf ... - Several plugins (``report -h reportportal``, ``report -h polarion``, - ``execute -h tmt``) allow selected variables to be processed, - even when plugin is not specified on the command line. .. _regular-expressions: @@ -756,7 +733,7 @@ Releases: https://tmt.readthedocs.io/en/stable/releases.html Copr: -https://copr.fedorainfracloud.org/coprs/g/teemtee/stable/ +https://copr.fedorainfracloud.org/coprs/g/teemtee/tmt/ PIP: https://pypi.org/project/tmt/ @@ -800,9 +777,7 @@ James Molet, Cristian Le, Lili Nie, Martin Čermák, Michael Vogt, Qinghua Cheng, Michael Engel, Anatoli Babenia, Colin Walters, Link Dupont, Mario Casquero, Martin Klusoň, Pavel Holica, Otto Šabart, Ismail Ibrahim Quwarah, Sergei Petrosian, Tom -Koscielniak, Han Han, Luigi Pellecchia, Siteshwar Vashisht, -Chris Kyrouac, Xiaofeng Wang, Coiby Xu, Michal Pospíšil, Wayne -Sun, Evgeni Vakhonin and Mike Stowell. +Koscielniak, Han Han and Luigi Pellecchia. Copyright diff --git a/docs/plugins/finish-header.inc.rst b/docs/plugins/finish-header.inc.rst index 21b4a2a747..e69de29bb2 100644 --- a/docs/plugins/finish-header.inc.rst +++ b/docs/plugins/finish-header.inc.rst @@ -1,4 +0,0 @@ -The :ref:`/spec/plans/finish` step plugins implement actions to be -performed after the test execution has been completed. These can -be useful for finishing tasks such as an additional guest cleanup -or uploading specific logs to an external server. diff --git a/docs/plugins/prepare-feature-header.inc.rst b/docs/plugins/prepare-feature-header.inc.rst deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/docs/plugins/prepare-header.inc.rst b/docs/plugins/prepare-header.inc.rst index 19058d6bc0..e69de29bb2 100644 --- a/docs/plugins/prepare-header.inc.rst +++ b/docs/plugins/prepare-header.inc.rst @@ -1,3 +0,0 @@ -The ``prepare`` step is used to define how the guest -environment should be prepared so that the tests can be -successfully executed. diff --git a/docs/plugins/provision-header.inc.rst b/docs/plugins/provision-header.inc.rst index 4e45611215..3b342fbf12 100644 --- a/docs/plugins/provision-header.inc.rst +++ b/docs/plugins/provision-header.inc.rst @@ -1,13 +1,3 @@ -Describes what environment is needed for testing and how it should be -provisioned. There are several provision plugins supporting multiple ways -to provision the environment for testing, for example: - -* :ref:`/plugins/provision/virtual.testcloud` -* :ref:`/plugins/provision/container` -* :ref:`/plugins/provision/connect` -* :ref:`/plugins/provision/local` -* :ref:`/plugins/provision/artemis` - .. _/plugins/provision/hard-reboot: Hard reboot @@ -26,30 +16,5 @@ Following plugins fully implement hard reboot: * :ref:`/plugins/provision/beaker` * :ref:`/plugins/provision/container` * :ref:`virtual` -* :ref:`/plugins/provision/artemis` .. include:: hardware-matrix.rst - - -.. _/plugins/provision/ssh-options: - -SSH options ------------ - -When communicating with guests over SSH, tmt adds several SSH options by -default to relevant commands: - -.. code-block:: - - # Try establishing connection multiple times before giving up. - ConnectionAttempts=5 - ConnectTimeout=60 - - # Prevent SSH from disconnecting if no data has been - # received from the server for a long time. - ServerAliveInterval=5 - ServerAliveCountMax=60 - -Additional SSH options can be specified either via ``ssh-option`` key -of respective plugins, or by setting ``TMT_SSH_*`` environment -variables; see :ref:`command-variables` for details. diff --git a/docs/plugins/report-header.inc.rst b/docs/plugins/report-header.inc.rst index c172e76c19..e69de29bb2 100644 --- a/docs/plugins/report-header.inc.rst +++ b/docs/plugins/report-header.inc.rst @@ -1 +0,0 @@ -Report test results according to user preferences. diff --git a/docs/questions.rst b/docs/questions.rst index 451ffb4432..f4dbd7e2e6 100644 --- a/docs/questions.rst +++ b/docs/questions.rst @@ -30,19 +30,6 @@ __ https://tmt.readthedocs.io/en/latest/ .. _libvirt: -Who is using tmt? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Are there any example projects which are using ``tmt`` which I -could use as an inspiration for my initial configuration setup? - -* The `HealthTrio Success Story`__ with CentOS Stream and tmt -* The `AlmaLinux`__ community is using tmt for its compose testing - -__ https://blog.centos.org/2024/01/managing-internal-ci-tests-with-tmt-for-centos-stream-updates/ -__ https://github.com/AlmaLinux/compose-tests - - Using tmt outside of Fedora, CentOS and RHEL distribution ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -59,11 +46,11 @@ On the other hand - when tmt is used to execute tests on provisioned guest it depends if the plan will try to install any packages (either by test :ref:`/spec/tests/require`, :ref:`/spec/tests/recommend` or using prepare -:ref:`/plugins/prepare/install` plugin) it will fail as tmt +:ref:`/spec/plans/prepare/install` plugin) it will fail as tmt currently doesn't work with other package management tools. This can be worked around by installing the test dependencies (as well -as the ``rsync`` command) using :ref:`/plugins/prepare/ansible` -or :ref:`/plugins/prepare/shell` prepare plugins. +as the ``rsync`` command) using :ref:`/spec/plans/prepare/ansible` +or :ref:`/spec/plans/prepare/shell` prepare plugins. Virtualization Tips @@ -171,85 +158,6 @@ Metadata Specification and are supposed to be synced temporarily to keep backward compatibility. -.. _restraint-compatibility: - -Restraint Compatibility -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -For backward-compatibility ``tmt`` provides selected commands -of the `restraint`__ framework so that existing tests can be more -easily migrated. Currently the following scripts are supported: - -* ``rhts-abort`` and ``rstrnt-abort`` — :ref:`/stories/features/abort` -* ``rhts-reboot`` and ``rstrnt-reboot`` — :ref:`/stories/features/reboot` -* ``rhts-submit-log`` and ``rstrnt-report-log`` — :ref:`/stories/features/report-log` -* ``rhts-report-result`` and ``rstrnt-report-result`` — :ref:`/stories/features/report-result` - -Note that these scripts cover only the common use cases and some -of their irrelevant options, such as ``--server`` used for the -restraint server, are ignored. - -.. warning:: - - Currently this functionality is enabled by default. If your - tests depend on these compatibility scripts, please ensure - that the ``restraint-compatible`` option is enabled under the - :ref:`/plugins/execute/tmt` execute step. In the future these - scripts will be available on the guest only if this option is - enabled. - - .. code-block:: yaml - - execute: - how: tmt - restraint-compatible: true - - If possible, we recommend to update your existing tests and - use ``tmt-abort``, ``tmt-reboot``, ``tmt-file-submit`` and - ``tmt-report-result`` scripts instead. These are not planned - to be removed and will be supported in the future. - -__ https://restraint.readthedocs.io/ - - -.. _mulithost-compatibility: - -Multihost Compatibility -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Some older tests might be using the ``CLIENTS`` and ``SERVERS`` -environment variables to get the information about the guests -involved in the multihost testing. In order to provide these -variables to all tests in a tmt plan it is possible to use the -``TMT_PLAN_ENVIRONMENT_FILE`` variable and set them based on the -:ref:`/spec/plans/guest-topology`. The example below demonstrates -the usage on a simple tmt plan: - -.. code-block:: yaml - - provision: - - name: server - how: virtual - connection: system - - name: client - how: virtual - connection: system - - prepare: - - summary: Export client and server hostname for all tests - how: shell - script: | - source "$TMT_TOPOLOGY_BASH" - echo "CLIENTS=${TMT_GUESTS[client.hostname]}" >> "$TMT_PLAN_ENVIRONMENT_FILE" - echo "SERVERS=${TMT_GUESTS[server.hostname]}" >> "$TMT_PLAN_ENVIRONMENT_FILE" - - execute: - how: tmt - script: | - echo "clients: $CLIENTS" - echo "servers: $SERVERS" - - Why is the 'id' key added to my test during export? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/releases.rst b/docs/releases.rst index 6c75401c7e..1d3ee3002e 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -4,444 +4,17 @@ Releases ====================== -tmt-1.50.0 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -It is now possible to use ``extra-*`` metadata keys in tests, plans -and stories for arbitrary user-defined data, within the limits of -what YAML allows. These keys are always ignored by ``tmt lint``. -See the :ref:`/spec/core/extra` key specification for details and -examples. - -Added ``--dry`` option for the :ref:`/plugins/provision/bootc` plugin. - -Added a specification for :ref:`policies ` that allow CI -system and CI workflow maintainers to modify tests and plans to include -mandatory checks and phases as required by their testing process. - -Initial implementation for the test-level policies has been added as -well, aiming at CI workflows that need to enforce AVC checks across the -whole component portfolio. - -The ``results.yaml`` file will now contain the log path for -``journal.xml``. - - -tmt-1.49.0 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The :ref:`import of remote plans` support has -been extended to allow import of multiple plans. New keys, -``scope`` and ``importing``, allow users to control which plans to -import and how to connect them with the importing plans. - -New :ref:`/plugins/prepare/feature` prepare plugin ``crb`` has -been implemented which allows to easily enable or disable the -CodeReady Builder repository on common test environments. - -The console log content is now available for guests provisioned by -the :ref:`/plugins/provision/virtual.testcloud` plugin. - -Failures from tests and their checks were previously not fully -saved or reported. Now, a separate ``failures.yaml`` file is -created for each failed test and check, stored within their -respective directories. When a failure occurs, the path to this -file is included in the result logs. Check failures are now also -being reported to ReportPortal. - -Output of the :ref:`/plugins/execute/tmt` and -:ref:`/plugins/report/display` is changing in this release, to -provide slightly more details, headers and timestamps. The -``execute`` step now starts using ``display`` for its own progress -reporting, providing the unified formatting and simplified code. - -When the login step was called in a separate command after the -guest has been provisioned, the connection seemed to be stuck. -This has been caused by the SSH master process not being -terminated together with tmt, new tmt command would then spawn its -own and conflict with the forgotten one. tmt no longer leaves the -SSH master process running, preventing the issue. - -An issue in the :ref:`/plugins/provision/beaker` provision plugin -prevented reconnecting to running guests. This has been fixed so -now it's possible to fully work with existing tmt runs as well. - -A bug causing executed tests to remain in the ``pending`` state -when the machine became unresponsive has been fixed. Tests will -now correctly transition to the ``error`` state. - - -tmt-1.48.0 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -A new ``tmt about`` command has been introduced, -initially providing information about the :ref:`tmt plugins `. - -The :ref:`HTML report plugin ` now supports a -new ``file`` key, allowing users to specify a custom output path for -the generated HTML report. - -When using ``and``/``or`` groups in combination with -:ref:`hardware requirements `, ``tmt`` will now emit -a warning to alert users about potential ambiguity in how these -constraints are applied. - -For users of the :ref:`testcloud provisioner `, -``PermitRootLogin`` is now enabled by default for Red Hat CoreOS (RHCOS) -guests, simplifying access. - -An issue with saving remote :ref:`Ansible playbooks ` -to the correct directory during provisioning and preparation has been fixed. - -The internal representation of an imported plan has been improved, -though this should be largely transparent to users. - -Several internal improvements and updates to development tooling and -CI processes have been made to enhance stability and maintainability. - - -tmt-1.47.0 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -When ``tmt`` works with image mode, it now uses the native -package installation method instead of ``rpm-ostree``. -``tmt`` creates a ``Containerfile`` based on the booted image, -adds the required packages, builds a new image, and reboots the -system to use the updated image with the necessary packages. - -If applicable, the ``crb`` repository is now automatically enabled -when enabling ``epel`` repository. - -If a mixture of local and remote plans is detected, ``tmt`` now -prints a warning and skips the ``local`` plan. - -In the ``execute`` step, the documentation of the ``duration`` -option was enhanced to correctly describe the effect of the -option. - -The ``execute`` plugin now explicitly requires ``awk`` to be -installed on the machine, due to its recent removal from -Fedora containers. - -The documentation of the ``feature`` plugins now includes a list -of required Ansible modules. - -The documentation of plugins was improved to include examples -of keys with actual values. - -The default unit of the ``memory`` hardware requirement is now -``MiB``. It is used if no unit was specified. - -The steps documentation was deduplicated, and all information -from the specs was moved to the ``plugins`` section. - - -tmt-1.46.0 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The :ref:`/plugins/report/junit` report plugin now supports a new -experimental ``subresults`` JUnit flavor. This flavor introduces -support for tmt subresults and adjusts the hierarchy of -```` and ```` tags. With this flavor, test -results are represented as ```` tags, each containing a -```` tag for the main result, along with additional -```` tags for any subresults. - -As a tech preview, a new :ref:`/plugins/test-checks/coredump` check -plugin has been added to detect system crashes using systemd-coredump -during test execution. The plugin monitors for any segmentation -faults and other crashes that produce core dumps. It can be configured -to ignore specific crash patterns and crash details are saved for -further investigation. - -When reporting results to ReportPortal, each test result can now -directly link to a URL. To achieve this, a new key ``link-template`` -was added to the :ref:`/plugins/report/reportportal` plugin, which -can be used to provide a template that will be rendered for each test -result and appended to the end of its description. In cooperation with -Testing Farm, this will allow ReportPortal test results to directly -point to their respective artifacts. - -A new ``restraint-compatible`` key has been implemented for the -:ref:`/plugins/execute/tmt` execute plugin which allows to enable -and disable the :ref:`restraint-compatibility` features. For now -it only affects whether the ``$OUTPUTFILE`` variable is respected -or not. In the future this will allow users to enable/disable all -restraint compatibility features. Please, update your plans with -``restraint-compatibility: true`` as soon as possible if your -tests depend on the restraint features. - -A new :ref:`system.management-controller` -hardware property has been proposed to allow specifying the desired -system management interface (e.g., IPMI) when provisioning hardware. -While not yet implemented, this feature aims to support more precise -hardware selection in the future. - - -tmt-1.45.0 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -FIPS mode can now be enabled for RHEL or CentosStream 8, 9 or 10 -by a prepare step feature ``fips``. Moreover, the ``tmt try`` -command now supports the new :ref:`/stories/cli/try/option/fips` -option backed by the :ref:`/plugins/prepare/feature` plugin. - -New option ``--build-disk-image-only`` is now supported by the -:ref:`/plugins/provision/bootc` plugin and can be used for just -building the disk image without actually provisioning the guest. - -When running ``tmt try``, failure in ``prepare`` phase drops the -user to the menu to be able to login to the machine and possibly -try it again. - -When working with an existing run which involved executing only a -subset of plans, commands such as ``tmt run --last report`` will -load the respective plans only instead of all available plans to -save disk space and speed up the execution. - -Aborted tests and tests that failed when -:ref:`/spec/plans/execute/exit-first` was enabled did not skip all -remaining tests, only tests from the current ``discover`` phase. -Plans with multiple ``discover`` phases would start ``execute`` -step for remaining ``discover`` phases. This is now fixed, aborted -test and :ref:`/spec/plans/execute/exit-first` will skip **all** -remaining tests. - -Added support for translating hardware constraints using a config -file for the :ref:`/plugins/provision/beaker` provision plugin. It -will try to get the config file, and find translations that would -match the constraints. See -:py:class:`tmt.config.models.hardware.MrackTranslation` for an -example translation config. - -When pruning a repository with a specified ``path``, the -``discover`` step now saves the data to the correct temporary -directory and respects the structure of the original repository. -This ensures that the test attributes have correct paths. - -The latest ``fmf`` package is now required to ensure that the -``deployment-mode`` context :ref:`/spec/context/dimension` is -fully supported. - -The default :ref:`/plugins/provision/ssh-options` used for -connecting to provisioned guests are now documented. - - -tmt-1.44.0 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The ``results.yaml`` file is now populated with test results -right after the ``discover`` step is finished and the file is -continuously updated during test execution to provide the latest -results. This change also adds a new ``pending`` result outcome -to the :ref:`/spec/results` specification for tests that were -discovered but not yet executed. - -Execute tmt option ``--ignore-duration`` makes tmt to execute -the test as long as it needs. Execute plugin doesn't need to be -specified on the commandline for :ref:`plugin-variables` to work -for this option. - -Add the ``--command`` option for the ``tmt run reboot`` so that -users specify the command to run on guest to trigger the reboot. - -A new plan shaping plugin has been implemented to repeat a plan N times, -demonstrating how one plan can be turned into many plans. - -The ``deployment-mode`` context dimension is now included in test run -exports to Polarion. - - -tmt-1.43.0 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Add the ``--workdir-root`` option for the ``tmt clean images`` -command so that users can specify the directory they want. - -A new ``upload-subresults`` key has been introduced for the -:ref:`/plugins/report/reportportal` plugin, allowing the import of -tmt subresults as child test items into ReportPortal. This -behavior is optional and is disabled by default. - -Option ``tmt run --max N`` can split plan to multiple plans to -include N tests at max. - -Test name is logged in kernel buffer before and after the -:ref:`/plugins/test-checks/dmesg` check is executed. - - -tmt-1.42.1 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The ``tmt show`` command now prints in verbose mode manual test -instructions as well. - -A new context :ref:`/spec/context/dimension` ``deployment-mode`` -has been added to the specification. It can be used to -:ref:`/spec/core/adjust` test and plan metadata for the -``package`` or ``image`` mode context. - -The ``ansible-core`` package is now a recommended dependency package -for tmt. It is used by plugins that use Ansible under the hood, -:ref:`prepare/ansible`, -:ref:`finish/ansible`, -and :ref:`prepare/feature`. - -A new core attribute :ref:`/spec/core/author` has been implemented -for tracking the original author of the test, plan or story. In -contrast to the :ref:`/spec/core/contact` key, this field is not -supposed to be updated and can be useful when trying to track down -the original author for consultation. - -The ``container`` executor now works in `Fedora Toolbx`__ when Podman is run -using ``flatpak-spawn --host`` on the host system. - -__ https://docs.fedoraproject.org/en-US/fedora-silverblue/toolbox/ - -Add support for running playbooks from Ansible collections specified -using the ``namespace.collection.playbook`` notation. - -Added ``--dry`` option for the ``beaker`` provision plugin. When -used it prints the Beaker Job XML without submitting it. - -:ref:`Results specification documentation` has now -a dedicated place in the specification for improved discoverability. - -The ``rpm-ostree`` package installation now includes the -``--assumeyes`` option for improved compatibility. - -Verbosity levels in ``tmt * show`` commands are now honored. - -Added new traceback verbosity level, ``TMT_SHOW_TRACEBACK=2``, which -prints local variables in every frame, shorterning long values. See -:ref:`command-variables` for details. - -Fixed an issue where ``execute`` step incorrectly attempted to run -disabled ``discover`` phases. - -Pre-defined order values of :ref:`prepare phases` -were documented. - - -tmt-1.41.1 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Fedora Rawhide transitioned files from ``/usr/sbin`` to -``/usr/bin``, breaking path-based requirements installation for -the AVC check. This update adjusts the check to rely on packages, -restoring the functionality on Fedora Rawhide. - - -tmt-1.41.0 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Tests defined using the :ref:`/plugins/discover/shell` discover -method are now executed in the exact order as listed in the config -file. This fixes a problem which has been introduced in the recent -``fmf`` update. - -The :ref:`/plugins/report/reportportal` plugin now exports all -test contact information, rather than just the first contact -instance. - -The :ref:`/plugins/provision/beaker` provision plugin gains -support for submitting jobs on behalf of a group through the -``beaker-job-group`` key. The submitting user must be a member of -the given job group. - -The ``note`` field of tmt :ref:`/spec/results` changes from -a string to a list of strings, to better accommodate multiple notes. - -The ``Node`` alias for the ``Core`` class has been dropped as it -has been deprecated a long time ago. - -Previously when the test run was interrupted in the middle of the -test execution the :ref:`/spec/plans/report` step would be skipped -and no results would be reported. Now the report step is performed -always so that users can access results of those tests which were -successfully executed. - -The ``tmt try`` command now accepts the whole action word in -addition to just a first letter, i.e. ``l`` and ``login`` now -both work. - - -tmt-1.40.0 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The execution of individual step configurations can be controlled -using the new :ref:`when` key. Enable and disable -selected step phase easily with the same syntax as used for the -context :ref:`/spec/core/adjust` rules. - -When the ``login`` command is used to enter an interactive session -on the guest, for example during a ``tmt try`` session, the -current working directory is set to the path of the last executed -test, so that users can easily investigate the test code there and -experiment with it directly on the guest. - -A new ``--workdir-root`` option is now supported in the ``tmt -clean`` and ``tmt run`` commands so that users can specify the -directory which should be cleaned up and where new test runs -should be stored. - -New ``--keep`` option has been implemented for the ``tmt clean -guests`` and ``tmt clean`` commands. Users can now choose to keep -the selected number of latest guests, and maybe also runs, clean -the rest to release the resources. - -The log file paths of tmt subresults created by shell tests by -calling the ``tmt-report-result`` or by calling beakerlib's -``rlPhaseEnd`` saved in ``results.yaml`` are now relative to the -``execute`` directory. - -The :ref:`/plugins/report/reportportal` plugin now handles the -timestamps for ``custom`` and ``restraint`` results correctly. It -should prevent the ``start-time`` of a result being higher than -the ``end-time``. It should be also ensured that the end time of -all launch items is the same or higher than the start time of a -parent item/launch. - -The :ref:`/plugins/provision/beaker` provision plugin gained -support for adding public keys to the guest instance by populating -the kickstart file. - -Documentation pages now use the `new tmt logo`__ designed by Maria -Leonova. - -__ https://github.com/teemtee/docs/tree/main/logo - tmt-1.39.0 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The :ref:`/plugins/provision/beaker` provision plugin gains -support for :ref:`system.model-name`, -:ref:`system.vendor-name`, -:ref:`cpu.family` and -:ref:`cpu.frequency` hardware requirements. +support for :ref:`system.model-name` and +:ref:`system.vendor-name` hardware requirements. The ``tmt lint`` command now reports a failure if empty environment files are found. -The ``tmt try`` command now supports the new -:ref:`/stories/cli/try/option/arch` option. - -As a tech preview, a new :ref:`/plugins/provision/bootc` provision -plugin has been implemented. It takes a container image as input, -builds a bootc disk image from the container image, then uses the -:ref:`/plugins/provision/virtual.testcloud` plugin to create a -virtual machine using the bootc disk image. - -The ``tmt reportportal`` plugin has newly introduced size limit -for logs uploaded to ReportPortal because large logs decreases -ReportPortal UI usability. Default limit are 1 MB for a test -output and 50 kB for a traceback (error log). -Limits can be controlled using the newly introduced -``reportportal`` plugin options ``--log-size-limit`` and -``--traceback-size-limit`` or the respective environment -variables. - tmt-1.38.0 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -576,7 +149,7 @@ committish reference, either branch, tag, git-describe, or if all fails the commit hash. You may encounter this in the verbose log of ``tmt tests show`` or plan/test imports. -:ref:`Result specification` now defines +:ref:`Result specification` now defines ``original-result`` key holding the original outcome of a test, subtest or test checks. The effective outcome, stored in ``result`` key, is computed from the original outcome, and it is @@ -606,7 +179,7 @@ tmt-1.36.0 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ tmt will now emit a warning when :ref:`custom test results` -file does not follow the :ref:`result specification`. +file does not follow the :ref:`result specification`. We have started to use ``warnings.deprecated`` to advertise upcoming API deprecations. @@ -867,7 +440,7 @@ changes. ``info`` results are now treated as ``pass`` results, and would be counted towards the successful exit code, ``0``, instead of the exit code ``2`` in older releases. -The :ref:`/plugins/report/polarion` report now supports the +The :ref:`/spec/plans/report/polarion` report now supports the ``fips`` field to store information about whether the FIPS mode was enabled or disabled on the guest during the test execution. @@ -916,7 +489,7 @@ option which allows to use a user account and execute ``prepare``, ``execute`` and ``finish`` steps using ``sudo -E`` when necessary. -The :ref:`/plugins/report/html` report plugin now shows +The :ref:`/spec/plans/report/html` report plugin now shows :ref:`/spec/tests/check` results so that it's possible to inspect detected AVC denials directly from the report. @@ -933,7 +506,7 @@ can be used to update step phase fields only when not set in the ``fmf`` files. In this way it's possible to easily fill the gaps in the plans, for example provide the default distro image. -The :ref:`/plugins/report/html` report plugin now shows +The :ref:`/spec/plans/report/html` report plugin now shows provided :ref:`/spec/plans/context` and link to the test ``data`` directory so that additional logs can be easily checked. @@ -941,7 +514,7 @@ The **avc** :ref:`/spec/tests/check` allows to detect avc denials which appear during the test execution. A new ``skip`` custom result outcome has been added to the -:ref:`/spec/results` specification. +:ref:`/spec/plans/results` specification. All context :ref:`/spec/context/dimension` values are now handled in a case insensitive way. diff --git a/docs/scripts/generate-hardware-matrix.py b/docs/scripts/generate-hardware-matrix.py index af12763bf0..163a71f999 100755 --- a/docs/scripts/generate-hardware-matrix.py +++ b/docs/scripts/generate-hardware-matrix.py @@ -32,8 +32,7 @@ def main() -> None: tmt.plugins._explore_packages(logger) known_methods = sorted( - tmt.steps.provision.ProvisionPlugin._supported_methods.iter_plugin_ids() - ) + tmt.steps.provision.ProvisionPlugin._supported_methods.iter_plugin_ids()) tree = tmt.Tree(logger=logger, path=Path.cwd()) @@ -48,9 +47,9 @@ def main() -> None: if hw_requirement == 'arch': continue - # C420: current implementation creates a new tuple for each method - # https://github.com/teemtee/tmt/pull/3662#discussion_r2040669353 - matrix[hw_requirement] = {method: (False, int) for method in known_methods} # noqa: C420 + matrix[hw_requirement] = { + method: (False, int) for method in known_methods + } if not story.link: pass @@ -75,14 +74,11 @@ def main() -> None: else: matrix[hw_requirement][implemented_by_method] = (True, None) - output_filepath.write_text( - render_template_file( - template_filepath, - LOGGER=logger, - MATRIX=matrix, - NOTES=notes, - ) - ) + output_filepath.write_text(render_template_file( + template_filepath, + LOGGER=logger, + MATRIX=matrix, + NOTES=notes)) if __name__ == '__main__': diff --git a/docs/scripts/generate-lint-checks.py b/docs/scripts/generate-lint-checks.py index 231f76394a..c47e942116 100755 --- a/docs/scripts/generate-lint-checks.py +++ b/docs/scripts/generate-lint-checks.py @@ -16,9 +16,7 @@ def _sort_linters(linters: list[Linter]) -> list[Linter]: - """ - Sort a list of linters by their ID - """ + """ Sort a list of linters by their ID """ return sorted(linters, key=lambda x: x.id) @@ -36,7 +34,7 @@ def main() -> None: 'PLAN_LINTERS': _sort_linters(Plan.get_linter_registry()), 'STORY_LINTERS': _sort_linters(Story.get_linter_registry()), 'COLLECTION_LINTERS': _sort_linters(LintableCollection.get_linter_registry()), - } + } output_filepath.write_text(render_template_file(template_filepath, **linters)) diff --git a/docs/scripts/generate-plugins.py b/docs/scripts/generate-plugins.py index 42b6d26393..9368f7311d 100755 --- a/docs/scripts/generate-plugins.py +++ b/docs/scripts/generate-plugins.py @@ -7,7 +7,6 @@ from typing import Any import tmt.checks -import tmt.container import tmt.log import tmt.plugins import tmt.steps @@ -15,21 +14,17 @@ import tmt.steps.execute import tmt.steps.finish import tmt.steps.prepare -import tmt.steps.prepare.feature import tmt.steps.provision import tmt.steps.report import tmt.utils -import tmt.utils.hints -from tmt.container import ContainerClass -from tmt.utils import Path +from tmt.utils import ContainerClass, Path from tmt.utils.templates import render_template_file REVIEWED_PLUGINS: tuple[str, ...] = ( 'prepare/ansible', 'test-checks/avc', - 'test-checks/dmesg', - 'test-checks/watchdog', -) + 'test-checks/dmesg' + ) HELP = textwrap.dedent(""" @@ -40,13 +35,10 @@ def _is_ignored( - container: ContainerClass, - field: dataclasses.Field[Any], - metadata: tmt.container.FieldMetadata, -) -> bool: - """ - Check whether a given field is to be ignored in documentation - """ + container: ContainerClass, + field: dataclasses.Field[Any], + metadata: tmt.utils.FieldMetadata) -> bool: + """ Check whether a given field is to be ignored in documentation """ if field.name in ('how', '_OPTIONLESS_FIELDS'): return True @@ -58,28 +50,23 @@ def _is_ignored( def _is_inherited( - container: ContainerClass, - field: dataclasses.Field[Any], - metadata: tmt.container.FieldMetadata, -) -> bool: - """ - Check whether a given field is inherited from step data base class - """ + container: ContainerClass, + field: dataclasses.Field[Any], + metadata: tmt.utils.FieldMetadata) -> bool: + """ Check whether a given field is inherited from step data base class """ # TODO: for now, it's a list, but inspecting the actual tree of classes # would be more generic. It's good enough for now. - return field.name in ('name', 'where', 'when', 'order', 'summary', 'enabled', 'result') + return field.name in ('name', 'where', 'order', 'summary', 'enabled', 'result') def container_ignored_fields(container: ContainerClass) -> list[str]: - """ - Collect container field names that are never displayed - """ + """ Collect container field names that are never displayed """ field_names: list[str] = [] - for field in tmt.container.container_fields(container): - _, _, _, _, metadata = tmt.container.container_field(container, field.name) + for field in tmt.utils.container_fields(container): + _, _, _, _, metadata = tmt.utils.container_field(container, field.name) if _is_ignored(container, field, metadata): field_names.append(field.name) @@ -88,14 +75,12 @@ def container_ignored_fields(container: ContainerClass) -> list[str]: def container_inherited_fields(container: ContainerClass) -> list[str]: - """ - Collect container field names that are inherited from step data base class - """ + """ Collect container field names that are inherited from step data base class """ field_names: list[str] = [] - for field in tmt.container.container_fields(container): - _, _, _, _, metadata = tmt.container.container_field(container, field.name) + for field in tmt.utils.container_fields(container): + _, _, _, _, metadata = tmt.utils.container_field(container, field.name) if _is_inherited(container, field, metadata): field_names.append(field.name) @@ -104,14 +89,12 @@ def container_inherited_fields(container: ContainerClass) -> list[str]: def container_intrinsic_fields(container: ContainerClass) -> list[str]: - """ - Collect container fields specific for the given step data - """ + """ Collect container fields specific for the given step data """ field_names: list[str] = [] - for field in tmt.container.container_fields(container): - _, _, _, _, metadata = tmt.container.container_field(container, field.name) + for field in tmt.utils.container_fields(container): + _, _, _, _, metadata = tmt.utils.container_field(container, field.name) if _is_ignored(container, field, metadata): continue @@ -125,52 +108,28 @@ def container_intrinsic_fields(container: ContainerClass) -> list[str]: def is_enum(value: Any) -> bool: - """ - Find out whether a given value is an enum member - """ + """ Find out whether a given value is an enum member """ return isinstance(value, enum.Enum) def _create_step_plugin_iterator(registry: tmt.plugins.PluginRegistry[tmt.steps.Method]): - """ - Create iterator over plugins of a given registry - """ + """ Create iterator over plugins of a given registry """ def plugin_iterator(): - for plugin_id in sorted(registry.iter_plugin_ids()): + for plugin_id in registry.iter_plugin_ids(): plugin = registry.get_plugin(plugin_id).class_ - if hasattr(plugin, 'get_data_class'): - yield plugin_id, plugin, plugin.get_data_class() - - else: - yield plugin_id, plugin, plugin._data_class - - return plugin_iterator - - -def _create_feature_plugin_iterator( - registry: tmt.plugins.PluginRegistry[tmt.steps.prepare.feature.FeatureClass], -): - """Create iterator over plugins of a feature plugin registry""" - - def plugin_iterator(): - for plugin_id in sorted(registry.iter_plugin_ids()): - plugin = registry.get_plugin(plugin_id) - yield plugin_id, plugin, plugin._data_class return plugin_iterator def _create_test_check_plugin_iterator(registry: tmt.plugins.PluginRegistry[tmt.steps.Method]): - """ - Create iterator over plugins of a test check registry - """ + """ Create iterator over plugins of a test check registry """ def plugin_iterator(): - for plugin_id in sorted(registry.iter_plugin_ids()): + for plugin_id in registry.iter_plugin_ids(): plugin = registry.get_plugin(plugin_id) yield plugin_id, plugin, plugin._check_class @@ -197,38 +156,27 @@ def main() -> None: if step_name == 'discover': plugin_generator = _create_step_plugin_iterator( - tmt.steps.discover.DiscoverPlugin._supported_methods - ) + tmt.steps.discover.DiscoverPlugin._supported_methods) elif step_name == 'execute': plugin_generator = _create_step_plugin_iterator( - tmt.steps.execute.ExecutePlugin._supported_methods - ) + tmt.steps.execute.ExecutePlugin._supported_methods) elif step_name == 'finish': plugin_generator = _create_step_plugin_iterator( - tmt.steps.finish.FinishPlugin._supported_methods - ) + tmt.steps.finish.FinishPlugin._supported_methods) elif step_name == 'prepare': plugin_generator = _create_step_plugin_iterator( - tmt.steps.prepare.PreparePlugin._supported_methods - ) + tmt.steps.prepare.PreparePlugin._supported_methods) elif step_name == 'provision': plugin_generator = _create_step_plugin_iterator( - tmt.steps.provision.ProvisionPlugin._supported_methods - ) + tmt.steps.provision.ProvisionPlugin._supported_methods) elif step_name == 'report': plugin_generator = _create_step_plugin_iterator( - tmt.steps.report.ReportPlugin._supported_methods - ) - - elif step_name == 'prepare-feature': - plugin_generator = _create_feature_plugin_iterator( - tmt.steps.prepare.feature._FEATURE_PLUGIN_REGISTRY - ) + tmt.steps.report.ReportPlugin._supported_methods) elif step_name == 'test-checks': plugin_generator = _create_test_check_plugin_iterator(tmt.checks._CHECK_PLUGIN_REGISTRY) @@ -237,22 +185,18 @@ def main() -> None: raise tmt.utils.GeneralError(f"Unhandled step name '{step_name}'.") # ... and render the template. - output_filepath.write_text( - render_template_file( - template_filepath, - LOGGER=logger, - STEP=step_name, - PLUGINS=plugin_generator, - REVIEWED_PLUGINS=REVIEWED_PLUGINS, - HINTS=tmt.utils.hints.HINTS, - is_enum=is_enum, - container_fields=tmt.container.container_fields, - container_field=tmt.container.container_field, - container_ignored_fields=container_ignored_fields, - container_inherited_fields=container_inherited_fields, - container_intrinsic_fields=container_intrinsic_fields, - ) - ) + output_filepath.write_text(render_template_file( + template_filepath, + LOGGER=logger, + STEP=step_name, + PLUGINS=plugin_generator, + REVIEWED_PLUGINS=REVIEWED_PLUGINS, + is_enum=is_enum, + container_fields=tmt.utils.container_fields, + container_field=tmt.utils.container_field, + container_ignored_fields=container_ignored_fields, + container_inherited_fields=container_inherited_fields, + container_intrinsic_fields=container_intrinsic_fields)) if __name__ == '__main__': diff --git a/docs/scripts/generate-stories.py b/docs/scripts/generate-stories.py index 86f88bd0f3..e7cfaae5ed 100755 --- a/docs/scripts/generate-stories.py +++ b/docs/scripts/generate-stories.py @@ -37,10 +37,8 @@ def __getattr__(cls, name: str) -> 'Mock': '/spec/plans': 'Plans', '/spec/stories': 'Stories', '/spec/context': 'Context', - '/spec/policy': 'Policy', '/spec/hardware': 'Hardware', - '/spec/results': 'Results', -} + } def main() -> None: @@ -55,19 +53,19 @@ def main() -> None: logger = tmt.Logger.create() logger.add_console_handler() - # Explore available plugins - tmt.plugins.explore(logger) + # Explore available *export* plugins - do not import other plugins, we don't need them. + tmt.plugins.explore_export_package(logger) # Generate stories tree = tmt.Tree(logger=logger, path=Path.cwd()) - for area, title in AREA_TITLES.items(): + for area in AREA_TITLES: logger.info(f'Generating rst files from {area}') with open(f"{area.lstrip('/')}.rst", 'w') as doc: # Anchor and title doc.write(f'.. _{area}:\n\n') - doc.write(f"{title}\n{'=' * len(title)}\n") + doc.write(f"{AREA_TITLES[area]}\n{'=' * len(AREA_TITLES[area])}\n") # Included stories for story in tree.stories(names=[area], whole=True): if not story.enabled: @@ -76,8 +74,7 @@ def main() -> None: rendered = story.export( format='rst', include_title=story.name != area, - template=story_template_filepath, - ) + template=story_template_filepath) doc.write(rendered) doc.write('\n\n') diff --git a/docs/scripts/generate-template-extensions.py b/docs/scripts/generate-template-extensions.py deleted file mode 100755 index 3979afec58..0000000000 --- a/docs/scripts/generate-template-extensions.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import textwrap - -from tmt.utils import Path -from tmt.utils.templates import TEMPLATE_FILTERS, TEMPLATE_TESTS, render_template_file - -HELP = textwrap.dedent(""" -Usage: generate-template-extensions.py - -Generate docs for all custom Jinja2 template filters and tests. -""").strip() - - -def main() -> None: - if len(sys.argv) != 3: - print(HELP) - - sys.exit(1) - - template_filepath = Path(sys.argv[1]) - output_filepath = Path(sys.argv[2]) - - output_filepath.write_text( - render_template_file( - template_filepath, - FILTERS=TEMPLATE_FILTERS, - TESTS=TEMPLATE_TESTS, - ) - ) - - -if __name__ == '__main__': - main() diff --git a/docs/scripts/generate-template-filters.py b/docs/scripts/generate-template-filters.py new file mode 100755 index 0000000000..890cdc6840 --- /dev/null +++ b/docs/scripts/generate-template-filters.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +import sys +import textwrap + +from tmt.utils import Path +from tmt.utils.templates import TEMPLATE_FILTERS, render_template_file + +HELP = textwrap.dedent(""" +Usage: generate-template-filters.py + +Generate docs for all known Jinja2 template filters. +""").strip() + + +def main() -> None: + if len(sys.argv) != 3: + print(HELP) + + sys.exit(1) + + template_filepath = Path(sys.argv[1]) + output_filepath = Path(sys.argv[2]) + + output_filepath.write_text(render_template_file(template_filepath, TEMPLATES=TEMPLATE_FILTERS)) + + +if __name__ == '__main__': + main() diff --git a/docs/scripts/generate-test-runner-guest-matrix.py b/docs/scripts/generate-test-runner-guest-matrix.py deleted file mode 100755 index 14341d99c5..0000000000 --- a/docs/scripts/generate-test-runner-guest-matrix.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python3 - -import re -import sys -import textwrap -from re import Pattern -from typing import Any - -import tmt.plugins -import tmt.steps.provision -from tmt.utils import GeneralError, Path, yaml_to_dict -from tmt.utils.templates import render_template_file - -HELP = textwrap.dedent(""" -Usage: generate-runner-guest-matrix.py - -Generate page with runner vs guest compatibility matrix. -""").strip() - - -def main() -> None: - if len(sys.argv) != 4: - print(HELP) - - sys.exit(1) - - template_filepath = Path(sys.argv[1]) - definitions_filepath = Path(sys.argv[2]) - output_filepath = Path(sys.argv[3]) - - # We will need a logger... - logger = tmt.Logger.create() - logger.add_console_handler() - - # Explore available *export* plugins - do not import other plugins, we don't need them. - tmt.plugins._explore_packages(logger) - - logger.info('Generating reST file for runner/guest compatibility matrix') - - definitions = yaml_to_dict(definitions_filepath.read_text()) - - environment_names: list[str] = definitions['environments'] - - try: - unsupported_environments: list[Pattern[str]] = [ - re.compile(pattern) for pattern in definitions['unsupported'] - ] - - except re.error as exc: - raise GeneralError(f"Invalid 'unsupported' pattern '{exc.pattern}'.") from exc - - try: - unknown_environments: list[Pattern[str]] = [ - re.compile(pattern) for pattern in definitions['unknown'] - ] - - except re.error as exc: - raise GeneralError(f"Invalid 'unknown' pattern '{exc.pattern}'.") from exc - - try: - notes: dict[Pattern[str], Any] = { - re.compile(note['pattern']): note for note in definitions['notes'] - } - - except re.error as exc: - raise GeneralError(f"Invalid 'notes' pattern '{exc.pattern}'.") from exc - - matrix: dict[str, list[tuple[str, str]]] = {} - - for runner in environment_names: - matrix[runner] = [] - - for guest in environment_names: - combination_key = f'{runner} + {guest}' - - for pattern, note in notes.items(): - if not pattern.match(combination_key): - continue - - matrix[runner].append(('supported-with-caveats', note)) - - break - - else: - if any(pattern.match(combination_key) for pattern in unsupported_environments): - matrix[runner].append(('unsupported', None)) - - elif any(pattern.match(combination_key) for pattern in unknown_environments): - matrix[runner].append(('unknown', None)) - - else: - matrix[runner].append(('supported', None)) - - output_filepath.write_text( - render_template_file( - template_filepath, - LOGGER=logger, - MATRIX=matrix, - NOTES=notes.values(), - ), - ) - - -if __name__ == '__main__': - main() diff --git a/docs/spec.rst b/docs/spec.rst index b30a1cd663..7150dd289e 100644 --- a/docs/spec.rst +++ b/docs/spec.rst @@ -53,7 +53,5 @@ Level 3: Stories spec/plans spec/stories spec/context - spec/policy spec/hardware - spec/results spec/lint diff --git a/docs/templates/plugins.rst.j2 b/docs/templates/plugins.rst.j2 index 626afe0a8e..995c9b4d09 100644 --- a/docs/templates/plugins.rst.j2 +++ b/docs/templates/plugins.rst.j2 @@ -1,45 +1,17 @@ -{% if STEP != 'prepare-feature' %} :tocdepth: 0 -{% endif %} - -{% if STEP == 'prepare-feature' %} - {% set page_header_char = '-' %} - {% set plugin_header_char = '^' %} - {% set plugin_configuration_header_char = '#' %} - - {% macro render_page_header() %}{% endmacro %} - -{% else %} - {% set page_header_char = '~' %} - {% set plugin_header_char = '-' %} - {% set plugin_configuration_header_char = '^' %} - - {% macro render_page_header() %} -.. _/plugins/{{ STEP }}: - -{{ render_header((STEP | replace('-', ' ') | capitalize) + ' Plugins', page_header_char) }} - {% endmacro %} - -{% endif %} - -{% macro render_header(name, char) %} -{{ name }} -{{ char * (name | length) }} -{% endmacro %} - -{% macro render_plugin_header(name) %} -{{ render_header(name, plugin_header_char) }} -{% endmacro %} - -{% macro render_plugin_configuration_header(name) %} -{{ render_header(name, plugin_configuration_header_char) }} -{% endmacro %} {% macro render_field(plugin_id, plugin_data_class, field_name) %} {% set _, option, _, _, metadata = container_field(plugin_data_class, field_name) %} - {% set envvar = metadata.envvar or "TMT_PLUGIN_%s_%s_%s" | format(STEP.upper(), plugin_id.upper(), field_name.replace('-', '_').upper()) %} -{{ option }} + {% if metadata.metavar %} +{{ option }}: ``{{ metadata.metavar }}`` + {% elif metadata.default is boolean %} +{{ option }}: ``true|false`` + {% elif metadata.choices %} +{{ option }}: ``{{ metadata.choices }}`` + {% else %} +{{ option }}: + {% endif %} {% if metadata.help %} {{ metadata.help | trim | indent(4, first=true) }} {% endif %} @@ -51,8 +23,6 @@ {% elif actual_default is string %} Default: ``{{ actual_default }}`` {% elif actual_default is integer %} - Default: ``{{ actual_default }}`` - {% elif actual_default is unit %} Default: ``{{ actual_default }}`` {% elif actual_default is none %} Default: *not set* @@ -65,68 +35,25 @@ {% elif is_enum(actual_default) %} Default: ``{{ actual_default.value }}`` {% else %} - {% set error_message = "%s/%s.%s: could not render default value, '%s'" | format(STEP, plugin_id, field_name, actual_default) %} - {{ raise_error(error_message) }} + {% set _ = LOGGER.warn("%s/%s.%s: could not render default value, '%s'" | format(STEP, plugin_id, field_name, actual_default), shift=0) %} + Default: *could not render default value correctly* {% endif %} {% endif %} - - Environment variable: ``{{ envvar }}`` - - In plan metadata: - - .. code-block:: yaml - - {% if metadata.metavar %} - {{ option }}: {{ metadata.metavar }} - - {% for example in metadata.help_example_values %} - {{ option }}: {{ example }} - {% endfor %} - {% elif metadata.default is boolean %} - {{ option }}: true|false - {% else %} - {{ option }}: - {% endif %} - - On command-line: - - .. code-block:: shell - - {% if metadata.metavar %} - --{{ option }} {{ metadata.metavar | shell_quote }} - export {{ envvar }}={{ metadata.metavar | shell_quote }} - - {% for example in metadata.help_example_values %} - --{{ option }} {{ example | shell_quote }} - {% endfor %} - {% for example in metadata.help_example_values %} - export {{ envvar }}={{ example | shell_quote }} - {% endfor %} - {% elif metadata.default is boolean %} - --{{ option }} - export {{ envvar }}=1|0 - {% else %} - --{{ option }} ... - export {{ envvar }}=... - - {% for example in metadata.help_example_values %} - --{{ option }} {{ example | shell_quote }} - {% endfor %} - {% for example in metadata.help_example_values %} - export {{ envvar }}={{ example | shell_quote }} - {% endfor %} - {% endif %} {% endmacro %} -{{ render_page_header() }} +.. _/plugins/{{ STEP }}: + +{{ STEP | replace('-', ' ') | capitalize }} Plugins +{{ '~' * (8 + (STEP | length)) }} .. include:: {{ STEP }}-header.inc.rst {% for PLUGIN_ID, PLUGIN, PLUGIN_DATA_CLASS in PLUGINS() %} - {% if loop.first and STEP != 'prepare-feature' %} + {% if loop.first %} .. _/plugins/{{ STEP }}/common-keys: -{{ render_plugin_header('Common Keys') }} +Common Keys +----------- The following keys are accepted by all plugins of the ``{{ STEP }}`` step. @@ -135,11 +62,10 @@ The following keys are accepted by all plugins of the ``{{ STEP }}`` step. {% endfor %} {% endif %} -{% if STEP != 'prepare-feature' %} .. _/plugins/{{ STEP }}/{{ PLUGIN_ID | trim }}: -{% endif %} -{{ render_plugin_header(PLUGIN_ID) }} +{{ PLUGIN_ID }} +{{ '-' * (PLUGIN_ID | length) }} {# Emit the warning only for plugins that have not been reviewed yet. #} {% set plugin_full_id = STEP + "/" + PLUGIN_ID %} @@ -156,63 +82,20 @@ The following keys are accepted by all plugins of the ``{{ STEP }}`` step. {{ PLUGIN.__doc__ | dedent | trim }} {% endif %} -{% if plugin_full_id in HINTS %} -.. note:: - -{{ HINTS[plugin_full_id].text | indent(3, first=true) }} -{% endif %} - -{% if STEP == 'prepare-feature' %} - {# - When rendering prepare/feature plugins, emit only a reduced version of - plugin configuration: no shared keys, just the intrinsic ones. - #} - - {% set intrinsic_fields = container_intrinsic_fields(PLUGIN_DATA_CLASS) | sort %} - - {% if intrinsic_fields %} - {% for field_name in intrinsic_fields %} -{{ render_field(PLUGIN_ID, PLUGIN_DATA_CLASS, field_name) }} - {% endfor %} - - {% if not loop.last %} ----- - {% endif %} - - {% endif %} - -{% elif STEP == 'prepare' and PLUGIN_ID == 'feature' %} - {# - Since prepare/feature does not have intrinsic keys of its own, - and all its keys are keys of its plugins, include the generated - document describing the plugins instead of rendering any keys. - #} - -See also :ref:`Common Keys{# djlint:off H025 #}{# djlint:on H025 #}` accepted by the plugin. - -.. include:: prepare-feature.rst +{% set intrinsic_fields = container_intrinsic_fields(PLUGIN_DATA_CLASS) | sort %} -{% else %} - {# - And for all other plugins, render their intrinsic keys, and don't - forget to point to the shared ones. - #} - - {% set intrinsic_fields = container_intrinsic_fields(PLUGIN_DATA_CLASS) | sort %} - - {% if intrinsic_fields %} -{{ render_plugin_configuration_header('Configuration') }} +{% if intrinsic_fields %} +Configuration +^^^^^^^^^^^^^ See also :ref:`Common Keys{# djlint:off H025 #}{# djlint:on H025 #}` accepted by the plugin. - {% for field_name in intrinsic_fields %} +{% for field_name in intrinsic_fields %} {{ render_field(PLUGIN_ID, PLUGIN_DATA_CLASS, field_name) }} - {% endfor %} +{% endfor %} - {% if not loop.last %} +{% if not loop.last %} ---- - {% endif %} - {% endif %} {% endif %} - +{% endif %} {% endfor %} diff --git a/docs/templates/template-extensions.rst.j2 b/docs/templates/template-filters.rst.j2 similarity index 50% rename from docs/templates/template-extensions.rst.j2 rename to docs/templates/template-filters.rst.j2 index 405921aa1e..f47e1aa53d 100644 --- a/docs/templates/template-extensions.rst.j2 +++ b/docs/templates/template-filters.rst.j2 @@ -6,49 +6,29 @@ #} .. - Please, do not edit this file, is is rendered from template-extensions.rst.j2, + Please, do not edit this file, is is rendered from template-filters.rst.j2, and all your changes would be overwritten. -Template Extensions +Template filters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ When it comes to creating text documents, like :ref:`custom_templates` or exporting tests, plans and stories into ReST or Markdown, tmt relies on the well-known `Jinja`__ package. All Jinja templating features are supported, and, on -top of `filters`__ and `tests`__ shipped with Jinja, tmt adds several -custom ones. +top of `filters`__ shipped with Jinja, tmt adds several custom ones. __ https://palletsprojects.com/p/jinja/ __ https://jinja.palletsprojects.com/en/3.1.x/templates/#filters -__ https://jinja.palletsprojects.com/en/stable/templates/#tests -Filters ------------------------------------------------------------------- - -{% for filter_name in FILTERS.keys() | sort %} - {% set filter_callable = FILTERS[filter_name] %} +{% for filter_name in TEMPLATES.keys() | sort %} + {% set filter_callable = TEMPLATES[filter_name] %} {{ filter_name }} -{{ '^' * (filter_name | length) }} +{{ '-' * (filter_name | length) }} {% if filter_callable.__doc__ %} {{ filter_callable.__doc__ | dedent | trim }} {% endif %} {% endfor %} - -Tests ------------------------------------------------------------------- - -{% for test_name in TESTS.keys() | sort %} - {% set test_callable = TESTS[test_name] %} - -{{ test_name }} -{{ '^' * (test_name | length) }} - -{% if test_callable.__doc__ %} -{{ test_callable.__doc__ | dedent | trim }} -{% endif %} - -{% endfor %} diff --git a/docs/templates/test-runner-guest-compatibility-matrix.inc.rst.j2 b/docs/templates/test-runner-guest-compatibility-matrix.inc.rst.j2 deleted file mode 100644 index 27c2368985..0000000000 --- a/docs/templates/test-runner-guest-compatibility-matrix.inc.rst.j2 +++ /dev/null @@ -1,48 +0,0 @@ -{# - A Jinja2 template for rendering runner vs guest compatibility matrix in tmt's own docs. -#} - -.. - Please, do not edit this file, is is rendered from runner-guest-compatibility.yaml - and test-runner-guest-compatibility-matrix.rst.j2, and all your changes would be overwritten. - -Guest Compatibility -------------------- - -.. list-table:: - :header-rows: 1 - - * - Runner / Guest - {% for runner in MATRIX.keys() %} - - **{{ runner }}** - {% endfor %} - -{% for runner, guests in MATRIX.items() %} - * - {{ runner }} - {% for state, note in guests %} - {% if state == 'supported' %} - - ✅ - - {% elif state == 'supported-with-caveats' %} - - ❗{% if note %} See :ref:`note{# djlint:off H025 #}{# djlint:on H025 #}`{% endif %} - - {% elif state == 'unknown' %} - - ❓ - - {% elif state == 'unsupported' %} - - ❌ - - {% endif %} - {% endfor %} - -{% endfor %} - -{% for note in NOTES %} -.. _note-{{ note.title | urlencode }}: - -{{ note.title }} -{{ '^' * (note.title | length) }} - -{{ note.text | dedent }} - -{% endfor %} diff --git a/docs/test-runner-guest-compatibility.yaml b/docs/test-runner-guest-compatibility.yaml deleted file mode 100644 index 01ab38b826..0000000000 --- a/docs/test-runner-guest-compatibility.yaml +++ /dev/null @@ -1,52 +0,0 @@ ---- - -# A list of known, recognized environments for runners and guests -environments: - - CentOS 7 - - CentOS Stream 8 - - CentOS Stream 9 - - CentOS Stream 10 - - Fedora 39 - - Fedora 40 - - Fedora 41 - - Fedora Rawhide - - Alpine 3 - - Debian 12 - - Ubuntu 22 - -# -# Keys and list entries below are treated as regular expressions tested -# against a "{runner} + {guest}" string. -# - -# A list of patterns matching unsupported runner/guest combinations. -unsupported: [] - -# A list of patterns matching runner/guest combinations where the state -# of support is unknown. -unknown: - # The following should work as runners, with tmt installed from PyPI, - # but we do not know for sure. - - 'Alpine .*' - - 'Ubuntu .*' - - 'Debian .*' - -notes: - - pattern: 'Fedora Rawhide \+ (CentOS 7|Centos Stream 8)' - title: Fedora Rawhide and Ansible 2.17+ - text: > - The :ref:`prepare/ansible`, - :ref:`prepare/feature` - and :ref:`finish/ansible` plugins will - not work out of the box. See their documentation for details and - possible workarounds. - - - pattern: '.*? \+ (Alpine|Debian|Ubuntu)' - title: Beakerlib not packaged for Linux distributions outside of Fedora family - text: > - Tests using the :ref:`beakerlib` framework - will not work because it requires ``beakerlib`` package that can - be installed from repositories available on the guest. At this - moment, ``beakerlib`` is available as RPM packages, and it is not - installable by a package manager beyond the Fedora family (Fedora, - CentOS, CentOS Stream, RHEL). diff --git a/docs/toolbelt-catalog.yaml b/docs/toolbelt-catalog.yaml index e28814a9f6..6eece7e480 100644 --- a/docs/toolbelt-catalog.yaml +++ b/docs/toolbelt-catalog.yaml @@ -27,7 +27,7 @@ metadata: - title: pypi url: https://pypi.org/project/tmt - title: copr - url: https://copr.fedorainfracloud.org/coprs/g/teemtee/stable/ + url: https://copr.fedorainfracloud.org/coprs/g/teemtee/tmt/ - title: "#tmt" url: https://app.slack.com/client/E030G10V24F/C04LRPNDZ4Y icon: chat diff --git a/examples/ansible/test.yml b/examples/ansible/test.yml index caf6997c51..076f271086 100644 --- a/examples/ansible/test.yml +++ b/examples/ansible/test.yml @@ -1,5 +1,3 @@ ---- - - name: A simple test hosts: localhost tasks: diff --git a/examples/plugins/discover.py b/examples/plugins/discover.py index f00a81dfe7..0541e5be02 100644 --- a/examples/plugins/discover.py +++ b/examples/plugins/discover.py @@ -17,10 +17,7 @@ class DiscoverExample(tmt.steps.discover.DiscoverPlugin): """ def show(self): - """ - Show plugin details for given or all available keys - """ - + """ Show plugin details for given or all available keys """ super().show([]) print("show() called") @@ -31,7 +28,6 @@ def wake(self): If a list of option names is provided, their value will be checked and stored in self.data unless empty or undefined. """ - print("wake() called") # Check provided tests, default to an empty list @@ -45,7 +41,6 @@ def go(self): Discover available tests in this case. """ - super().go() print("go() called") @@ -62,5 +57,4 @@ def tests(self): Each DiscoverPlugin has to implement this method. Should return a list of Test() objects. """ - return self._tests diff --git a/examples/plugins/provision.py b/examples/plugins/provision.py index 4ba91815b4..bb50f4aeea 100644 --- a/examples/plugins/provision.py +++ b/examples/plugins/provision.py @@ -1,3 +1,4 @@ + import tmt import tmt.steps from tmt.options import option @@ -30,42 +31,26 @@ class ProvisionExample(tmt.steps.provision.ProvisionPlugin): @classmethod def options(cls, how=None): - """ - Prepare command line options for example - """ - + """ Prepare command line options for example """ return [ option( - '-w', - '--what', - metavar='WHAT', - help="Example how to pass value.", - ), + '-w', '--what', metavar='WHAT', + help="Example how to pass value."), option( - '-s', - '--switch', - is_flag=True, - help="Example how to enable something.", - ), - *super().options(how), - ] + '-s', '--switch', is_flag=True, + help="Example how to enable something."), + *super().options(how)] def default(self, option, default=None): - """ - Return the default value for the given option - """ - + """ Return the default value for the given option """ defaults = { 'what': 'default value', 'switch': False, - } + } return defaults.get(option, default) def show(self): - """ - Show provision details - """ - + """ Show provision details """ super().show(['what', 'switch']) def wake(self, data=None): @@ -75,7 +60,6 @@ def wake(self, data=None): Override data with command line options. Wake up the guest based on provided guest data. """ - super().wake(['what', 'switch']) print("wake() called") @@ -88,10 +72,7 @@ def wake(self, data=None): self._guest.wake() def go(self): - """ - Provision the container - """ - + """ Provision the container """ super().go() print("go() called") @@ -116,7 +97,6 @@ def guest(self): Each ProvisionPlugin has to implement this method. Should return a provisioned Guest() instance. """ - return self._guest @@ -158,7 +138,6 @@ def load(self, data): line options / L2 metadata / user configuration and wake up data stored by the save() method below. """ - super().load(data) self.what = data.get('what') self.switch = data.get('switch') @@ -171,7 +150,6 @@ def wake(self): attach to a running guest instance and execute commands. Called after load() is completed so all guest data should be prepared. """ - print("wake() called") def save(self): @@ -183,17 +161,13 @@ def save(self): the guest. Everything needed to attach to a running instance should be added into the data dictionary by child classes. """ - data = super().save() data['what'] = self.what data['switch'] = self.switch return data def _some_your_internal_stuff(self): - """ - Do some heavy lifting - """ - + """ Do some heavy lifting """ return True def start(self): @@ -215,7 +189,8 @@ def start(self): if self._some_your_internal_stuff(): return - raise tmt.utils.ProvisionError("All attempts to provision a machine with example failed.") + raise tmt.utils.ProvisionError( + "All attempts to provision a machine with example failed.") def setup(self): """ @@ -225,7 +200,6 @@ def setup(self): to get the plans to work. For example, ensure the permissions in TMT_WORKDIR_ROOT will work if the user is non-root. """ - print("setup() called") # For advanced development @@ -251,17 +225,11 @@ def execute(self, command, **kwargs): return ["Fedora", "whatever"] def delete(self): - """ - Remove the example instance - """ - + """ Remove the example instance """ self.debug("You should place code for cleanup here.") def remove(self): - """ - Remove the guest - """ - + """ Remove the guest """ if self.what: self.info('guest', 'removed', 'green') self.delete() diff --git a/examples/redis/ansible/tasks/redis_variables.yml b/examples/redis/ansible/tasks/redis_variables.yml index a10a8bc30e..c6707ae479 100644 --- a/examples/redis/ansible/tasks/redis_variables.yml +++ b/examples/redis/ansible/tasks/redis_variables.yml @@ -1,5 +1,3 @@ ---- - - name: Set variables into redis database ansible.builtin.command: | redis-cli set key value1 diff --git a/plans/provision/bootc.fmf b/plans/provision/bootc.fmf deleted file mode 100644 index 31c35e53e1..0000000000 --- a/plans/provision/bootc.fmf +++ /dev/null @@ -1,34 +0,0 @@ -summary: Bootc virtual machine via testcloud - -description: | - Verify functionality of the bootc provision plugin. - -discover: - how: fmf - filter: 'tag:provision-bootc' - -prepare+: - - name: start-libvirtd - script: | - systemctl start libvirtd - systemctl status libvirtd - -adjust+: - - enabled: true - when: how == provision - - - provision: - hardware: - virtualization: - is-supported: true - memory: ">= 8 GB" - when: trigger == commit - - - prepare+: - - name: Disable IPv6 - how: shell - script: - - sysctl -w net.ipv6.conf.all.disable_ipv6=1 - - sysctl -w net.ipv6.conf.default.disable_ipv6=1 - because: Disable IPv6 in CI to avoid IPv6 connections that are disabled in CI - when: trigger == commit diff --git a/plans/provision/container.fmf b/plans/provision/container.fmf index 380006ed0e..b4d1237294 100644 --- a/plans/provision/container.fmf +++ b/plans/provision/container.fmf @@ -25,15 +25,3 @@ environment: PROVISION_HOW: container enabled: true - -/install: - discover+: - test: install - -/ansible: - discover+: - test: ansible - -/the-rest: - discover+: - exclude: (install|ansible) diff --git a/plans/provision/virtual.fmf b/plans/provision/virtual.fmf index 98c7847b14..6f671bc45c 100644 --- a/plans/provision/virtual.fmf +++ b/plans/provision/virtual.fmf @@ -24,15 +24,13 @@ prepare+: sudo systemctl start libvirtd sudo systemctl status libvirtd - -# TODO hotfix disable the image preparation because of weird dns issues -# - name: prepare-image -# summary: Fetch the image, refresh dnf cache, install beakerlib -# script: | -# tmt run --remove plan --default provision --how virtual finish -# for image in /var/tmp/tmt/testcloud/images/*qcow2; do -# virt-customize --add $image --run-command 'dnf --refresh install -y beakerlib' -# done + - name: prepare-image + summary: Fetch the image, refresh dnf cache, install beakerlib + script: | + tmt run --remove plan --default provision --how virtual finish + for image in /var/tmp/tmt/testcloud/images/*qcow2; do + virt-customize --add $image --run-command 'dnf --refresh install -y beakerlib' + done context+: provision_how: virtual @@ -48,7 +46,7 @@ adjust+: hardware: virtualization: is-supported: true - memory: ">= 8 GB" + memory: ">= 4 GB" when: trigger == commit - prepare+: diff --git a/pyproject.toml b/pyproject.toml index 70090ba1b5..0f125de355 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ dynamic = ["version"] description = "Test Management Tool" readme = "README.rst" license = "MIT" -license-file = "LICENSE" +license-files = { paths = ["LICENSE"] } requires-python = ">=3.9" authors = [ { name = "Petr Splichal", email = "psplicha@redhat.com" }, @@ -24,51 +24,42 @@ classifiers = [ "License :: OSI Approved :: MIT License", "Natural Language :: English", "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", "Topic :: Utilities", "Topic :: Software Development :: Testing", "Operating System :: POSIX :: Linux", ] -dependencies = [ - # Minimal versions based on c9s rpms - "click>=8.0.3", - "docutils>=0.16", - "fmf>=1.7.0", - "jinja2>=2.11.3", - "packaging>=20.9", - "pint>=0.16.1", - "pydantic>=1.10.14", - "pygments>=2.7.4", - "requests>=2.25.1", - "ruamel.yaml>=0.16.6", - "urllib3>=1.26.5, <3.0", - "typing-extensions>=4.12.2; python_version < '3.13'", +dependencies = [ # F39 / PyPI + "click>=8.0.3,!=8.1.4", # 8.1.3 / 8.1.6 TODO type annotations tmt.cli.Context -> click.core.Context click/issues/2558 + "docutils>=0.16", # 0.16 is the current one available for RHEL9 + "fmf>=1.4.0", + "jinja2>=2.11.3", # 3.1.2 / 3.1.2 + "packaging>=20", # 20 seems to be available with RHEL8 + "pint>=0.16.1", # 0.16.1 + "pygments>=2.7.4", # 2.7.4 is the current one available for RHEL9 + "requests>=2.25.1", # 2.28.2 / 2.31.0 + "ruamel.yaml>=0.16.6", # 0.17.32 / 0.17.32 + "urllib3>=1.26.5, <3.0", # 1.26.16 / 2.0.4 + "typing-extensions>=4.9.0; python_version < '3.13'", ] [project.optional-dependencies] -ansible = [ - "ansible-core>=2.14.18", - ] test-convert = [ "html2text>=2020.1.16", "markdown>=3.3.4", - "nitrate>=1.9.0", + "nitrate>=1.8.2", "python-bugzilla>=3.2.0", ] export-polarion = [ - "pylero>=0.1.0", + "pylero>=0.0.8", ] provision-beaker = [ - "mrack>=1.23.2", + "mrack>=1.15.1", ] provision-virtual = [ - "testcloud>=0.11.7", + "testcloud>=0.11.3", ] provision-container = [] -provision-bootc = [] report-junit = [ # Required to support XML parsing and checking the XSD schemas. "lxml>=4.6.5", @@ -84,7 +75,6 @@ all = [ "tmt[test-convert]", "tmt[export-polarion]", "tmt[provision-container]", - "tmt[provision-bootc]", "tmt[provision-virtual]", "tmt[provision-beaker]", "tmt[report-junit]", @@ -97,7 +87,7 @@ docs = [ "readthedocs-sphinx-ext", "docutils>=0.18.1", "Sphinx==7.3.7", - "fmf>=1.7.0", + "fmf>=1.4.0", ] [project.scripts] @@ -163,7 +153,7 @@ lint = ["autopep8 {args:.}", "ruff --fix {args:.}"] type = ["mypy {args:tmt}"] check = ["lint", "type"] -unit = "make images/test && pytest -vvv -ra --showlocals -n 0 tests/unit" +unit = "make images-tests && pytest -vvv -ra --showlocals -n 0 tests/unit" smoke = "pytest -vvv -ra --showlocals -n 0 tests/unit/test_cli.py" cov = [ "coverage run --source=tmt -m pytest -vvv -ra --showlocals -n 0 tests", @@ -186,7 +176,7 @@ template = "dev" description = "Run scripts with multiple Python versions" [[tool.hatch.envs.test.matrix]] -python = ["3.9", "3.12", "3.13"] +python = ["3.9", "3.11", "3.12"] [tool.hatch.envs.docs] dependencies = ["tmt[docs]"] @@ -242,11 +232,6 @@ module = [ ] ignore_missing_imports = true -[[tool.mypy.overrides]] -# Workaround for https://github.com/python/mypy/issues/12664 -module = "ruamel.*" -cache_dir = "/dev/null" - [tool.pyright] include = [ "tmt/**/*.py", @@ -255,14 +240,13 @@ ignore = [ "docs/**", "examples/**", "tests/**", - "tmt/_compat/pydantic.py", "tmt/export/*.py", "tmt/plugins/*.py", "tmt/steps/*.py", "tmt/steps/discover/*.py", "tmt/steps/execute/*.py", "tmt/steps/finish/*.py", - "tmt/steps/prepare/feature/__init__.py", + "tmt/steps/prepare/feature.py", "tmt/steps/prepare/__init__.py", "tmt/steps/prepare/install.py", "tmt/steps/provision/*.py", @@ -292,22 +276,26 @@ reportPrivateUsage = false reportUnknownMemberType = false reportUnnecessaryCast = false # Unnecessary "cast" call; type is already... +[tool.autopep8] +max_line_length = 99 +in-place = true +recursive = true +hang-closing = true +aggressive = 2 + +[tool.pep8] +max-line-length = "99" + [tool.ruff] line-length = 99 src = ["tmt", "tests"] target-version = "py39" - -[tool.ruff.format] -quote-style = "preserve" # TODO: Convert to a specific quote style - -[tool.ruff.lint] -select = [ +lint.select = [ "F", # pyflakes "E", # pycodestyle error "W", # pycodestyle warning "I", # isort "N", # pep8-naming - "PERF", # Perflint "D", # pydocstyle "UP", # pyupgrade "YTT", # flake8-2020 @@ -342,10 +330,8 @@ select = [ "FURB", # refurb "RUF", # ruff "D", # pydocstyle - # Override docstring convention - "D213", # multi-line-summary-second-line ] -ignore = [ +lint.ignore = [ "B904", # Within an `except` clause, raise exceptions with `raise ... from err` or `raise ... "COM812", # Trailing comma missing "G004", # Logging statement uses f-string @@ -362,9 +348,6 @@ ignore = [ "S607", # Starting a process with a partial executable path "S105", # Possible hardcoded password assigned to: "PASS" "SIM103", # Return the condition directly - can hurt readability - "PERF203", # `try`-`except` within a loop incurs performance overhead - "D200", # One-line docstring should fit on one line - "D212", # Multi-line docstring summary should start at the first line # pydocstyle # TODO: the permanent list (drop this comment once the temporary list @@ -382,14 +365,16 @@ ignore = [ "D107", # Missing docstring in __init__ "D202", # No blank lines allowed after function docstring "D205", # 1 blank line required between summary line and description + "D210", # No whitespaces allowed surrounding docstring text + "D212", # Multi-line docstring summary should start at the first line "D301", # Use r""" if any backslashes in a docstring "D400", # First line should end with a period "D401", # First line of docstring should be in imperative mood "D415", # First line should end with a period, question mark, or exclamation point ] -logger-objects = ["tmt.log.Logger"] -typing-modules = ["tmt._compat.typing"] +lint.logger-objects = ["tmt.log.Logger"] +lint.typing-modules = ["tmt._compat.typing"] [tool.ruff.lint.per-file-ignores] # Less strict security checks in tests @@ -400,13 +385,6 @@ typing-modules = ["tmt._compat.typing"] "S108", # Probable insecure usage of temporary file or directory: "{}" "FLY002", # Use f-string instead of .join ] -# The naming scheme of tmt/_compat will overshadow stdlib modules -"tmt/_compat/pathlib.py" = ["A005"] -"tmt/_compat/typing.py" = ["A005"] -"tmt/_compat/warnings.py" = ["A005"] -# Following files also overshadow stdlib modules -"tmt/queue.py" = ["A005"] -"tmt/steps/report/html.py" = ["A005"] # The purpose of tmt/_compat is to be used with TID251 (banned imports) "tmt/_compat/**.py" = ["TID251"] "docs/conf.py" = ["TID251"] @@ -433,17 +411,10 @@ builtins-ignorelist = ["help", "format", "input", "filter", "copyright", "max"] "typing_extensions.Self".msg = "Use tmt._compat.typing.Self instead." "pathlib.Path".msg = "Use tmt._compat.pathlib.Path instead." "pathlib.PosixPath".msg = "Use tmt._compat.pathlib.Path instead." -"pydantic".msg = "Use tmt._compat.pydantic instead." "warnings.deprecated".msg = "Use tmt._compat.warnings.deprecated instead." "os.path".msg = "Use tmt._compat.pathlib.Path and pathlib instead." # Banning builtins is not yet supported: https://github.com/astral-sh/ruff/issues/10079 # "builtins.open".msg = "Use Path.{write_text,append_text,read_text,write_bytes,read_bytes} instead." -"dataclasses.dataclass".msg = "Use tmt.container.container instead." -"dataclasses.field".msg = "Use tmt.container.field instead." -"click.style".msg = "Use tmt.log.style instead." - -[tool.ruff.lint.flake8-type-checking] -runtime-evaluated-base-classes = ["tmt.config.models.BaseConfig"] [tool.ruff.lint.isort] known-first-party = ["tmt"] diff --git a/scripts/list-new-contributors b/scripts/list-new-contributors deleted file mode 100755 index 504e288bd2..0000000000 --- a/scripts/list-new-contributors +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# Script to list new contributors since the last release -# This identifies contributors whose first ever commit to the repository -# occurred after the last release tag. - -set -e - -# Get the last release tag -LAST_TAG_REF=$(git describe --tags --abbrev=0 HEAD 2>/dev/null) - -if [ -z "$LAST_TAG_REF" ]; then - # No tags found, list all contributors - echo "No release tags found. Listing all contributors:" - git log HEAD --format='%aN' --no-merges | sort -u -else - # Compare contributors before and after the last tag - echo "New contributors since release $LAST_TAG_REF:" - comm -13 \ - <(git log --no-merges --format='%aN' "$LAST_TAG_REF" | sort -u) \ - <(git log --no-merges --format='%aN' HEAD | sort -u) -fi diff --git a/spec/context/dimension.fmf b/spec/context/dimension.fmf index f0a929679c..8f82d6392a 100644 --- a/spec/context/dimension.fmf +++ b/spec/context/dimension.fmf @@ -14,11 +14,6 @@ description: | runs (fedora, fedora-33, centos, centos-8, centos-8.4, centos-stream, centos-stream-9, rhel, rhel-8, rhel-8.4). - deployment-mode - Deployment mode of the distribution, ``package`` for the - standard way using packages or ``image`` for the new way - of distributing using container images. - variant The distribution variant (Client, Desktop, Server Workstation, Silverblue, CoreOS). diff --git a/spec/core/author.fmf b/spec/core/author.fmf deleted file mode 100644 index 740907289e..0000000000 --- a/spec/core/author.fmf +++ /dev/null @@ -1,32 +0,0 @@ -summary: Author contact - -story: - As a user I would like to contact the original author of the - test, plan or story without having to dig through the git - history or check various random files. - -description: | - This dedicated field provides a standard location for - the original author's information, as it can be scattered - across various places and may be difficult to locate. - Additionally, when a test is moved to a different Git - repository, the original author's information can be lost. - - .. versionadded:: 1.42 - -example: - - | - # Single author - author: Name Surname - - - | - # Multiple authors - author: - - First Person - - Second Person - -link: - - implemented-by: /tmt/base.py - - verified-by: /tests/plan/show - - verified-by: /tests/story/show - - verified-by: /tests/test/show diff --git a/spec/core/extra.fmf b/spec/core/extra.fmf deleted file mode 100644 index 9beddc6af5..0000000000 --- a/spec/core/extra.fmf +++ /dev/null @@ -1,39 +0,0 @@ -summary: Additional user-specified metadata - -story: - As a developer or tester I want to specify additional test or - plan properties for use by external systems or tmt plugins. - -description: | - When a test or a plan contains metadata keys with names starting - with ``extra-``, the keys are automatically ignored by ``tmt lint``. - This allows the user to define ie. ``extra-myproject`` as a top - level YAML key and specify any arbitrary YAML-compliant metadata - under that key. - - tmt does not otherwise interact with these keys except for a short - list of names used by ``tmt tests convert`` or ``tmt tests export``: - - .. code-block:: - - extra-nitrate - extra-hardware - extra-pepa - extra-summary - extra-task - - .. versionadded:: 1.50 - -example: | - # Additional classification of a test, parsed by - # user scripts built on top of fmf - extra-compliance: - verifies-profile: stig - complete: no - stig-ids: - - RHEL-09-214015 - - RHEL-09-215025 - -link: - - implemented-by: /tmt/base.py - - verified-by: /tests/core/extra diff --git a/spec/hardware/cpu.fmf b/spec/hardware/cpu.fmf index 67b4ed9c59..eea3b5d58a 100644 --- a/spec/hardware/cpu.fmf +++ b/spec/hardware/cpu.fmf @@ -65,12 +65,9 @@ description: | See e.g. https://virtual-dba.com/blog/sockets-cores-and-threads/ for the socket, core and thread distinction. - See e.g. https://www.cpu-world.com/ for information on family, model, - stepping and corresponding names. ``/proc/cpuinfo`` and ``lscpu`` are - also useful resources. - - .. versionchanged:: 1.39 - ``beaker`` plugins supports ``family`` and ``frequency`` + See e.g. https://en.wikichip.org/wiki/WikiChip for information on family, + model, stepping and corresponding names. ``/proc/cpuinfo`` and ``lscpu`` + are also useful resources. .. versionchanged:: 1.38 ``beaker`` plugins supports ``stepping`` @@ -125,8 +122,8 @@ example: link: - implemented-by: /tmt/steps/provision/artemis.py - note: "``cpu.vendor``, ``cpu.vendor-name``, ``cpu.frequency`` and ``cpu.hyper-threading`` not implemented yet" + note: "``cpu.vendor``, ``cpu.vendor-name`` and ``cpu.hyper-threading`` not implemented yet" - implemented-by: /tmt/steps/provision/mrack.py - note: "``cpu.sockets``, ``cpu.threads``, ``cpu.cores-per-socket``, ``cpu.threads-per-core``, ``cpu.family-name``, ``cpu.vendor`` not implemented yet" + note: "``cpu.flag``, ``cpu.processors``, ``cpu.model``, ``cpu.cores``, ``cpu.model-name``, ``cpu.hyper-threading``, ``cpu.stepping`` and ``cpu.vendor-name`` only" - implemented-by: /tmt/steps/provision/testcloud.py note: "``cpu.processors`` only" diff --git a/spec/hardware/main.fmf b/spec/hardware/main.fmf index 894aced7cc..e9184d8634 100644 --- a/spec/hardware/main.fmf +++ b/spec/hardware/main.fmf @@ -123,28 +123,6 @@ description: | - cpu: model: 69 - .. note:: - - ``and`` and ``or`` operators cannot be combined with HW - requirements on the same level: - - .. code-block:: yaml - - # Incorrect: - hardware: - memory: 1 GB - or: - - hostname: foo.redhat.com - - hostname: bar.redhat.com - - # Use this instead: - hardware: - and: - - memory: 1 GB - - or: - - hostname: foo.redhat.com - - hostname: bar.redhat.com - Units ----- @@ -218,7 +196,7 @@ description: | .. note:: Some plugins may support requirements that are impossible - to satisfy, e.g. the :ref:`/plugins/provision/local` + to satisfy, e.g. the :ref:`/spec/plans/provision/local` plugin can support the ``cpu.family`` requirement, but it is hard-locked to the CPU of one's own machine. diff --git a/spec/hardware/memory.fmf b/spec/hardware/memory.fmf index 3c8197f100..99115e64d5 100644 --- a/spec/hardware/memory.fmf +++ b/spec/hardware/memory.fmf @@ -5,7 +5,7 @@ description: | .. code-block:: # Number or string, the amount of memory requested. - # MiB is assumed when no unit is specified. + # Bytes are assumed when no unit is specified. memory: 1234|"2 GiB" example: diff --git a/spec/hardware/system.fmf b/spec/hardware/system.fmf index c5e9352084..1300f789f1 100644 --- a/spec/hardware/system.fmf +++ b/spec/hardware/system.fmf @@ -20,27 +20,6 @@ description: | # Integer or string, required number of NUMA nodes. numa-nodes: 2|">= 2" - # System management controller - management-controller: - - # String, name of the management interface protocol - protocol: IPMI - - # Number or string, an ID of the management interface vendor. - vendor: "0x04b3" - - # String, a name of the management interface vendor. - vendor-name: "~ MontaVista Software, Inc." - - # Number or string, an ID of the management interface device (product). - device: "0x04b3" - - # String, a name of the management interface device (product). - device-name: "~ Some device name" - - .. versionchanged:: 1.46 - added ``management-controller`` specification - .. versionchanged:: 1.39 ``beaker`` plugin supports ``vendor-name`` @@ -64,12 +43,6 @@ example: vendor-name: "~ HPE" numa-nodes: ">= 4" - - | - # Select any system with available Intelligent Platform Management Interface (IPMI). - system: - management-controller: - protocol: IPMI - link: - implemented-by: /tmt/steps/provision/mrack.py - note: "``system.vendor``, ``system.model`` and ``management-controller`` not implemented yet" + note: "``system.vendor`` and ``system.model`` not implemented yet" diff --git a/spec/plans/discover.fmf b/spec/plans/discover.fmf index 19d8477120..d96cf4d75c 100644 --- a/spec/plans/discover.fmf +++ b/spec/plans/discover.fmf @@ -39,6 +39,212 @@ description: | description: Long test description. ... +/shell: + summary: Provide a manual list of shell test cases + description: | + List of test cases to be executed can be defined manually + directly in the plan as a list of dictionaries containing + test ``name`` and actual ``test`` script. Optionally it is + possible to define any other :ref:`/spec/tests` attributes + such as ``path`` or ``duration`` here as well. The default + :ref:`/spec/tests/duration` for tests defined directly in + the discover step is ``1h``. + + It is possible to fetch code from a remote git repository + using ``url``. In that case repository is cloned and all + paths are relative to the remote git root. Using remote + repo and local test code at the same time is not possible + within the same discover config, use + :ref:`multiple-configs` instead. + + url + Git repository, used directly as a git clone argument. + + ref + Branch, tag or commit specifying the desired git + revision. Defaults to the remote repository's default + branch. + + keep-git-metadata + By default the ``.git`` directory is removed to save + disk space. Set to ``true`` to sync the git metadata + to guest as well. Implicit if ``dist-git-source`` is + used. + + Use the :ref:`/spec/plans/discover/dist-git-source` + options to download rpm sources for dist-git repositories. + + example: + - | + # Define several local tests + discover: + how: shell + tests: + - name: /help/main + test: tmt --help + - name: /help/test + test: tmt test --help + - name: /help/smoke + test: ./smoke.sh + path: /tests/shell + duration: 1m + - | + # Fetch tests from a remote repository + discover: + how: shell + url: https://github.com/teemtee/tmt + tests: + - name: Use tests/full/test.sh from the remote repo + path: /tests/full + test: ./test.sh + + link: + - implemented-by: /tmt/steps/discover/shell.py + +/fmf: + summary: Discover available tests using the fmf format + description: | + Use the `Flexible Metadata Format`_ to explore all + available tests in given repository. The following + parameters are supported: + + url + Git repository containing the metadata tree. + Current git repository used by default. + ref + Branch, tag or commit specifying the desired git + revision. Defaults to the remote repository's default + branch if url given or to the current ``HEAD`` if url + not provided. + + Additionally, one can set ``ref`` dynamically. This is + possible using a special file in tmt format stored in + the *default* branch of a tests repository. This special + file should contain rules assigning attribute ``ref`` + in an `adjust` block, for example depending on a test + run context. + + Dynamic ``ref`` assignment is enabled whenever a test + plan reference has the format ``ref: @FILEPATH``. + path + Path to the metadata tree root. Must be relative to + the git repository root if url provided, absolute + local filesystem path otherwise. By default ``.`` is + used. + + See also the `fmf identifier`_ documentation for details. + Use the following keys to limit the test discovery by test + name, an advanced filter or link: + + test + List of test names or regular expressions used to + select tests by name. Duplicate test names are allowed + to enable repetitive test execution, preserving the + listed test order. + link + Select tests using the :ref:`/spec/core/link` keys. + Values must be in the form of ``RELATION:TARGET``, + tests containing at least one of them are selected. + Regular expressions are supported for both relation + and target. Relation part can be omitted to match all + relations. + filter + Apply advanced filter based on test metadata + attributes. See ``pydoc fmf.filter`` for more info. + exclude + Exclude tests which match a regular expression. + prune + Copy only immediate directories of executed tests and + their required files. + + .. versionadded:: 1.29 + + It is also possible to limit tests only to those that have + changed in git since a given revision. This can be + particularly useful when testing changes to tests + themselves (e.g. in a pull request CI). Related config + options (all optional) are: + + modified-only + Set to ``true`` if you want to filter modified tests + only. The test is modified if its name starts with + the name of any directory modified since modified-ref. + modified-url + An additional remote repository to be used as the + reference for comparison. Will be fetched as a + ``reference`` remote in the test dir. + modified-ref + The ref to compare against. Defaults to the local + repository's default branch. Note that you need to + specify ``reference/`` to compare to a branch + from the repository specified in ``modified-url``. + + Use the :ref:`/spec/plans/discover/dist-git-source` + options to download rpm sources for dist-git repositories. + In addition to the common dist-git-source features, the + ``fmf`` plugin also supports the following options: + + dist-git-init + Set to ``true`` to initialize fmf root inside + extracted sources at ``dist-git-extract`` location or + top directory. To be used when the sources contain fmf + files (for example tests) but do not have an + associated fmf root. + + dist-git-remove-fmf-root + Set to ``true`` to remove any fmf root present in the + sources. ``dist-git-init`` can be used to create it + later at desired location. + + dist-git-merge + Set to ``true`` to combine fmf root from the sources + and fmf root from the plan. It allows to have plans + and tests defined in the DistGit repo which use tests + and other resources from the downloaded sources. Any + plans in extracted sources will not be processed. + + dist-git-extract + Path specifying what should be copied from the + sources. Defaults to top fmf root or top directory + ``/``. + + .. _fmf identifier: https://fmf.readthedocs.io/en/latest/concept.html#identifiers + .. _Flexible Metadata Format: https://fmf.readthedocs.io/ + + example: + - | + # Discover all fmf tests in the current repository + discover: + how: fmf + - | + # Fetch tests from a remote repo, filter by name/tier + discover: + how: fmf + url: https://github.com/teemtee/tmt + ref: main + path: /metadata/tree/path + test: [regexp] + filter: tier:1 + - | + # Choose tests verifying given issue + discover: + how: fmf + link: verifies:issues/123$ + - | + # Select only tests which have been modified + discover: + how: fmf + modified-only: true + modified-url: https://github.com/teemtee/tmt-official + modified-ref: reference/main + - | + # Extract tests from the distgit sources + discover: + how: fmf + dist-git-source: true + link: + - implemented-by: /tmt/steps/discover/fmf.py + /dist-git-source: summary: Download rpm sources for dist-git repositories description: | @@ -56,7 +262,35 @@ description: | order ``60``. All created files and directories by this command are directly in ``TMT_SOURCE_DIR``. - The :ref:`/plugins/discover/fmf` plugin supports + .. versionadded:: 1.32 + + dist-git-source + Download and (optionally) extract the sources. + + dist-git-type + Use the provided DistGit handler instead of the auto + detection. Useful when running from forked + repositories. + + dist-git-download-only + When ``true`` sources are just downloaded. + No ``rpmbuild -bp``, nor installation of require + or buildddeps happens. + + .. versionadded:: 1.29 + + dist-git-require + List of packages to install before ``rpmbuild -bp`` is attempted + on the sources. The rpm-build package itself is installed automatically. + + .. versionadded:: 1.32 + + dist-git-install-builddeps + When set to ``true`` build requirements are installed. + + .. versionadded:: 1.32 + + The :ref:`/spec/plans/discover/fmf` plugin supports additional dist-git options, see its documentation for details. @@ -107,7 +341,6 @@ description: | /where: summary: Execute tests on selected guests - description: | In the :ref:`/spec/plans/provision/multihost` scenarios it is often necessary to execute test code on selected guests @@ -115,27 +348,31 @@ description: | The ``where`` key allows to select guests where the tests should be executed by providing their ``name`` or the ``role`` they play in the scenario. Use a list to specify - multiple names or roles. By default, when the ``where`` - key is not defined, tests are executed on all provisioned + multiple names or roles. By default, when the ``where`` key + is not defined, tests are executed on all provisioned guests. + There is also an alternative to the syntax using a ``where`` + dictionary encapsulating the ``discover`` config under keys + corresponding to guest names or roles. This can result in + much more concise config especially when defining several + shell scripts for each guest or role. + example: - | - # Run a different script for each guest or role + # Run different script for each guest or role discover: - - how: shell - where: client + how: shell tests: - name: run-the-client-code test: client.py - - how: shell - where: server - tests: + where: client - name: run-the-server-code test: server.py + where: server - | - # Filter different sets of tests for each guest or role + # Filter different set of tests for each guest or role discover: - how: fmf filter: tag:client-tests @@ -144,36 +381,37 @@ description: | filter: tag:server-tests where: server + - | + # Alternative syntax using the 'where' dictionary + # encapsulating for tests defined by fmf + discover: + where: + client: + - how: fmf + filter: tag:client-tests + server: + - how: fmf + filter: tag:server-tests + + - | + # Alternative syntax using the 'where' dictionary + # encapsulating for shell script tests + discover: + where: + server: + how: shell + tests: + - test: first server script + - test: second server script + - test: third server script + client: + how: shell + tests: + - test: first client script + - test: second client script + - test: third client script link: - implemented-by: /tmt/steps - verified-by: /tests/multihost/complete - verified-by: /tests/multihost/web - documented-by: /docs/guide.rst#multihost-testing - -/when: - summary: Conditional step configuration - description: | - Using the ``when`` key makes it easier to restrict a step configuration - to run only if any of the specified rules matches. - The syntax is the same as in ``adjust`` and :ref:`/spec/context`. - - Values can be single string with the rule or list of rules. - example: | - discover: - - name: Private tests, applicable only for RHEL - when: distro == rhel - how: fmf - url: https://secret - - name: Public tests - how: fmf - url: https://public - - name: Just a demo of more rules in the 'when' key - how: shell - script: ./something.sh - when: - - initiator == konflux && distro == fedora - - initiator == human && distro == fedora - link: - - implemented-by: /tmt/steps - - verified-by: /tests/steps/when - - documented-by: /docs/guide.rst diff --git a/spec/plans/execute.fmf b/spec/plans/execute.fmf index 9c6630bf75..fc3d2d3b43 100644 --- a/spec/plans/execute.fmf +++ b/spec/plans/execute.fmf @@ -20,7 +20,114 @@ description: | In each plan, the execute step must produce a ``results.yaml`` file with results for executed tests. The format of the file is described - at :ref:`/spec/results`. + at :ref:`/spec/plans/results`. + +/upgrade: + summary: Perform system upgrades during testing + story: + As a tester I want to verify that a configured application + or service still correctly works after the system upgrade. + description: | + In order to enable developing tests for upgrade testing, we + need to provide a way how to execute these tests easily. + This does not cover unit tests for individual actors but + rather system tests which verify the whole upgrade story. + + The ``upgrade`` executor runs the discovered tests (using + the internal executor, hence the same config options can + be used), then performs a set of upgrade tasks from + a remote repository, and finally, re-runs the tests on + the upgraded system. + + The ``IN_PLACE_UPGRADE`` environment variable is set during + the test execution to differentiate between the stages of + the test. It is set to ``old`` during the first execution + and ``new`` during the second execution. Test names are + prefixed with this value to make the names unique. + Based on this variable, the test can perform appropriate + actions. + + old + setup, test + new + test, cleanup + without + setup, test, cleanup + + The upgrade tasks performing the actual system upgrade are + taken from a remote repository (specified by the ``url`` key) + based on an upgrade path (specified by the ``upgrade-path`` key) + or other filters (e.g. specified by the ``filter`` key). + If both ``upgrade-path`` and extra filters are specified, + the discover keys in the remote upgrade path plan are overridden + by the filters specified in the local plan. + + The upgrade path must correspond to a plan name in the remote + repository whose discover selects tests (upgrade tasks). + The environment variables defined in the upgrade path are passed + to the upgrade tasks. If the ``url`` is not provided, + upgrade path and upgrade tasks are taken from the current + repository. + example: + - | + # Main testing plan + discover: + how: fmf + execute: + how: upgrade + url: https://github.com/teemtee/upgrade + upgrade-path: /paths/fedora35to36 + + - | + # Upgrade path /paths/fedora35to36.fmf in the remote repository + discover: # Selects appropriate upgrade tasks (L1 tests) + how: fmf + filter: "tag:fedora" + environment: # This is passed to upgrade tasks + SOURCE: 35 + TARGET: 36 + execute: + how: tmt + + - | + # Alternative main testing plan, without upgrade path + execute: + how: upgrade + url: https://github.com/teemtee/upgrade + filter: "tag:fedora" + + - | + # A simple beakerlib test using the $IN_PLACE_UPGRADE variable + . /usr/share/beakerlib/beakerlib.sh || exit 1 + + VENV_PATH=/var/tmp/venv_test + + rlJournalStart + # Perform the setup only for the old distro + if [[ "$IN_PLACE_UPGRADE" != "new" ]]; then + rlPhaseStartSetup + rlRun "python3.9 -m venv $VENV_PATH" + rlRun "$VENV_PATH/bin/pip install pyjokes" + rlPhaseEnd + fi + + # Execute the test for both old & new distro + rlPhaseStartTest + rlAsssertExists "$VENV_PATH/bin/pyjoke" + rlRun "$VENV_PATH/bin/pyjoke" + rlPhaseEnd + + # Skip the cleanup phase when on the old distro + if [[ "$IN_PLACE_UPGRADE" != "old" ]]; then + rlPhaseStartCleanup + rlRun "rm -rf $VENV_PATH" + rlPhaseEnd + fi + rlJournalEnd + + link: + - implemented-by: /tmt/steps/execute/upgrade.py + - verified-by: /tests/execute/upgrade /isolate: @@ -34,14 +141,14 @@ description: | isolate: true /exit-first: - summary: Stop execution after the first test failure or error + summary: Stop execution after a test fails story: As a user I want to avoid waiting for all discovered tests to finish if one of them fails. description: Optional boolean attribute `exit-first` can be used to make the executor stop executing tests once a test - failure or error is encountered. + failure is encountered. example: | execute: how: tmt @@ -50,9 +157,86 @@ description: | - implemented-by: /tmt/steps/execute/internal.py - verified-by: /tests/execute/exit-first +/tmt: + summary: Internal test executor + story: As a user I want to execute tests directly from tmt. + description: | + The internal ``tmt`` executor runs tests in the + provisioned environment one by one directly from the + tmt code which allows features such as showing live + :ref:`/stories/cli/steps/execute/progress` or the + :ref:`/stories/cli/steps/execute/interactive` session . + This is the default execute step implementation. + + The executor provides following shell scripts which can + be used by the tests for certain operations. + + ``tmt-file-submit`` + Archive the given file in the tmt test data directory. + See the :ref:`/stories/features/report-log` section + for more details. + + ``tmt-reboot`` + Soft reboot the machine from inside the test. After reboot + the execution starts from the test which rebooted the machine. + An environment variable ``TMT_REBOOT_COUNT`` is provided which + the test can use to handle the reboot. The variable holds the + number of reboots performed by the test. For more information + see the :ref:`/stories/features/reboot` feature documentation. + + ``tmt-report-result`` + Generate a result report file from inside the test. Can be + called multiple times by the test. The generated report + file will be overwritten if a higher hierarchical result is + reported by the test. The hierarchy is as follows: + SKIP, PASS, WARN, FAIL. + For more information see the + :ref:`/stories/features/report-result` feature documentation. + + ``tmt-abort`` + Generate an abort file from inside the test. This will + set the current test result to failed and terminate + the execution of subsequent tests. + For more information see the + :ref:`/stories/features/abort` feature documentation. + + The scripts are hosted by default in the ``/usr/local/bin`` + directory, except for guests using ``rpm-ostree``, where + ``/var/lib/tmt/scripts`` is used. The directory can be forced + using the ``TMT_SCRIPTS_DIR`` environment variable. + Note that for guests using ``rpm-ostree``, the directory is added + to executable paths using the system-wide ``/etc/profile.d/tmt.sh`` + profile script. + + .. warning:: + + Please be aware that for guests using ``rpm-ostree`` + the provided scripts will only be available in a shell that + loads the profile scripts. This is the default for + ``bash``-like shells, but might not work for others. + + example: | + execute: + how: tmt + link: + - implemented-by: /tmt/steps/execute/internal.py + - verified-by: /tests/execute/framework + /script: summary: Execute shell scripts story: As a user I want to easily run shell script as a test. + description: | + Execute arbitrary shell commands and check their exit + code which is used as a test result. The ``script`` field + is provided to cover simple test use cases only and must + not be combined with the :ref:`/spec/plans/discover` step + which is more suitable for more complex test scenarios. + + Default shell options are applied to the script, see + :ref:`/spec/tests/test` for more details. The default + :ref:`/spec/tests/duration` for tests defined directly + under the execute step is ``1h``. Use the ``duration`` + attribute to modify the default limit. example: - | diff --git a/spec/plans/finish.fmf b/spec/plans/finish.fmf index 7a5be9ceed..866eb0e026 100644 --- a/spec/plans/finish.fmf +++ b/spec/plans/finish.fmf @@ -2,19 +2,57 @@ summary: Finishing tasks description: Additional actions to be performed after the test execution - has been completed. Counterpart of the - :ref:`/spec/plans/prepare` step, useful for various cleanup - actions. For more information about available options see the - :ref:`/plugins/finish` section. + has been completed. Counterpart of the ``prepare`` step + useful for various cleanup actions. Use the + :ref:`/spec/core/order` attribute to select in which order + finishing tasks should happen if there are multiple configs. + Default order is ``50``. example: | finish: - - how: shell + how: shell script: upload-logs.sh - - how: ansible - playbook: playbooks/cleanup.yaml -link: - - implemented-by: /tmt/steps/finish + +/shell: + summary: + Perform finishing tasks using shell (bash) scripts + description: + Execute arbitrary shell commands to finish the testing. + + Default shell options are applied to the script, see the + :ref:`/spec/tests/test` key specification for more + details. + + example: | + finish: + how: shell + script: + - upload-logs.sh || true + - rm -rf /tmp/temporary-files + link: + - implemented-by: /tmt/steps/finish/shell.py + +/ansible: + summary: + Perform finishing tasks using ansible + description: | + One or more playbooks can be provided as a list under the + ``playbook`` attribute. Each of them will be applied + using ``ansible-playbook`` in the given order. The path + must be relative to the metadata tree root. + + Remote playbooks can be referenced as well as the local ones, + and both kinds can be used at the same time. + + example: | + finish: + how: ansible + playbook: + - playbooks/common.yml + - playbooks/os/rhel7.yml + - https://foo.bar/rhel7-final-touches.yml + link: + - implemented-by: /tmt/steps/finish/ansible.py /where: summary: Perform finishing tasks on selected guests @@ -36,31 +74,3 @@ link: link: - implemented-by: /tmt/steps - documented-by: /docs/guide.rst - -/when: - summary: Conditional step configuration - description: | - Using the ``when`` key makes it easier to restrict a step configuration - to run only if any of the specified rules matches. - The syntax is the same as in ``adjust`` and :ref:`/spec/context`. - - Values can be single string with the rule or list of rules. - example: | - finish: - - name: finish config to run only on Fedora - when: distro == fedora - how: shell - script: ./fedora_specific. - - name: Runs always - how: shell - script: ./setup.sh - - name: Just a demo of more rules in the 'when' key - how: shell - script: ./something.sh - when: - - initiator == konflux && distro == fedora - - initiator == human && distro == fedora - link: - - implemented-by: /tmt/steps - - verified-by: /tests/steps/when - - documented-by: /docs/guide.rst diff --git a/spec/plans/import.fmf b/spec/plans/import.fmf index 3a42b39c49..a2e8ff28b0 100644 --- a/spec/plans/import.fmf +++ b/spec/plans/import.fmf @@ -17,70 +17,10 @@ description: | Remote plans are identified by the ``plan`` key which must contain an ``import`` dictionary with an `fmf identifier`__ of - the remote plan: - - .. code-block:: yaml - - /some-plan: - plan: - import: - url: ... - name: ... - ref: ... - path: ... - - The ``url`` and ``name`` keys are required, ``ref`` and ``path`` - are optional. If ``ref`` is set, the given git ref will be - checked out before looking for plans; ``path`` defines where in - the remote repository the metadata tree lives. - - By default, only a single remote plan is allowed to be imported, - and it always replaces the importing plan. To import multiple plans, - set ``scope`` key to ``all-plans``: - - .. code-block:: yaml - - /some-plan: - plan: - import: - ... - - # This is the default, only the first discovered plan is imported, - # the rest is ignored. - scope: first-plan-only - - # To limit the import to one and one plan only, and fail if - # more plans would match the criteria: - scope: single-plan-only - - # To allow import of multiple plans: - scope: all-plans - - If, instead of replacing the current plan, you want to make the imported - plans "children" of the current plan with the ``importing`` - key: - - .. code-block:: yaml - - /some-plan: - plan: - import: - ... - - # This is the default, replace the current plan. - importing: replace - - # All imported plans will get their names to begin with `/some-plan/...`. - importing: become-parent - - The ``name`` key is treated as a - :ref:`regular expression `, and only plans with - matching names will be imported. - - .. note:: - - Regular expression accepted by the ``name`` key is applied in the - "match" mode, i.e. it must match from the start of the string. + the remote plan. The ``url`` and ``name`` keys have to be + defined, ``ref`` and ``path`` are optional. Only one remote + plan can be referenced and a full plan ``name`` must be used + (no string matching is applied). Additionally, one can utilize dynamic ``ref`` assignment when importing a plan in order to avoid hardcoding ``ref`` value in @@ -126,13 +66,6 @@ example: path: /examples/httpd name: /smoke - - | - # Use 'name' as a regular expression to import multiple plans - plan: - import: - url: https://github.com/teemtee/tmt - name: /plans/provision/(connect|local) - link: - relates: https://github.com/teemtee/tmt/issues/975 - verified-by: /tests/plan/import diff --git a/spec/plans/prepare.fmf b/spec/plans/prepare.fmf index 450cffe4a6..cb9e820d6b 100644 --- a/spec/plans/prepare.fmf +++ b/spec/plans/prepare.fmf @@ -5,38 +5,21 @@ description: | environment should be prepared so that the tests can be successfully executed. - The :ref:`/plugins/prepare/install` plugin provides an easy + The :ref:`/spec/plans/prepare/install` plugin provides an easy way to install required or recommended packages from disk and from the official distribution or copr repositories. Use the - :ref:`/plugins/prepare/ansible` plugin for applying custom - playbooks or execute :ref:`/plugins/prepare/shell` scripts + :ref:`/spec/plans/prepare/ansible` plugin for applying custom + playbooks or execute :ref:`/spec/plans/prepare/shell` scripts to perform arbitrary preparation tasks. - See the :ref:`/plugins/prepare` section for the full list of - available prepare plugins and their supported options. - Use the ``order`` attribute to select in which order the preparation should happen if there are multiple configs. - The following are predefined ``order`` values of various - preparations by tmt: - - 30 - Installation of essential plugin and check requirements. - - 50 - The default order of any object. - - 70 - Installation of packages :ref:`required` by tests. - - 75 - Installation of packages :ref:`recommended` by tests. - - .. note:: - - Individual plugins may define their own special ``order`` values, - and you shall find the relevant information in :ref:`plugins` - documentation. + Default order is ``50``. For installation of essential plugin + and check requirements ``30`` is used, for installation of + required packages gathered from the :ref:`/spec/tests/require` + attribute of individual tests order ``70`` is used, for recommended + packages it is ``75``. The Dist-git prepare happens before those, + with order ``60``. .. note:: @@ -61,10 +44,138 @@ example: | how: shell script: systemctl start httpd -link: - - implemented-by: /tmt/steps/prepare - - verified-by: /tests/prepare +/shell: + summary: + Prepare system using shell (bash) commands + description: + Execute arbitrary shell commands to set up the system. + Default shell options are applied to the script, see the + :ref:`/spec/tests/test` key specification for more + details. + + example: | + prepare: + how: shell + script: dnf install -y httpd + link: + - implemented-by: /tmt/steps/provision + +/ansible: + summary: + Apply ansible playbook to get the desired final state. + description: | + One or more playbooks can be provided as a list under the + ``playbook`` attribute. Each of them will be applied + using ``ansible-playbook`` in the given order. The path + must be relative to the metadata tree root. + + Use ``extra-args`` attribute to enable optional arguments for + ``ansible-playbook``. + + Remote playbooks can be referenced as well as the local ones, + and both kinds can be used at the same time. + + .. note:: + + Depending on the specific setup of the test runner, + there might be limitations affecting which playbooks + can be used, for example the list of available ansible + collections. See the :ref:`test-runner` section for + details. + + example: | + prepare: + how: ansible + playbook: + - playbooks/common.yml + - playbooks/os/rhel7.yml + - https://foo.bar/rhel7-final-touches.yml + extra-args: '-vvv' + link: + - implemented-by: /tmt/steps/provision + - verified-by: /tests/prepare/ansible + +/feature: + summary: + Easily enable and disable common features + description: | + The ``feature`` plugin provides a comfortable way to + enable and disable some commonly used functionality. As + for now enabling and disabling the ``epel`` repository is + supported, ``crb`` and ``fips`` are coming in the near + future. + + .. versionadded:: 1.31 + example: | + prepare: + how: feature + epel: enabled + link: + - implemented-by: /tmt/steps/prepare/feature + - verified-by: /tests/prepare/feature + +/install: + summary: + Install packages on the guest + description: | + One or more RPM packages can be specified under the + ``package`` attribute. The packages will be installed + on the guest. They can either be specified using their + names, paths to local rpm files or urls to remote rpms. + + Additionally, the ``directory`` attribute can be used to + install all packages from the given directory. Copr + repositories can be enabled using the ``copr`` attribute. + Use the ``exclude`` option to skip selected packages from + installation (globbing characters are supported as well). + + It's possible to change the behaviour when a package is + missing using the ``missing`` attribute. The missing + packages can either be silently ignored ('skip') or a + preparation error is thrown ('fail'), which is the default + behaviour. + + example: + - | + # Install local rpms using file path + prepare: + how: install + package: + - tmp/RPMS/noarch/tmt-0.15-1.fc31.noarch.rpm + - tmp/RPMS/noarch/python3-tmt-0.15-1.fc31.noarch.rpm + + - | + # Install remote packages using url + prepare: + how: install + package: + - https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm + - https://dl.fedoraproject.org/pub/epel/epel-next-release-latest-8.noarch.rpm + + - | + # Install the whole directory, exclude selected packages + prepare: + how: install + directory: + - tmp/RPMS/noarch + exclude: + - tmt+all + - tmt+provision-virtual + + - | + # Enable copr repository, skip missing packages + prepare: + how: install + # Repository with a group owner (@ prefixed) requires quotes, e.g. + # copr: "@osci/rpminspect" + copr: psss/tmt + package: tmt-all + missing: skip + + link: + - implemented-by: /tmt/steps/provision + - verified-by: /tests/prepare/install /where: summary: Apply preparation on selected guests @@ -95,31 +206,3 @@ link: - verified-by: /tests/multihost/web - verified-by: /tests/multihost/corner-cases - documented-by: /docs/guide.rst - -/when: - summary: Conditional step configuration - description: | - Using the ``when`` key makes it easier to restrict a step configuration - to run only if any of the specified rules matches. - The syntax is the same as in ``adjust`` and :ref:`/spec/context`. - - Values can be single string with the rule or list of rules. - example: | - prepare: - - name: Prepare config to run only on Fedora - when: distro == fedora - how: shell - script: ./fedora_specific. - - name: Runs always - how: shell - script: ./setup.sh - - name: Just a demo of more rules in the 'when' key - how: shell - script: ./something.sh - when: - - initiator == konflux && distro == fedora - - initiator == human && distro == fedora - link: - - implemented-by: /tmt/steps - - verified-by: /tests/steps/when - - documented-by: /docs/guide.rst diff --git a/spec/plans/provision.fmf b/spec/plans/provision.fmf index ef49efd7e4..00c0d0bc60 100644 --- a/spec/plans/provision.fmf +++ b/spec/plans/provision.fmf @@ -1,7 +1,16 @@ summary: Provision a system for testing description: | - See the :ref:`/plugins/provision` documentation for details about + Describes what environment is needed for testing and how it + should be provisioned. There are several provision plugins + supporting multiple ways to provision the environment for + testing, for example + :ref:`/spec/plans/provision/virtual`, + :ref:`/spec/plans/provision/container`, + :ref:`/spec/plans/provision/connect`, + :ref:`/spec/plans/provision/local` or + :ref:`/spec/plans/provision/artemis`. + See individual plugin documentation for details about supported options. As part of the provision step it is also possible to specify @@ -20,8 +29,172 @@ example: | how: virtual image: fedora -link: - - implemented-by: /tmt/steps/provision +/virtual: + summary: Provision a virtual machine (default) + description: + Create a new virtual machine on the localhost using + testcloud (libvirt). Testcloud takes care of downloading + an image and making necessary changes to it for optimal + experience (such as disabling UseDNS and GSSAPI for SSH). + + example: + - | + # Provision a fedora virtual machine + provision: + how: virtual + image: fedora + - | + # Provision a virtual machine from a specific QCOW2 file, + # using specific memory and disk settings, using the fedora user, + # and using sudo to run scripts. + provision: + how: virtual + image: https://download.fedoraproject.org/pub/fedora/linux/releases/39/Cloud/x86_64/images/Fedora-Cloud-Base-39-1.5.x86_64.qcow2 + user: fedora + become: true + # in MB + memory: 2048 + # in GB + disk: 30 + + link: + - implemented-by: /tmt/steps/provision/testcloud.py + +/local: + summary: Use the localhost for testing + description: + Do not provision any system. Tests will be executed + directly on the localhost. Note that for some actions like + installing additional packages you need root permission or + enabled sudo. + example: | + provision: + how: local + link: + - implemented-by: /tmt/steps/provision/local.py + - verified-by: /tests/init/base + +/container: + summary: Provision a container + description: + Download (if necessary) and start a new container using + podman or docker. + + example: + - | + # Use the fedora:latest image + provision: + how: container + image: fedora:latest + - | + # Use an image with a non-root user with sudo privileges, + # and run scripts with sudo. + provision: + how: container + image: image with non-root user with sudo privileges + user: tester + become: true + link: + - implemented-by: /tmt/steps/provision/podman.py + +/openstack: + summary: Provision a virtual machine in OpenStack + description: + Create a virtual machine using OpenStack. + example: | + provision: + how: openstack + image: f31 + +/beaker: + summary: Provision a machine in Beaker + description: | + Reserve a machine from the Beaker pool using the ``mrack`` + plugin. `mrack`__ is a multicloud provisioning library + supporting multiple cloud services including Beaker. + + The following two files are used for configuration: + + /etc/tmt/mrack.conf + for basic configuration + + /etc/tmt/provisioning-config.yaml + configuration per supported provider + + Beaker installs distribution specified by the ``image`` + key. If the image can not be translated using the + ``provisioning-config.yaml`` file mrack passes the image + value to Beaker hub and tries to request distribution + based on the image value. This way we can bypass default + translations and use desired distribution specified like + the one in the example below. + + __ https://github.com/neoave/mrack + + .. versionadded:: 1.22 + + example: + - | + # Use image name translation + provision: + how: beaker + image: fedora + - | + # Specify the distro directly + provision: + how: beaker + image: Fedora-37% + - | + # Set custom whiteboard description (added in 1.30) + provision: + how: beaker + whiteboard: Just a smoke test for now + + link: + - implemented-by: /tmt/steps/provision/mrack.py + - verified-by: /plans/provision/beaker + + +/connect: + summary: Connect to a provisioned box + description: | + Do not provision a new system. Instead, use provided + authentication data to connect to a running machine. + + guest + hostname or ip address of the box + user + user name to be used to log in, ``root`` by default + become + whether to run scripts with ``sudo``, ignored if user + is already root, ``false`` by default + password + user password to be used to log in + key + path to the file with private key + port + use specific port to connect to + link: + - implemented-by: /tmt/steps/provision/connect.py + + example: + - | + # Connect to existing box with username and password + provision: + how: connect + guest: hostname or ip address + user: username + password: password + + - | + # Connect with user ``fedora`` using a key, and use + # ``sudo`` to run scripts + provision: + how: connect + guest: hostname or ip address + user: fedora + become: true + key: private-key-path /multihost: summary: Multihost testing specification diff --git a/spec/plans/provision/artemis.fmf b/spec/plans/provision/artemis.fmf new file mode 100644 index 0000000000..3d8a3c6231 --- /dev/null +++ b/spec/plans/provision/artemis.fmf @@ -0,0 +1,62 @@ +summary: Provision a guest via an Artemis service +description: | + Reserve a machine using the Artemis service. + Users can specify many requirements, mostly regarding the + desired OS, RAM, disk size and more. Most of the HW specifications + defined in the :ref:`/spec/hardware` are supported. Including the + :ref:`/spec/plans/provision/kickstart`. + + Artemis takes machines from AWS, OpenStack, Beaker or Azure. + By default, Artemis handles the selection of a cloud provider + to its best abilities and the required specification. However, it + is possible to specify the keyword ``pool`` and select the + desired cloud provider. + + Artemis project: + https://gitlab.com/testing-farm/artemis + + .. note:: + + When used together with TF infrastructure + some of the options from the first example below + will be filled for you by the TF service. + +example: + - | + provision: + how: artemis + # Specify the Artemis URL where the service is running. + # Here is an example of a local Artemis instance + api-url: "http://127.0.0.1:8001/v0.0.56" + api-version: "0.0.56" + image: Fedora-37 + # ssh key used to connect to the machine + keyname: master-key + + - | + provision: + how: artemis + # How long (seconds) to wait for guest provisioning to complete + provision-timeout: 300 + # How often (seconds) check Artemis API for provisioning status + provision-tick: 40 + # How long (seconds) to wait for API operations to complete + api-timeout: 15 + # How many attempts to use when talking to API + api-retries: 5 + # How long (seconds) before the guest "is-alive" watchdog is dispatched + watchdog-dispatch-delay: 200 + # How often (seconds) check that the guest "is-alive" + watchdog-period-delay : 500 + - | + provision: + how: artemis + arch: x86_64 + pool: beaker + hardware: + # Pick a guest with at least 8 GB RAM + memory: ">= 8 GB" + +link: + - implemented-by: /tmt/steps/provision/artemis.py + - implemented-by: /tmt/schemas/provision/artemis.yaml diff --git a/spec/plans/provision/kickstart.fmf b/spec/plans/provision/kickstart.fmf index d522279f54..cde646e841 100644 --- a/spec/plans/provision/kickstart.fmf +++ b/spec/plans/provision/kickstart.fmf @@ -70,7 +70,8 @@ example: %post systemctl disable firewalld %end - metadata: "no_autopart harness=restraint" + metadata: | + "no-autopart harness=restraint" kernel-options: "ksdevice=eth1" kernel-options-post: "quiet" diff --git a/spec/plans/report.fmf b/spec/plans/report.fmf index 335df4e554..f7be175e08 100644 --- a/spec/plans/report.fmf +++ b/spec/plans/report.fmf @@ -1,30 +1,263 @@ summary: Report test results +story: + As a tester I want to have a nice overview of results once + the testing is finished. description: - Report test results according to user preferences. For more information - about the supported report methods, check the :ref:`/plugins/report` - documentation. + Report test results according to user preferences. example: [] -link: - - implemented-by: /tmt/steps/report -/when: - summary: Conditional step configuration - description: | - Using the ``when`` key makes it easier to restrict a step configuration - to run only if any of the specified rules matches. - The syntax is the same as in ``adjust`` and :ref:`/spec/context`. +/display: + summary: Show results in the terminal window + story: + As a tester I want to see test results in the plain text + form in my shell session. + description: + Test results will be displayed as part of the command line + tool output directly in the terminal. Allows to select the + desired level of verbosity + example: | + tmt run -l report # overall summary only + tmt run -l report -v # individual test results + tmt run -l report -vv # show full paths to logs + tmt run -l report -vvv # provide complete test output + link: + - implemented-by: /tmt/steps/report/display.py - Values can be single string with the rule or list of rules. +/html: + summary: Generate a web page with test results + story: + As a tester I want to review results in a nicely arranged + web page with links to detailed test output. + description: + Create a local ``html`` file with test results arranged in + a table. Optionally open the page in the default browser. example: | + # Enable html report from the command line + tmt run --all report --how html + tmt run --all report --how html --open + tmt run -l report -h html -o + + # Use html as the default report for given plan report: - - name: Open html report - when: - - trigger is not defined - - initiator == human how: html open: true - - how: display link: - - implemented-by: /tmt/steps - - verified-by: /tests/steps/when - - documented-by: /docs/guide.rst + - implemented-by: /tmt/steps/report/html.py + +/junit: + summary: Generate a JUnit report file + story: + As a tester I want to review results in a JUnit xml file. + description: + Create a JUnit file ``junit.xml`` with test results. + example: | + # Enable junit report from the command line + tmt run --all report --how junit + tmt run --all report --how junit --file test.xml + + # Use junit as the default report for given plan + report: + how: junit + file: test.xml + link: + - implemented-by: /tmt/steps/report/junit.py + +/polarion: + summary: Generate an xUnit file and export it into Polarion + story: + As a tester I want to review tests in Polarion + and have all results linked to existing test cases there. + description: | + Write test results into an xUnit file and upload to Polarion. + + In order to get quickly started create a pylero config + file ``~/.pylero`` in your home directory with the + following content: + + .. code-block:: ini + + [webservice] + url=https://{your polarion web URL}/polarion + svn_repo=https://{your polarion web URL}/repo + default_project={your project name} + user={your username} + password={your password} + + See the `Pylero Documentation`__ for more details on how + to configure the ``pylero`` module. + + __ https://github.com/RedHatQE/pylero + + .. note:: + + For Polarion report to export correctly you need to + use password authentication, since exporting the + report happens through Polarion XUnit importer which + does not support using tokens. You can still + authenticate with token to only generate the report + using ``--no-upload`` argument. + + .. note:: + + Your Polarion project might need a custom value format + for the ``arch``, ``planned-in`` and other fields. The + format of these fields might differ across Polarion + projects, for example, ``x8664`` can be used instead + of ``x86_64`` for the architecture. + + example: + - | + # Enable polarion report from the command line + tmt run --all report --how polarion --project-id tmt + tmt run --all report --how polarion --project-id tmt --no-upload --file test.xml + + - | + # Use polarion as the default report for given plan + report: + how: polarion + file: test.xml + project-id: tmt + title: tests_that_pass + planned-in: RHEL-9.1.0 + pool-team: sst_tmt + link: + - implemented-by: /tmt/steps/report/polarion.py + +/reportportal: + summary: Report test results to a ReportPortal instance + story: + As a tester I want to review results in a nicely arranged + web page, filter them via context attributes and get links + to detailed test output and other test information. + description: + Provide test results and fmf data per each plan, + and send it to a Report Portal instance via its API + with token, url and project name given. + + Note that all options can be passed as environment variables + in the format TMT_PLUGIN_REPORT_REPORTPORTAL_${OPTION} + to enable execution purely via metadata and/or + environment variables (e.g. in Testing Farm). + + example: + - | + # Optionally set environment variables according to TMT_PLUGIN_REPORT_REPORTPORTAL_${OPTION} + export TMT_PLUGIN_REPORT_REPORTPORTAL_URL=${url-to-RP-instance} + export TMT_PLUGIN_REPORT_REPORTPORTAL_TOKEN=${token-from-RP-profile} + + # Boolean options are activated with value of 1: + TMT_PLUGIN_REPORT_REPORTPORTAL_SUITE_PER_PLAN=1 + + - | + # Enable ReportPortal report from the command line depending on the use case: + + ## Simple upload with all project, url endpoint and user token passed in command line + tmt run --all report --how reportportal --project=baseosqe --url="https://reportportal.xxx.com" --token="abc...789" + + ## Simple upload with url and token exported in environment variable + tmt run --all report --how reportportal --project=baseosqe + + ## Upload with project name in fmf data, filtering out parameters (environment variables) that tend to be unique and break the history aggregation + tmt run --all report --how reportportal --exclude-variables="^(TMT|PACKIT|TESTING_FARM).*" + + ## Upload all plans as suites into one ReportPortal launch + tmt run --all report --how reportportal --suite-per-plan --launch=Errata --launch-description="..." + + ## Rerun the launch with suite structure for the test results to be uploaded into the latest launch with the same name as a new 'Retry' tab (mapping based on unique paths) + tmt run --all report --how reportportal --suite-per-plan --launch=Errata --launch-rerun + + ## Rerun the tmt run and append the new result logs under the previous one uploaded in ReportPortal (precise mapping) + tmt run --id run-012 --all report --how reportportal --again + + ## Additional upload of new suites into given launch with suite structure + tmt run --all report --how reportportal --suite-per-plan --upload-to-launch=4321 + + ## Additional upload of new tests into given launch with non-suite structure + tmt run --all report --how reportportal --launch-per-plan --upload-to-launch=1234 + + ## Additional upload of new tests into given suite + tmt run --all report --how reportportal --upload-to-suite=123456 + + ## Upload Idle tests, then execute it and add result logs into prepared empty tests + tmt run discover report --how reportportal --defect-type=Idle + tmt run --last --all report --how reportportal --again + - | + # Use ReportPortal as the default report for given plan + report: + how: reportportal + project: baseosqe + + # Report context attributes for given plan + context: + ... + - | + # Report description, contact, id and environment variables for given test + summary: ... + contact: ... + id: ... + environment: + ... + link: + - implemented-by: /tmt/steps/report/reportportal.py + +/file: + description: | + + Save the report into a ``report.yaml`` file. + + The ``OVERALL_RESULT`` is the overall result of all plan + results. It is counted the same way as ``PLAN_RESULT``. + + The ``TEST_RESULT`` is the same as in `execute`_ step + definition: + + * info - test finished and produced only information + message + * passed - test finished and passed + * failed - test finished and failed + * error - a problem encountered during test execution + + Note the priority of test results is as written above, + with ``info`` having the lowest priority and ``error`` has + the highest. This is important for ``PLAN_RESULT``. + + The ``PLAN_RESULT`` is the overall result or all test + results for the plan run. It has the same values as + ``TEST_RESULT``. Plan result is counted according to the + priority of the test outcome values. For example: + + * if the test results are info, passed, passed - the + plan result will be passed + * if the test results are info, passed, failed - the + plan result will be failed + * if the test results are failed, error, passed - the + plan result will be error + + The ``LOG_PATH`` is the test log output path, relative + to the execute step plan run directory. The ``log`` key + will be a list of such paths, even if there is just a single + log. + + example: | + result: OVERALL_RESULT + plans: + /plan/one: + result: PLAN_RESULT + tests: + /test/one: + result: TEST_RESULT + log: + - LOG_PATH + + /test/two: + result: TEST_RESULT + log: + - LOG_PATH + - LOG_PATH + - LOG_PATH + /plan/two: + result: PLAN_RESULT + /test/one: + result: TEST_RESULT + log: + - LOG_PATH diff --git a/spec/results.fmf b/spec/plans/results.fmf similarity index 89% rename from spec/results.fmf rename to spec/plans/results.fmf index f60cf86c65..732ae155c6 100644 --- a/spec/results.fmf +++ b/spec/plans/results.fmf @@ -1,6 +1,6 @@ -story: - To integrate tmt with other tools, tmt should standardize and - document the format in which test results are saved on storage. +summary: Define format of on-disk storage of results +title: Results Format +order: 90 description: | The following text defines a YAML file structure tmt uses for storing @@ -21,21 +21,20 @@ description: | name: /test/one # fmf ID of the test. - fmf-id: + fmf_id: url: http://some.git.host.com/project/tests.git ref: main name: /test/one path: / # String, the effective outcome of the test execution. - result: "pass"|"fail"|"info"|"warn"|"error"|"skip"|"pending" + result: "pass"|"fail"|"info"|"warn"|"error"|"skip" # String, the original outcome of the test execution. - original-result: "pass"|"fail"|"info"|"warn"|"error"|"skip"|"pending" + original-result: "pass"|"fail"|"info"|"warn"|"error"|"skip" - # List of strings, optional comments to report with the result. - note: - - Things were great. + # String, optional comment to report with the result. + note: "Things were great." # List of strings, paths to log files. log: @@ -84,14 +83,13 @@ description: | # relate to each check alone. check: # String, the effective outcome of the check execution. - - result: "pass"|"fail"|"info"|"warn"|"error"|"skip"|"pending" + - result: "pass"|"fail"|"info"|"warn"|"error"|"skip" # String, the original outcome of the check execution. - original-result: "pass"|"fail"|"info"|"warn"|"error"|"skip"|"pending" + original-result: "pass"|"fail"|"info"|"warn"|"error"|"skip" - # List of strings, optional comments to report with the result. - note: - - Things were great. + # String, optional comment to report with the result. + note: "Things were great." # List of strings, paths to logs files. log: @@ -123,14 +121,13 @@ description: | - name: First test case # String, the effective outcome of the phase execution. - result: "pass"|"fail"|"info"|"warn"|"error"|"skip"|"pending" + result: "pass"|"fail"|"info"|"warn"|"error"|"skip" # String, the original outcome of the phase execution. - original-result: "pass"|"fail"|"info"|"warn"|"error"|"skip"|"pending" + original-result: "pass"|"fail"|"info"|"warn"|"error"|"skip" - # List of strings, optional comments to report with the result. - note: - - Things were great. + # String, optional comment to report with the result. + note: "Things were great." # List of strings, paths to log files. log: @@ -153,7 +150,7 @@ description: | check: ... - .. _/spec/results/outcomes: + .. _/spec/plans/results/outcomes: The ``result`` key can have the following values: @@ -187,10 +184,6 @@ description: | a single process produces multiple results but not all tests were run. - pending - Test was discovered and is waiting for execution. - - The ``name`` and ``result`` keys are required. Also, ``name``, ``result``, and ``event`` keys are required for entries under ``check`` key, and ``name`` and ``result`` keys are required for entries under ``subresult`` @@ -210,7 +203,7 @@ description: | The first ``log`` item is considered to be the "main" log, presented to the user by default. - The ``serial-number``, ``guest`` and ``fmf-id`` keys, if present in the + The ``serial-number``, ``guest`` and ``fmf_id`` keys, if present in the custom results file, will be overwritten by tmt during their import after test completes. This happens on purpose, to assure this vital information is correct. @@ -263,9 +256,6 @@ description: | .. versionchanged:: 1.37 original result of test, subtest and check is stored in ``original-result`` key. - .. versionchanged:: 1.41 - ``note`` field changed from a string to a list of strings. - __ https://github.com/teemtee/tmt/blob/main/tmt/schemas/results.yaml example: @@ -279,8 +269,7 @@ example: start-time: "2023-03-10T09:44:14.439120+00:00" end-time: "2023-03-10T09:44:24.242227+00:00" duration: 00:00:09 - note: - - good result + note: good result ids: extra-nitrate: some-nitrate-id guest: @@ -295,8 +284,7 @@ example: start-time: "2023-03-10T09:44:14.439120+00:00" end-time: "2023-03-10T09:44:24.242227+00:00" duration: 00:00:09 - note: - - fail result + note: fail result guest: name: default-0 @@ -307,8 +295,7 @@ example: log: - pass_log duration: 00:11:22 - note: - - good result + note: good result ids: extra-nitrate: some-nitrate-id @@ -318,8 +305,7 @@ example: - fail_log - another_log duration: 00:22:33 - note: - - fail result + note: fail result - | # Example of a perfectly valid, yet stingy custom results file @@ -339,8 +325,7 @@ example: start-time: "2023-03-10T09:44:14.439120+00:00" end-time: "2023-03-10T09:44:24.242227+00:00" duration: 00:00:09 - note: - - good result + note: good result ids: extra-nitrate: some-nitrate-id guest: @@ -350,12 +335,12 @@ example: event: after-test result: pass log: [] - note: [] + note: - name: kernel-panic event: after-test result: pass log: [] - note: [] + note: - | # syntax: json @@ -366,7 +351,7 @@ example: "result": "pass", "log": ["pass_log"], "duration": "00:11:22", - "note": ["good result"] + "note": "good result" } ] diff --git a/spec/policy.fmf b/spec/policy.fmf deleted file mode 100644 index 11294503dc..0000000000 --- a/spec/policy.fmf +++ /dev/null @@ -1,152 +0,0 @@ -story: | - As an owner of a CI system or product CI workflow, I would like to - modify all component plans and tests to include phases and checks - that are deemed mandatory for the given CI workflow. - -description: | - .. note:: - - As of now, the specification focuses and applies to - :ref:`test metadata` only. In the future, we plan - to support plans and stories as well. - - tmt policies offer a powerful and flexible way to dynamically - modify test metadata using templates. Instead of maintaining multiple - similar test versions, you can define a base test and then apply - policy to adjust its properties for different scenarios (e.g., - different environments, feature flags, or CI runs). - - A policy is stored in a YAML file, and consists of one or more - rules, each mapping a test key to a template that provides new - content for the said key. - - .. code-block:: yaml - - test-policy: - - :