diff --git a/.github/generate_charmcraft_extensions.py b/.github/generate_charmcraft_extensions.py new file mode 100755 index 000000000..d3781676d --- /dev/null +++ b/.github/generate_charmcraft_extensions.py @@ -0,0 +1,180 @@ +#!/usr/bin/env -S uv run --script --no-project +# /// script +# requires-python = ">=3.10" +# dependencies = ["pyyaml"] +# /// + +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Generate testing/src/scenario/_charmcraft_extensions.py. + +For each charmcraft extension profile (django-framework, fastapi-framework, etc.), +this script runs `charmcraft init` and `charmcraft expand-extensions` in a temp +directory, then extracts the metadata, config, and actions that the extension adds. + +The output module contains three dictionaries per extension: + METADATA: dict[str, dict] - containers, peers, provides, requires, resources, assumes + CONFIG: dict[str, dict] - config options added by each extension + ACTIONS: dict[str, dict] - actions added by each extension +""" + +from __future__ import annotations + +import pathlib +import subprocess +import sys +import tempfile +from typing import Any + +import yaml + +# Keys from the expanded YAML that belong to "metadata" (the charm's metadata.yaml equivalent). +METADATA_KEYS = { + 'assumes', + 'containers', + 'peers', + 'provides', + 'requires', + 'resources', +} + +# Keys that are user-provided or build-related (not extension-contributed metadata). +IGNORED_KEYS = { + 'name', + 'summary', + 'description', + 'type', + 'bases', + 'base', + 'platforms', + 'extensions', + 'parts', + 'charm-libs', +} + +OUTPUT_FILE = ( + pathlib.Path(__file__).resolve().parent.parent + / 'testing' + / 'src' + / 'scenario' + / '_charmcraft_extensions.py' +) + + +def get_extensions() -> list[str]: + """Get the list of available charmcraft extensions via ``charmcraft list-extensions``.""" + result = subprocess.check_output(['charmcraft', 'list-extensions'], text=True) + extensions = [] + for line in result.splitlines(): + # Skip header and separator lines. + if not line or line.startswith('Extension') or line.startswith('---'): + continue + name = line.split()[0] + extensions.append(name) + return sorted(extensions) + + +def run_charmcraft(profile: str, workdir: pathlib.Path) -> dict[str, Any]: + """Run charmcraft init + expand-extensions and return the expanded YAML as a dict.""" + subprocess.check_call( + ['charmcraft', 'init', '--profile', profile, '--name', 'test-charm'], + cwd=workdir, + ) + output = subprocess.check_output( + ['charmcraft', 'expand-extensions'], + cwd=workdir, + text=True, + ) + return yaml.safe_load(output) + + +def extract_extension_data( + expanded: dict[str, Any], +) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: + """Extract metadata, config options, and actions from expanded charmcraft YAML.""" + metadata = {key: expanded[key] for key in METADATA_KEYS if key in expanded} + config = expanded.get('config', {}).get('options', {}) + actions: dict[str, Any] = expanded.get('actions', {}) + + return metadata, config, actions + + +def generate_module( + all_data: dict[str, tuple[dict[str, Any], dict[str, Any], dict[str, Any]]], +) -> str: + """Generate the Python module source code.""" + lines = [ + '# Copyright 2026 Canonical Ltd.', + '# See LICENSE file for licensing details.', + '"""Charmcraft extension metadata, config, and actions.', + '', + 'Auto-generated by .github/generate_charmcraft_extensions.py', + 'Do not edit manually.', + '"""', + '', + 'from __future__ import annotations', + '', + 'from typing import Any, TypedDict', + '', + '', + 'class _ExtensionMetadata(TypedDict, total=False):', + ' assumes: list[str]', + ' containers: dict[str, Any]', + ' peers: dict[str, Any]', + ' provides: dict[str, Any]', + ' requires: dict[str, Any]', + ' resources: dict[str, Any]', + '', + '', + ] + + # Build the three top-level dicts, pre-sorting keys. + metadata_entries: dict[str, dict[str, Any]] = {} + config_entries: dict[str, dict[str, Any]] = {} + action_entries: dict[str, dict[str, Any]] = {} + for profile, (metadata, config, actions) in sorted(all_data.items()): + metadata_entries[profile] = {k: metadata[k] for k in sorted(metadata)} + config_entries[profile] = {k: config[k] for k in sorted(config)} + action_entries[profile] = {k: actions[k] for k in sorted(actions)} + + for var_name, data, type_str, doc_name in [ + ('METADATA', metadata_entries, 'dict[str, _ExtensionMetadata]', 'Metadata'), + ('CONFIG', config_entries, 'dict[str, dict[str, Any]]', 'Config options'), + ('ACTIONS', action_entries, 'dict[str, dict[str, Any]]', 'Actions'), + ]: + lines.append(f'# {doc_name} added by each charmcraft extension.') + lines.append(f'{var_name}: {type_str} = {data!r}') + lines.append('') + + return '\n'.join(lines) + + +def main() -> int: # noqa: D103 + extensions = get_extensions() + print(f'Found extensions: {", ".join(extensions)}') + + all_data: dict[str, tuple[dict[str, Any], dict[str, Any], dict[str, Any]]] = {} + + for profile in extensions: + print(f'Processing {profile}...') + with tempfile.TemporaryDirectory() as tmpdir: + expanded = run_charmcraft(profile, pathlib.Path(tmpdir)) + all_data[profile] = extract_extension_data(expanded) + + module_source = generate_module(all_data) + OUTPUT_FILE.write_text(module_source) + print(f'Written to {OUTPUT_FILE}') + + print('Running tox -e format...') + subprocess.run( + ['tox', '-e', 'format', '--', str(OUTPUT_FILE)], + cwd=OUTPUT_FILE.parent, + check=True, + ) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/docs/explanation/state-transition-testing.md b/docs/explanation/state-transition-testing.md index f1e6a8d51..9275bca1d 100644 --- a/docs/explanation/state-transition-testing.md +++ b/docs/explanation/state-transition-testing.md @@ -231,6 +231,32 @@ ctx = testing.Context( state = ctx.run(ctx.on.start(), testing.State()) ``` +## Charmcraft extensions + +If your `charmcraft.yaml` uses a charmcraft extension such as `flask-framework` +or `django-framework`, the testing framework will automatically expand it when +automatically loading metadata. The extension's metadata (containers, relations, resources, +and so on), config options, and actions are merged into the charm spec, simulating what +`charmcraft expand-extensions` does at pack time. + +This means you can write tests against a charm that uses extensions without having +to manually specify all the metadata the extension adds: + +```python +# Given a charmcraft.yaml with: +# extensions: +# - flask-framework + +ctx = testing.Context(MyFlaskCharm) +# The 'ingress' relation is provided by the flask-framework extension. +state = ctx.run(ctx.on.start(), testing.State(relations={testing.Relation('ingress')})) +``` + +If your `charmcraft.yaml` defines keys that overlap with what the extension +provides (for example, a config option or relation with the same name), both +`charmcraft pack` and the testing framework will raise an error. Rename or +remove the overlapping keys to fix this. + ## Immutability All of the data structures in the state, (`State`, `Relation`, `Container`, and diff --git a/docs/howto/write-unit-tests-for-a-charm.md b/docs/howto/write-unit-tests-for-a-charm.md index bb26ca6e6..9b0013dda 100644 --- a/docs/howto/write-unit-tests-for-a-charm.md +++ b/docs/howto/write-unit-tests-for-a-charm.md @@ -122,6 +122,18 @@ def test_peer_changed(): > See more: [](ops.testing.State.from_context) +```{note} +If your `charmcraft.yaml` uses a charmcraft extension (e.g. +`extensions: [flask-framework]`), the metadata, config, and actions that +the extension adds are automatically merged in when the testing framework +loads the charm spec. You do not need to manually specify them. + +If your `charmcraft.yaml` defines keys that overlap with what the extension +provides (e.g. a config option with the same name), the testing framework +will raise a `ValueError`, matching the behaviour of `charmcraft pack`. +Rename or remove the overlapping keys to fix this. +``` + ## Mock beyond the State If you wish to use the framework to test an existing charm type, you will probably need to mock out certain calls that are not covered by the `State` data structure. In that case, you will have to manually mock, patch or otherwise simulate those calls. diff --git a/pyproject.toml b/pyproject.toml index d26c28d89..26bd51042 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -227,6 +227,12 @@ exclude = ["tracing/ops_tracing/vendor/*"] # All documentation linting. "D", ] +"testing/src/scenario/_charmcraft_extensions.py" = [ + "E501", # Auto-generated file; long description strings are unavoidable. +] +".github/generate_charmcraft_extensions.py" = [ + "S607", # Intentionally calls charmcraft/tox by name. +] "ops/_private/timeconv.py" = [ "RUF001", # String contains ambiguous `µ` (MICRO SIGN). Did you mean `μ` (GREEK SMALL LETTER MU)? "RUF002", # Docstring contains ambiguous `µ` (MICRO SIGN). Did you mean `μ` (GREEK SMALL LETTER MU)? diff --git a/testing/src/scenario/_charmcraft_extensions.py b/testing/src/scenario/_charmcraft_extensions.py new file mode 100644 index 000000000..b697c8d37 --- /dev/null +++ b/testing/src/scenario/_charmcraft_extensions.py @@ -0,0 +1,427 @@ +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. +"""Charmcraft extension metadata, config, and actions. + +Auto-generated by .github/generate_charmcraft_extensions.py +Do not edit manually. +""" + +from __future__ import annotations + +from typing import Any, TypedDict + + +class _ExtensionMetadata(TypedDict, total=False): + assumes: list[str] + containers: dict[str, Any] + peers: dict[str, Any] + provides: dict[str, Any] + requires: dict[str, Any] + resources: dict[str, Any] + + +# Metadata added by each charmcraft extension. +METADATA: dict[str, _ExtensionMetadata] = { + 'django-framework': { + 'assumes': ['k8s-api'], + 'containers': {'django-app': {'resource': 'django-app-image'}}, + 'peers': {'secret-storage': {'interface': 'secret-storage'}}, + 'provides': { + 'grafana-dashboard': {'interface': 'grafana_dashboard'}, + 'metrics-endpoint': {'interface': 'prometheus_scrape'}, + }, + 'requires': { + 'ingress': {'interface': 'ingress', 'limit': 1}, + 'logging': {'interface': 'loki_push_api'}, + }, + 'resources': { + 'django-app-image': {'description': 'django application image.', 'type': 'oci-image'} + }, + }, + 'expressjs-framework': { + 'assumes': ['k8s-api'], + 'containers': {'app': {'resource': 'app-image'}}, + 'peers': {'secret-storage': {'interface': 'secret-storage'}}, + 'provides': { + 'grafana-dashboard': {'interface': 'grafana_dashboard'}, + 'metrics-endpoint': {'interface': 'prometheus_scrape'}, + }, + 'requires': { + 'ingress': {'interface': 'ingress', 'limit': 1}, + 'logging': {'interface': 'loki_push_api'}, + }, + 'resources': { + 'app-image': {'description': 'expressjs application image.', 'type': 'oci-image'} + }, + }, + 'fastapi-framework': { + 'assumes': ['k8s-api'], + 'containers': {'app': {'resource': 'app-image'}}, + 'peers': {'secret-storage': {'interface': 'secret-storage'}}, + 'provides': { + 'grafana-dashboard': {'interface': 'grafana_dashboard'}, + 'metrics-endpoint': {'interface': 'prometheus_scrape'}, + }, + 'requires': { + 'ingress': {'interface': 'ingress', 'limit': 1}, + 'logging': {'interface': 'loki_push_api'}, + }, + 'resources': { + 'app-image': {'description': 'fastapi application image.', 'type': 'oci-image'} + }, + }, + 'flask-framework': { + 'assumes': ['k8s-api'], + 'containers': {'flask-app': {'resource': 'flask-app-image'}}, + 'peers': {'secret-storage': {'interface': 'secret-storage'}}, + 'provides': { + 'grafana-dashboard': {'interface': 'grafana_dashboard'}, + 'metrics-endpoint': {'interface': 'prometheus_scrape'}, + }, + 'requires': { + 'ingress': {'interface': 'ingress', 'limit': 1}, + 'logging': {'interface': 'loki_push_api'}, + }, + 'resources': { + 'flask-app-image': {'description': 'flask application image.', 'type': 'oci-image'} + }, + }, + 'go-framework': { + 'assumes': ['k8s-api'], + 'containers': {'app': {'resource': 'app-image'}}, + 'peers': {'secret-storage': {'interface': 'secret-storage'}}, + 'provides': { + 'grafana-dashboard': {'interface': 'grafana_dashboard'}, + 'metrics-endpoint': {'interface': 'prometheus_scrape'}, + }, + 'requires': { + 'ingress': {'interface': 'ingress', 'limit': 1}, + 'logging': {'interface': 'loki_push_api'}, + }, + 'resources': {'app-image': {'description': 'go application image.', 'type': 'oci-image'}}, + }, + 'spring-boot-framework': { + 'assumes': ['k8s-api'], + 'containers': {'app': {'resource': 'app-image'}}, + 'peers': {'secret-storage': {'interface': 'secret-storage'}}, + 'provides': { + 'grafana-dashboard': {'interface': 'grafana_dashboard'}, + 'metrics-endpoint': {'interface': 'prometheus_scrape'}, + }, + 'requires': { + 'ingress': {'interface': 'ingress', 'limit': 1}, + 'logging': {'interface': 'loki_push_api'}, + }, + 'resources': { + 'app-image': {'description': 'spring-boot application image.', 'type': 'oci-image'} + }, + }, +} + +# Config options added by each charmcraft extension. +CONFIG: dict[str, dict[str, Any]] = { + 'django-framework': { + 'django-allowed-hosts': { + 'description': 'A comma-separated list of host/domain names that this Django site can serve. This ' + 'configuration will set the DJANGO_ALLOWED_HOSTS environment variable with its ' + 'content being a JSON encoded list.', + 'type': 'string', + }, + 'django-debug': { + 'default': False, + 'description': 'Whether Django debug mode is enabled.', + 'type': 'boolean', + }, + 'django-secret-key': { + 'description': 'The secret key used for securely signing the session cookie and for any other ' + 'security related needs by your Django application. This configuration will set ' + 'the DJANGO_SECRET_KEY environment variable.', + 'type': 'string', + }, + 'django-secret-key-id': { + 'description': 'This configuration is similar to `django-secret-key`, but instead accepts a Juju ' + 'user secret ID. The secret should contain a single key, "value", which maps to ' + 'the actual Django secret key. To create the secret, run the following command: ' + '`juju add-secret my-django-secret-key value= && juju grant-secret ' + 'my-django-secret-key django-k8s`, and use the output secret ID to configure this ' + 'option.', + 'type': 'secret', + }, + 'webserver-keepalive': { + 'description': 'Time in seconds for webserver to wait for requests on a Keep-Alive connection.', + 'type': 'int', + }, + 'webserver-threads': { + 'description': 'Run each webserver worker with the specified number of threads.', + 'type': 'int', + }, + 'webserver-timeout': { + 'description': 'Time in seconds to kill and restart silent webserver workers.', + 'type': 'int', + }, + 'webserver-worker-class': { + 'description': "The webserver worker process class for handling requests. Can be either 'gevent' " + "or 'sync'.", + 'type': 'string', + }, + 'webserver-workers': { + 'description': 'The number of webserver worker processes for handling requests.', + 'type': 'int', + }, + }, + 'expressjs-framework': { + 'app-port': { + 'default': 8080, + 'description': 'Default port where the application will listen on.', + 'type': 'int', + }, + 'app-secret-key': { + 'description': 'Long secret you can use for sessions, csrf or any other thing where you need a ' + 'random secret shared by all units', + 'type': 'string', + }, + 'app-secret-key-id': { + 'description': 'This configuration is similar to `app-secret-key`, but instead accepts a Juju ' + 'user secret ID. The secret should contain a single key, "value", which maps to ' + 'the actual application secret key. To create the secret, run the following ' + 'command: `juju add-secret my-app-secret-key value= && juju ' + 'grant-secret my-app-secret-key my-app`, and use the output secret ID to configure ' + 'this option.', + 'type': 'secret', + }, + 'metrics-path': { + 'default': '/metrics', + 'description': 'Path where the prometheus metrics will be scraped.', + 'type': 'string', + }, + 'metrics-port': { + 'default': 8080, + 'description': 'Port where the prometheus metrics will be scraped.', + 'type': 'int', + }, + }, + 'fastapi-framework': { + 'app-secret-key': { + 'description': 'Long secret you can use for sessions, csrf or any other thing where you need a ' + 'random secret shared by all units', + 'type': 'string', + }, + 'app-secret-key-id': { + 'description': 'This configuration is similar to `app-secret-key`, but instead accepts a Juju ' + 'user secret ID. The secret should contain a single key, "value", which maps to ' + 'the actual application secret key. To create the secret, run the following ' + 'command: `juju add-secret my-app-secret-key value= && juju ' + 'grant-secret my-app-secret-key my-app`, and use the output secret ID to configure ' + 'this option.', + 'type': 'secret', + }, + 'metrics-path': { + 'default': '/metrics', + 'description': 'Path where the prometheus metrics will be scraped.', + 'type': 'string', + }, + 'metrics-port': { + 'default': 8080, + 'description': 'Port where the prometheus metrics will be scraped.', + 'type': 'int', + }, + 'webserver-log-level': { + 'default': 'info', + 'description': "Set the log level. Options: 'critical', 'error', 'warning', 'info', 'debug', " + "'trace'. Sets the env variable UVICORN_LOG_LEVEL.", + 'type': 'string', + }, + 'webserver-port': { + 'default': 8080, + 'description': 'Bind to a socket with this port. Default: 8000. Sets env variable UVICORN_PORT.', + 'type': 'int', + }, + 'webserver-workers': { + 'default': 1, + 'description': 'Number of workers for uvicorn. Sets env variable WEB_CONCURRENCY. See ' + 'https://www.uvicorn.org/#command-line-options.', + 'type': 'int', + }, + }, + 'flask-framework': { + 'flask-application-root': { + 'description': 'Path in which the application / web server is mounted. This configuration will ' + 'set the FLASK_APPLICATION_ROOT environment variable. Run ' + '`app.config.from_prefixed_env()` in your Flask application in order to receive ' + 'this configuration.', + 'type': 'string', + }, + 'flask-debug': {'description': 'Whether Flask debug mode is enabled.', 'type': 'boolean'}, + 'flask-env': { + 'description': "What environment the Flask app is running in, by default it's 'production'.", + 'type': 'string', + }, + 'flask-permanent-session-lifetime': { + 'description': 'Time in seconds for the cookie to expire in the Flask application permanent ' + 'sessions. This configuration will set the FLASK_PERMANENT_SESSION_LIFETIME ' + 'environment variable. Run `app.config.from_prefixed_env()` in your Flask ' + 'application in order to receive this configuration.', + 'type': 'int', + }, + 'flask-preferred-url-scheme': { + 'default': 'HTTPS', + 'description': 'Scheme for generating external URLs when not in a request context in the Flask ' + 'application. By default, it\'s "HTTPS". This configuration will set the ' + 'FLASK_PREFERRED_URL_SCHEME environment variable. Run ' + '`app.config.from_prefixed_env()` in your Flask application in order to receive ' + 'this configuration.', + 'type': 'string', + }, + 'flask-secret-key': { + 'description': 'The secret key used for securely signing the session cookie and for any other ' + 'security related needs by your Flask application. This configuration will set the ' + 'FLASK_SECRET_KEY environment variable. Run `app.config.from_prefixed_env()` in ' + 'your Flask application in order to receive this configuration.', + 'type': 'string', + }, + 'flask-secret-key-id': { + 'description': 'This configuration is similar to `flask-secret-key`, but instead accepts a Juju ' + 'user secret ID. The secret should contain a single key, "value", which maps to ' + 'the actual Flask secret key. To create the secret, run the following command: ' + '`juju add-secret my-flask-secret-key value= && juju grant-secret ' + 'my-flask-secret-key flask-k8s`, and use the output secret ID to configure this ' + 'option.', + 'type': 'secret', + }, + 'flask-session-cookie-secure': { + 'description': 'Set the secure attribute in the Flask application cookies. This configuration ' + 'will set the FLASK_SESSION_COOKIE_SECURE environment variable. Run ' + '`app.config.from_prefixed_env()` in your Flask application in order to receive ' + 'this configuration.', + 'type': 'boolean', + }, + 'webserver-keepalive': { + 'description': 'Time in seconds for webserver to wait for requests on a Keep-Alive connection.', + 'type': 'int', + }, + 'webserver-threads': { + 'description': 'Run each webserver worker with the specified number of threads.', + 'type': 'int', + }, + 'webserver-timeout': { + 'description': 'Time in seconds to kill and restart silent webserver workers.', + 'type': 'int', + }, + 'webserver-worker-class': { + 'description': "The webserver worker process class for handling requests. Can be either 'gevent' " + "or 'sync'.", + 'type': 'string', + }, + 'webserver-workers': { + 'description': 'The number of webserver worker processes for handling requests.', + 'type': 'int', + }, + }, + 'go-framework': { + 'app-port': { + 'default': 8080, + 'description': 'Default port where the application will listen on.', + 'type': 'int', + }, + 'app-secret-key': { + 'description': 'Long secret you can use for sessions, csrf or any other thing where you need a ' + 'random secret shared by all units', + 'type': 'string', + }, + 'app-secret-key-id': { + 'description': 'This configuration is similar to `app-secret-key`, but instead accepts a Juju ' + 'user secret ID. The secret should contain a single key, "value", which maps to ' + 'the actual application secret key. To create the secret, run the following ' + 'command: `juju add-secret my-app-secret-key value= && juju ' + 'grant-secret my-app-secret-key my-app`, and use the output secret ID to configure ' + 'this option.', + 'type': 'secret', + }, + 'metrics-path': { + 'default': '/metrics', + 'description': 'Path where the prometheus metrics will be scraped.', + 'type': 'string', + }, + 'metrics-port': { + 'default': 8080, + 'description': 'Port where the prometheus metrics will be scraped.', + 'type': 'int', + }, + }, + 'spring-boot-framework': { + 'app-port': { + 'default': 8080, + 'description': 'Default port where the application will listen on.', + 'type': 'int', + }, + 'app-secret-key': { + 'description': 'Long secret you can use for sessions, csrf or any other thing where you need a ' + 'random secret shared by all units', + 'type': 'string', + }, + 'app-secret-key-id': { + 'description': 'This configuration is similar to `app-secret-key`, but instead accepts a Juju ' + 'user secret ID. The secret should contain a single key, "value", which maps to ' + 'the actual application secret key. To create the secret, run the following ' + 'command: `juju add-secret my-app-secret-key value= && juju ' + 'grant-secret my-app-secret-key my-app`, and use the output secret ID to configure ' + 'this option.', + 'type': 'secret', + }, + 'metrics-path': { + 'default': '/actuator/prometheus', + 'description': 'Path where the prometheus metrics will be scraped.', + 'type': 'string', + }, + 'metrics-port': { + 'default': 8080, + 'description': 'Port where the prometheus metrics will be scraped.', + 'type': 'int', + }, + }, +} + +# Actions added by each charmcraft extension. +ACTIONS: dict[str, dict[str, Any]] = { + 'django-framework': { + 'create-superuser': { + 'description': 'Create a new Django superuser account.', + 'params': {'email': {'type': 'string'}, 'username': {'type': 'string'}}, + 'required': ['username', 'email'], + }, + 'rotate-secret-key': { + 'description': 'Rotate the secret key. Users will be forced to log in again. This might be useful ' + 'if a security breach occurs.' + }, + }, + 'expressjs-framework': { + 'rotate-secret-key': { + 'description': 'Rotate the secret key. Users will be forced to log in again. This might be useful ' + 'if a security breach occurs.' + }, + }, + 'fastapi-framework': { + 'rotate-secret-key': { + 'description': 'Rotate the secret key. Users will be forced to log in again. This might be useful ' + 'if a security breach occurs.' + }, + }, + 'flask-framework': { + 'rotate-secret-key': { + 'description': 'Rotate the secret key. Users will be forced to log in again. This might be useful ' + 'if a security breach occurs.' + }, + }, + 'go-framework': { + 'rotate-secret-key': { + 'description': 'Rotate the secret key. Users will be forced to log in again. This might be useful ' + 'if a security breach occurs.' + }, + }, + 'spring-boot-framework': { + 'rotate-secret-key': { + 'description': 'Rotate the secret key. Users will be forced to log in again. This might be useful ' + 'if a security breach occurs.' + }, + }, +} diff --git a/testing/src/scenario/context.py b/testing/src/scenario/context.py index afeb7fd21..f9012d697 100644 --- a/testing/src/scenario/context.py +++ b/testing/src/scenario/context.py @@ -656,6 +656,9 @@ def __init__( :arg charm_type: the :class:`ops.CharmBase` subclass to handle the event. :arg meta: charm metadata to use. Needs to be a valid metadata.yaml format (as a dict). If none is provided, we will search for a ``metadata.yaml`` file in the charm root. + If the ``charmcraft.yaml`` contains an ``extensions`` key (e.g. + ``extensions: [flask-framework]``), the extension's metadata, + config, and actions will be automatically merged in. :arg actions: charm actions to use. Needs to be a valid actions.yaml format (as a dict). If none is provided, we will search for a ``actions.yaml`` file in the charm root. :arg config: charm config to use. Needs to be a valid config.yaml format (as a dict). diff --git a/testing/src/scenario/state.py b/testing/src/scenario/state.py index a5274ebf6..2081384ed 100644 --- a/testing/src/scenario/state.py +++ b/testing/src/scenario/state.py @@ -13,6 +13,7 @@ import random import re import string +import warnings from collections.abc import Callable, Iterable, Mapping, Sequence from enum import Enum from itertools import chain @@ -35,6 +36,7 @@ from ops import CloudCredential as CloudCredential_Ops from ops import CloudSpec as CloudSpec_Ops +from . import _charmcraft_extensions from .errors import MetadataNotFoundError, StateValidationError from .logger import logger as scenario_logger @@ -1860,6 +1862,87 @@ def _is_valid_charmcraft_25_metadata(meta: dict[str, Any]): return True +def _apply_extensions(meta: dict[str, Any], extensions: list[str]) -> None: + """Merge charmcraft extension defaults into the charm metadata in place. + + Extension defaults are applied first, then the local charmcraft.yaml + values are merged on top, simulating what ``charmcraft expand-extensions`` + does. + + Raises: + ValueError: if the local charmcraft.yaml defines keys that overlap + with what the extension provides (matching ``charmcraft pack`` + behaviour). + """ + for ext_name in extensions: + ext_meta = _charmcraft_extensions.METADATA.get(ext_name, {}) + ext_config = _charmcraft_extensions.CONFIG.get(ext_name, {}) + ext_actions = _charmcraft_extensions.ACTIONS.get(ext_name, {}) + + if not ext_meta and not ext_config and not ext_actions: + warnings.warn( + f'Unknown charmcraft extension {ext_name!r}; ignoring. ' + "Ensure you're running the latest ops, and open an issue if this persists.", + stacklevel=2, + ) + continue + + # Merge metadata: for dicts, error on overlapping keys + # (matching charmcraft behaviour). For lists, combine them. + for key, ext_value in ext_meta.items(): + if key not in meta: + meta[key] = copy.deepcopy(ext_value) + elif isinstance(ext_value, dict) and isinstance(meta[key], dict): + overlap = set(ext_value) & set(meta[key]) + if overlap: + raise ValueError( + f'overlapping keys {overlap} in {key} of ' + f'charmcraft.yaml which conflict with the ' + f'{ext_name} extension, please rename or remove them' + ) + merged = copy.deepcopy(ext_value) + merged.update(meta[key]) + meta[key] = merged + elif isinstance(ext_value, list) and isinstance(meta[key], list): + merged = copy.deepcopy(ext_value) + merged.extend(i for i in meta[key] if i not in merged) + meta[key] = merged + else: + raise ValueError( + 'Conflict between local and extension metadata. ' + 'Please check that your charmcraft.yaml is valid' + ) + + # Merge config options; error on overlapping keys. + if ext_config: + local_config = meta.get('config', {}) + local_options = local_config.get('options', {}) + overlap = set(ext_config) & set(local_options) + if overlap: + raise ValueError( + f'overlapping keys {overlap} in config.options of ' + f'charmcraft.yaml which conflict with the ' + f'{ext_name} extension, please rename or remove them' + ) + merged_options = copy.deepcopy(ext_config) + merged_options.update(local_options) + meta['config'] = {'options': merged_options} + + # Merge actions; error on overlapping keys. + if ext_actions: + local_actions = meta.get('actions', {}) + overlap = set(ext_actions) & set(local_actions) + if overlap: + raise ValueError( + f'overlapping keys {overlap} in actions of ' + f'charmcraft.yaml which conflict with the ' + f'{ext_name} extension, please rename or remove them' + ) + merged_actions = copy.deepcopy(ext_actions) + merged_actions.update(local_actions) + meta['actions'] = merged_actions + + @dataclasses.dataclass(frozen=True) class _CharmSpec(Generic[CharmType]): """Charm spec.""" @@ -1892,19 +1975,31 @@ def _load_metadata_legacy(charm_root: pathlib.Path): @staticmethod def _load_metadata(charm_root: pathlib.Path): - """Load metadata from charm projects created with Charmcraft >= 2.5.""" + """Load metadata from charm projects created with Charmcraft >= 2.5. + + If the ``charmcraft.yaml`` contains an ``extensions`` key (e.g. + ``extensions: [flask-framework]``), the extension's metadata, config, + and actions are merged in before the values are returned, simulating + what ``charmcraft expand-extensions`` does at pack time. + """ metadata_path = charm_root / 'charmcraft.yaml' meta: dict[str, Any] = ( yaml.safe_load(metadata_path.open()) if metadata_path.exists() else {} ) if not _is_valid_charmcraft_25_metadata(meta): meta = {} + + # Apply charmcraft extensions before extracting config/actions. + extensions = meta.pop('extensions', None) + if extensions: + _apply_extensions(meta, extensions) + config = meta.pop('config', None) actions = meta.pop('actions', None) return meta, config, actions @staticmethod - def autoload(charm_type: type[CharmBase]) -> _CharmSpec[CharmType]: + def autoload(charm_type: type[CharmBase]) -> _CharmSpec[CharmBase]: """Construct a ``_CharmSpec`` object by looking up the metadata from the charm's repo root. Will attempt to load the metadata off the ``charmcraft.yaml`` file diff --git a/testing/tests/test_charm_spec_autoload.py b/testing/tests/test_charm_spec_autoload.py index 01b7df1fc..b087668c4 100644 --- a/testing/tests/test_charm_spec_autoload.py +++ b/testing/tests/test_charm_spec_autoload.py @@ -30,7 +30,7 @@ def import_name(name: str, source: Path) -> Iterator[type[CharmBase]]: pkg_path = str(source.parent) sys.path.append(pkg_path) charm = importlib.import_module('mycharm') - obj = getattr(charm, name) + obj: type[CharmBase] = getattr(charm, name) sys.path.remove(pkg_path) yield obj del sys.modules['mycharm'] @@ -169,3 +169,276 @@ def test_config_defaults(tmp_path: Path, legacy: bool): with ctx(ctx.on.start(), State()) as mgr: mgr.run() assert mgr.charm.config['foo'] is True + + +class TestExtensions: + """Tests for charmcraft extension autoloading.""" + + def test_extension_adds_metadata(self, tmp_path: Path): + """An extension injects its metadata (containers, requires, etc.).""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + }, + ) as charm: + spec = _CharmSpec.autoload(charm) + assert 'flask-app' in spec.meta.get('containers', {}) + assert 'ingress' in spec.meta.get('requires', {}) + assert 'secret-storage' in spec.meta.get('peers', {}) + assert 'metrics-endpoint' in spec.meta.get('provides', {}) + assert 'flask-app-image' in spec.meta.get('resources', {}) + assert 'k8s-api' in spec.meta.get('assumes', []) + + def test_extension_adds_config(self, tmp_path: Path): + """An extension injects its config options.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + }, + ) as charm: + spec = _CharmSpec.autoload(charm) + assert spec.config is not None + options = spec.config.get('options', {}) + assert 'flask-debug' in options + assert 'webserver-workers' in options + + def test_extension_adds_actions(self, tmp_path: Path): + """An extension injects its actions.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + }, + ) as charm: + spec = _CharmSpec.autoload(charm) + assert spec.actions is not None + assert 'rotate-secret-key' in spec.actions + + def test_local_meta_overlaps_extension_errors(self, tmp_path: Path): + """Overlapping metadata keys with extension cause an error.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + 'requires': { + 'ingress': {'interface': 'custom-ingress', 'limit': 5}, + }, + }, + ) as charm: + with pytest.raises(ValueError, match=r'overlapping keys.*requires.*flask-framework'): + _CharmSpec.autoload(charm) + + def test_local_meta_no_overlap_with_extension(self, tmp_path: Path): + """Non-overlapping local metadata keys merge with extension.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + 'requires': { + 'my-custom-relation': {'interface': 'custom'}, + }, + }, + ) as charm: + spec = _CharmSpec.autoload(charm) + # Local-only entry is present. + assert 'my-custom-relation' in spec.meta['requires'] + # Extension entries are still present. + assert 'ingress' in spec.meta['requires'] + assert 'logging' in spec.meta['requires'] + + def test_local_config_overlaps_extension_errors(self, tmp_path: Path): + """Overlapping config options with extension cause an error.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + }, + config={ + 'options': { + 'flask-debug': {'type': 'boolean', 'default': True}, + 'my-custom-option': {'type': 'string'}, + }, + }, + ) as charm: + with pytest.raises( + ValueError, match=r'overlapping keys.*config\.options.*flask-framework' + ): + _CharmSpec.autoload(charm) + + def test_local_config_no_overlap_with_extension(self, tmp_path: Path): + """Non-overlapping local config options merge with extension.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + }, + config={ + 'options': { + 'my-custom-option': {'type': 'string'}, + }, + }, + ) as charm: + spec = _CharmSpec.autoload(charm) + assert spec.config is not None + options = spec.config['options'] + # Local-only option is present. + assert 'my-custom-option' in options + # Extension-only options are still present. + assert 'flask-debug' in options + assert 'webserver-workers' in options + + def test_local_actions_overlap_extension_errors(self, tmp_path: Path): + """Overlapping actions with extension cause an error.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + }, + actions={ + 'rotate-secret-key': {'description': 'custom rotate'}, + 'my-action': {'description': 'custom action'}, + }, + ) as charm: + with pytest.raises(ValueError, match=r'overlapping keys.*actions.*flask-framework'): + _CharmSpec.autoload(charm) + + def test_local_actions_no_overlap_with_extension(self, tmp_path: Path): + """Non-overlapping local actions merge with extension.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + }, + actions={ + 'my-action': {'description': 'custom action'}, + }, + ) as charm: + spec = _CharmSpec.autoload(charm) + assert spec.actions is not None + # Local-only action is present. + assert 'my-action' in spec.actions + # Extension action is still present. + assert 'rotate-secret-key' in spec.actions + + def test_local_assumes_merged_with_extension(self, tmp_path: Path): + """Local assumes list is merged with extension assumes.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + 'assumes': ['juju >= 3.1'], + }, + ) as charm: + spec = _CharmSpec.autoload(charm) + assumes = spec.meta['assumes'] + assert 'k8s-api' in assumes + assert 'juju >= 3.1' in assumes + + def test_django_extension_has_create_superuser(self, tmp_path: Path): + """Django extension adds the create-superuser action.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-django', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['django-framework'], + }, + ) as charm: + spec = _CharmSpec.autoload(charm) + assert spec.actions is not None + assert 'create-superuser' in spec.actions + assert 'rotate-secret-key' in spec.actions + + def test_unknown_extension_warns(self, tmp_path: Path): + """An unknown extension name emits a warning and is skipped.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-app', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['nonexistent-extension'], + }, + ) as charm: + with pytest.warns(UserWarning, match='Unknown charmcraft extension'): + spec = _CharmSpec.autoload(charm) + # No extension data merged, but the charm still loads. + assert spec.meta['name'] == 'my-app' + + def test_extension_stripped_from_meta(self, tmp_path: Path): + """The 'extensions' key should not remain in the loaded meta.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + }, + ) as charm: + spec = _CharmSpec.autoload(charm) + assert 'extensions' not in spec.meta + + def test_extension_with_relations_in_context(self, tmp_path: Path): + """Relations from an extension can be used in a Context run.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + }, + ) as charm: + ctx = Context(charm) + ctx.run( + ctx.on.start(), + State(relations={Relation('ingress')}), + )