From 241dfff9c0fe64f2536c80e0e378fe4af2b61fd6 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Fri, 6 Mar 2026 18:12:37 +1300 Subject: [PATCH 01/19] feat: autoload support for charmcraft extensions --- .github/generate_charmcraft_extensions.py | 198 ++++++++++ pyproject.toml | 7 + .../src/scenario/_charmcraft_extensions.py | 362 ++++++++++++++++++ testing/src/scenario/state.py | 65 ++++ testing/tests/test_charm_spec_autoload.py | 217 +++++++++++ 5 files changed, 849 insertions(+) create mode 100755 .github/generate_charmcraft_extensions.py create mode 100644 testing/src/scenario/_charmcraft_extensions.py diff --git a/.github/generate_charmcraft_extensions.py b/.github/generate_charmcraft_extensions.py new file mode 100755 index 000000000..9604bd597 --- /dev/null +++ b/.github/generate_charmcraft_extensions.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10" +# dependencies = ["pyyaml"] +# /// + +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Generate testing/src/scenario/_charmcraft_extensions.py. + +For each charmcraft extension profile (django-framework, fastapi-framework, etc.), +this script runs `charmcraft init` and `charmcraft expand-extensions` in a temp +directory, then extracts the metadata, config, and actions that the extension adds. + +The output module contains three dictionaries per extension: + METADATA: dict[str, dict] - containers, peers, provides, requires, resources, assumes + CONFIG: dict[str, dict] - config options added by each extension + ACTIONS: dict[str, dict] - actions added by each extension +""" + +from __future__ import annotations + +import pathlib +import pprint +import subprocess +import tempfile + +import yaml + +PROFILES = [ + 'django-framework', + 'fastapi-framework', + 'flask-framework', + 'go-framework', + 'spring-boot-framework', +] + +# Keys from the expanded YAML that belong to "metadata" (the charm's metadata.yaml equivalent). +METADATA_KEYS = { + 'assumes', + 'containers', + 'peers', + 'provides', + 'requires', + 'resources', +} + +# Keys that are user-provided or build-related (not extension-contributed metadata). +IGNORED_KEYS = { + 'name', + 'summary', + 'description', + 'type', + 'bases', + 'base', + 'platforms', + 'extensions', + 'parts', + 'charm-libs', +} + +OUTPUT_FILE = ( + pathlib.Path(__file__).resolve().parent.parent + / 'testing' + / 'src' + / 'scenario' + / '_charmcraft_extensions.py' +) + + +def run_charmcraft(profile: str, workdir: pathlib.Path) -> dict: + """Run charmcraft init + expand-extensions and return the expanded YAML as a dict.""" + subprocess.run( + ['charmcraft', 'init', '--profile', profile, '--name', 'test-charm'], + cwd=workdir, + check=True, + capture_output=True, + text=True, + ) + result = subprocess.run( + ['charmcraft', 'expand-extensions'], + cwd=workdir, + check=True, + capture_output=True, + text=True, + ) + return yaml.safe_load(result.stdout) + + +def extract_extension_data(expanded: dict) -> tuple[dict, dict, dict]: + """Extract metadata, config options, and actions from expanded charmcraft YAML.""" + metadata = {} + for key in METADATA_KEYS: + if key in expanded: + metadata[key] = expanded[key] + + config = {} + if 'config' in expanded and 'options' in expanded['config']: + config = expanded['config']['options'] + + actions = expanded.get('actions', {}) + + return metadata, config, actions + + +def format_dict(d: dict, indent: int = 4) -> str: + """Format a dict as a Python literal string.""" + formatted = pprint.pformat(d, width=100, sort_dicts=True) + if '\n' in formatted: + lines = formatted.splitlines() + result = lines[0] + for line in lines[1:]: + result += '\n' + ' ' * indent + line + return result + return formatted + + +def generate_module(all_data: dict[str, tuple[dict, dict, dict]]) -> str: + """Generate the Python module source code.""" + lines = [ + '# Copyright 2026 Canonical Ltd.', + '# See LICENSE file for licensing details.', + '"""Charmcraft extension metadata, config, and actions.', + '', + 'Auto-generated by .github/generate_charmcraft_extensions.py', + 'Do not edit manually.', + '"""', + '', + 'from __future__ import annotations', + '', + ] + + # Build the three top-level dicts. + metadata_entries = {} + config_entries = {} + action_entries = {} + for profile, (metadata, config, actions) in sorted(all_data.items()): + metadata_entries[profile] = metadata + config_entries[profile] = config + action_entries[profile] = actions + + for var_name, data, docstring in [ + ( + 'METADATA', + metadata_entries, + 'Metadata added by each charmcraft extension.', + ), + ('CONFIG', config_entries, 'Config options added by each charmcraft extension.'), + ('ACTIONS', action_entries, 'Actions added by each charmcraft extension.'), + ]: + lines.append(f'# {docstring}') + lines.append(f'{var_name}: dict[str, dict] = {{') + for profile in sorted(data): + lines.append(f' {profile!r}: {{') + for key in sorted(data[profile]): + val = data[profile][key] + formatted = format_dict(val, indent=8) + lines.append(f' {key!r}: {formatted},') + lines.append(' },') + lines.append('}') + lines.append('') + + return '\n'.join(lines) + + +def main() -> int: # noqa: D103 + all_data: dict[str, tuple[dict, dict, dict]] = {} + + for profile in PROFILES: + print(f'Processing {profile}...') + with tempfile.TemporaryDirectory() as tmpdir: + expanded = run_charmcraft(profile, pathlib.Path(tmpdir)) + all_data[profile] = extract_extension_data(expanded) + + module_source = generate_module(all_data) + + if OUTPUT_FILE.exists(): + response = input(f'{OUTPUT_FILE} already exists. Overwrite? [y/N] ').strip().lower() + if response != 'y': + print('Aborted.') + return 1 + + OUTPUT_FILE.write_text(module_source) + print(f'Written to {OUTPUT_FILE}') + + print('Running tox -e format...') + subprocess.run( + ['tox', '-e', 'format', '--', str(OUTPUT_FILE)], + cwd=OUTPUT_FILE.parent, + check=True, + ) + + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index 47e44f043..0293b0886 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -227,6 +227,13 @@ exclude = ["tracing/ops_tracing/vendor/*"] # All documentation linting. "D", ] +"testing/src/scenario/_charmcraft_extensions.py" = [ + "CPY001", # Auto-generated file. + "E501", # Auto-generated file; long description strings are unavoidable. +] +".github/generate_charmcraft_extensions.py" = [ + "S607", # Intentionally calls charmcraft/tox by name. +] "ops/_private/timeconv.py" = [ "RUF001", # String contains ambiguous `µ` (MICRO SIGN). Did you mean `μ` (GREEK SMALL LETTER MU)? "RUF002", # Docstring contains ambiguous `µ` (MICRO SIGN). Did you mean `μ` (GREEK SMALL LETTER MU)? diff --git a/testing/src/scenario/_charmcraft_extensions.py b/testing/src/scenario/_charmcraft_extensions.py new file mode 100644 index 000000000..9d7130e70 --- /dev/null +++ b/testing/src/scenario/_charmcraft_extensions.py @@ -0,0 +1,362 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. +"""Charmcraft extension metadata, config, and actions. + +Auto-generated by .github/generate_charmcraft_extensions.py +Do not edit manually. +""" + +from __future__ import annotations + +# Metadata added by each charmcraft extension. +METADATA: dict[str, dict] = { + 'django-framework': { + 'assumes': ['k8s-api'], + 'containers': {'django-app': {'resource': 'django-app-image'}}, + 'peers': {'secret-storage': {'interface': 'secret-storage'}}, + 'provides': { + 'grafana-dashboard': {'interface': 'grafana_dashboard'}, + 'metrics-endpoint': {'interface': 'prometheus_scrape'}, + }, + 'requires': { + 'ingress': {'interface': 'ingress', 'limit': 1}, + 'logging': {'interface': 'loki_push_api'}, + }, + 'resources': { + 'django-app-image': {'description': 'django application image.', 'type': 'oci-image'} + }, + }, + 'fastapi-framework': { + 'assumes': ['k8s-api'], + 'containers': {'app': {'resource': 'app-image'}}, + 'peers': {'secret-storage': {'interface': 'secret-storage'}}, + 'provides': { + 'grafana-dashboard': {'interface': 'grafana_dashboard'}, + 'metrics-endpoint': {'interface': 'prometheus_scrape'}, + }, + 'requires': { + 'ingress': {'interface': 'ingress', 'limit': 1}, + 'logging': {'interface': 'loki_push_api'}, + }, + 'resources': { + 'app-image': {'description': 'fastapi application image.', 'type': 'oci-image'} + }, + }, + 'flask-framework': { + 'assumes': ['k8s-api'], + 'containers': {'flask-app': {'resource': 'flask-app-image'}}, + 'peers': {'secret-storage': {'interface': 'secret-storage'}}, + 'provides': { + 'grafana-dashboard': {'interface': 'grafana_dashboard'}, + 'metrics-endpoint': {'interface': 'prometheus_scrape'}, + }, + 'requires': { + 'ingress': {'interface': 'ingress', 'limit': 1}, + 'logging': {'interface': 'loki_push_api'}, + }, + 'resources': { + 'flask-app-image': {'description': 'flask application image.', 'type': 'oci-image'} + }, + }, + 'go-framework': { + 'assumes': ['k8s-api'], + 'containers': {'app': {'resource': 'app-image'}}, + 'peers': {'secret-storage': {'interface': 'secret-storage'}}, + 'provides': { + 'grafana-dashboard': {'interface': 'grafana_dashboard'}, + 'metrics-endpoint': {'interface': 'prometheus_scrape'}, + }, + 'requires': { + 'ingress': {'interface': 'ingress', 'limit': 1}, + 'logging': {'interface': 'loki_push_api'}, + }, + 'resources': {'app-image': {'description': 'go application image.', 'type': 'oci-image'}}, + }, + 'spring-boot-framework': { + 'assumes': ['k8s-api'], + 'containers': {'app': {'resource': 'app-image'}}, + 'peers': {'secret-storage': {'interface': 'secret-storage'}}, + 'provides': { + 'grafana-dashboard': {'interface': 'grafana_dashboard'}, + 'metrics-endpoint': {'interface': 'prometheus_scrape'}, + }, + 'requires': { + 'ingress': {'interface': 'ingress', 'limit': 1}, + 'logging': {'interface': 'loki_push_api'}, + }, + 'resources': { + 'app-image': {'description': 'spring-boot application image.', 'type': 'oci-image'} + }, + }, +} + +# Config options added by each charmcraft extension. +CONFIG: dict[str, dict] = { + 'django-framework': { + 'django-allowed-hosts': { + 'description': 'A comma-separated list of host/domain names that this Django site can serve. This ' + 'configuration will set the DJANGO_ALLOWED_HOSTS environment variable with its ' + 'content being a JSON encoded list.', + 'type': 'string', + }, + 'django-debug': { + 'default': False, + 'description': 'Whether Django debug mode is enabled.', + 'type': 'boolean', + }, + 'django-secret-key': { + 'description': 'The secret key used for securely signing the session cookie and for any other ' + 'security related needs by your Django application. This configuration will set ' + 'the DJANGO_SECRET_KEY environment variable.', + 'type': 'string', + }, + 'django-secret-key-id': { + 'description': 'This configuration is similar to `django-secret-key`, but instead accepts a Juju ' + 'user secret ID. The secret should contain a single key, "value", which maps to ' + 'the actual Django secret key. To create the secret, run the following command: ' + '`juju add-secret my-django-secret-key value= && juju grant-secret ' + 'my-django-secret-key django-k8s`, and use the output secret ID to configure this ' + 'option.', + 'type': 'secret', + }, + 'webserver-keepalive': { + 'description': 'Time in seconds for webserver to wait for requests on a Keep-Alive connection.', + 'type': 'int', + }, + 'webserver-threads': { + 'description': 'Run each webserver worker with the specified number of threads.', + 'type': 'int', + }, + 'webserver-timeout': { + 'description': 'Time in seconds to kill and restart silent webserver workers.', + 'type': 'int', + }, + 'webserver-worker-class': { + 'description': "The webserver worker process class for handling requests. Can be either 'gevent' " + "or 'sync'.", + 'type': 'string', + }, + 'webserver-workers': { + 'description': 'The number of webserver worker processes for handling requests.', + 'type': 'int', + }, + }, + 'fastapi-framework': { + 'app-secret-key': { + 'description': 'Long secret you can use for sessions, csrf or any other thing where you need a ' + 'random secret shared by all units', + 'type': 'string', + }, + 'app-secret-key-id': { + 'description': 'This configuration is similar to `app-secret-key`, but instead accepts a Juju ' + 'user secret ID. The secret should contain a single key, "value", which maps to ' + 'the actual application secret key. To create the secret, run the following ' + 'command: `juju add-secret my-app-secret-key value= && juju ' + 'grant-secret my-app-secret-key my-app`, and use the output secret ID to configure ' + 'this option.', + 'type': 'secret', + }, + 'metrics-path': { + 'default': '/metrics', + 'description': 'Path where the prometheus metrics will be scraped.', + 'type': 'string', + }, + 'metrics-port': { + 'default': 8080, + 'description': 'Port where the prometheus metrics will be scraped.', + 'type': 'int', + }, + 'webserver-log-level': { + 'default': 'info', + 'description': "Set the log level. Options: 'critical', 'error', 'warning', 'info', 'debug', " + "'trace'. Sets the env variable UVICORN_LOG_LEVEL.", + 'type': 'string', + }, + 'webserver-port': { + 'default': 8080, + 'description': 'Bind to a socket with this port. Default: 8000. Sets env variable UVICORN_PORT.', + 'type': 'int', + }, + 'webserver-workers': { + 'default': 1, + 'description': 'Number of workers for uvicorn. Sets env variable WEB_CONCURRENCY. See ' + 'https://www.uvicorn.org/#command-line-options.', + 'type': 'int', + }, + }, + 'flask-framework': { + 'flask-application-root': { + 'description': 'Path in which the application / web server is mounted. This configuration will ' + 'set the FLASK_APPLICATION_ROOT environment variable. Run ' + '`app.config.from_prefixed_env()` in your Flask application in order to receive ' + 'this configuration.', + 'type': 'string', + }, + 'flask-debug': {'description': 'Whether Flask debug mode is enabled.', 'type': 'boolean'}, + 'flask-env': { + 'description': "What environment the Flask app is running in, by default it's 'production'.", + 'type': 'string', + }, + 'flask-permanent-session-lifetime': { + 'description': 'Time in seconds for the cookie to expire in the Flask application permanent ' + 'sessions. This configuration will set the FLASK_PERMANENT_SESSION_LIFETIME ' + 'environment variable. Run `app.config.from_prefixed_env()` in your Flask ' + 'application in order to receive this configuration.', + 'type': 'int', + }, + 'flask-preferred-url-scheme': { + 'default': 'HTTPS', + 'description': 'Scheme for generating external URLs when not in a request context in the Flask ' + 'application. By default, it\'s "HTTPS". This configuration will set the ' + 'FLASK_PREFERRED_URL_SCHEME environment variable. Run ' + '`app.config.from_prefixed_env()` in your Flask application in order to receive ' + 'this configuration.', + 'type': 'string', + }, + 'flask-secret-key': { + 'description': 'The secret key used for securely signing the session cookie and for any other ' + 'security related needs by your Flask application. This configuration will set the ' + 'FLASK_SECRET_KEY environment variable. Run `app.config.from_prefixed_env()` in ' + 'your Flask application in order to receive this configuration.', + 'type': 'string', + }, + 'flask-secret-key-id': { + 'description': 'This configuration is similar to `flask-secret-key`, but instead accepts a Juju ' + 'user secret ID. The secret should contain a single key, "value", which maps to ' + 'the actual Flask secret key. To create the secret, run the following command: ' + '`juju add-secret my-flask-secret-key value= && juju grant-secret ' + 'my-flask-secret-key flask-k8s`, and use the output secret ID to configure this ' + 'option.', + 'type': 'secret', + }, + 'flask-session-cookie-secure': { + 'description': 'Set the secure attribute in the Flask application cookies. This configuration ' + 'will set the FLASK_SESSION_COOKIE_SECURE environment variable. Run ' + '`app.config.from_prefixed_env()` in your Flask application in order to receive ' + 'this configuration.', + 'type': 'boolean', + }, + 'webserver-keepalive': { + 'description': 'Time in seconds for webserver to wait for requests on a Keep-Alive connection.', + 'type': 'int', + }, + 'webserver-threads': { + 'description': 'Run each webserver worker with the specified number of threads.', + 'type': 'int', + }, + 'webserver-timeout': { + 'description': 'Time in seconds to kill and restart silent webserver workers.', + 'type': 'int', + }, + 'webserver-worker-class': { + 'description': "The webserver worker process class for handling requests. Can be either 'gevent' " + "or 'sync'.", + 'type': 'string', + }, + 'webserver-workers': { + 'description': 'The number of webserver worker processes for handling requests.', + 'type': 'int', + }, + }, + 'go-framework': { + 'app-port': { + 'default': 8080, + 'description': 'Default port where the application will listen on.', + 'type': 'int', + }, + 'app-secret-key': { + 'description': 'Long secret you can use for sessions, csrf or any other thing where you need a ' + 'random secret shared by all units', + 'type': 'string', + }, + 'app-secret-key-id': { + 'description': 'This configuration is similar to `app-secret-key`, but instead accepts a Juju ' + 'user secret ID. The secret should contain a single key, "value", which maps to ' + 'the actual application secret key. To create the secret, run the following ' + 'command: `juju add-secret my-app-secret-key value= && juju ' + 'grant-secret my-app-secret-key my-app`, and use the output secret ID to configure ' + 'this option.', + 'type': 'secret', + }, + 'metrics-path': { + 'default': '/metrics', + 'description': 'Path where the prometheus metrics will be scraped.', + 'type': 'string', + }, + 'metrics-port': { + 'default': 8080, + 'description': 'Port where the prometheus metrics will be scraped.', + 'type': 'int', + }, + }, + 'spring-boot-framework': { + 'app-port': { + 'default': 8080, + 'description': 'Default port where the application will listen on.', + 'type': 'int', + }, + 'app-secret-key': { + 'description': 'Long secret you can use for sessions, csrf or any other thing where you need a ' + 'random secret shared by all units', + 'type': 'string', + }, + 'app-secret-key-id': { + 'description': 'This configuration is similar to `app-secret-key`, but instead accepts a Juju ' + 'user secret ID. The secret should contain a single key, "value", which maps to ' + 'the actual application secret key. To create the secret, run the following ' + 'command: `juju add-secret my-app-secret-key value= && juju ' + 'grant-secret my-app-secret-key my-app`, and use the output secret ID to configure ' + 'this option.', + 'type': 'secret', + }, + 'metrics-path': { + 'default': '/actuator/prometheus', + 'description': 'Path where the prometheus metrics will be scraped.', + 'type': 'string', + }, + 'metrics-port': { + 'default': 8080, + 'description': 'Port where the prometheus metrics will be scraped.', + 'type': 'int', + }, + }, +} + +# Actions added by each charmcraft extension. +ACTIONS: dict[str, dict] = { + 'django-framework': { + 'create-superuser': { + 'description': 'Create a new Django superuser account.', + 'params': {'email': {'type': 'string'}, 'username': {'type': 'string'}}, + 'required': ['username', 'email'], + }, + 'rotate-secret-key': { + 'description': 'Rotate the secret key. Users will be forced to log in again. This might be useful ' + 'if a security breach occurs.' + }, + }, + 'fastapi-framework': { + 'rotate-secret-key': { + 'description': 'Rotate the secret key. Users will be forced to log in again. This might be useful ' + 'if a security breach occurs.' + }, + }, + 'flask-framework': { + 'rotate-secret-key': { + 'description': 'Rotate the secret key. Users will be forced to log in again. This might be useful ' + 'if a security breach occurs.' + }, + }, + 'go-framework': { + 'rotate-secret-key': { + 'description': 'Rotate the secret key. Users will be forced to log in again. This might be useful ' + 'if a security breach occurs.' + }, + }, + 'spring-boot-framework': { + 'rotate-secret-key': { + 'description': 'Rotate the secret key. Users will be forced to log in again. This might be useful ' + 'if a security breach occurs.' + }, + }, +} diff --git a/testing/src/scenario/state.py b/testing/src/scenario/state.py index a5274ebf6..b598b65c1 100644 --- a/testing/src/scenario/state.py +++ b/testing/src/scenario/state.py @@ -1860,6 +1860,65 @@ def _is_valid_charmcraft_25_metadata(meta: dict[str, Any]): return True +def _apply_extensions( + meta: dict[str, Any], + extensions: list[str], +) -> dict[str, Any]: + """Merge charmcraft extension defaults into the charm metadata. + + Extension defaults are applied first, then the local charmcraft.yaml + values are merged on top, simulating what ``charmcraft expand-extensions`` + does. + """ + from . import _charmcraft_extensions + + for ext_name in extensions: + ext_meta = _charmcraft_extensions.METADATA.get(ext_name, {}) + ext_config = _charmcraft_extensions.CONFIG.get(ext_name, {}) + ext_actions = _charmcraft_extensions.ACTIONS.get(ext_name, {}) + + if not ext_meta and not ext_config and not ext_actions: + logger.warning( + f'Unknown charmcraft extension {ext_name!r}; ' + f'ignoring. You may need to regenerate ' + f'_charmcraft_extensions.py.', + ) + continue + + # Merge metadata: for dicts, extension provides defaults that + # the local yaml overrides. For lists, combine them. + for key, ext_value in ext_meta.items(): + if key not in meta: + meta[key] = copy.deepcopy(ext_value) + elif isinstance(ext_value, dict) and isinstance(meta[key], dict): + merged = copy.deepcopy(ext_value) + merged.update(meta[key]) + meta[key] = merged + elif isinstance(ext_value, list) and isinstance(meta[key], list): + merged = copy.deepcopy(ext_value) + for item in meta[key]: + if item not in merged: + merged.append(item) + meta[key] = merged + + # Merge config options. + if ext_config: + local_config = meta.get('config', {}) + local_options = local_config.get('options', {}) + merged_options = copy.deepcopy(ext_config) + merged_options.update(local_options) + meta['config'] = {'options': merged_options} + + # Merge actions. + if ext_actions: + local_actions = meta.get('actions', {}) + merged_actions = copy.deepcopy(ext_actions) + merged_actions.update(local_actions) + meta['actions'] = merged_actions + + return meta + + @dataclasses.dataclass(frozen=True) class _CharmSpec(Generic[CharmType]): """Charm spec.""" @@ -1899,6 +1958,12 @@ def _load_metadata(charm_root: pathlib.Path): ) if not _is_valid_charmcraft_25_metadata(meta): meta = {} + + # Apply charmcraft extensions before extracting config/actions. + extensions = meta.pop('extensions', None) + if extensions: + meta = _apply_extensions(meta, extensions) + config = meta.pop('config', None) actions = meta.pop('actions', None) return meta, config, actions diff --git a/testing/tests/test_charm_spec_autoload.py b/testing/tests/test_charm_spec_autoload.py index 1d77caf6e..79dabfca9 100644 --- a/testing/tests/test_charm_spec_autoload.py +++ b/testing/tests/test_charm_spec_autoload.py @@ -165,3 +165,220 @@ def test_config_defaults(tmp_path, legacy): with ctx(ctx.on.start(), State()) as mgr: mgr.run() assert mgr.charm.config['foo'] is True + + +class TestExtensions: + """Tests for charmcraft extension autoloading.""" + + def test_extension_adds_metadata(self, tmp_path): + """An extension injects its metadata (containers, requires, etc.).""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + }, + ) as charm: + spec = _CharmSpec.autoload(charm) + assert 'flask-app' in spec.meta.get('containers', {}) + assert 'ingress' in spec.meta.get('requires', {}) + assert 'secret-storage' in spec.meta.get('peers', {}) + assert 'metrics-endpoint' in spec.meta.get('provides', {}) + assert 'flask-app-image' in spec.meta.get('resources', {}) + assert 'k8s-api' in spec.meta.get('assumes', []) + + def test_extension_adds_config(self, tmp_path): + """An extension injects its config options.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + }, + ) as charm: + spec = _CharmSpec.autoload(charm) + assert spec.config is not None + options = spec.config.get('options', {}) + assert 'flask-debug' in options + assert 'webserver-workers' in options + + def test_extension_adds_actions(self, tmp_path): + """An extension injects its actions.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + }, + ) as charm: + spec = _CharmSpec.autoload(charm) + assert spec.actions is not None + assert 'rotate-secret-key' in spec.actions + + def test_local_meta_overrides_extension(self, tmp_path): + """Local charmcraft.yaml values take precedence over extension defaults.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + 'requires': { + 'ingress': {'interface': 'custom-ingress', 'limit': 5}, + }, + }, + ) as charm: + spec = _CharmSpec.autoload(charm) + # local override wins + assert spec.meta['requires']['ingress'] == { + 'interface': 'custom-ingress', + 'limit': 5, + } + # extension-only entries still present + assert 'logging' in spec.meta['requires'] + + def test_local_config_overrides_extension(self, tmp_path): + """Local config options override extension config defaults.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + }, + config={ + 'options': { + 'flask-debug': {'type': 'boolean', 'default': True}, + 'my-custom-option': {'type': 'string'}, + }, + }, + ) as charm: + spec = _CharmSpec.autoload(charm) + options = spec.config['options'] + # local override wins + assert options['flask-debug'] == {'type': 'boolean', 'default': True} + # local-only option present + assert 'my-custom-option' in options + # extension-only options still present + assert 'webserver-workers' in options + + def test_local_actions_override_extension(self, tmp_path): + """Local actions override extension action defaults.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + }, + actions={ + 'rotate-secret-key': {'description': 'custom rotate'}, + 'my-action': {'description': 'custom action'}, + }, + ) as charm: + spec = _CharmSpec.autoload(charm) + # local override wins + assert spec.actions['rotate-secret-key'] == { + 'description': 'custom rotate', + } + # local-only action present + assert 'my-action' in spec.actions + + def test_local_assumes_merged_with_extension(self, tmp_path): + """Local assumes list is merged with extension assumes.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + 'assumes': ['juju >= 3.1'], + }, + ) as charm: + spec = _CharmSpec.autoload(charm) + assumes = spec.meta['assumes'] + assert 'k8s-api' in assumes + assert 'juju >= 3.1' in assumes + + def test_django_extension_has_create_superuser(self, tmp_path): + """Django extension adds the create-superuser action.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-django', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['django-framework'], + }, + ) as charm: + spec = _CharmSpec.autoload(charm) + assert 'create-superuser' in spec.actions + assert 'rotate-secret-key' in spec.actions + + def test_unknown_extension_warns(self, tmp_path, caplog): + """An unknown extension name logs a warning and is skipped.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-app', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['nonexistent-extension'], + }, + ) as charm: + spec = _CharmSpec.autoload(charm) + assert 'Unknown charmcraft extension' in caplog.text + # no extension data merged, but charm still loads + assert spec.meta['name'] == 'my-app' + + def test_extension_stripped_from_meta(self, tmp_path): + """The 'extensions' key should not remain in the loaded meta.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + }, + ) as charm: + spec = _CharmSpec.autoload(charm) + assert 'extensions' not in spec.meta + + def test_extension_with_relations_in_context(self, tmp_path): + """Relations from an extension can be used in a Context run.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + }, + ) as charm: + ctx = Context(charm) + ctx.run( + ctx.on.start(), + State(relations={Relation('ingress')}), + ) From 7fb5c239034111f337ff725613c58a24f4951215 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Fri, 6 Mar 2026 18:21:50 +1300 Subject: [PATCH 02/19] Load extension list from charmcraft. --- .github/generate_charmcraft_extensions.py | 31 ++++++++--- .../src/scenario/_charmcraft_extensions.py | 55 ++++++++++++++++++- 2 files changed, 76 insertions(+), 10 deletions(-) diff --git a/.github/generate_charmcraft_extensions.py b/.github/generate_charmcraft_extensions.py index 9604bd597..bb5b3dbf5 100755 --- a/.github/generate_charmcraft_extensions.py +++ b/.github/generate_charmcraft_extensions.py @@ -28,14 +28,6 @@ import yaml -PROFILES = [ - 'django-framework', - 'fastapi-framework', - 'flask-framework', - 'go-framework', - 'spring-boot-framework', -] - # Keys from the expanded YAML that belong to "metadata" (the charm's metadata.yaml equivalent). METADATA_KEYS = { 'assumes', @@ -69,6 +61,24 @@ ) +def get_extensions() -> list[str]: + """Get the list of available charmcraft extensions via ``charmcraft list-extensions``.""" + result = subprocess.run( + ['charmcraft', 'list-extensions'], + check=True, + capture_output=True, + text=True, + ) + extensions = [] + for line in result.stdout.splitlines(): + # Skip header and separator lines. + if not line or line.startswith('Extension') or line.startswith('---'): + continue + name = line.split()[0] + extensions.append(name) + return sorted(extensions) + + def run_charmcraft(profile: str, workdir: pathlib.Path) -> dict: """Run charmcraft init + expand-extensions and return the expanded YAML as a dict.""" subprocess.run( @@ -165,9 +175,12 @@ def generate_module(all_data: dict[str, tuple[dict, dict, dict]]) -> str: def main() -> int: # noqa: D103 + extensions = get_extensions() + print(f'Found extensions: {", ".join(extensions)}') + all_data: dict[str, tuple[dict, dict, dict]] = {} - for profile in PROFILES: + for profile in extensions: print(f'Processing {profile}...') with tempfile.TemporaryDirectory() as tmpdir: expanded = run_charmcraft(profile, pathlib.Path(tmpdir)) diff --git a/testing/src/scenario/_charmcraft_extensions.py b/testing/src/scenario/_charmcraft_extensions.py index 9d7130e70..3a67008cf 100644 --- a/testing/src/scenario/_charmcraft_extensions.py +++ b/testing/src/scenario/_charmcraft_extensions.py @@ -1,4 +1,4 @@ -# Copyright 2024 Canonical Ltd. +# Copyright 2026 Canonical Ltd. # See LICENSE file for licensing details. """Charmcraft extension metadata, config, and actions. @@ -26,6 +26,22 @@ 'django-app-image': {'description': 'django application image.', 'type': 'oci-image'} }, }, + 'expressjs-framework': { + 'assumes': ['k8s-api'], + 'containers': {'app': {'resource': 'app-image'}}, + 'peers': {'secret-storage': {'interface': 'secret-storage'}}, + 'provides': { + 'grafana-dashboard': {'interface': 'grafana_dashboard'}, + 'metrics-endpoint': {'interface': 'prometheus_scrape'}, + }, + 'requires': { + 'ingress': {'interface': 'ingress', 'limit': 1}, + 'logging': {'interface': 'loki_push_api'}, + }, + 'resources': { + 'app-image': {'description': 'expressjs application image.', 'type': 'oci-image'} + }, + }, 'fastapi-framework': { 'assumes': ['k8s-api'], 'containers': {'app': {'resource': 'app-image'}}, @@ -141,6 +157,37 @@ 'type': 'int', }, }, + 'expressjs-framework': { + 'app-port': { + 'default': 8080, + 'description': 'Default port where the application will listen on.', + 'type': 'int', + }, + 'app-secret-key': { + 'description': 'Long secret you can use for sessions, csrf or any other thing where you need a ' + 'random secret shared by all units', + 'type': 'string', + }, + 'app-secret-key-id': { + 'description': 'This configuration is similar to `app-secret-key`, but instead accepts a Juju ' + 'user secret ID. The secret should contain a single key, "value", which maps to ' + 'the actual application secret key. To create the secret, run the following ' + 'command: `juju add-secret my-app-secret-key value= && juju ' + 'grant-secret my-app-secret-key my-app`, and use the output secret ID to configure ' + 'this option.', + 'type': 'secret', + }, + 'metrics-path': { + 'default': '/metrics', + 'description': 'Path where the prometheus metrics will be scraped.', + 'type': 'string', + }, + 'metrics-port': { + 'default': 8080, + 'description': 'Port where the prometheus metrics will be scraped.', + 'type': 'int', + }, + }, 'fastapi-framework': { 'app-secret-key': { 'description': 'Long secret you can use for sessions, csrf or any other thing where you need a ' @@ -335,6 +382,12 @@ 'if a security breach occurs.' }, }, + 'expressjs-framework': { + 'rotate-secret-key': { + 'description': 'Rotate the secret key. Users will be forced to log in again. This might be useful ' + 'if a security breach occurs.' + }, + }, 'fastapi-framework': { 'rotate-secret-key': { 'description': 'Rotate the secret key. Users will be forced to log in again. This might be useful ' From 1ea90f8ad55f2f6537816512fdf72860ef5ab793 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Fri, 6 Mar 2026 18:27:30 +1300 Subject: [PATCH 03/19] Improve the type annotations. --- .github/generate_charmcraft_extensions.py | 31 ++++++++++++------- .../src/scenario/_charmcraft_extensions.py | 8 +++-- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/.github/generate_charmcraft_extensions.py b/.github/generate_charmcraft_extensions.py index bb5b3dbf5..f1d3b6b5a 100755 --- a/.github/generate_charmcraft_extensions.py +++ b/.github/generate_charmcraft_extensions.py @@ -25,6 +25,7 @@ import pprint import subprocess import tempfile +from typing import Any import yaml @@ -79,7 +80,7 @@ def get_extensions() -> list[str]: return sorted(extensions) -def run_charmcraft(profile: str, workdir: pathlib.Path) -> dict: +def run_charmcraft(profile: str, workdir: pathlib.Path) -> dict[str, Any]: """Run charmcraft init + expand-extensions and return the expanded YAML as a dict.""" subprocess.run( ['charmcraft', 'init', '--profile', profile, '--name', 'test-charm'], @@ -98,23 +99,25 @@ def run_charmcraft(profile: str, workdir: pathlib.Path) -> dict: return yaml.safe_load(result.stdout) -def extract_extension_data(expanded: dict) -> tuple[dict, dict, dict]: +def extract_extension_data( + expanded: dict[str, Any], +) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: """Extract metadata, config options, and actions from expanded charmcraft YAML.""" - metadata = {} + metadata: dict[str, Any] = {} for key in METADATA_KEYS: if key in expanded: metadata[key] = expanded[key] - config = {} + config: dict[str, Any] = {} if 'config' in expanded and 'options' in expanded['config']: config = expanded['config']['options'] - actions = expanded.get('actions', {}) + actions: dict[str, Any] = expanded.get('actions', {}) return metadata, config, actions -def format_dict(d: dict, indent: int = 4) -> str: +def format_dict(d: dict[str, Any], indent: int = 4) -> str: """Format a dict as a Python literal string.""" formatted = pprint.pformat(d, width=100, sort_dicts=True) if '\n' in formatted: @@ -126,7 +129,9 @@ def format_dict(d: dict, indent: int = 4) -> str: return formatted -def generate_module(all_data: dict[str, tuple[dict, dict, dict]]) -> str: +def generate_module( + all_data: dict[str, tuple[dict[str, Any], dict[str, Any], dict[str, Any]]], +) -> str: """Generate the Python module source code.""" lines = [ '# Copyright 2026 Canonical Ltd.', @@ -139,12 +144,14 @@ def generate_module(all_data: dict[str, tuple[dict, dict, dict]]) -> str: '', 'from __future__ import annotations', '', + 'from typing import Any', + '', ] # Build the three top-level dicts. - metadata_entries = {} - config_entries = {} - action_entries = {} + metadata_entries: dict[str, dict[str, Any]] = {} + config_entries: dict[str, dict[str, Any]] = {} + action_entries: dict[str, dict[str, Any]] = {} for profile, (metadata, config, actions) in sorted(all_data.items()): metadata_entries[profile] = metadata config_entries[profile] = config @@ -160,7 +167,7 @@ def generate_module(all_data: dict[str, tuple[dict, dict, dict]]) -> str: ('ACTIONS', action_entries, 'Actions added by each charmcraft extension.'), ]: lines.append(f'# {docstring}') - lines.append(f'{var_name}: dict[str, dict] = {{') + lines.append(f'{var_name}: dict[str, dict[str, Any]] = {{') for profile in sorted(data): lines.append(f' {profile!r}: {{') for key in sorted(data[profile]): @@ -178,7 +185,7 @@ def main() -> int: # noqa: D103 extensions = get_extensions() print(f'Found extensions: {", ".join(extensions)}') - all_data: dict[str, tuple[dict, dict, dict]] = {} + all_data: dict[str, tuple[dict[str, Any], dict[str, Any], dict[str, Any]]] = {} for profile in extensions: print(f'Processing {profile}...') diff --git a/testing/src/scenario/_charmcraft_extensions.py b/testing/src/scenario/_charmcraft_extensions.py index 3a67008cf..ad0d05e1b 100644 --- a/testing/src/scenario/_charmcraft_extensions.py +++ b/testing/src/scenario/_charmcraft_extensions.py @@ -8,8 +8,10 @@ from __future__ import annotations +from typing import Any + # Metadata added by each charmcraft extension. -METADATA: dict[str, dict] = { +METADATA: dict[str, dict[str, Any]] = { 'django-framework': { 'assumes': ['k8s-api'], 'containers': {'django-app': {'resource': 'django-app-image'}}, @@ -107,7 +109,7 @@ } # Config options added by each charmcraft extension. -CONFIG: dict[str, dict] = { +CONFIG: dict[str, dict[str, Any]] = { 'django-framework': { 'django-allowed-hosts': { 'description': 'A comma-separated list of host/domain names that this Django site can serve. This ' @@ -370,7 +372,7 @@ } # Actions added by each charmcraft extension. -ACTIONS: dict[str, dict] = { +ACTIONS: dict[str, dict[str, Any]] = { 'django-framework': { 'create-superuser': { 'description': 'Create a new Django superuser account.', From ff2140ade56bdb81e54f46ed2c33d79bfd01f021 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Fri, 6 Mar 2026 18:33:00 +1300 Subject: [PATCH 04/19] Don't lazy import. --- testing/src/scenario/state.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/testing/src/scenario/state.py b/testing/src/scenario/state.py index b598b65c1..e57fca205 100644 --- a/testing/src/scenario/state.py +++ b/testing/src/scenario/state.py @@ -35,6 +35,7 @@ from ops import CloudCredential as CloudCredential_Ops from ops import CloudSpec as CloudSpec_Ops +from . import _charmcraft_extensions from .errors import MetadataNotFoundError, StateValidationError from .logger import logger as scenario_logger @@ -1870,8 +1871,6 @@ def _apply_extensions( values are merged on top, simulating what ``charmcraft expand-extensions`` does. """ - from . import _charmcraft_extensions - for ext_name in extensions: ext_meta = _charmcraft_extensions.METADATA.get(ext_name, {}) ext_config = _charmcraft_extensions.CONFIG.get(ext_name, {}) From 04a296d2d63516f0d319ff69797b495ff68cc283 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Fri, 6 Mar 2026 18:34:44 +1300 Subject: [PATCH 05/19] Make the warning more user-focused. --- testing/src/scenario/state.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/testing/src/scenario/state.py b/testing/src/scenario/state.py index e57fca205..cb8b6b770 100644 --- a/testing/src/scenario/state.py +++ b/testing/src/scenario/state.py @@ -1879,8 +1879,7 @@ def _apply_extensions( if not ext_meta and not ext_config and not ext_actions: logger.warning( f'Unknown charmcraft extension {ext_name!r}; ' - f'ignoring. You may need to regenerate ' - f'_charmcraft_extensions.py.', + f'ignoring. You may need to updste to a newer ops.' ) continue From 85bc5246ac2418724c4901589df81475202d7f50 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Fri, 6 Mar 2026 18:35:52 +1300 Subject: [PATCH 06/19] Make the warning more user-focused. --- testing/src/scenario/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/src/scenario/state.py b/testing/src/scenario/state.py index cb8b6b770..1ffdebdd6 100644 --- a/testing/src/scenario/state.py +++ b/testing/src/scenario/state.py @@ -1879,7 +1879,7 @@ def _apply_extensions( if not ext_meta and not ext_config and not ext_actions: logger.warning( f'Unknown charmcraft extension {ext_name!r}; ' - f'ignoring. You may need to updste to a newer ops.' + f'ignoring. You may need to update to a newer ops.' ) continue From 48359d66849756ee18761a09d3982f0f5fbe6d94 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Fri, 6 Mar 2026 18:39:46 +1300 Subject: [PATCH 07/19] Remove unnecessary skip. --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0293b0886..9a22d388c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -228,7 +228,6 @@ exclude = ["tracing/ops_tracing/vendor/*"] "D", ] "testing/src/scenario/_charmcraft_extensions.py" = [ - "CPY001", # Auto-generated file. "E501", # Auto-generated file; long description strings are unavoidable. ] ".github/generate_charmcraft_extensions.py" = [ From 617728362b6404c204fe03c89a48281b6f4413b5 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Fri, 6 Mar 2026 18:46:36 +1300 Subject: [PATCH 08/19] Minor cleanup. --- testing/src/scenario/state.py | 6 ++++-- testing/tests/test_charm_spec_autoload.py | 24 +++++++++++------------ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/testing/src/scenario/state.py b/testing/src/scenario/state.py index 1ffdebdd6..ffb202721 100644 --- a/testing/src/scenario/state.py +++ b/testing/src/scenario/state.py @@ -13,6 +13,7 @@ import random import re import string +import warnings from collections.abc import Callable, Iterable, Mapping, Sequence from enum import Enum from itertools import chain @@ -1877,9 +1878,10 @@ def _apply_extensions( ext_actions = _charmcraft_extensions.ACTIONS.get(ext_name, {}) if not ext_meta and not ext_config and not ext_actions: - logger.warning( + warnings.warn( f'Unknown charmcraft extension {ext_name!r}; ' - f'ignoring. You may need to update to a newer ops.' + f'ignoring. You may need to update to a newer ops.', + stacklevel=2, ) continue diff --git a/testing/tests/test_charm_spec_autoload.py b/testing/tests/test_charm_spec_autoload.py index 79dabfca9..9d32b3431 100644 --- a/testing/tests/test_charm_spec_autoload.py +++ b/testing/tests/test_charm_spec_autoload.py @@ -240,12 +240,12 @@ def test_local_meta_overrides_extension(self, tmp_path): }, ) as charm: spec = _CharmSpec.autoload(charm) - # local override wins + # Local override wins. assert spec.meta['requires']['ingress'] == { 'interface': 'custom-ingress', 'limit': 5, } - # extension-only entries still present + # Extension-only entries are still present. assert 'logging' in spec.meta['requires'] def test_local_config_overrides_extension(self, tmp_path): @@ -268,11 +268,11 @@ def test_local_config_overrides_extension(self, tmp_path): ) as charm: spec = _CharmSpec.autoload(charm) options = spec.config['options'] - # local override wins + # Local override wins. assert options['flask-debug'] == {'type': 'boolean', 'default': True} - # local-only option present + # Local-only option is present. assert 'my-custom-option' in options - # extension-only options still present + # Extension-only options are still present. assert 'webserver-workers' in options def test_local_actions_override_extension(self, tmp_path): @@ -292,11 +292,11 @@ def test_local_actions_override_extension(self, tmp_path): }, ) as charm: spec = _CharmSpec.autoload(charm) - # local override wins + # Local override wins. assert spec.actions['rotate-secret-key'] == { 'description': 'custom rotate', } - # local-only action present + # Local-only action is present. assert 'my-action' in spec.actions def test_local_assumes_merged_with_extension(self, tmp_path): @@ -333,8 +333,8 @@ def test_django_extension_has_create_superuser(self, tmp_path): assert 'create-superuser' in spec.actions assert 'rotate-secret-key' in spec.actions - def test_unknown_extension_warns(self, tmp_path, caplog): - """An unknown extension name logs a warning and is skipped.""" + def test_unknown_extension_warns(self, tmp_path): + """An unknown extension name emits a warning and is skipped.""" with create_tempcharm( tmp_path, meta={ @@ -345,9 +345,9 @@ def test_unknown_extension_warns(self, tmp_path, caplog): 'extensions': ['nonexistent-extension'], }, ) as charm: - spec = _CharmSpec.autoload(charm) - assert 'Unknown charmcraft extension' in caplog.text - # no extension data merged, but charm still loads + with pytest.warns(UserWarning, match='Unknown charmcraft extension'): + spec = _CharmSpec.autoload(charm) + # No extension data merged, but the charm still loads. assert spec.meta['name'] == 'my-app' def test_extension_stripped_from_meta(self, tmp_path): From 1f46d6b0eca7a138e1886cb6b0f1f818af6495b3 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Fri, 6 Mar 2026 19:59:18 +1300 Subject: [PATCH 09/19] Also update the docs. --- docs/explanation/state-transition-testing.md | 25 ++++++++++++++++++++ docs/howto/write-unit-tests-for-a-charm.md | 7 ++++++ testing/src/scenario/context.py | 3 +++ testing/src/scenario/state.py | 8 ++++++- 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/docs/explanation/state-transition-testing.md b/docs/explanation/state-transition-testing.md index f1e6a8d51..d26a0eb31 100644 --- a/docs/explanation/state-transition-testing.md +++ b/docs/explanation/state-transition-testing.md @@ -231,6 +231,31 @@ ctx = testing.Context( state = ctx.run(ctx.on.start(), testing.State()) ``` +## Charmcraft extensions + +If your `charmcraft.yaml` uses a charmcraft extension such as `flask-framework` +or `django-framework`, the testing framework will automatically expand it when +autoloading metadata. The extension's metadata (containers, relations, resources, +etc.), config options, and actions are merged into the charm spec, simulating what +`charmcraft expand-extensions` does at pack time. + +This means you can write tests against a charm that uses extensions without having +to manually specify all the metadata the extension adds: + +```python +# Given a charmcraft.yaml with: +# extensions: +# - flask-framework + +ctx = testing.Context(MyFlaskCharm) +# The 'ingress' relation is provided by the flask-framework extension. +state = ctx.run(ctx.on.start(), testing.State(relations={testing.Relation('ingress')})) +``` + +Local values in `charmcraft.yaml` take precedence over extension defaults. For +example, if you define a custom `ingress` relation in your `charmcraft.yaml`, it +will override the one provided by the extension. + ## Immutability All of the data structures in the state, (`State`, `Relation`, `Container`, and diff --git a/docs/howto/write-unit-tests-for-a-charm.md b/docs/howto/write-unit-tests-for-a-charm.md index bb26ca6e6..f47968c9d 100644 --- a/docs/howto/write-unit-tests-for-a-charm.md +++ b/docs/howto/write-unit-tests-for-a-charm.md @@ -122,6 +122,13 @@ def test_peer_changed(): > See more: [](ops.testing.State.from_context) +```{note} +If your `charmcraft.yaml` uses a charmcraft extension (e.g. +`extensions: [flask-framework]`), the metadata, config, and actions that +the extension adds are automatically merged in when the testing framework +loads the charm spec. You do not need to manually specify them. +``` + ## Mock beyond the State If you wish to use the framework to test an existing charm type, you will probably need to mock out certain calls that are not covered by the `State` data structure. In that case, you will have to manually mock, patch or otherwise simulate those calls. diff --git a/testing/src/scenario/context.py b/testing/src/scenario/context.py index 15ca8259d..d9c65c518 100644 --- a/testing/src/scenario/context.py +++ b/testing/src/scenario/context.py @@ -655,6 +655,9 @@ def __init__( :arg charm_type: the :class:`ops.CharmBase` subclass to handle the event. :arg meta: charm metadata to use. Needs to be a valid metadata.yaml format (as a dict). If none is provided, we will search for a ``metadata.yaml`` file in the charm root. + If the ``charmcraft.yaml`` contains an ``extensions`` key (e.g. + ``extensions: [flask-framework]``), the extension's metadata, + config, and actions will be automatically merged in. :arg actions: charm actions to use. Needs to be a valid actions.yaml format (as a dict). If none is provided, we will search for a ``actions.yaml`` file in the charm root. :arg config: charm config to use. Needs to be a valid config.yaml format (as a dict). diff --git a/testing/src/scenario/state.py b/testing/src/scenario/state.py index ffb202721..b92a27a8d 100644 --- a/testing/src/scenario/state.py +++ b/testing/src/scenario/state.py @@ -1951,7 +1951,13 @@ def _load_metadata_legacy(charm_root: pathlib.Path): @staticmethod def _load_metadata(charm_root: pathlib.Path): - """Load metadata from charm projects created with Charmcraft >= 2.5.""" + """Load metadata from charm projects created with Charmcraft >= 2.5. + + If the ``charmcraft.yaml`` contains an ``extensions`` key (e.g. + ``extensions: [flask-framework]``), the extension's metadata, config, + and actions are merged in before the values are returned, simulating + what ``charmcraft expand-extensions`` does at pack time. + """ metadata_path = charm_root / 'charmcraft.yaml' meta: dict[str, Any] = ( yaml.safe_load(metadata_path.open()) if metadata_path.exists() else {} From 572a70f7d9789fabfb671e70a202fd79dbe94ecd Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Fri, 6 Mar 2026 20:01:25 +1300 Subject: [PATCH 10/19] Avoid 'autoload'. --- docs/explanation/state-transition-testing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/explanation/state-transition-testing.md b/docs/explanation/state-transition-testing.md index d26a0eb31..5b9bb997c 100644 --- a/docs/explanation/state-transition-testing.md +++ b/docs/explanation/state-transition-testing.md @@ -235,8 +235,8 @@ state = ctx.run(ctx.on.start(), testing.State()) If your `charmcraft.yaml` uses a charmcraft extension such as `flask-framework` or `django-framework`, the testing framework will automatically expand it when -autoloading metadata. The extension's metadata (containers, relations, resources, -etc.), config options, and actions are merged into the charm spec, simulating what +automatically loading metadata. The extension's metadata (containers, relations, resources, +and so on), config options, and actions are merged into the charm spec, simulating what `charmcraft expand-extensions` does at pack time. This means you can write tests against a charm that uses extensions without having From 44103c0a821a319792c0cdbf6fe5768433ecd4c0 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 10 Mar 2026 13:20:53 +1300 Subject: [PATCH 11/19] Apply suggestion from @james-garner-canonical Co-authored-by: James Garner --- .github/generate_charmcraft_extensions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/generate_charmcraft_extensions.py b/.github/generate_charmcraft_extensions.py index f1d3b6b5a..e05627d0d 100755 --- a/.github/generate_charmcraft_extensions.py +++ b/.github/generate_charmcraft_extensions.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env -S uv run --script --no-project # /// script # requires-python = ">=3.10" # dependencies = ["pyyaml"] From 88853be96e8cf8728ae1b31659758a421e6afa11 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 10 Mar 2026 13:24:18 +1300 Subject: [PATCH 12/19] Suggestion from review. --- .github/generate_charmcraft_extensions.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/generate_charmcraft_extensions.py b/.github/generate_charmcraft_extensions.py index e05627d0d..9dd9606ac 100755 --- a/.github/generate_charmcraft_extensions.py +++ b/.github/generate_charmcraft_extensions.py @@ -82,21 +82,19 @@ def get_extensions() -> list[str]: def run_charmcraft(profile: str, workdir: pathlib.Path) -> dict[str, Any]: """Run charmcraft init + expand-extensions and return the expanded YAML as a dict.""" - subprocess.run( + subprocess.check_call( ['charmcraft', 'init', '--profile', profile, '--name', 'test-charm'], cwd=workdir, - check=True, - capture_output=True, - text=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, ) - result = subprocess.run( + output = subprocess.check_output( ['charmcraft', 'expand-extensions'], cwd=workdir, - check=True, - capture_output=True, text=True, + stderr=subprocess.DEVNULL, ) - return yaml.safe_load(result.stdout) + return yaml.safe_load(output) def extract_extension_data( From c064515ca39ab486501b38eee5c8948de2094c29 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 10 Mar 2026 13:26:25 +1300 Subject: [PATCH 13/19] Apply suggestions from code review Co-authored-by: James Garner --- .github/generate_charmcraft_extensions.py | 27 +++++++---------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/.github/generate_charmcraft_extensions.py b/.github/generate_charmcraft_extensions.py index 9dd9606ac..371abfacd 100755 --- a/.github/generate_charmcraft_extensions.py +++ b/.github/generate_charmcraft_extensions.py @@ -101,15 +101,8 @@ def extract_extension_data( expanded: dict[str, Any], ) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: """Extract metadata, config options, and actions from expanded charmcraft YAML.""" - metadata: dict[str, Any] = {} - for key in METADATA_KEYS: - if key in expanded: - metadata[key] = expanded[key] - - config: dict[str, Any] = {} - if 'config' in expanded and 'options' in expanded['config']: - config = expanded['config']['options'] - + metadata = {key: expanded[key] for key in METADATA_KEYS if key in expanded} + config = expanded.get('config', {}).get('options', {}) actions: dict[str, Any] = expanded.get('actions', {}) return metadata, config, actions @@ -155,16 +148,12 @@ def generate_module( config_entries[profile] = config action_entries[profile] = actions - for var_name, data, docstring in [ - ( - 'METADATA', - metadata_entries, - 'Metadata added by each charmcraft extension.', - ), - ('CONFIG', config_entries, 'Config options added by each charmcraft extension.'), - ('ACTIONS', action_entries, 'Actions added by each charmcraft extension.'), + for var_name, data, doc_name in [ + ('METADATA', metadata_entries, 'Metadata'), + ('CONFIG', config_entries, 'Config options'), + ('ACTIONS', action_entries, 'Actions'), ]: - lines.append(f'# {docstring}') + lines.append(f'# {doc_name} added by each charmcraft extension.') lines.append(f'{var_name}: dict[str, dict[str, Any]] = {{') for profile in sorted(data): lines.append(f' {profile!r}: {{') @@ -213,4 +202,4 @@ def main() -> int: # noqa: D103 if __name__ == '__main__': - raise SystemExit(main()) + sys.exit(main()) From 174601ac32048d25a49f2b047d44268420d7da16 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 10 Mar 2026 13:29:36 +1300 Subject: [PATCH 14/19] Apply suggestion from @james-garner-canonical Co-authored-by: James Garner --- testing/src/scenario/state.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/testing/src/scenario/state.py b/testing/src/scenario/state.py index b92a27a8d..38c6b3039 100644 --- a/testing/src/scenario/state.py +++ b/testing/src/scenario/state.py @@ -1862,10 +1862,7 @@ def _is_valid_charmcraft_25_metadata(meta: dict[str, Any]): return True -def _apply_extensions( - meta: dict[str, Any], - extensions: list[str], -) -> dict[str, Any]: +def _apply_extensions(meta: dict[str, Any], extensions: list[str]) -> dict[str, Any]: """Merge charmcraft extension defaults into the charm metadata. Extension defaults are applied first, then the local charmcraft.yaml From c95768b0097b983b5ebd8b00eacd35bfa606dc01 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 10 Mar 2026 16:10:53 +1300 Subject: [PATCH 15/19] More closely match the charmcraft behaviour of clashing values. --- docs/explanation/state-transition-testing.md | 7 +- docs/howto/write-unit-tests-for-a-charm.md | 5 ++ testing/src/scenario/state.py | 34 +++++++- testing/tests/test_charm_spec_autoload.py | 89 ++++++++++++++++---- 4 files changed, 110 insertions(+), 25 deletions(-) diff --git a/docs/explanation/state-transition-testing.md b/docs/explanation/state-transition-testing.md index 5b9bb997c..df6dbaf4d 100644 --- a/docs/explanation/state-transition-testing.md +++ b/docs/explanation/state-transition-testing.md @@ -252,9 +252,10 @@ ctx = testing.Context(MyFlaskCharm) state = ctx.run(ctx.on.start(), testing.State(relations={testing.Relation('ingress')})) ``` -Local values in `charmcraft.yaml` take precedence over extension defaults. For -example, if you define a custom `ingress` relation in your `charmcraft.yaml`, it -will override the one provided by the extension. +If your `charmcraft.yaml` defines keys that overlap with what the extension +provides (for example, a config option or relation with the same name), the +testing framework will raise a `ValueError`, matching the behaviour of +`charmcraft pack`. Rename or remove the overlapping keys to fix this. ## Immutability diff --git a/docs/howto/write-unit-tests-for-a-charm.md b/docs/howto/write-unit-tests-for-a-charm.md index f47968c9d..9b0013dda 100644 --- a/docs/howto/write-unit-tests-for-a-charm.md +++ b/docs/howto/write-unit-tests-for-a-charm.md @@ -127,6 +127,11 @@ If your `charmcraft.yaml` uses a charmcraft extension (e.g. `extensions: [flask-framework]`), the metadata, config, and actions that the extension adds are automatically merged in when the testing framework loads the charm spec. You do not need to manually specify them. + +If your `charmcraft.yaml` defines keys that overlap with what the extension +provides (e.g. a config option with the same name), the testing framework +will raise a `ValueError`, matching the behaviour of `charmcraft pack`. +Rename or remove the overlapping keys to fix this. ``` ## Mock beyond the State diff --git a/testing/src/scenario/state.py b/testing/src/scenario/state.py index 38c6b3039..4257000af 100644 --- a/testing/src/scenario/state.py +++ b/testing/src/scenario/state.py @@ -1868,6 +1868,11 @@ def _apply_extensions(meta: dict[str, Any], extensions: list[str]) -> dict[str, Extension defaults are applied first, then the local charmcraft.yaml values are merged on top, simulating what ``charmcraft expand-extensions`` does. + + Raises: + ValueError: if the local charmcraft.yaml defines keys that overlap + with what the extension provides (matching ``charmcraft pack`` + behaviour). """ for ext_name in extensions: ext_meta = _charmcraft_extensions.METADATA.get(ext_name, {}) @@ -1882,12 +1887,19 @@ def _apply_extensions(meta: dict[str, Any], extensions: list[str]) -> dict[str, ) continue - # Merge metadata: for dicts, extension provides defaults that - # the local yaml overrides. For lists, combine them. + # Merge metadata: for dicts, error on overlapping keys + # (matching charmcraft behaviour). For lists, combine them. for key, ext_value in ext_meta.items(): if key not in meta: meta[key] = copy.deepcopy(ext_value) elif isinstance(ext_value, dict) and isinstance(meta[key], dict): + overlap = set(ext_value) & set(meta[key]) + if overlap: + raise ValueError( + f'overlapping keys {overlap} in {key} of ' + f'charmcraft.yaml which conflict with the ' + f'{ext_name} extension, please rename or remove them' + ) merged = copy.deepcopy(ext_value) merged.update(meta[key]) meta[key] = merged @@ -1898,17 +1910,31 @@ def _apply_extensions(meta: dict[str, Any], extensions: list[str]) -> dict[str, merged.append(item) meta[key] = merged - # Merge config options. + # Merge config options; error on overlapping keys. if ext_config: local_config = meta.get('config', {}) local_options = local_config.get('options', {}) + overlap = set(ext_config) & set(local_options) + if overlap: + raise ValueError( + f'overlapping keys {overlap} in config.options of ' + f'charmcraft.yaml which conflict with the ' + f'{ext_name} extension, please rename or remove them' + ) merged_options = copy.deepcopy(ext_config) merged_options.update(local_options) meta['config'] = {'options': merged_options} - # Merge actions. + # Merge actions; error on overlapping keys. if ext_actions: local_actions = meta.get('actions', {}) + overlap = set(ext_actions) & set(local_actions) + if overlap: + raise ValueError( + f'overlapping keys {overlap} in actions of ' + f'charmcraft.yaml which conflict with the ' + f'{ext_name} extension, please rename or remove them' + ) merged_actions = copy.deepcopy(ext_actions) merged_actions.update(local_actions) meta['actions'] = merged_actions diff --git a/testing/tests/test_charm_spec_autoload.py b/testing/tests/test_charm_spec_autoload.py index 9d32b3431..269631f05 100644 --- a/testing/tests/test_charm_spec_autoload.py +++ b/testing/tests/test_charm_spec_autoload.py @@ -224,8 +224,8 @@ def test_extension_adds_actions(self, tmp_path): assert spec.actions is not None assert 'rotate-secret-key' in spec.actions - def test_local_meta_overrides_extension(self, tmp_path): - """Local charmcraft.yaml values take precedence over extension defaults.""" + def test_local_meta_overlaps_extension_errors(self, tmp_path): + """Overlapping metadata keys with extension cause an error.""" with create_tempcharm( tmp_path, meta={ @@ -238,18 +238,34 @@ def test_local_meta_overrides_extension(self, tmp_path): 'ingress': {'interface': 'custom-ingress', 'limit': 5}, }, }, + ) as charm: + with pytest.raises(ValueError, match=r'overlapping keys.*requires.*flask-framework'): + _CharmSpec.autoload(charm) + + def test_local_meta_no_overlap_with_extension(self, tmp_path): + """Non-overlapping local metadata keys merge with extension.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + 'requires': { + 'my-custom-relation': {'interface': 'custom'}, + }, + }, ) as charm: spec = _CharmSpec.autoload(charm) - # Local override wins. - assert spec.meta['requires']['ingress'] == { - 'interface': 'custom-ingress', - 'limit': 5, - } - # Extension-only entries are still present. + # Local-only entry is present. + assert 'my-custom-relation' in spec.meta['requires'] + # Extension entries are still present. + assert 'ingress' in spec.meta['requires'] assert 'logging' in spec.meta['requires'] - def test_local_config_overrides_extension(self, tmp_path): - """Local config options override extension config defaults.""" + def test_local_config_overlaps_extension_errors(self, tmp_path): + """Overlapping config options with extension cause an error.""" with create_tempcharm( tmp_path, meta={ @@ -265,18 +281,39 @@ def test_local_config_overrides_extension(self, tmp_path): 'my-custom-option': {'type': 'string'}, }, }, + ) as charm: + with pytest.raises( + ValueError, match=r'overlapping keys.*config\.options.*flask-framework' + ): + _CharmSpec.autoload(charm) + + def test_local_config_no_overlap_with_extension(self, tmp_path): + """Non-overlapping local config options merge with extension.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + }, + config={ + 'options': { + 'my-custom-option': {'type': 'string'}, + }, + }, ) as charm: spec = _CharmSpec.autoload(charm) options = spec.config['options'] - # Local override wins. - assert options['flask-debug'] == {'type': 'boolean', 'default': True} # Local-only option is present. assert 'my-custom-option' in options # Extension-only options are still present. + assert 'flask-debug' in options assert 'webserver-workers' in options - def test_local_actions_override_extension(self, tmp_path): - """Local actions override extension action defaults.""" + def test_local_actions_overlap_extension_errors(self, tmp_path): + """Overlapping actions with extension cause an error.""" with create_tempcharm( tmp_path, meta={ @@ -290,14 +327,30 @@ def test_local_actions_override_extension(self, tmp_path): 'rotate-secret-key': {'description': 'custom rotate'}, 'my-action': {'description': 'custom action'}, }, + ) as charm: + with pytest.raises(ValueError, match=r'overlapping keys.*actions.*flask-framework'): + _CharmSpec.autoload(charm) + + def test_local_actions_no_overlap_with_extension(self, tmp_path): + """Non-overlapping local actions merge with extension.""" + with create_tempcharm( + tmp_path, + meta={ + 'type': 'charm', + 'name': 'my-flask', + 'summary': 'foo', + 'description': 'foo', + 'extensions': ['flask-framework'], + }, + actions={ + 'my-action': {'description': 'custom action'}, + }, ) as charm: spec = _CharmSpec.autoload(charm) - # Local override wins. - assert spec.actions['rotate-secret-key'] == { - 'description': 'custom rotate', - } # Local-only action is present. assert 'my-action' in spec.actions + # Extension action is still present. + assert 'rotate-secret-key' in spec.actions def test_local_assumes_merged_with_extension(self, tmp_path): """Local assumes list is merged with extension assumes.""" From b173a9084867101cf3325f5cf5dd9287014e28aa Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 10 Mar 2026 18:36:08 +1300 Subject: [PATCH 16/19] Address review comments. --- .github/generate_charmcraft_extensions.py | 61 +++++++------------ docs/explanation/state-transition-testing.md | 6 +- .../src/scenario/_charmcraft_extensions.py | 14 ++++- testing/src/scenario/state.py | 16 ++--- 4 files changed, 43 insertions(+), 54 deletions(-) diff --git a/.github/generate_charmcraft_extensions.py b/.github/generate_charmcraft_extensions.py index 371abfacd..4c4e97558 100755 --- a/.github/generate_charmcraft_extensions.py +++ b/.github/generate_charmcraft_extensions.py @@ -22,8 +22,8 @@ from __future__ import annotations import pathlib -import pprint import subprocess +import sys import tempfile from typing import Any @@ -108,18 +108,6 @@ def extract_extension_data( return metadata, config, actions -def format_dict(d: dict[str, Any], indent: int = 4) -> str: - """Format a dict as a Python literal string.""" - formatted = pprint.pformat(d, width=100, sort_dicts=True) - if '\n' in formatted: - lines = formatted.splitlines() - result = lines[0] - for line in lines[1:]: - result += '\n' + ' ' * indent + line - return result - return formatted - - def generate_module( all_data: dict[str, tuple[dict[str, Any], dict[str, Any], dict[str, Any]]], ) -> str: @@ -135,34 +123,36 @@ def generate_module( '', 'from __future__ import annotations', '', - 'from typing import Any', + 'from typing import Any, TypedDict', + '', + '', + 'class _ExtensionMetadata(TypedDict, total=False):', + ' assumes: list[str]', + ' containers: dict[str, Any]', + ' peers: dict[str, Any]', + ' provides: dict[str, Any]', + ' requires: dict[str, Any]', + ' resources: dict[str, Any]', + '', '', ] - # Build the three top-level dicts. + # Build the three top-level dicts, pre-sorting keys. metadata_entries: dict[str, dict[str, Any]] = {} config_entries: dict[str, dict[str, Any]] = {} action_entries: dict[str, dict[str, Any]] = {} for profile, (metadata, config, actions) in sorted(all_data.items()): - metadata_entries[profile] = metadata - config_entries[profile] = config - action_entries[profile] = actions - - for var_name, data, doc_name in [ - ('METADATA', metadata_entries, 'Metadata'), - ('CONFIG', config_entries, 'Config options'), - ('ACTIONS', action_entries, 'Actions'), + metadata_entries[profile] = {k: metadata[k] for k in sorted(metadata)} + config_entries[profile] = {k: config[k] for k in sorted(config)} + action_entries[profile] = {k: actions[k] for k in sorted(actions)} + + for var_name, data, type_str, doc_name in [ + ('METADATA', metadata_entries, 'dict[str, _ExtensionMetadata]', 'Metadata'), + ('CONFIG', config_entries, 'dict[str, dict[str, Any]]', 'Config options'), + ('ACTIONS', action_entries, 'dict[str, dict[str, Any]]', 'Actions'), ]: lines.append(f'# {doc_name} added by each charmcraft extension.') - lines.append(f'{var_name}: dict[str, dict[str, Any]] = {{') - for profile in sorted(data): - lines.append(f' {profile!r}: {{') - for key in sorted(data[profile]): - val = data[profile][key] - formatted = format_dict(val, indent=8) - lines.append(f' {key!r}: {formatted},') - lines.append(' },') - lines.append('}') + lines.append(f'{var_name}: {type_str} = {data!r}') lines.append('') return '\n'.join(lines) @@ -181,13 +171,6 @@ def main() -> int: # noqa: D103 all_data[profile] = extract_extension_data(expanded) module_source = generate_module(all_data) - - if OUTPUT_FILE.exists(): - response = input(f'{OUTPUT_FILE} already exists. Overwrite? [y/N] ').strip().lower() - if response != 'y': - print('Aborted.') - return 1 - OUTPUT_FILE.write_text(module_source) print(f'Written to {OUTPUT_FILE}') diff --git a/docs/explanation/state-transition-testing.md b/docs/explanation/state-transition-testing.md index df6dbaf4d..9275bca1d 100644 --- a/docs/explanation/state-transition-testing.md +++ b/docs/explanation/state-transition-testing.md @@ -253,9 +253,9 @@ state = ctx.run(ctx.on.start(), testing.State(relations={testing.Relation('ingre ``` If your `charmcraft.yaml` defines keys that overlap with what the extension -provides (for example, a config option or relation with the same name), the -testing framework will raise a `ValueError`, matching the behaviour of -`charmcraft pack`. Rename or remove the overlapping keys to fix this. +provides (for example, a config option or relation with the same name), both +`charmcraft pack` and the testing framework will raise an error. Rename or +remove the overlapping keys to fix this. ## Immutability diff --git a/testing/src/scenario/_charmcraft_extensions.py b/testing/src/scenario/_charmcraft_extensions.py index ad0d05e1b..b697c8d37 100644 --- a/testing/src/scenario/_charmcraft_extensions.py +++ b/testing/src/scenario/_charmcraft_extensions.py @@ -8,10 +8,20 @@ from __future__ import annotations -from typing import Any +from typing import Any, TypedDict + + +class _ExtensionMetadata(TypedDict, total=False): + assumes: list[str] + containers: dict[str, Any] + peers: dict[str, Any] + provides: dict[str, Any] + requires: dict[str, Any] + resources: dict[str, Any] + # Metadata added by each charmcraft extension. -METADATA: dict[str, dict[str, Any]] = { +METADATA: dict[str, _ExtensionMetadata] = { 'django-framework': { 'assumes': ['k8s-api'], 'containers': {'django-app': {'resource': 'django-app-image'}}, diff --git a/testing/src/scenario/state.py b/testing/src/scenario/state.py index 4257000af..ef68a373e 100644 --- a/testing/src/scenario/state.py +++ b/testing/src/scenario/state.py @@ -1862,8 +1862,8 @@ def _is_valid_charmcraft_25_metadata(meta: dict[str, Any]): return True -def _apply_extensions(meta: dict[str, Any], extensions: list[str]) -> dict[str, Any]: - """Merge charmcraft extension defaults into the charm metadata. +def _apply_extensions(meta: dict[str, Any], extensions: list[str]) -> None: + """Merge charmcraft extension defaults into the charm metadata in place. Extension defaults are applied first, then the local charmcraft.yaml values are merged on top, simulating what ``charmcraft expand-extensions`` @@ -1881,8 +1881,8 @@ def _apply_extensions(meta: dict[str, Any], extensions: list[str]) -> dict[str, if not ext_meta and not ext_config and not ext_actions: warnings.warn( - f'Unknown charmcraft extension {ext_name!r}; ' - f'ignoring. You may need to update to a newer ops.', + f'Unknown charmcraft extension {ext_name!r}; ignoring. ' + "Ensure you're running the latest ops, and open an issue if this persists.", stacklevel=2, ) continue @@ -1905,9 +1905,7 @@ def _apply_extensions(meta: dict[str, Any], extensions: list[str]) -> dict[str, meta[key] = merged elif isinstance(ext_value, list) and isinstance(meta[key], list): merged = copy.deepcopy(ext_value) - for item in meta[key]: - if item not in merged: - merged.append(item) + merged.extend(i for i in meta[key] if i not in merged) meta[key] = merged # Merge config options; error on overlapping keys. @@ -1939,8 +1937,6 @@ def _apply_extensions(meta: dict[str, Any], extensions: list[str]) -> dict[str, merged_actions.update(local_actions) meta['actions'] = merged_actions - return meta - @dataclasses.dataclass(frozen=True) class _CharmSpec(Generic[CharmType]): @@ -1991,7 +1987,7 @@ def _load_metadata(charm_root: pathlib.Path): # Apply charmcraft extensions before extracting config/actions. extensions = meta.pop('extensions', None) if extensions: - meta = _apply_extensions(meta, extensions) + _apply_extensions(meta, extensions) config = meta.pop('config', None) actions = meta.pop('actions', None) From 9b196697657ced9881b2de981ff04c8e37e95a20 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 10 Mar 2026 18:45:15 +1300 Subject: [PATCH 17/19] More refiew adjustments. --- .github/generate_charmcraft_extensions.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/generate_charmcraft_extensions.py b/.github/generate_charmcraft_extensions.py index 4c4e97558..d3781676d 100755 --- a/.github/generate_charmcraft_extensions.py +++ b/.github/generate_charmcraft_extensions.py @@ -64,14 +64,9 @@ def get_extensions() -> list[str]: """Get the list of available charmcraft extensions via ``charmcraft list-extensions``.""" - result = subprocess.run( - ['charmcraft', 'list-extensions'], - check=True, - capture_output=True, - text=True, - ) + result = subprocess.check_output(['charmcraft', 'list-extensions'], text=True) extensions = [] - for line in result.stdout.splitlines(): + for line in result.splitlines(): # Skip header and separator lines. if not line or line.startswith('Extension') or line.startswith('---'): continue @@ -85,14 +80,11 @@ def run_charmcraft(profile: str, workdir: pathlib.Path) -> dict[str, Any]: subprocess.check_call( ['charmcraft', 'init', '--profile', profile, '--name', 'test-charm'], cwd=workdir, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, ) output = subprocess.check_output( ['charmcraft', 'expand-extensions'], cwd=workdir, text=True, - stderr=subprocess.DEVNULL, ) return yaml.safe_load(output) From 235cbc25888428438c8f65fe28b4d35300e1489c Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Fri, 13 Mar 2026 10:37:20 +1300 Subject: [PATCH 18/19] Add an explicit else as per code review. --- testing/src/scenario/state.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/testing/src/scenario/state.py b/testing/src/scenario/state.py index ef68a373e..72b4d166d 100644 --- a/testing/src/scenario/state.py +++ b/testing/src/scenario/state.py @@ -1907,6 +1907,11 @@ def _apply_extensions(meta: dict[str, Any], extensions: list[str]) -> None: merged = copy.deepcopy(ext_value) merged.extend(i for i in meta[key] if i not in merged) meta[key] = merged + else: + raise ValueError( + 'Conflict between local and extension metadata. ' + 'Please check that your charmcraft.yaml is valid' + ) # Merge config options; error on overlapping keys. if ext_config: From 3a633db1ef02afa6ebe39c0fec288de7ede8e01a Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Fri, 13 Mar 2026 11:00:55 +1300 Subject: [PATCH 19/19] Scenario tests get type checked now :) --- testing/src/scenario/state.py | 2 +- testing/tests/test_charm_spec_autoload.py | 33 ++++++++++++----------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/testing/src/scenario/state.py b/testing/src/scenario/state.py index 72b4d166d..2081384ed 100644 --- a/testing/src/scenario/state.py +++ b/testing/src/scenario/state.py @@ -1999,7 +1999,7 @@ def _load_metadata(charm_root: pathlib.Path): return meta, config, actions @staticmethod - def autoload(charm_type: type[CharmBase]) -> _CharmSpec[CharmType]: + def autoload(charm_type: type[CharmBase]) -> _CharmSpec[CharmBase]: """Construct a ``_CharmSpec`` object by looking up the metadata from the charm's repo root. Will attempt to load the metadata off the ``charmcraft.yaml`` file diff --git a/testing/tests/test_charm_spec_autoload.py b/testing/tests/test_charm_spec_autoload.py index 9648fbadc..b087668c4 100644 --- a/testing/tests/test_charm_spec_autoload.py +++ b/testing/tests/test_charm_spec_autoload.py @@ -30,7 +30,7 @@ def import_name(name: str, source: Path) -> Iterator[type[CharmBase]]: pkg_path = str(source.parent) sys.path.append(pkg_path) charm = importlib.import_module('mycharm') - obj = getattr(charm, name) + obj: type[CharmBase] = getattr(charm, name) sys.path.remove(pkg_path) yield obj del sys.modules['mycharm'] @@ -174,7 +174,7 @@ def test_config_defaults(tmp_path: Path, legacy: bool): class TestExtensions: """Tests for charmcraft extension autoloading.""" - def test_extension_adds_metadata(self, tmp_path): + def test_extension_adds_metadata(self, tmp_path: Path): """An extension injects its metadata (containers, requires, etc.).""" with create_tempcharm( tmp_path, @@ -194,7 +194,7 @@ def test_extension_adds_metadata(self, tmp_path): assert 'flask-app-image' in spec.meta.get('resources', {}) assert 'k8s-api' in spec.meta.get('assumes', []) - def test_extension_adds_config(self, tmp_path): + def test_extension_adds_config(self, tmp_path: Path): """An extension injects its config options.""" with create_tempcharm( tmp_path, @@ -212,7 +212,7 @@ def test_extension_adds_config(self, tmp_path): assert 'flask-debug' in options assert 'webserver-workers' in options - def test_extension_adds_actions(self, tmp_path): + def test_extension_adds_actions(self, tmp_path: Path): """An extension injects its actions.""" with create_tempcharm( tmp_path, @@ -228,7 +228,7 @@ def test_extension_adds_actions(self, tmp_path): assert spec.actions is not None assert 'rotate-secret-key' in spec.actions - def test_local_meta_overlaps_extension_errors(self, tmp_path): + def test_local_meta_overlaps_extension_errors(self, tmp_path: Path): """Overlapping metadata keys with extension cause an error.""" with create_tempcharm( tmp_path, @@ -246,7 +246,7 @@ def test_local_meta_overlaps_extension_errors(self, tmp_path): with pytest.raises(ValueError, match=r'overlapping keys.*requires.*flask-framework'): _CharmSpec.autoload(charm) - def test_local_meta_no_overlap_with_extension(self, tmp_path): + def test_local_meta_no_overlap_with_extension(self, tmp_path: Path): """Non-overlapping local metadata keys merge with extension.""" with create_tempcharm( tmp_path, @@ -268,7 +268,7 @@ def test_local_meta_no_overlap_with_extension(self, tmp_path): assert 'ingress' in spec.meta['requires'] assert 'logging' in spec.meta['requires'] - def test_local_config_overlaps_extension_errors(self, tmp_path): + def test_local_config_overlaps_extension_errors(self, tmp_path: Path): """Overlapping config options with extension cause an error.""" with create_tempcharm( tmp_path, @@ -291,7 +291,7 @@ def test_local_config_overlaps_extension_errors(self, tmp_path): ): _CharmSpec.autoload(charm) - def test_local_config_no_overlap_with_extension(self, tmp_path): + def test_local_config_no_overlap_with_extension(self, tmp_path: Path): """Non-overlapping local config options merge with extension.""" with create_tempcharm( tmp_path, @@ -309,6 +309,7 @@ def test_local_config_no_overlap_with_extension(self, tmp_path): }, ) as charm: spec = _CharmSpec.autoload(charm) + assert spec.config is not None options = spec.config['options'] # Local-only option is present. assert 'my-custom-option' in options @@ -316,7 +317,7 @@ def test_local_config_no_overlap_with_extension(self, tmp_path): assert 'flask-debug' in options assert 'webserver-workers' in options - def test_local_actions_overlap_extension_errors(self, tmp_path): + def test_local_actions_overlap_extension_errors(self, tmp_path: Path): """Overlapping actions with extension cause an error.""" with create_tempcharm( tmp_path, @@ -335,7 +336,7 @@ def test_local_actions_overlap_extension_errors(self, tmp_path): with pytest.raises(ValueError, match=r'overlapping keys.*actions.*flask-framework'): _CharmSpec.autoload(charm) - def test_local_actions_no_overlap_with_extension(self, tmp_path): + def test_local_actions_no_overlap_with_extension(self, tmp_path: Path): """Non-overlapping local actions merge with extension.""" with create_tempcharm( tmp_path, @@ -351,12 +352,13 @@ def test_local_actions_no_overlap_with_extension(self, tmp_path): }, ) as charm: spec = _CharmSpec.autoload(charm) + assert spec.actions is not None # Local-only action is present. assert 'my-action' in spec.actions # Extension action is still present. assert 'rotate-secret-key' in spec.actions - def test_local_assumes_merged_with_extension(self, tmp_path): + def test_local_assumes_merged_with_extension(self, tmp_path: Path): """Local assumes list is merged with extension assumes.""" with create_tempcharm( tmp_path, @@ -374,7 +376,7 @@ def test_local_assumes_merged_with_extension(self, tmp_path): assert 'k8s-api' in assumes assert 'juju >= 3.1' in assumes - def test_django_extension_has_create_superuser(self, tmp_path): + def test_django_extension_has_create_superuser(self, tmp_path: Path): """Django extension adds the create-superuser action.""" with create_tempcharm( tmp_path, @@ -387,10 +389,11 @@ def test_django_extension_has_create_superuser(self, tmp_path): }, ) as charm: spec = _CharmSpec.autoload(charm) + assert spec.actions is not None assert 'create-superuser' in spec.actions assert 'rotate-secret-key' in spec.actions - def test_unknown_extension_warns(self, tmp_path): + def test_unknown_extension_warns(self, tmp_path: Path): """An unknown extension name emits a warning and is skipped.""" with create_tempcharm( tmp_path, @@ -407,7 +410,7 @@ def test_unknown_extension_warns(self, tmp_path): # No extension data merged, but the charm still loads. assert spec.meta['name'] == 'my-app' - def test_extension_stripped_from_meta(self, tmp_path): + def test_extension_stripped_from_meta(self, tmp_path: Path): """The 'extensions' key should not remain in the loaded meta.""" with create_tempcharm( tmp_path, @@ -422,7 +425,7 @@ def test_extension_stripped_from_meta(self, tmp_path): spec = _CharmSpec.autoload(charm) assert 'extensions' not in spec.meta - def test_extension_with_relations_in_context(self, tmp_path): + def test_extension_with_relations_in_context(self, tmp_path: Path): """Relations from an extension can be used in a Context run.""" with create_tempcharm( tmp_path,