diff --git a/lib/charms/data_platform_libs/v1/data_interfaces.py b/lib/charms/data_platform_libs/v1/data_interfaces.py index 0d156cd4..199b7bb6 100644 --- a/lib/charms/data_platform_libs/v1/data_interfaces.py +++ b/lib/charms/data_platform_libs/v1/data_interfaces.py @@ -312,7 +312,7 @@ def _on_resource_requested(self, event: ResourceRequestedEvent) -> None: # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 3 +LIBPATCH = 4 PYDEPS = ["ops>=2.0.0", "pydantic>=2.11"] @@ -496,6 +496,9 @@ def store_new_data( MtlsSecretStr = Annotated[OptionalSecretStr, Field(exclude=True, default=None), "mtls"] ExtraSecretStr = Annotated[OptionalSecretStr, Field(exclude=True, default=None), "extra"] EntitySecretStr = Annotated[OptionalSecretStr, Field(exclude=True, default=None), "entity"] +RequestedEntitySecretStr = Annotated[ + OptionalSecretStr, Field(exclude=True, default=None), "requested-entity" +] class Scope(Enum): @@ -833,9 +836,16 @@ def extract_secrets(self, info: ValidationInfo): if not secret_uri: continue - secret = repository.get_secret( - secret_group, secret_uri=secret_uri, short_uuid=short_uuid - ) + try: + secret = repository.get_secret( + secret_group, secret_uri=secret_uri, short_uuid=short_uuid + ) + except SecretNotFoundError: + # v0 deletes the requested entity secret + if secret_group == "requested-entity": + logger.debug("Missing requested entity secret") + continue + raise if not secret: logger.info(f"No secret for group {secret_group} and short uuid {short_uuid}") @@ -1002,6 +1012,13 @@ class RequirerCommonModel(CommonModel): entity_permissions: list[EntityPermissionModel] | None = Field(default=None) secret_mtls: SecretString | None = Field(default=None) mtls_cert: MtlsSecretStr = Field(default=None) + secret_requested_entity: SecretString | None = Field( + default=None, + validation_alias=AliasChoices("requested-entity-secret", "secret-requested-entity"), + ) + entity_name: RequestedEntitySecretStr = Field(default=None) + entity_password: RequestedEntitySecretStr = Field(default=None, serialization_alias="password") + prefix_matching: Literal["all", "only-existing"] | None = Field(default=None) @model_validator(mode="after") def validate_fields(self): @@ -1015,6 +1032,9 @@ def validate_fields(self): if self.entity_type == "GROUP" and self.extra_user_roles: raise ValueError("Inconsistent entity information. Use extra_group_roles instead") + if self.entity_password and not self.entity_name: + raise ValueError("Unable to set entity password without an entity name") + return self @@ -1047,6 +1067,7 @@ class ResourceProviderModel(ProviderCommonModel): entity_name: EntitySecretStr = Field(default=None) entity_password: EntitySecretStr = Field(default=None) version: str | None = Field(default=None) + prefix_resources: str | None = Field(default=None) class RequirerDataContractV0(RequirerCommonModel): @@ -2090,6 +2111,12 @@ class AuthenticationUpdatedEvent(ResourceRequirerEvent[TResourceProviderModel]): pass +class ResourcePrefixResourcesChangedEvent(ResourceRequirerEvent[TResourceProviderModel]): + """Prefix resources have changed.""" + + pass + + # Error Propagation Events @@ -2148,6 +2175,7 @@ class ResourceRequiresEvents(CharmEvents, Generic[TResourceProviderModel]): authentication_updated = EventSource(AuthenticationUpdatedEvent) status_raised = EventSource(StatusRaisedEvent) status_resolved = EventSource(StatusResolvedEvent) + prefix_resources_changed = EventSource(ResourcePrefixResourcesChangedEvent) ############################################################################## @@ -2632,6 +2660,11 @@ def set_response(self, relation_id: int, response: ResourceProviderModel): self.interface.write_model( relation_id, response, context={"version": "v0"} ) # {"database": "database-name", "secret-user": "uri", ...} + # Set expected prefix field if present + if response.prefix_resources: + self.interface.repository(relation_id).write_field( + "prefix-databases", response.prefix_resources + ) return model = self.interface.build_model(relation_id, DataContractV1[response.__class__]) @@ -2875,6 +2908,10 @@ def __init__( f"{relation_alias}_read_only_endpoints_changed", ResourceReadOnlyEndpointsChangedEvent, ) + self.on.define_event( + f"{relation_alias}_prefix_resources_changed", + ResourcePrefixResourcesChangedEvent, + ) ############################################################################## # Extra useful functions @@ -3241,3 +3278,11 @@ def _handle_event( ) self._emit_aliased_event(event, "authentication_updated", response) return + + if "prefix-resources" in _diff.added or "prefix-resources" in _diff.changed: + logger.info(f"prefix resources updated for {response.resource} at {datetime.now()}") + getattr(self.on, "prefix_resources_changed").emit( + event.relation, app=event.app, unit=event.unit, response=response + ) + self._emit_aliased_event(event, "prefix_resources_changed", response) + return diff --git a/tests/v0/integration/database-charm/src/charm.py b/tests/v0/integration/database-charm/src/charm.py index 9c1e23fb..50e5b2c4 100755 --- a/tests/v0/integration/database-charm/src/charm.py +++ b/tests/v0/integration/database-charm/src/charm.py @@ -194,7 +194,7 @@ def _on_database_pebble_ready(self, event: WorkloadEvent) -> None: "command": "/usr/local/bin/docker-entrypoint.sh postgres", "startup": "enabled", "environment": { - "PGDATA": "/var/lib/postgresql/data/pgdata", + "PGDATA": "/var/lib/postgresql/data/pgdata/data", "POSTGRES_PASSWORD": self._stored.password, }, } diff --git a/tests/v1/integration/application-charm/metadata.yaml b/tests/v1/integration/application-charm/metadata.yaml index 46cd4de1..bfdd79c3 100644 --- a/tests/v1/integration/application-charm/metadata.yaml +++ b/tests/v1/integration/application-charm/metadata.yaml @@ -12,6 +12,8 @@ requires: interface: database_client first-database-roles: interface: database_client + first-database-username: + interface: database_client second-database-db: interface: database_client multiple-database-clusters: @@ -36,4 +38,6 @@ requires: etcd-client: interface: etcd_client certificates: - interface: tls-certificates \ No newline at end of file + interface: tls-certificates + database-with-prefix: + interface: database_client diff --git a/tests/v1/integration/application-charm/src/charm.py b/tests/v1/integration/application-charm/src/charm.py index 44b0d69d..5a87db36 100755 --- a/tests/v1/integration/application-charm/src/charm.py +++ b/tests/v1/integration/application-charm/src/charm.py @@ -120,6 +120,20 @@ def __init__(self, *args): self._on_first_database_auth_updated, ) + self.first_database_username = ResourceRequirerEventHandler( + self, + "first-database-username", + requests=[ + RequirerCommonModel( + resource=database_name, entity_type="USER", entity_name="testuser" + ) + ], + response_model=ExtendedResponseModel, + ) + self.framework.observe( + self.first_database.on.resource_created, self._on_first_database_created + ) + # Events related to the second database that is requested # (these events are defined in the database requires charm library). database_name = f"{self.app.name.replace('-', '_')}_second_database_db" @@ -163,6 +177,17 @@ def __init__(self, *args): self._on_cluster_endpoints_changed, ) + self.database_prefixes = ResourceRequirerEventHandler( + charm=self, + relation_name="database-with-prefix", + requests=[ + RequirerCommonModel( + resource="testdb*", extra_user_roles=EXTRA_USER_ROLES, prefix_matching="all" + ) + ], + response_model=ExtendedResponseModel, + ) + # Multiple database clusters charm events (defined dynamically # in the database requires charm library, using the provided cluster/relation aliases). database_name = f"{self.app.name.replace('-', '_')}_aliased_multiple_database_clusters" diff --git a/tests/v1/integration/backward-compatibility-charm/metadata.yaml b/tests/v1/integration/backward-compatibility-charm/metadata.yaml index 898da718..68a5d11d 100644 --- a/tests/v1/integration/backward-compatibility-charm/metadata.yaml +++ b/tests/v1/integration/backward-compatibility-charm/metadata.yaml @@ -10,3 +10,5 @@ summary: | requires: backward-database: interface: database_client + backward-prefix-database: + interface: database_client diff --git a/tests/v1/integration/backward-compatibility-charm/src/charm.py b/tests/v1/integration/backward-compatibility-charm/src/charm.py index e0f025c2..faa75fed 100755 --- a/tests/v1/integration/backward-compatibility-charm/src/charm.py +++ b/tests/v1/integration/backward-compatibility-charm/src/charm.py @@ -35,6 +35,16 @@ def __init__(self, *args): self.database = DatabaseRequires(self, "backward-database", "bwclient") self.framework.observe(self.database.on.database_created, self._on_resource_created) + self.database_prefixes = DatabaseRequires( + charm=self, + relation_name="backward-prefix-database", + database_name="testdb*", + requested_entity_name="testuser", + ) + self.framework.observe( + self.database_prefixes.on.database_created, self._on_resource_created + ) + def _on_start(self, _) -> None: """Only sets an active status.""" self.unit.status = ActiveStatus("Backward compatibility charm ready!") diff --git a/tests/v1/integration/database-charm/src/charm.py b/tests/v1/integration/database-charm/src/charm.py index e200083a..d86e6397 100755 --- a/tests/v1/integration/database-charm/src/charm.py +++ b/tests/v1/integration/database-charm/src/charm.py @@ -208,7 +208,7 @@ def _on_database_pebble_ready(self, event: WorkloadEvent) -> None: "command": "/usr/local/bin/docker-entrypoint.sh postgres", "startup": "enabled", "environment": { - "PGDATA": "/var/lib/postgresql/data/pgdata", + "PGDATA": "/var/lib/postgresql/data/pgdata/data", "POSTGRES_PASSWORD": self._stored.password, }, } @@ -228,9 +228,15 @@ def _on_resource_requested(self, event: ResourceRequestedEvent) -> None: resource = request.resource extra_user_roles = request.extra_user_roles + username = request.entity_name + password = request.entity_password - username = f"relation_{relation_id}_{request.request_id}" - password = self._new_password() + if resource[-1] == "*": + resources = [f"{resource[:-1]}1", f"{resource[:-1]}2"] + else: + resources = [resource] + username = username or f"relation_{relation_id}_{request.request_id}" + password = password or self._new_password() connection_string = ( "dbname='postgres' user='postgres' host='localhost' " f"password='{self._stored.password}' connect_timeout=10" @@ -238,13 +244,14 @@ def _on_resource_requested(self, event: ResourceRequestedEvent) -> None: connection = psycopg2.connect(connection_string) connection.autocommit = True cursor = connection.cursor() - # Create the database, user and password. Also gives the user access to the database. - cursor.execute(f"CREATE DATABASE {resource};") cursor.execute(f"CREATE USER {username} WITH ENCRYPTED PASSWORD '{password}';") - cursor.execute(f"GRANT ALL PRIVILEGES ON DATABASE {resource} TO {username};") - # Add the roles to the user. - if extra_user_roles: - cursor.execute(f"ALTER USER {username} {extra_user_roles};") + # Create the database, user and password. Also gives the user access to the database. + for resource in resources: + cursor.execute(f"CREATE DATABASE {resource};") + cursor.execute(f"GRANT ALL PRIVILEGES ON DATABASE {resource} TO {username};") + # Add the roles to the user. + if extra_user_roles: + cursor.execute(f"ALTER USER {username} {extra_user_roles};") # Get the database version. cursor.execute("SELECT version();") version = cursor.fetchone()[0] @@ -272,6 +279,7 @@ def _on_resource_requested(self, event: ResourceRequestedEvent) -> None: username=username, endpoints=f"{self.model.get_binding('database').network.bind_address}:5432", version=version, + prefix_resources=",".join(resources) if len(resources) > 1 else None, ) self.database.set_response(event.relation.id, response) self.unit.status = ActiveStatus() @@ -294,8 +302,11 @@ def _on_resource_entity_requested(self, event: ResourceEntityRequestedEvent) -> entity_type = request.entity_type # Generate a entity-name and a entity-password for the application. - rolename = self._new_rolename() - password = self._new_password() + rolename = request.entity_name + password = request.entity_password + + rolename = rolename or self._new_rolename() + password = password or self._new_password() # Connect to the database. connection_string = ( diff --git a/tests/v1/integration/test_backward_compatibility_charm.py b/tests/v1/integration/test_backward_compatibility_charm.py index bde843f7..84a08247 100644 --- a/tests/v1/integration/test_backward_compatibility_charm.py +++ b/tests/v1/integration/test_backward_compatibility_charm.py @@ -89,3 +89,39 @@ async def test_backward_relation_with_charm_libraries_secrets(ops_test: OpsTest) assert len(password) == 16 assert endpoints assert database == "bwclient" + + +@pytest.mark.abort_on_fail +async def test_backward_relation_with_prefix_and_username(ops_test: OpsTest): + prefix_relation = "backward-prefix-database" + # Relate the charms and wait for them exchanging some connection data. + await ops_test.model.add_relation( + DATABASE_APP_NAME, f"{APPLICATION_APP_NAME}:{prefix_relation}" + ) + await ops_test.model.wait_for_idle(apps=APP_NAMES, status="active") + + # check unit message to check if the topic_created_event is triggered + for unit in ops_test.model.applications[APPLICATION_APP_NAME].units: + assert unit.workload_status_message == "backward_database_created" + + # Get the requests + secret_uri = ( + await get_application_relation_data( + ops_test, APPLICATION_APP_NAME, prefix_relation, f"{PROV_SECRET_PREFIX}user" + ) + or "" + ) + secret_data = await get_juju_secret(ops_test, secret_uri) + username = secret_data["username"] + password = secret_data["password"] + endpoints = await get_application_relation_data( + ops_test, APPLICATION_APP_NAME, prefix_relation, "endpoints" + ) + prefix_databases = await get_application_relation_data( + ops_test, APPLICATION_APP_NAME, prefix_relation, "prefix-databases" + ) + + assert username == "testuser" + assert len(password) == 16 + assert endpoints + assert prefix_databases == "testdb1,testdb2" diff --git a/tests/v1/integration/test_charm.py b/tests/v1/integration/test_charm.py index 1529adc9..f56711d8 100644 --- a/tests/v1/integration/test_charm.py +++ b/tests/v1/integration/test_charm.py @@ -41,6 +41,8 @@ DB_FIRST_DATABASE_RELATION_NAME = "first-database-db" DB_SECOND_DATABASE_RELATION_NAME = "second-database-db" ROLES_FIRST_DATABASE_RELATION_NAME = "first-database-roles" +USERNAME_FIRST_DATABASE_RELATION_NAME = "first-database-username" +PREFIX_DATABASE_RELATION_NAME = "database-with-prefix" MULTIPLE_DATABASE_CLUSTERS_RELATION_NAME = "multiple-database-clusters" ALIASED_MULTIPLE_DATABASE_CLUSTERS_RELATION_NAME = "aliased-multiple-database-clusters" @@ -765,6 +767,33 @@ async def test_database_roles_relation_with_charm_libraries_secrets(ops_test: Op assert entity_pass is not None +@pytest.mark.abort_on_fail +@pytest.mark.usefixtures("only_with_juju_secrets") +async def test_database_username(ops_test: OpsTest): + """Test basic functionality of database-roles relation interface.""" + # Relate the charms and wait for them exchanging some connection data. + + pytest.first_database_relation = await ops_test.model.add_relation( + f"{APPLICATION_APP_NAME}:{USERNAME_FIRST_DATABASE_RELATION_NAME}", DATABASE_APP_NAME + ) + await ops_test.model.wait_for_idle(apps=APP_NAMES, status="active") + + requests = json.loads( + await get_application_relation_data( + ops_test, APPLICATION_APP_NAME, USERNAME_FIRST_DATABASE_RELATION_NAME, "requests" + ) + or "[]" + ) + request = requests[0] + secret_uri = request.get(f"{SECRET_REF_PREFIX}entity") + secret_content = await get_juju_secret(ops_test, secret_uri) + entity_name = secret_content["entity-name"] + entity_pass = secret_content["entity-password"] + + assert entity_name == "testuser" + assert entity_pass is not None + + async def test_an_application_can_connect_to_multiple_database_clusters( ops_test: OpsTest, database_charm ): @@ -1370,3 +1399,30 @@ async def test_scaling_requires_can_access_shared_secrets(ops_test): await action.wait() new_password = action.results.get("value") assert new_password == orig_password + + +@pytest.mark.abort_on_fail +async def test_database_prefix(ops_test: OpsTest): + """Test basic functionality of database-roles relation interface.""" + # Relate the charms and wait for them exchanging some connection data. + + pytest.first_database_relation = await ops_test.model.add_relation( + f"{APPLICATION_APP_NAME}:{PREFIX_DATABASE_RELATION_NAME}", DATABASE_APP_NAME + ) + await ops_test.model.wait_for_idle(apps=APP_NAMES, status="active") + + requests = json.loads( + await get_application_relation_data( + ops_test, APPLICATION_APP_NAME, PREFIX_DATABASE_RELATION_NAME, "requests" + ) + or "[]" + ) + request = requests[0] + + assert request["prefix-resources"] == "testdb1,testdb2" + data = json.loads( + await get_application_relation_data( + ops_test, APPLICATION_APP_NAME, PREFIX_DATABASE_RELATION_NAME, "data" + ) + ) + assert data[request["request-id"]]["prefix-matching"] == "all" diff --git a/tests/v1/unit/test_data_interfaces.py b/tests/v1/unit/test_data_interfaces.py index 30f49ef3..85899494 100644 --- a/tests/v1/unit/test_data_interfaces.py +++ b/tests/v1/unit/test_data_interfaces.py @@ -950,6 +950,10 @@ def __init__(self, *args): self.requirer.on.cluster1_resource_created, self._on_cluster1_resource_created, ) + self.framework.observe( + self.requirer.on.prefix_resources_changed, + self._on_prefix_resources_changed, + ) def log_relation_size(self, prefix=""): logger.info(f"ยง{prefix} relations: {len(self.requirer.interface.relations)}") @@ -993,6 +997,9 @@ def _on_endpoints_changed(self, _) -> None: def _on_read_only_endpoints_changed(self, _) -> None: self.log_relation_size("on_read_only_endpoints_changed") + def _on_prefix_resources_changed(self, _) -> None: + self.log_relation_size("on_prefix_resources_changed") + def _on_cluster1_resource_created(self, _) -> None: self.log_relation_size("on_cluster1_resource_created") @@ -1010,6 +1017,7 @@ def reset_aliases(): delattr(ResourceRequiresEvents, f"{cluster_alias}_resource_entity_created") delattr(ResourceRequiresEvents, f"{cluster_alias}_endpoints_changed") delattr(ResourceRequiresEvents, f"{cluster_alias}_read_only_endpoints_changed") + delattr(ResourceRequiresEvents, f"{cluster_alias}_prefix_resources_changed") except AttributeError: # Ignore the events not existing before the first test. pass @@ -1090,6 +1098,13 @@ def test_relation_interface_consistency(self): with pytest.raises(ValidationError): KafkaRequestModel(consumer_group_prefix="*") + def test_requested_entity_consistency(self): + """Test consistency restrictions on direct secret usage and helper values.""" + # Try to set password without entity name + with pytest.raises(ValueError) as e: + RequirerCommonModel(entity_password="testpass") + assert "Unable to set entity password without an entity name" in str(e.value) + class TestDatabaseRequiresNoRelations(DataRequirerBaseTests, unittest.TestCase): metadata = METADATA @@ -1197,10 +1212,12 @@ def test_requires_interface_functions_secrets(self): "external-node-connectivity": False, "extra-group-roles": None, "extra-user-roles": "CREATEDB,CREATEROLE", + "prefix-matching": None, "request-id": "c759221a6c14c72a", "resource": "data_platform", "salt": "kkkkkkkk", "secret-mtls": None, + "secret-requested-entity": None, }, { "entity-permissions": None, @@ -1208,10 +1225,12 @@ def test_requires_interface_functions_secrets(self): "external-node-connectivity": False, "extra-group-roles": None, "extra-user-roles": None, + "prefix-matching": None, "request-id": "9ecfabfbb5258f88", "resource": "", "salt": "xxxxxxxx", "secret-mtls": None, + "secret-requested-entity": None, }, ], } @@ -1459,16 +1478,19 @@ def test_fetch_my_relation_data_and_fields_secrets(self): "entity-permissions": None, "entity-type": None, "salt": "kkkkkkkk", + "prefix-matching": None, "request-id": "c759221a6c14c72a", "resource": "data_platform", "extra-group-roles": None, "extra-user-roles": "CREATEDB,CREATEROLE", "external-node-connectivity": False, "secret-mtls": None, + "secret-requested-entity": None, }, { "entity-permissions": None, "salt": "xxxxxxxx", + "prefix-matching": None, "request-id": "9ecfabfbb5258f88", "resource": "", "entity-type": "USER", @@ -1476,6 +1498,7 @@ def test_fetch_my_relation_data_and_fields_secrets(self): "extra-user-roles": None, "external-node-connectivity": False, "secret-mtls": None, + "secret-requested-entity": None, }, ], } @@ -1702,6 +1725,87 @@ def test_additional_fields_are_accessible(self, _on_resource_created): assert event.response.uris == "host1:port,host2:port" assert event.response.version == "1.0" + @patch.object(charm, "_on_prefix_resources_changed") + @pytest.mark.usefixtures("only_with_juju_secrets") + def test_on_prefix_resources_changed(self, _on_prefix_resources_changed): + """Asserts that the prefix resources are in the relation databag when they change.""" + # Simulate adding endpoints to the relation. + self.harness.update_relation_data( + self.rel_id, + self.provider, + { + "requests": json.dumps( + [ + { + "salt": "kkkkkkkk", + "request-id": "c759221a6c14c72a", + "prefix-resources": "db1,db2", + } + ] + ), + "data": json.dumps( + { + "c759221a6c14c72a": { + "salt": "kkkkkkkk", + "request-id": "c759221a6c14c72a", + "resource": DATABASE, + } + } + ), + }, + ) + + # Assert the correct hook is called. + _on_prefix_resources_changed.assert_called_once() + + # Check that the endpoints are present in the relation + # using the requires charm library event. + event = _on_prefix_resources_changed.call_args[0][0] + assert event.response.prefix_resources == "db1,db2" + + # Reset the mock call count. + _on_prefix_resources_changed.reset_mock() + + # Set the same data in the relation (no change in the endpoints). + self.harness.update_relation_data( + self.rel_id, + self.provider, + { + "requests": json.dumps( + [ + { + "salt": "kkkkkkkk", + "request-id": "c759221a6c14c72a", + "prefix-resources": "db1,db2", + } + ] + ) + }, + ) + + # Assert the hook was not called again. + _on_prefix_resources_changed.assert_not_called() + + # Then, change the endpoints in the relation. + self.harness.update_relation_data( + self.rel_id, + self.provider, + { + "requests": json.dumps( + [ + { + "salt": "kkkkkkkk", + "request-id": "c759221a6c14c72a", + "prefix-resources": "db1,db2,db3", + } + ] + ) + }, + ) + + # Assert the hook is called now. + _on_prefix_resources_changed.assert_called_once() + def test_assign_relation_alias(self): """Asserts the correct relation alias is assigned to the relation.""" unit_name = f"{self.app_name}/0"