diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 296816ba..f94c00b9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -71,6 +71,7 @@ jobs: - integration-s3 - integration-opensearch - integration-kafka + - integration-spark-sa juju-version: - juju-bootstrap-option: "2.9.51" juju-snap-channel: "2.9/stable" @@ -99,6 +100,10 @@ jobs: ubuntu-versions: {series: focal} - tox-environments: integration-kafka ubuntu-versions: {series: focal} + - tox-environments: integration-spark-sa + juju-version: {juju-snap-channel: "2.9/stable"} + - tox-environments: integration-spark-sa + ubuntu-versions: {series: focal} name: ${{ matrix.tox-environments }} Juju ${{ matrix.juju-version.juju-snap-channel}} -- ${{ matrix.ubuntu-versions.series }} needs: - lint diff --git a/lib/charms/data_platform_libs/v0/spark_service_account.py b/lib/charms/data_platform_libs/v0/spark_service_account.py new file mode 100644 index 00000000..0dbd12ff --- /dev/null +++ b/lib/charms/data_platform_libs/v0/spark_service_account.py @@ -0,0 +1,498 @@ +# Copyright 2025 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +r"""Library for creating service accounts that are configured to run Spark jobs. + +This library contains the SparkServiceAccountProvider and SparkServiceAccountRequirer +classes for handling the relation between charms that require Spark Service Account +to be created in order to function, and charms that create and provide them. + +### SparkServiceAccountRequirer + +Following is an example of using the SparkServiceAccountRequirer class in the context +of the application charm code: + +```python +import json + +from charms.data_platform_libs.v0.spark_service_account import ( + SparkServiceAccountRequirer, + ServiceAccountGrantedEvent, + ServiceAccountPropertyChangedEvent, + ServiceAccountGoneEvent +) +from ops.model import ActiveStatus, BlockedStatus + + +class RequirerCharm(CharmBase): + + def __init__(self, *args): + super().__init__(*args) + + namespace, username = "default", "test" + self.spark_service_account_requirer = SparkServiceAccountRequirer(self, relation_name="service-account", service_account=f"{namespace}:{username}", ) + self.framework.observe( + self.spark_service_account_requirer.on.account_granted, self._on_account_granted + ) + self.framework.observe( + self.spark_service_account_requirer.on.account_gone, self._on_account_gone + ) + self.framework.observe( + self.spark_service_account_requirer.on.properties_changed, self._on_spark_properties_changed + ) + + def _on_account_granted(self, event: ServiceAccountGrantedEvent): + # Handle the account_granted event + + namespace, username = event.service_account.split(":") + props_string = self.service_account_requirer.relation_data.fetch_relation_field(event.relation.id, "spark-properties") + props = json.loads(props_string) + + resource_manifest = self.service_account_requirer.relation_data.fetch_relation_field(event.relation.id, "resource-manifest") + + # Create configuration file for app + config_file = self._render_app_config_file( + namespace=namespace, + username=username, + spark_properties=props, + resource_manifest=resource_manifest + ) + + # Start application with rendered configuration + self._start_application(config_file) + + # Set appropriate status + self.unit.status = ActiveStatus("Received Spark service account") + + def _on_spark_properties_changed(self, event: ServiceAccountPropertyChangedEvent): + # Handle the properties_changed event + namespace, username = event.service_account.split(":") + + # Fetch the Spark properties from event data + props_string = self.service_account_requirer.relation_data.fetch_relation_field(event.relation.id, "spark-properties") + props = json.loads(props_string) + + resource_manifest = self.service_account_requirer.relation_data.fetch_relation_field(event.relation.id, "resource-manifest") + + # Create configuration file for app + config_file = self._render_app_config_file( + namespace=namespace, + username=username, + spark_properties=props, + resource_manifest=resource_manifest + ) + + # Start application with rendered configuration + self._start_application(config_file) + + # Set appropriate status + self.unit.status = ActiveStatus("Spark service account properties changed") + + def _on_account_gone(self, event: ServiceAccountGoneEvent): + # Handle the account_gone event + + # Create configuration file for app + config_file = self._render_app_config_file( + namespace=None, + username=None, + spark_properties=None, + resource_manifest=None, + ) + + # Start application with rendered configuration + self._start_application(config_file) + + # Set appropriate status + self.unit.status = BlockedStatus("Missing spark service account") +``` + +### SparkServiceAccountProvider +Following is an example of using the SparkServiceAccountProvider class in the context +of the application charm code: + +```python +from charms.data_platform_libs.v0.spark_service_account import ( + SparkServiceAccountProvider, + ServiceAccountRequestedEvent, + ServiceAccountReleasedEvent, +) + + +class ProviderCharm(CharmBase): + + def __init__(self, *args): + super().__init__(*args) + + self.spark_service_account_provider = SparkServiceAccountProvider(self, relation_name="service-account") + self.framework.observe(self.sa.on.account_requested, self._on_service_account_requested) + self.framework.observe(self.sa.on.account_released, self._on_service_account_released) + + + def _on_service_account_requested(self, event: ServiceAccountRequestedEvent): + # Handle the account_requested event + + namespace, username = event.service_account.split(":") + skip_creation = event.skip_creation + + if not skip_creation: + # Create the service account + self.create_service_account(namespace, username) + + resource_manifest = self.generate_resource_manifest(namespace, username) + spark_properties = self.generate_spark_properties(namespace, username) + + # Write the service account, Spark properties and resource manifest to relation data + self.spark_service_account_provider.set_service_account(event.relation.id, f"{namespace}:{username}") + self.spark_service_account_provider.set_spark_properties(event.relation.id, spark_properties) + self.spark_service_account_provider.set_resource_manifest(event.relation.id, resource_manifest) + + + def _on_service_account_released(self, event: ServiceAccountReleasedEvent): + # Handle account_released event + + namespace, username = event.service_account.split(":") + skip_creation = event.skip_creation + + if not skip_creation: + # Delete the service account + self.delete_service_account(namespace, username) +``` + +""" + + +import logging +from typing import List, Optional + +from ops import Model, RelationCreatedEvent, SecretChangedEvent +from ops.charm import ( + CharmBase, + CharmEvents, + RelationBrokenEvent, + RelationChangedEvent, + RelationEvent, +) +from ops.framework import EventSource, ObjectEvents + +from charms.data_platform_libs.v0.data_interfaces import ( + SECRET_GROUPS, + EventHandlers, + ProviderData, + RelationEventWithSecret, + RequirerData, + RequirerEventHandlers, +) + +# The unique Charmhub library identifier, never change it +LIBID = "1f402a9b0ec547788b185c167ab9b5fe" + +# Increment this major API version when introducing breaking changes +LIBAPI = 0 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 1 + +PYDEPS = ["ops>=2.0.0"] + +SPARK_PROPERTIES_RELATION_FIELD = "spark-properties" + +logger = logging.getLogger(__name__) + + +class ServiceAccountEvent(RelationEventWithSecret): + """Base class for Service account events.""" + + @property + def service_account(self) -> Optional[str]: + """Returns the service account was requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("service-account", "") + + @property + def spark_properties(self) -> Optional[str]: + """Returns the Spark properties associated with service account.""" + if not self.relation.app: + return None + + if self.secrets_enabled: + secret = self._get_secret("extra") + if secret: + return secret.get("spark-properties", "{}") + + return self.relation.data[self.relation.app].get("spark-properties", "{}") + + @property + def resource_manifest(self) -> Optional[str]: + """Returns the resource manifest associated with service account.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("resource-manifest", "{}") + + @property + def skip_creation(self) -> bool: + """Returns the skip-creation flag associated with service account.""" + if not self.relation.app: + return False + + skip = self.relation.data[self.relation.app].get("skip-creation", "false") + return skip.lower() == "true" + + +class ServiceAccountRequestedEvent(ServiceAccountEvent): + """Event emitted when a set of service account is requested for use on this relation.""" + + +class ServiceAccountReleasedEvent(ServiceAccountEvent): + """Event emitted when a set of service account is released.""" + + +class SparkServiceAccountProviderEvents(CharmEvents): + """Event descriptor for events raised by ServiceAccountProvider.""" + + account_requested = EventSource(ServiceAccountRequestedEvent) + account_released = EventSource(ServiceAccountReleasedEvent) + + +class ServiceAccountGrantedEvent(ServiceAccountEvent): + """Event emitted when service account are granted on this relation.""" + + +class ServiceAccountGoneEvent(RelationEvent): + """Event emitted when service account are removed from this relation.""" + + +class ServiceAccountPropertyChangedEvent(ServiceAccountEvent): + """Event emitted when Spark properties for the service account are changed in this relation.""" + + +class SparkServiceAccountRequirerEvents(ObjectEvents): + """Event descriptor for events raised by the Requirer.""" + + account_granted = EventSource(ServiceAccountGrantedEvent) + account_gone = EventSource(ServiceAccountGoneEvent) + properties_changed = EventSource(ServiceAccountPropertyChangedEvent) + + +class SparkServiceAccountProviderData(ProviderData): + """Implementation of ProviderData for the Spark Service Account relation.""" + + RESOURCE_FIELD = "service-account" + + def __init__(self, model: Model, relation_name: str) -> None: + super().__init__(model, relation_name) + + def set_service_account(self, relation_id: int, service_account: str) -> None: + """Set the service account name in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + service_account: the service account name. + """ + self.update_relation_data(relation_id, {"service-account": service_account}) + + def set_spark_properties(self, relation_id: int, spark_properties: str) -> None: + """Set the Spark properties in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + spark_properties: the dictionary that contains key-value for Spark properties. + """ + self.update_relation_data(relation_id, {SPARK_PROPERTIES_RELATION_FIELD: spark_properties}) + + def set_resource_manifest(self, relation_id: int, resource_manifest: str) -> None: + """Set the resource manifest in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + resource_manifest: the dictionary that contains key-value for resource manifest. + """ + self.update_relation_data(relation_id, {"resource-manifest": resource_manifest}) + + +class SparkServiceAccountProviderEventHandlers(EventHandlers): + """Provider-side of the Spark Service Account relation.""" + + on = SparkServiceAccountProviderEvents() # pyright: ignore [reportAssignmentType] + + def __init__(self, charm: CharmBase, relation_data: SparkServiceAccountProviderData) -> None: + super().__init__(charm, relation_data) + # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above + self.relation_data = relation_data + self.framework.observe( + charm.on[self.relation_data.relation_name].relation_broken, + self._on_relation_broken, + ) + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation has changed.""" + # Leader only + if not self.relation_data.local_unit.is_leader(): + return + + diff = self._diff(event) + # emit on account requested if service account name is provided by the requirer application + if "service-account" in diff.added: + getattr(self.on, "account_requested").emit( + event.relation, app=event.app, unit=event.unit + ) + + def _on_relation_broken(self, event: RelationBrokenEvent) -> None: + """React to the relation broken event by releasing the service account.""" + # Leader only + if not self.relation_data.local_unit.is_leader(): + return + + getattr(self.on, "account_released").emit(event.relation, app=event.app, unit=event.unit) + + +class SparkServiceAccountProvider( + SparkServiceAccountProviderData, SparkServiceAccountProviderEventHandlers +): + """Provider-side of the Spark Service Account relation.""" + + def __init__(self, charm: CharmBase, relation_name: str) -> None: + SparkServiceAccountProviderData.__init__(self, charm.model, relation_name) + SparkServiceAccountProviderEventHandlers.__init__(self, charm, self) + + +class SparkServiceAccountRequirerData(RequirerData): + """Implementation of RequirerData for the Spark Service Account relation.""" + + def __init__( + self, + model: Model, + relation_name: str, + service_account: str, + skip_creation: bool = False, + additional_secret_fields: Optional[List[str]] = [], + ): + """Manager of Spark Service Account relations.""" + if not additional_secret_fields: + additional_secret_fields = [] + if SPARK_PROPERTIES_RELATION_FIELD not in additional_secret_fields: + additional_secret_fields.append(SPARK_PROPERTIES_RELATION_FIELD) + super().__init__(model, relation_name, additional_secret_fields=additional_secret_fields) + self.service_account = service_account + self.skip_creation = "true" if skip_creation else "false" + + @property + def service_account(self): + """Service account used for Spark.""" + return self._service_account + + @service_account.setter + def service_account(self, value): + self._service_account = value + + +class SparkServiceAccountRequirerEventHandlers(RequirerEventHandlers): + """Requirer-side event handlers of the Spark Service Account relation.""" + + on = SparkServiceAccountRequirerEvents() # pyright: ignore [reportAssignmentType] + + def __init__(self, charm: CharmBase, relation_data: SparkServiceAccountRequirerData) -> None: + super().__init__(charm, relation_data) + # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above + self.relation_data = relation_data + self.framework.observe( + charm.on[self.relation_data.relation_name].relation_broken, + self._on_relation_broken, + ) + + def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: + """Event emitted when the Spark Service Account relation is created.""" + super()._on_relation_created_event(event) + + if not self.relation_data.local_unit.is_leader(): + return + + # Sets service_account in the relation + relation_data = { + f: getattr(self.relation_data, f.replace("-", "_"), "") + for f in ["service-account", "skip-creation"] + } + + self.relation_data.update_relation_data(event.relation.id, relation_data) + + def _on_secret_changed_event(self, event: SecretChangedEvent): + """Event notifying about a new value of a secret.""" + if not event.secret.label: + return + + relation = self.relation_data._relation_from_secret_label(event.secret.label) + if not relation: + logging.info( + f"Received secret {event.secret.label} but couldn't parse, seems irrelevant" + ) + return + + if relation.app == self.charm.app: + logging.info("Secret changed event ignored for Secret Owner") + + remote_unit = None + for unit in relation.units: + if unit.app != self.charm.app: + remote_unit = unit + + getattr(self.on, "properties_changed").emit(relation, app=relation.app, unit=remote_unit) + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the Spark Service Account relation has changed.""" + logger.info("On Spark Service Account relation changed") + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # Register all new secrets with their labels + if any(newval for newval in diff.added if self.relation_data._is_secret_field(newval)): + self.relation_data._register_secrets_to_relation(event.relation, diff.added) + + secret_field_user = self.relation_data._generate_secret_field_name(SECRET_GROUPS.USER) + + if ("service-account" in diff.added) or secret_field_user in diff.added: + getattr(self.on, "account_granted").emit( + event.relation, app=event.app, unit=event.unit + ) + + def _on_relation_broken(self, event: RelationBrokenEvent) -> None: + """Notify the charm about a broken service account relation.""" + logger.info("On Spark Service Account relation gone") + getattr(self.on, "account_gone").emit(event.relation, app=event.app, unit=event.unit) + + +class SparkServiceAccountRequirer( + SparkServiceAccountRequirerData, SparkServiceAccountRequirerEventHandlers +): + """Requirer side of the Spark Service Account relation.""" + + def __init__( + self, + charm: CharmBase, + relation_name: str, + service_account: str, + skip_creation: bool = False, + additional_secret_fields: Optional[List[str]] = [], + ) -> None: + SparkServiceAccountRequirerData.__init__( + self, + charm.model, + relation_name, + service_account, + skip_creation=skip_creation, + additional_secret_fields=additional_secret_fields, + ) + SparkServiceAccountRequirerEventHandlers.__init__(self, charm, self) diff --git a/tests/integration/application-spark-service-account-charm/actions.yaml b/tests/integration/application-spark-service-account-charm/actions.yaml new file mode 100644 index 00000000..9edfe421 --- /dev/null +++ b/tests/integration/application-spark-service-account-charm/actions.yaml @@ -0,0 +1,5 @@ +get-spark-properties: + description: Get spark properties + +get-resource-manifest: + description: Get K8s resource manifest \ No newline at end of file diff --git a/tests/integration/application-spark-service-account-charm/charmcraft.yaml b/tests/integration/application-spark-service-account-charm/charmcraft.yaml new file mode 100644 index 00000000..4a0e40bf --- /dev/null +++ b/tests/integration/application-spark-service-account-charm/charmcraft.yaml @@ -0,0 +1,11 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +type: charm +bases: + - build-on: + - name: "ubuntu" + channel: "22.04" + run-on: + - name: "ubuntu" + channel: "22.04" diff --git a/tests/integration/application-spark-service-account-charm/lib/charms/data_platform_libs/v0/.gitkeep b/tests/integration/application-spark-service-account-charm/lib/charms/data_platform_libs/v0/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/application-spark-service-account-charm/metadata.yaml b/tests/integration/application-spark-service-account-charm/metadata.yaml new file mode 100644 index 00000000..707e2b31 --- /dev/null +++ b/tests/integration/application-spark-service-account-charm/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +name: spark-sa-app +description: | + A requirer charm for spark-service-account relation +summary: | + A requirer charm for spark-service-account relation + + +requires: + spark-service-account: + interface: spark_service_account + + +peers: + spark-properties: + interface: spark-properties \ No newline at end of file diff --git a/tests/integration/application-spark-service-account-charm/requirements.txt b/tests/integration/application-spark-service-account-charm/requirements.txt new file mode 100644 index 00000000..b05b783d --- /dev/null +++ b/tests/integration/application-spark-service-account-charm/requirements.txt @@ -0,0 +1 @@ +ops>=2.4.1 \ No newline at end of file diff --git a/tests/integration/application-spark-service-account-charm/src/charm.py b/tests/integration/application-spark-service-account-charm/src/charm.py new file mode 100755 index 00000000..12a7f9b3 --- /dev/null +++ b/tests/integration/application-spark-service-account-charm/src/charm.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 + +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Spark service account requirer charm.""" + +import json +import logging + +from ops import ActiveStatus, WaitingStatus +from ops.charm import ActionEvent, CharmBase +from ops.main import main + +from charms.data_platform_libs.v0.data_interfaces import DataPeer +from charms.data_platform_libs.v0.spark_service_account import ( + ServiceAccountGoneEvent, + ServiceAccountGrantedEvent, + ServiceAccountPropertyChangedEvent, + SparkServiceAccountRequirer, +) + +logger = logging.getLogger(__name__) + +NAMESPASCE = "default" +USERNAME1 = "user1" +SERVICE_ACCOUNT1 = f"{NAMESPASCE}:{USERNAME1}" +PEER_REL = "spark-properties" +REQUIRER_REL = "spark-service-account" + + +class SparkServiceAccountRequirerCharm(CharmBase): + def __init__(self, *args): + super().__init__(*args) + + # Default charm events. + self.framework.observe(self.on.start, self._on_start) + + self.peer_relation_data = DataPeer(self, relation_name=PEER_REL) + + # Charm events defined in the spark_service_account charm library. + self.service_account_requirer = SparkServiceAccountRequirer( + self, relation_name=REQUIRER_REL, service_account=SERVICE_ACCOUNT1, skip_creation=False + ) + + self.framework.observe( + self.service_account_requirer.on.account_granted, self._on_service_account_granted + ) + self.framework.observe( + self.service_account_requirer.on.account_gone, self._on_service_account_gone + ) + self.framework.observe( + self.service_account_requirer.on.properties_changed, + self._on_service_account_properties_changed, + ) + self.framework.observe( + self.on.get_spark_properties_action, self._on_get_spark_properties_action + ) + self.framework.observe( + self.on.get_resource_manifest_action, self._on_get_resource_manifest_action + ) + + def _on_start(self, _) -> None: + """Only sets a blocked status.""" + message = "Waiting for spark-service-account relation" + if self.unit.is_leader(): + self.app.status = WaitingStatus(message) + self.unit.status = WaitingStatus(message) + + def _on_service_account_granted(self, event: ServiceAccountGrantedEvent): + """Handle the `ServiceAccountGranted` event.""" + if not self.unit.is_leader() or not event.service_account: + return + message = f"Service account granted: {event.service_account}" + self.unit.status = ActiveStatus(message) + self.app.status = ActiveStatus(message) + spark_properties = json.loads(event.spark_properties or "{}") + + peer_relation = self.model.get_relation(PEER_REL) + if not peer_relation: + return + self.peer_relation_data.update_relation_data( + relation_id=peer_relation.id, data=spark_properties + ) + + def _on_service_account_gone(self, event: ServiceAccountGoneEvent): + """Handle the `ServiceAccountGone` event.""" + if not self.unit.is_leader(): + return + message = "Waiting for spark-service-account relation" + self.app.status = WaitingStatus(message) + self.unit.status = WaitingStatus(message) + + peer_relation = self.model.get_relation(PEER_REL) + if not peer_relation: + return + peer_relation.data[self.app].clear() + + def _on_service_account_properties_changed(self, event: ServiceAccountPropertyChangedEvent): + """Handle the `ServiceAccountPropertyChangedEvent`.""" + if not self.unit.is_leader() or not event.service_account: + return + + requirer_relation = self.model.get_relation(relation_name=REQUIRER_REL) + if not requirer_relation: + return + spark_properties = ( + self.service_account_requirer.fetch_relation_field( + relation_id=requirer_relation.id, field="spark-properties" + ) + or "{}" + ) + peer_relation = self.model.get_relation(PEER_REL) + if not peer_relation: + return + self.peer_relation_data.update_relation_data( + relation_id=peer_relation.id, data=json.loads(spark_properties) + ) + + def _on_get_spark_properties_action(self, event: ActionEvent): + peer_relation = self.model.get_relation(PEER_REL) + if not peer_relation: + logger.warning("No peer relation") + return + relation_data = self.peer_relation_data.fetch_my_relation_data([peer_relation.id]) + if not relation_data: + return + props = relation_data[peer_relation.id] + event.set_results({"spark-properties": json.dumps(props)}) + + def _on_get_resource_manifest_action(self, event: ActionEvent): + sa_relation = self.model.get_relation(REQUIRER_REL) + if not sa_relation: + logger.warning("No service account relation") + return + + resource_manifest = self.service_account_requirer.fetch_relation_field( + relation_id=sa_relation.id, field="resource-manifest" + ) + event.set_results({"resource-manifest": resource_manifest}) + + +if __name__ == "__main__": + main(SparkServiceAccountRequirerCharm) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 35da09bd..90109794 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -58,6 +58,10 @@ def copy_data_interfaces_library_into_charm(ops_test: OpsTest): shutil.copyfile(library_path, install_path) install_path = "tests/integration/opensearch-charm/" + library_path shutil.copyfile(library_path, install_path) + install_path = "tests/integration/spark-service-account-charm/" + library_path + shutil.copyfile(library_path, install_path) + install_path = "tests/integration/application-spark-service-account-charm/" + library_path + shutil.copyfile(library_path, install_path) @pytest.fixture(scope="module", autouse=True) @@ -70,6 +74,18 @@ def copy_s3_library_into_charm(ops_test: OpsTest): shutil.copyfile(library_path, install_path_requirer) +@pytest.fixture(scope="module", autouse=True) +def copy_spark_service_account_library_into_charm(ops_test: OpsTest): + """Copy the spark service account library to the applications charm folder.""" + library_path = "lib/charms/data_platform_libs/v0/spark_service_account.py" + install_path_provider = "tests/integration/spark-service-account-charm/" + library_path + install_path_requirer = ( + "tests/integration/application-spark-service-account-charm/" + library_path + ) + shutil.copyfile(library_path, install_path_provider) + shutil.copyfile(library_path, install_path_requirer) + + @pytest.fixture(scope="module") async def application_charm(ops_test: OpsTest): """Build the application charm.""" @@ -110,6 +126,22 @@ async def s3_charm(ops_test: OpsTest): return charm +@pytest.fixture(scope="module") +async def application_spark_service_account_charm(ops_test: OpsTest): + """Build the aplication-spark-service-account charm.""" + charm_path = "tests/integration/application-spark-service-account-charm" + charm = await ops_test.build_charm(charm_path) + return charm + + +@pytest.fixture(scope="module") +async def spark_service_account_charm(ops_test: OpsTest): + """Build the spark-service-account charm.""" + charm_path = "tests/integration/spark-service-account-charm" + charm = await ops_test.build_charm(charm_path) + return charm + + @pytest.fixture(scope="module") async def kafka_charm(ops_test: OpsTest): """Build the Kafka charm.""" diff --git a/tests/integration/spark-service-account-charm/actions.yaml b/tests/integration/spark-service-account-charm/actions.yaml new file mode 100644 index 00000000..f442ef6d --- /dev/null +++ b/tests/integration/spark-service-account-charm/actions.yaml @@ -0,0 +1,9 @@ +get-spark-properties: + description: Get spark properties + +add-spark-property: + description: Add spark property + params: + conf: + type: string + description: The spark property in format key=value \ No newline at end of file diff --git a/tests/integration/spark-service-account-charm/charmcraft.yaml b/tests/integration/spark-service-account-charm/charmcraft.yaml new file mode 100644 index 00000000..4a0e40bf --- /dev/null +++ b/tests/integration/spark-service-account-charm/charmcraft.yaml @@ -0,0 +1,11 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +type: charm +bases: + - build-on: + - name: "ubuntu" + channel: "22.04" + run-on: + - name: "ubuntu" + channel: "22.04" diff --git a/tests/integration/spark-service-account-charm/lib/charms/data_platform_libs/v0/.gitkeep b/tests/integration/spark-service-account-charm/lib/charms/data_platform_libs/v0/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/spark-service-account-charm/metadata.yaml b/tests/integration/spark-service-account-charm/metadata.yaml new file mode 100644 index 00000000..df622178 --- /dev/null +++ b/tests/integration/spark-service-account-charm/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +name: spark-sa-provider-charm +description: | + A charm that provides the provider side + of spark-service-account relation. +summary: | + A charm that provides the provider side + of spark-service-account relation. + +provides: + spark-service-account: + interface: spark_service_account + +peers: + spark-properties: + interface: spark-properties \ No newline at end of file diff --git a/tests/integration/spark-service-account-charm/requirements.txt b/tests/integration/spark-service-account-charm/requirements.txt new file mode 100644 index 00000000..3570c743 --- /dev/null +++ b/tests/integration/spark-service-account-charm/requirements.txt @@ -0,0 +1,3 @@ +ops>=2.4.1 +spark8t>=0.0.12 +pyyaml>=6.0.2 \ No newline at end of file diff --git a/tests/integration/spark-service-account-charm/src/charm.py b/tests/integration/spark-service-account-charm/src/charm.py new file mode 100755 index 00000000..1822f91c --- /dev/null +++ b/tests/integration/spark-service-account-charm/src/charm.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 + +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Spark service account provider charm.""" + +import json +import logging + +import yaml +from ops import ActiveStatus +from ops.charm import ActionEvent, CharmBase +from ops.main import main +from spark8t.domain import PropertyFile + +from charms.data_platform_libs.v0.data_interfaces import DataPeer +from charms.data_platform_libs.v0.spark_service_account import ( + ServiceAccountReleasedEvent, + ServiceAccountRequestedEvent, + SparkServiceAccountProvider, +) + +logger = logging.getLogger(__name__) + +NAMESPASCE = "default" +USERNAME = "user" +SERVICE_ACCOUNT = f"{NAMESPASCE}:{USERNAME}" +PEER_REL = "spark-properties" +PROVIDER_REL = "spark-service-account" +RESOURCE_MANIFEST = {"foo": "bar"} + + +class SparkServiceAccountProviderCharm(CharmBase): + def __init__(self, *args): + super().__init__(*args) + + # Default charm events. + self.framework.observe(self.on.start, self._on_start) + + self.peer_relation_data = DataPeer(self, relation_name=PEER_REL) + + # Charm events defined in the spark_service_account charm library. + self.service_account_provider = SparkServiceAccountProvider( + self, relation_name="spark-service-account" + ) + + self.framework.observe( + self.service_account_provider.on.account_requested, self._on_service_account_requested + ) + self.framework.observe( + self.service_account_provider.on.account_released, self._on_service_account_released + ) + self.framework.observe( + self.on.get_spark_properties_action, self._on_get_spark_properties_action + ) + self.framework.observe( + self.on.add_spark_property_action, self._on_add_spark_property_action + ) + self.framework.observe(self.on[PEER_REL].relation_changed, self._on_peer_relation_changed) + + def _on_start(self, _) -> None: + """Only sets an active status.""" + self.unit.status = ActiveStatus("") + peer_relation = self.model.get_relation(PEER_REL) + if not peer_relation: + return + self.peer_relation_data.update_relation_data( + relation_id=peer_relation.id, data={"default-key": "default-val"} + ) + + def _create_service_account(self, service_account: str) -> None: + logger.info(f"Created service account: {service_account}") + + def _delete_service_account(self, service_account: str) -> None: + logger.info(f"Deleted service account: {service_account}") + + def _add_spark_property(self, property_line: str) -> None: + key, val = PropertyFile.parse_property_line(property_line) + peer_relation = self.model.get_relation(PEER_REL) + if not peer_relation: + return + self.peer_relation_data.update_relation_data(peer_relation.id, {key: val}) + logger.info(f"Added property {property_line}.") + + def _get_spark_properties(self) -> dict[str, str]: + peer_relation = self.model.get_relation(PEER_REL) + if not peer_relation: + return {} + relation_data = self.peer_relation_data.fetch_my_relation_data([peer_relation.id]) + if not relation_data: + return {} + props = relation_data[peer_relation.id] + return props + + def _on_get_spark_properties_action(self, event: ActionEvent) -> None: + props = self._get_spark_properties() + event.set_results({"spark-properties": json.dumps(props)}) + + def _on_add_spark_property_action(self, event: ActionEvent) -> None: + conf = event.params["conf"] + self._add_spark_property(property_line=conf) + event.set_results({"success": "true"}) + + def _on_service_account_requested(self, event: ServiceAccountRequestedEvent): + """Handle the `ServiceAccountRequested` event for the Spark Integration hub.""" + if not self.unit.is_leader() or not event.service_account: + return + self._create_service_account(event.service_account) + properties = self._get_spark_properties() + + self.service_account_provider.set_service_account(event.relation.id, event.service_account) # type: ignore + self.service_account_provider.set_spark_properties( + event.relation.id, json.dumps(properties) + ) + self.service_account_provider.set_resource_manifest(event.relation.id, yaml.dump(RESOURCE_MANIFEST)) # type: ignore + + def _on_service_account_released(self, event: ServiceAccountReleasedEvent): + """Handle the `ServiceAccountReleased` event for the Spark Integration hub.""" + if not self.unit.is_leader() or not event.service_account: + return + self._delete_service_account(event.service_account) + + def _on_peer_relation_changed(self, _): + """Handle on PEER relation changed event.""" + provider_relation = self.model.get_relation(relation_name=PROVIDER_REL) + if not provider_relation: + return + relation_id = provider_relation.id + properties = self._get_spark_properties() + self.service_account_provider.set_spark_properties( + relation_id=relation_id, spark_properties=json.dumps(properties) + ) + + +if __name__ == "__main__": + main(SparkServiceAccountProviderCharm) diff --git a/tests/integration/test_spark_service_account.py b/tests/integration/test_spark_service_account.py new file mode 100644 index 00000000..a922d7de --- /dev/null +++ b/tests/integration/test_spark_service_account.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +import asyncio +import json +import logging + +import pytest +import yaml +from pytest_operator.plugin import OpsTest + +logger = logging.getLogger(__name__) + +APPLICATION_APP_NAME = "app" +SA_PROVIDER_APP_NAME = "sa-provider" +APP_NAMES = [APPLICATION_APP_NAME, SA_PROVIDER_APP_NAME] + +RELATION_NAME = "spark-service-account" + + +@pytest.mark.abort_on_fail +async def test_deploy_charms( + ops_test: OpsTest, + application_spark_service_account_charm, + spark_service_account_charm, + dp_libs_ubuntu_series, +): + """Deploy both charms (application and service account provider app) to use in the tests.""" + await asyncio.gather( + ops_test.model.deploy( + application_spark_service_account_charm, + application_name=APPLICATION_APP_NAME, + num_units=1, + series=dp_libs_ubuntu_series, + ), + ops_test.model.deploy( + spark_service_account_charm, + application_name=SA_PROVIDER_APP_NAME, + num_units=1, + series=dp_libs_ubuntu_series, + ), + ) + await ops_test.model.wait_for_idle(apps=[SA_PROVIDER_APP_NAME], status="active") + await ops_test.model.wait_for_idle(apps=[APPLICATION_APP_NAME], status="waiting") + + assert ops_test.model.applications[SA_PROVIDER_APP_NAME].status == "active" + assert ops_test.model.applications[APPLICATION_APP_NAME].status == "waiting" + assert ( + ops_test.model.applications[APPLICATION_APP_NAME].status_message + == "Waiting for spark-service-account relation" + ) + + +@pytest.mark.abort_on_fail +async def test_spark_service_account_relation_with_charm_libraries(ops_test: OpsTest): + """Test basic functionality of spark_service_account relation interface.""" + # Relate the charms and wait for them exchanging some data. + await ops_test.model.add_relation( + SA_PROVIDER_APP_NAME, f"{APPLICATION_APP_NAME}:{RELATION_NAME}" + ) + await ops_test.model.wait_for_idle(apps=APP_NAMES, status="active") + + assert ops_test.model.applications[APPLICATION_APP_NAME].status == "active" + assert ( + ops_test.model.applications[APPLICATION_APP_NAME].status_message + == "Service account granted: default:user1" + ) + + app_unit = ops_test.model.applications[APPLICATION_APP_NAME].units[0] + action = await app_unit.run_action( + action_name="get-spark-properties", + ) + result = await action.wait() + spark_properties = json.loads(result.results.get("spark-properties", "{}")) + assert spark_properties["default-key"] == "default-val" + + app_unit = ops_test.model.applications[APPLICATION_APP_NAME].units[0] + action = await app_unit.run_action( + action_name="get-resource-manifest", + ) + result = await action.wait() + resource_manifest = yaml.safe_load(result.results.get("resource-manifest", "")) + + assert resource_manifest["foo"] == "bar" + + +@pytest.mark.abort_on_fail +async def test_spark_properties_changed(ops_test: OpsTest): + """Test the change in spark properties get reflected in requirer charm.""" + provider_unit = ops_test.model.applications[SA_PROVIDER_APP_NAME].units[0] + action = await provider_unit.run_action( + action_name="add-spark-property", conf="new-key=new-val" + ) + result = await action.wait() + assert result.results.get("success") == "true" + + await ops_test.model.wait_for_idle(apps=APP_NAMES, status="active") + + app_unit = ops_test.model.applications[APPLICATION_APP_NAME].units[0] + action = await app_unit.run_action( + action_name="get-spark-properties", + ) + result = await action.wait() + spark_properties = json.loads(result.results.get("spark-properties", "{}")) + + assert spark_properties["default-key"] == "default-val" + assert spark_properties["new-key"] == "new-val" + + +@pytest.mark.abort_on_fail +async def test_spark_service_account_relation_broken(ops_test: OpsTest): + """Test what happens when spark service account relation is broken.""" + await ops_test.model.applications[SA_PROVIDER_APP_NAME].remove_relation( + f"{SA_PROVIDER_APP_NAME}:spark-service-account", f"{APPLICATION_APP_NAME}:{RELATION_NAME}" + ) + await ops_test.model.wait_for_idle(apps=[SA_PROVIDER_APP_NAME], status="active") + await ops_test.model.wait_for_idle(apps=[APPLICATION_APP_NAME], status="waiting") + + assert ops_test.model.applications[SA_PROVIDER_APP_NAME].status == "active" + assert ops_test.model.applications[APPLICATION_APP_NAME].status == "waiting" + assert ( + ops_test.model.applications[APPLICATION_APP_NAME].status_message + == "Waiting for spark-service-account relation" + ) + + app_unit = ops_test.model.applications[APPLICATION_APP_NAME].units[0] + action = await app_unit.run_action( + action_name="get-spark-properties", + ) + result = await action.wait() + spark_properties = json.loads(result.results.get("spark-properties", "{}")) + assert len(spark_properties) == 0 diff --git a/tests/unit/test_spark_service_account.py b/tests/unit/test_spark_service_account.py new file mode 100644 index 00000000..cbb8ee5b --- /dev/null +++ b/tests/unit/test_spark_service_account.py @@ -0,0 +1,339 @@ +import json +from unittest import mock + +import pytest +import yaml +from lib.charms.data_platform_libs.v0.spark_service_account import ( + ServiceAccountGoneEvent, + ServiceAccountGrantedEvent, + ServiceAccountPropertyChangedEvent, + ServiceAccountReleasedEvent, + ServiceAccountRequestedEvent, + SparkServiceAccountProvider, + SparkServiceAccountRequirer, +) +from ops.charm import ActionEvent, CharmBase +from ops.testing import Context, Relation, Secret, State + +PROVIDER_APP = "service-account-provider" +REQUIRER_APP = "service-account-requirer" +SERVICE_ACCOUNT = "default:user" +RELATION_INTERFACE = "spark_service_account" +RELATION_NAME = "spark-service-account" +SPARK_PROPS = {"foo": "bar"} +RESOURCE_MANIFEST = {"foo": "bar"} + + +class SparkServiceAccountProviderCharm(CharmBase): + """Mock the provider charm for Spark Service Account relation for testing.""" + + META = { + "name": PROVIDER_APP, + "provides": {RELATION_NAME: {"interface": RELATION_INTERFACE}}, + } + + ACTIONS = {"add-config": {"params": {"conf": {"type": "string"}}}} + + def __init__(self, *args): + super().__init__(*args) + self.provider = SparkServiceAccountProvider(self, relation_name=RELATION_NAME) + self.framework.observe(self.provider.on.account_requested, self._on_account_requested) + self.framework.observe(self.provider.on.account_released, self._on_account_released) + self.framework.observe(self.on.add_config_action, self._on_add_config_action) + + def _generate_service_account_manifest(self, service_account: str) -> dict: + print(f"Generated service account resource manifest: {service_account}") + return RESOURCE_MANIFEST + + def _create_service_account(self, service_account: str) -> None: + print(f"Created service account: {service_account}") + + def _delete_service_account(self, service_account: str) -> None: + print(f"Deleted service account: {service_account}") + + def _on_account_requested(self, event: ServiceAccountRequestedEvent) -> None: + if not event.service_account: + return + service_account = event.service_account + if not event.skip_creation: + self._create_service_account(service_account) + manifest = self._generate_service_account_manifest(service_account=service_account) + self.provider.set_service_account(event.relation.id, service_account) + self.provider.set_spark_properties(event.relation.id, json.dumps(SPARK_PROPS)) + self.provider.set_resource_manifest( + event.relation.id, resource_manifest=yaml.dump(manifest) + ) + + def _on_account_released(self, event: ServiceAccountReleasedEvent) -> None: + if not event.service_account: + return + service_account = event.service_account + skip_creation = event.skip_creation + if not skip_creation: + self._delete_service_account(service_account) + + def _on_add_config_action(self, event: ActionEvent) -> None: + conf = event.params["conf"] + key, val = conf.split("=", 1) + props = SPARK_PROPS.copy() + props.update({key: val}) + for rel in self.provider.relations: + self.provider.set_spark_properties(rel.id, json.dumps(props)) + + +class SparkServiceAccountRequirerCharm(CharmBase): + + META = {"name": REQUIRER_APP, "requires": {RELATION_NAME: {"interface": RELATION_INTERFACE}}} + + CONFIG = {"options": {"skip-creation": {"type": "string", "default": "false"}}} + + def __init__(self, *args): + super().__init__(*args) + skip_creation = self.config.get("skip-creation", "false") == "true" + self.requirer = SparkServiceAccountRequirer( + self, + relation_name=RELATION_NAME, + service_account=SERVICE_ACCOUNT, + skip_creation=skip_creation, + ) + self.framework.observe(self.requirer.on.account_granted, self._on_account_granted) + self.framework.observe(self.requirer.on.account_gone, self._on_account_gone) + self.framework.observe(self.requirer.on.properties_changed, self._on_properties_changed) + + def _consume_service_account( + self, service_account: str | None, spark_properties: str | None + ) -> None: + if not service_account or not spark_properties: + return + print(f"Consuming service account: {service_account}.") + props = json.loads(spark_properties) + print(f"Spark properties are: {props}") + + def _on_account_granted(self, event: ServiceAccountGrantedEvent) -> None: + service_account = event.service_account + spark_properties = event.spark_properties + self._consume_service_account(service_account, spark_properties) + + def _on_account_gone(self, event: ServiceAccountGoneEvent) -> None: + self._consume_service_account(None, None) + + def _on_properties_changed(self, event: ServiceAccountPropertyChangedEvent) -> None: + service_account = event.service_account + spark_properties = event.spark_properties + self._consume_service_account(service_account, spark_properties) + + +@pytest.mark.usefixtures("only_with_juju_secrets") +class TestSparkServiceAccountProvider: + + def get_relation(self): + return Relation( + endpoint=RELATION_NAME, + interface=RELATION_INTERFACE, + remote_app_name=REQUIRER_APP, + local_app_data={}, + remote_app_data={}, + ) + + @property + def context(self): + return Context( + charm_type=SparkServiceAccountProviderCharm, + meta=SparkServiceAccountProviderCharm.META, + actions=SparkServiceAccountProviderCharm.ACTIONS, + ) + + @mock.patch.object(SparkServiceAccountProviderCharm, "_create_service_account") + def test_service_account_created_by_provider(self, mock_create_sa): + relation = self.get_relation() + state1 = State(relations=[relation], leader=True) + relation.remote_app_data.update( + { + "service-account": SERVICE_ACCOUNT, + "requested-secrets": json.dumps(["spark-properties"]), + } + ) + + state2 = self.context.run(self.context.on.relation_changed(relation), state1) + mock_create_sa.assert_called_with(SERVICE_ACCOUNT) + + local_app_data = state2.get_relation(relation.id).local_app_data + assert local_app_data["service-account"] == SERVICE_ACCOUNT + assert local_app_data["resource-manifest"] == yaml.dump(RESOURCE_MANIFEST) + + assert "secret-extra" in local_app_data + + secret_id = local_app_data["secret-extra"] + secret_content = state2.get_secret(id=secret_id).latest_content + assert secret_content is not None + spark_properties = json.loads(secret_content["spark-properties"]) + assert spark_properties == SPARK_PROPS + + @mock.patch.object(SparkServiceAccountProviderCharm, "_create_service_account") + def test_service_account_creation_skipped(self, mock_create_sa): + relation = self.get_relation() + state1 = State(relations=[relation], leader=True) + relation.remote_app_data.update( + { + "service-account": SERVICE_ACCOUNT, + "requested-secrets": json.dumps(["spark-properties"]), + "skip-creation": "true", + } + ) + + state2 = self.context.run(self.context.on.relation_changed(relation), state1) + assert not mock_create_sa.called + + local_app_data = state2.get_relation(relation.id).local_app_data + assert local_app_data["service-account"] == SERVICE_ACCOUNT + assert local_app_data["resource-manifest"] == yaml.dump(RESOURCE_MANIFEST) + assert "secret-extra" in local_app_data + + secret_id = local_app_data["secret-extra"] + secret_content = state2.get_secret(id=secret_id).latest_content + assert secret_content is not None + spark_properties = json.loads(secret_content["spark-properties"]) + assert spark_properties == SPARK_PROPS + + def test_service_account_property_changed( + self, + ): + relation = self.get_relation() + state1 = State(relations=[relation], leader=True) + relation.remote_app_data.update( + { + "service-account": SERVICE_ACCOUNT, + "requested-secrets": json.dumps(["spark-properties"]), + } + ) + state2 = self.context.run(self.context.on.relation_changed(relation), state1) + state3 = self.context.run( + self.context.on.action("add-config", params={"conf": "newkey=newval"}), state2 + ) + + local_app_data = state3.get_relation(relation.id).local_app_data + assert local_app_data["service-account"] == SERVICE_ACCOUNT + assert "secret-extra" in local_app_data + + secret_id = local_app_data["secret-extra"] + secret_content = state3.get_secret(id=secret_id).latest_content + assert secret_content is not None + spark_properties = json.loads(secret_content["spark-properties"]) + assert spark_properties["newkey"] == "newval" + + @mock.patch.object(SparkServiceAccountProviderCharm, "_delete_service_account") + def test_service_account_released(self, mock_delete_sa): + relation = self.get_relation() + relation.remote_app_data.update( + { + "service-account": SERVICE_ACCOUNT, + "requested-secrets": json.dumps(["spark-properties"]), + } + ) + state1 = State(relations=[relation], leader=True) + self.context.run(self.context.on.relation_broken(relation), state1) + mock_delete_sa.assert_called_with(SERVICE_ACCOUNT) + + @mock.patch.object(SparkServiceAccountProviderCharm, "_delete_service_account") + def test_service_account_released_skip_deletion(self, mock_delete_sa): + relation = self.get_relation() + relation.remote_app_data.update( + { + "service-account": SERVICE_ACCOUNT, + "requested-secrets": json.dumps(["spark-properties"]), + "skip-creation": "true", + } + ) + state1 = State(relations=[relation], leader=True) + self.context.run(self.context.on.relation_broken(relation), state1) + assert not mock_delete_sa.called + + +@pytest.mark.usefixtures("only_with_juju_secrets") +class TestSparkServiceAccountRequirer: + + def get_relation(self) -> Relation: + return Relation( + endpoint=RELATION_NAME, + interface=RELATION_INTERFACE, + remote_app_name=PROVIDER_APP, + local_app_data={ + "service-account": SERVICE_ACCOUNT, + "requested-secrets": json.dumps(["spark-properties"]), + }, + remote_app_data={}, + ) + + @property + def context(self): + return Context( + charm_type=SparkServiceAccountRequirerCharm, + meta=SparkServiceAccountRequirerCharm.META, + config=SparkServiceAccountRequirerCharm.CONFIG, + ) + + @mock.patch.object(SparkServiceAccountRequirerCharm, "_consume_service_account") + def test_service_account_granted(self, mock_consume_sa): + relation = self.get_relation() + relation.remote_app_data.update( + { + "service-account": SERVICE_ACCOUNT, + } + ) + + state1 = State( + relations=[relation], + # secrets=[props_secret], + leader=True, + ) + self.context.run(self.context.on.relation_changed(relation), state1) + + args, kwargs = mock_consume_sa.call_args + service_account, spark_properties = args + assert service_account == SERVICE_ACCOUNT + assert spark_properties == "{}" + + @mock.patch.object(SparkServiceAccountRequirerCharm, "_consume_service_account") + def test_spark_properties_changed(self, mock_consume_sa): + relation = self.get_relation() + props_secret = Secret( + tracked_content={"spark-properties": json.dumps(SPARK_PROPS)}, + label=f"{RELATION_NAME}.{relation.id}.extra.secret", + ) + relation.remote_app_data.update( + {"service-account": SERVICE_ACCOUNT, "secret-extra": props_secret.id} + ) + state1 = State( + relations=[relation], + secrets=[props_secret], + leader=True, + config={"skip-creation": "true"}, + ) + + self.context.run(self.context.on.secret_changed(props_secret), state1) + + args, kwargs = mock_consume_sa.call_args + service_account, spark_properties = args + assert service_account == SERVICE_ACCOUNT + assert "foo" in json.loads(spark_properties) + + print(state1.config) + + @mock.patch.object(SparkServiceAccountRequirerCharm, "_consume_service_account") + def test_service_account_gone(self, mock_consume_sa): + relation = self.get_relation() + props_secret = Secret( + tracked_content={"spark-properties": json.dumps(SPARK_PROPS)}, + label=f"{RELATION_NAME}.{relation.id}.extra.secret", + ) + relation.remote_app_data.update( + {"service-account": SERVICE_ACCOUNT, "secret-extra": props_secret.id} + ) + state1 = State(relations=[relation], secrets=[props_secret], leader=True) + + self.context.run(self.context.on.relation_broken(relation), state1) + + args, kwargs = mock_consume_sa.call_args + service_account, spark_properties = args + assert service_account is None + assert spark_properties is None diff --git a/tox.ini b/tox.ini index a6e13e46..8b4b4c3c 100644 --- a/tox.ini +++ b/tox.ini @@ -72,6 +72,7 @@ deps = pytest<8.2.0 pytest-mock coverage[toml] + ops-scenario -r {tox_root}/requirements.txt commands = coverage run --source={[vars]src_path},{[vars]lib_path} \ @@ -140,6 +141,19 @@ deps = commands = pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/test_s3_charm.py +[testenv:integration-spark-sa] +description = Run Spark service account integration tests +deps = + psycopg2-binary + pytest<8.2.0 + juju{env:LIBJUJU_VERSION_SPECIFIER:==3.6.1.0} + pytest-operator + pytest-mock + pyyaml + -r {tox_root}/requirements.txt +commands = + pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/test_spark_service_account.py + [testenv:integration-opensearch] description = Run opensearch integration tests deps =