From bdf9930b7b2dcab8901d301c2d5d764038bb0f5f Mon Sep 17 00:00:00 2001 From: Dragomir Penev Date: Fri, 7 Nov 2025 19:02:47 +0200 Subject: [PATCH 1/7] Port charm test WIP --- .../test_async_replication_upgrade.py | 11 +- .../spaces/test_spaced_async_replication.py | 253 +++++++++++++++++ tests/integration/test_backups_aws.py | 2 +- tests/integration/test_backups_gcp.py | 2 +- tests/integration/test_backups_pitr_aws.py | 2 +- tests/integration/test_backups_pitr_gcp.py | 2 +- tests/integration/test_charm.py | 255 ++++-------------- tests/integration/test_tls.py | 2 +- 8 files changed, 314 insertions(+), 215 deletions(-) create mode 100644 tests/integration/spaces/test_spaced_async_replication.py diff --git a/tests/integration/high_availability/test_async_replication_upgrade.py b/tests/integration/high_availability/test_async_replication_upgrade.py index 75e2b5f97bc..98a47ed9889 100644 --- a/tests/integration/high_availability/test_async_replication_upgrade.py +++ b/tests/integration/high_availability/test_async_replication_upgrade.py @@ -42,6 +42,8 @@ def second_model(juju: Juju, request: pytest.FixtureRequest) -> Generator: logging.info(f"Creating model: {model_name}") juju.add_model(model_name) + model_2 = Juju(model=model_name) + model_2.cli("set-model-constraints", f"arch={architecture.architecture}") yield model_name if request.config.getoption("--keep-models"): @@ -84,24 +86,23 @@ def test_deploy(first_model: str, second_model: str, charm: str) -> None: configuration = {"profile": "testing"} constraints = {"arch": architecture.architecture} - # TODO Deploy from edge logging.info("Deploying postgresql clusters") model_1 = Juju(model=first_model) model_1.deploy( - charm=charm, + charm=DB_APP_NAME, app=DB_APP_1, base="ubuntu@24.04", + channel="16/edge", config=configuration, - constraints=constraints, num_units=3, ) model_2 = Juju(model=second_model) model_2.deploy( - charm=charm, + charm=DB_APP_NAME, app=DB_APP_2, base="ubuntu@24.04", + channel="16/edge", config=configuration, - constraints=constraints, num_units=3, ) diff --git a/tests/integration/spaces/test_spaced_async_replication.py b/tests/integration/spaces/test_spaced_async_replication.py new file mode 100644 index 00000000000..72e0846e9af --- /dev/null +++ b/tests/integration/spaces/test_spaced_async_replication.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +import logging +import time +from collections.abc import Generator + +import jubilant +import pytest +from jubilant import Juju +from tenacity import Retrying, stop_after_attempt + +from .. import architecture +from ..high_availability.high_availability_helpers_new import ( + get_app_leader, + get_app_units, + get_db_max_written_value, + wait_for_apps_status, +) + +DB_APP_1 = "db1" +DB_APP_2 = "db2" +DB_TEST_APP_NAME = "postgresql-test-app" +DB_TEST_APP_1 = "test-app1" +DB_TEST_APP_2 = "test-app2" + +MINUTE_SECS = 60 + +logging.getLogger("jubilant.wait").setLevel(logging.WARNING) + + +@pytest.fixture(scope="module") +def first_model(juju: Juju, lxd_spaces, request: pytest.FixtureRequest) -> Generator: + """Creates and return the first model.""" + yield juju.model + + +@pytest.fixture(scope="module") +def second_model(juju: Juju, lxd_spaces, request: pytest.FixtureRequest) -> Generator: + """Creates and returns the second model.""" + model_name = f"{juju.model}-other" + + logging.info(f"Creating model: {model_name}") + juju.add_model(model_name) + model_2 = Juju(model=model_name) + model_2.cli("reload-spaces") + model_2.cli("add-space", "client", "10.0.0.1/24") + model_2.cli("add-space", "peers", "10.10.10.1/24") + model_2.cli("add-space", "isolated", "10.20.20.1/24") + + yield model_name + if request.config.getoption("--keep-models"): + return + + logging.info(f"Destroying model: {model_name}") + juju.destroy_model(model_name, destroy_storage=True, force=True) + + +@pytest.fixture() +def first_model_continuous_writes(first_model: str) -> Generator: + """Starts continuous writes to the cluster for a test and clear the writes at the end.""" + model_1 = Juju(model=first_model) + application_unit = get_app_leader(model_1, DB_TEST_APP_1) + + logging.info("Clearing continuous writes") + model_1.run( + unit=application_unit, action="clear-continuous-writes", wait=120 + ).raise_on_failure() + + logging.info("Starting continuous writes") + + for attempt in Retrying(stop=stop_after_attempt(10), reraise=True): + with attempt: + result = model_1.run(unit=application_unit, action="start-continuous-writes") + result.raise_on_failure() + + assert result.results["result"] == "True" + + yield + + logging.info("Clearing continuous writes") + model_1.run( + unit=application_unit, action="clear-continuous-writes", wait=120 + ).raise_on_failure() + + +def test_deploy(first_model: str, second_model: str, lxd_spaces, charm) -> None: + """Simple test to ensure that the database application charms get deployed.""" + configuration = {"profile": "testing"} + constraints = {"arch": architecture.architecture, "spaces": "client,peers"} + bind = {"database-peers": "peers", "database": "client"} + + logging.info("Deploying postgresql clusters") + model_1 = Juju(model=first_model) + model_1.deploy( + charm=charm, + app=DB_APP_1, + base="ubuntu@24.04", + config=configuration, + constraints=constraints, + bind=bind, + num_units=3, + ) + + model_2 = Juju(model=second_model) + model_2.deploy( + charm=charm, + app=DB_APP_2, + base="ubuntu@24.04", + config=configuration, + constraints=constraints, + bind=bind, + num_units=3, + ) + + logging.info("Deploying tls operators") + constraints = {"arch": architecture.architecture} + model_1.deploy( + charm="self-signed-certificates", + channel="1/stable", + constraints=constraints, + base="ubuntu@22.04", + ) + + model_1.offer(f"{first_model}.self-signed-certificates", endpoint="certificates") + model_2.consume(f"{first_model}.self-signed-certificates", "certificates-offer") + + model_1.integrate(f"{DB_APP_1}:client-certificates", "self-signed-certificates") + model_1.integrate(f"{DB_APP_1}:peer-certificates", "self-signed-certificates") + model_2.integrate(f"{DB_APP_2}:client-certificates", "certificates-offer") + model_2.integrate(f"{DB_APP_2}:peer-certificates", "certificates-offer") + + logging.info("Deploying test application") + constraints = {"arch": architecture.architecture, "spaces": "client"} + bind = {"database": "client"} + model_1.deploy( + charm=DB_TEST_APP_NAME, + app=DB_TEST_APP_1, + base="ubuntu@22.04", + channel="latest/edge", + num_units=1, + constraints=constraints, + bind=bind, + ) + model_2.deploy( + charm=DB_TEST_APP_NAME, + app=DB_TEST_APP_2, + base="ubuntu@22.04", + channel="latest/edge", + num_units=1, + constraints=constraints, + bind=bind, + ) + + logging.info("Relating test application") + model_1.integrate(f"{DB_TEST_APP_1}:database", f"{DB_APP_1}:database") + model_2.integrate(f"{DB_TEST_APP_2}:database", f"{DB_APP_2}:database") + + logging.info("Waiting for the applications to settle") + model_1.wait( + ready=wait_for_apps_status(jubilant.all_active, DB_APP_1, DB_TEST_APP_1), + timeout=20 * MINUTE_SECS, + ) + model_2.wait( + ready=wait_for_apps_status(jubilant.all_active, DB_APP_2, DB_TEST_APP_2), + timeout=20 * MINUTE_SECS, + ) + + +def test_async_relate(first_model: str, second_model: str) -> None: + """Relate the two PostgreSQL clusters.""" + logging.info("Creating offers in first model") + model_1 = Juju(model=first_model) + model_1.offer(f"{first_model}.{DB_APP_1}", endpoint="replication-offer") + + logging.info("Consuming offer in second model") + model_2 = Juju(model=second_model) + model_2.consume(f"{first_model}.{DB_APP_1}") + + logging.info("Relating the two postgresql clusters") + model_2.integrate(f"{DB_APP_1}", f"{DB_APP_2}:replication") + + logging.info("Waiting for the applications to settle") + model_1.wait( + ready=wait_for_apps_status(jubilant.any_active, DB_APP_1), + timeout=10 * MINUTE_SECS, + ) + model_2.wait( + ready=wait_for_apps_status(jubilant.any_active, DB_APP_2), + timeout=10 * MINUTE_SECS, + ) + + +def test_create_replication(first_model: str, second_model: str) -> None: + """Run the create-replication action and wait for the applications to settle.""" + model_1 = Juju(model=first_model) + model_2 = Juju(model=second_model) + + logging.info("Running create replication action") + model_1.run( + unit=get_app_leader(model_1, DB_APP_1), action="create-replication", wait=5 * MINUTE_SECS + ).raise_on_failure() + + logging.info("Waiting for the applications to settle") + model_1.wait( + ready=wait_for_apps_status(jubilant.all_active, DB_APP_1), timeout=20 * MINUTE_SECS + ) + model_2.wait( + ready=wait_for_apps_status(jubilant.all_active, DB_APP_2), timeout=20 * MINUTE_SECS + ) + + +def test_data_replication( + first_model: str, second_model: str, first_model_continuous_writes +) -> None: + """Test to write to primary, and read the same data back from replicas.""" + logging.info("Testing data replication") + results = get_db_max_written_values(first_model, second_model, first_model, DB_TEST_APP_1) + + assert len(results) == 6 + assert all(results[0] == x for x in results), "Data is not consistent across units" + assert results[0] > 1, "No data was written to the database" + + +def get_db_max_written_values( + first_model: str, second_model: str, test_model: str, test_app: str +) -> list[int]: + """Return list with max written value from all units.""" + db_name = f"{test_app.replace('-', '_')}_database" + model_1 = Juju(model=first_model) + model_2 = Juju(model=second_model) + test_app_model = model_1 if test_model == first_model else model_2 + + logging.info("Stopping continuous writes") + test_app_model.run( + unit=get_app_leader(test_app_model, test_app), action="stop-continuous-writes" + ).raise_on_failure() + + time.sleep(5) + results = [] + + logging.info(f"Querying max value on all {DB_APP_1} units") + for unit_name in get_app_units(model_1, DB_APP_1): + unit_max_value = get_db_max_written_value(model_1, DB_APP_1, unit_name, db_name) + results.append(unit_max_value) + + logging.info(f"Querying max value on all {DB_APP_2} units") + for unit_name in get_app_units(model_2, DB_APP_2): + unit_max_value = get_db_max_written_value(model_2, DB_APP_2, unit_name, db_name) + results.append(unit_max_value) + + return results diff --git a/tests/integration/test_backups_aws.py b/tests/integration/test_backups_aws.py index ad456d8783a..14e61951751 100644 --- a/tests/integration/test_backups_aws.py +++ b/tests/integration/test_backups_aws.py @@ -25,7 +25,7 @@ ) S3_INTEGRATOR_APP_NAME = "s3-integrator" tls_certificates_app_name = "self-signed-certificates" -tls_channel = "latest/stable" +tls_channel = "1/stable" tls_config = {"ca-common-name": "Test CA"} logger = logging.getLogger(__name__) diff --git a/tests/integration/test_backups_gcp.py b/tests/integration/test_backups_gcp.py index 5be2a5891c5..ed19d3ff43f 100644 --- a/tests/integration/test_backups_gcp.py +++ b/tests/integration/test_backups_gcp.py @@ -26,7 +26,7 @@ FAILED_TO_INITIALIZE_STANZA_ERROR_MESSAGE = "failed to initialize stanza, check your S3 settings" S3_INTEGRATOR_APP_NAME = "s3-integrator" tls_certificates_app_name = "self-signed-certificates" -tls_channel = "latest/stable" +tls_channel = "1/stable" tls_config = {"ca-common-name": "Test CA"} logger = logging.getLogger(__name__) diff --git a/tests/integration/test_backups_pitr_aws.py b/tests/integration/test_backups_pitr_aws.py index a472251eb3b..1bdf2c475a7 100644 --- a/tests/integration/test_backups_pitr_aws.py +++ b/tests/integration/test_backups_pitr_aws.py @@ -20,7 +20,7 @@ CANNOT_RESTORE_PITR = "cannot restore PITR, juju debug-log for details" S3_INTEGRATOR_APP_NAME = "s3-integrator" TLS_CERTIFICATES_APP_NAME = "self-signed-certificates" -TLS_CHANNEL = "latest/stable" +TLS_CHANNEL = "1/stable" TLS_CONFIG = {"ca-common-name": "Test CA"} logger = logging.getLogger(__name__) diff --git a/tests/integration/test_backups_pitr_gcp.py b/tests/integration/test_backups_pitr_gcp.py index 0f49205114d..f1ce4b1213a 100644 --- a/tests/integration/test_backups_pitr_gcp.py +++ b/tests/integration/test_backups_pitr_gcp.py @@ -20,7 +20,7 @@ CANNOT_RESTORE_PITR = "cannot restore PITR, juju debug-log for details" S3_INTEGRATOR_APP_NAME = "s3-integrator" TLS_CERTIFICATES_APP_NAME = "self-signed-certificates" -TLS_CHANNEL = "latest/stable" +TLS_CHANNEL = "1/stable" TLS_CONFIG = {"ca-common-name": "Test CA"} logger = logging.getLogger(__name__) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index f9a3af31f31..d659da1fb8f 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -4,78 +4,67 @@ import logging -from time import sleep from typing import get_args +import jubilant import psycopg2 import pytest import requests +from jubilant import Juju from psycopg2 import sql -from pytest_operator.plugin import OpsTest -from tenacity import Retrying, stop_after_attempt, wait_exponential, wait_fixed from locales import SNAP_LOCALES -from .ha_tests.helpers import get_cluster_roles from .helpers import ( CHARM_BASE, DATABASE_APP_NAME, STORAGE_PATH, - check_cluster_members, convert_records_to_dict, db_connect, - find_unit, - get_password, - get_primary, - get_unit_address, - run_command_on_unit, - scale_application, - switchover, +) +from .high_availability_helpers_new import ( + get_unit_ip, + get_user_password, + wait_for_apps_status, ) -logger = logging.getLogger(__name__) - +DB_APP_NAME = "postgresql" +MINUTE_SECS = 60 UNIT_IDS = [0, 1, 2] -@pytest.mark.abort_on_fail -@pytest.mark.skip_if_deployed -async def test_deploy(ops_test: OpsTest, charm: str): - """Deploy the charm-under-test. - - Assert on the unit status before any relations/configurations take place. - """ - # Deploy the charm with Patroni resource. - await ops_test.model.deploy( - charm, - application_name=DATABASE_APP_NAME, - num_units=3, +def test_deploy(juju: Juju, charm) -> None: + """Simple test to ensure that the PostgreSQL and application charms get deployed.""" + logging.info("Deploying PostgreSQL cluster") + juju.deploy( + charm=charm, + app=DB_APP_NAME, base=CHARM_BASE, config={"profile": "testing"}, + num_units=3, ) - # Reducing the update status frequency to speed up the triggering of deferred events. - await ops_test.model.set_config({"update-status-hook-interval": "10s"}) - - await ops_test.model.wait_for_idle(apps=[DATABASE_APP_NAME], status="active", timeout=1500) - assert ops_test.model.applications[DATABASE_APP_NAME].units[0].workload_status == "active" + logging.info("Wait for applications to become active") + juju.wait( + ready=wait_for_apps_status(jubilant.all_active, DB_APP_NAME), + timeout=20 * MINUTE_SECS, + ) -@pytest.mark.abort_on_fail @pytest.mark.parametrize("unit_id", UNIT_IDS) -async def test_database_is_up(ops_test: OpsTest, unit_id: int): +def test_database_is_up(juju: Juju, unit_id: int): # Query Patroni REST API and check the status that indicates # both Patroni and PostgreSQL are up and running. - host = get_unit_address(ops_test, f"{DATABASE_APP_NAME}/{unit_id}") + host = get_unit_ip(juju, DB_APP_NAME, f"{DB_APP_NAME}/{unit_id}") result = requests.get(f"https://{host}:8008/health", verify=False) assert result.status_code == 200 @pytest.mark.parametrize("unit_id", UNIT_IDS) -async def test_exporter_is_up(ops_test: OpsTest, unit_id: int): +def test_exporter_is_up(juju: Juju, unit_id: int): # Query Patroni REST API and check the status that indicates # both Patroni and PostgreSQL are up and running. - host = get_unit_address(ops_test, f"{DATABASE_APP_NAME}/{unit_id}") + host = get_unit_ip(juju, DB_APP_NAME, f"{DB_APP_NAME}/{unit_id}") result = requests.get(f"http://{host}:9187/metrics") assert result.status_code == 200 assert "pg_exporter_last_scrape_error 0" in result.content.decode("utf8"), ( @@ -84,14 +73,14 @@ async def test_exporter_is_up(ops_test: OpsTest, unit_id: int): @pytest.mark.parametrize("unit_id", UNIT_IDS) -async def test_settings_are_correct(ops_test: OpsTest, unit_id: int): +def test_settings_are_correct(juju: Juju, unit_id: int): # Connect to the PostgreSQL instance. # Retrieving the operator user password using the action. - password = await get_password(ops_test) + password = get_user_password(juju, DB_APP_NAME, "operator") # Connect to PostgreSQL. - host = get_unit_address(ops_test, f"{DATABASE_APP_NAME}/{unit_id}") - logger.info("connecting to the database host: %s", host) + host = get_unit_ip(juju, DB_APP_NAME, f"{DB_APP_NAME}/{unit_id}") + logging.info("connecting to the database host: %s", host) with db_connect(host, password) as connection: assert connection.status == psycopg2.extensions.STATUS_READY @@ -160,20 +149,15 @@ async def test_settings_are_correct(ops_test: OpsTest, unit_id: int): assert settings["retry_timeout"] == 10 assert settings["maximum_lag_on_failover"] == 1048576 - logger.warning("Asserting port ranges") - unit = ops_test.model.applications[DATABASE_APP_NAME].units[unit_id] - assert unit.data["port-ranges"][0]["from-port"] == 5432 - assert unit.data["port-ranges"][0]["to-port"] == 5432 - assert unit.data["port-ranges"][0]["protocol"] == "tcp" + logging.warning("Asserting port ranges") + unit = juju.status.apps[DATABASE_APP_NAME].units[f"{DB_APP_NAME}/{unit_id}"] + assert unit.open_ports == ["5432/tcp"] -async def test_postgresql_locales(ops_test: OpsTest) -> None: - raw_locales = await run_command_on_unit( - ops_test, - ops_test.model.applications[DATABASE_APP_NAME].units[0].name, - "ls /snap/charmed-postgresql/current/usr/lib/locale", - ) - locales = raw_locales.splitlines() +def test_postgresql_locales(juju: Juju) -> None: + task = juju.exec("ls /snap/charmed-postgresql/current/usr/lib/locale", unit=f"{DB_APP_NAME}/0") + task.raise_on_failure() + locales = task.stdout.splitlines() locales.append("C") locales.sort() @@ -183,21 +167,24 @@ async def test_postgresql_locales(ops_test: OpsTest) -> None: assert locales == list(get_args(SNAP_LOCALES)) -async def test_postgresql_parameters_change(ops_test: OpsTest) -> None: +def test_postgresql_parameters_change(juju: Juju) -> None: """Test that's possible to change PostgreSQL parameters.""" - await ops_test.model.applications[DATABASE_APP_NAME].set_config({ - "memory_max_prepared_transactions": "100", - "memory_shared_buffers": "32768", # 2 * 128MB. Patroni may refuse the config if < 128MB - "response_lc_monetary": "en_GB.utf8", - "experimental_max_connections": "200", - }) - await ops_test.model.wait_for_idle(apps=[DATABASE_APP_NAME], status="active", idle_period=30) - password = await get_password(ops_test) + juju.config( + app=DATABASE_APP_NAME, + values={ + "memory_max_prepared_transactions": "100", + "memory_shared_buffers": "32768", # 2 * 128MB. Patroni may refuse the config if < 128MB + "response_lc_monetary": "en_GB.utf8", + "experimental_max_connections": "200", + }, + ) + juju.wait(ready=wait_for_apps_status(jubilant.all_active, DB_APP_NAME)) + password = get_user_password(juju, DB_APP_NAME, "operator") # Connect to PostgreSQL. for unit_id in UNIT_IDS: - host = get_unit_address(ops_test, f"{DATABASE_APP_NAME}/{unit_id}") - logger.info("connecting to the database host: %s", host) + host = get_unit_ip(juju, DB_APP_NAME, f"{DB_APP_NAME}/{unit_id}") + logging.info("connecting to the database host: %s", host) try: with ( psycopg2.connect( @@ -227,145 +214,3 @@ async def test_postgresql_parameters_change(ops_test: OpsTest) -> None: assert settings["max_connections"] == "200" finally: connection.close() - - -async def test_scale_down_and_up(ops_test: OpsTest): - """Test data is replicated to new units after a scale up.""" - # Ensure the initial number of units in the application. - initial_scale = len(UNIT_IDS) - await scale_application(ops_test, DATABASE_APP_NAME, initial_scale) - - # Scale down the application. - await scale_application(ops_test, DATABASE_APP_NAME, initial_scale - 1) - - # Ensure the member was correctly removed from the cluster - # (by comparing the cluster members and the current units). - await check_cluster_members(ops_test, DATABASE_APP_NAME) - - # Scale up the application (2 more units than the current scale). - await scale_application(ops_test, DATABASE_APP_NAME, initial_scale + 1) - - # Assert the correct members are part of the cluster. - await check_cluster_members(ops_test, DATABASE_APP_NAME) - - # Test the deletion of the unit that is both the leader and the primary. - any_unit_name = ops_test.model.applications[DATABASE_APP_NAME].units[0].name - primary = await get_primary(ops_test, any_unit_name) - leader_unit = await find_unit(ops_test, leader=True, application=DATABASE_APP_NAME) - - # Trigger a switchover if the primary and the leader are not the same unit. - patroni_password = await get_password(ops_test, "patroni") - - if primary != leader_unit.name: - switchover(ops_test, primary, patroni_password, leader_unit.name) - - # Get the new primary unit. - primary = await get_primary(ops_test, any_unit_name) - # Check that the primary changed. - for attempt in Retrying( - stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=30) - ): - with attempt: - assert primary == leader_unit.name - - await ops_test.model.applications[DATABASE_APP_NAME].destroy_units(leader_unit.name) - await ops_test.model.wait_for_idle( - apps=[DATABASE_APP_NAME], status="active", timeout=1000, wait_for_exact_units=initial_scale - ) - - # Assert the correct members are part of the cluster. - await check_cluster_members(ops_test, DATABASE_APP_NAME) - - # Scale up the application (2 more units than the current scale). - await scale_application(ops_test, DATABASE_APP_NAME, initial_scale + 2) - - # Test the deletion of both the unit that is the leader and the unit that is the primary. - any_unit_name = ops_test.model.applications[DATABASE_APP_NAME].units[0].name - primary = await get_primary(ops_test, any_unit_name) - leader_unit = await find_unit(ops_test, DATABASE_APP_NAME, True) - - # Trigger a switchover if the primary and the leader are the same unit. - if primary == leader_unit.name: - switchover(ops_test, primary, patroni_password) - - # Get the new primary unit. - primary = await get_primary(ops_test, any_unit_name) - # Check that the primary changed. - for attempt in Retrying( - stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=30) - ): - with attempt: - assert primary != leader_unit.name - - await ops_test.model.applications[DATABASE_APP_NAME].destroy_units(primary, leader_unit.name) - await ops_test.model.wait_for_idle( - apps=[DATABASE_APP_NAME], - status="active", - timeout=2000, - idle_period=30, - wait_for_exact_units=initial_scale, - ) - - # Wait some time to elect a new primary. - sleep(30) - - # Assert the correct members are part of the cluster. - await check_cluster_members(ops_test, DATABASE_APP_NAME) - - # End with the cluster having the initial number of units. - await scale_application(ops_test, DATABASE_APP_NAME, initial_scale) - - -async def test_switchover_sync_standby(ops_test: OpsTest): - original_roles = await get_cluster_roles( - ops_test, ops_test.model.applications[DATABASE_APP_NAME].units[0].name - ) - run_action = await ops_test.model.units[original_roles["sync_standbys"][0]].run_action( - "promote-to-primary", scope="unit" - ) - await run_action.wait() - - await ops_test.model.wait_for_idle(status="active", timeout=200) - - new_roles = await get_cluster_roles( - ops_test, ops_test.model.applications[DATABASE_APP_NAME].units[0].name - ) - assert new_roles["primaries"][0] == original_roles["sync_standbys"][0] - - -async def test_persist_data_through_primary_deletion(ops_test: OpsTest): - """Test data persists through a primary deletion.""" - # Set a composite application name in order to test in more than one series at the same time. - any_unit_name = ops_test.model.applications[DATABASE_APP_NAME].units[0].name - for attempt in Retrying(stop=stop_after_attempt(3), wait=wait_fixed(5), reraise=True): - with attempt: - primary = await get_primary(ops_test, any_unit_name) - password = await get_password(ops_test) - - # Write data to primary IP. - host = get_unit_address(ops_test, primary) - logger.info(f"connecting to primary {primary} on {host}") - with db_connect(host, password) as connection: - connection.autocommit = True - with connection.cursor() as cursor: - cursor.execute("CREATE TABLE primarydeletiontest (testcol INT);") - connection.close() - - # Remove one unit. - await ops_test.model.destroy_units( - primary, - ) - await ops_test.model.wait_for_idle(apps=[DATABASE_APP_NAME], status="active", timeout=1500) - - # Add the unit again. - await ops_test.model.applications[DATABASE_APP_NAME].add_unit(count=1) - await ops_test.model.wait_for_idle(apps=[DATABASE_APP_NAME], status="active", timeout=2000) - - # Testing write occurred to every postgres instance by reading from them - for unit in ops_test.model.applications[DATABASE_APP_NAME].units: - host = unit.public_address - logger.info("connecting to the database host: %s", host) - with db_connect(host, password) as connection, connection.cursor() as cursor: - # Ensure we can read from "primarydeletiontest" table - cursor.execute("SELECT * FROM primarydeletiontest;") - connection.close() diff --git a/tests/integration/test_tls.py b/tests/integration/test_tls.py index 8d92185f045..123f03297b8 100644 --- a/tests/integration/test_tls.py +++ b/tests/integration/test_tls.py @@ -30,7 +30,7 @@ APP_NAME = METADATA["name"] tls_certificates_app_name = "self-signed-certificates" -tls_channel = "latest/stable" +tls_channel = "1/stable" tls_config = {"ca-common-name": "Test CA"} From 70d9eec49795832b5b202d3abf7188bfe646cdd8 Mon Sep 17 00:00:00 2001 From: Dragomir Penev Date: Fri, 7 Nov 2025 19:43:55 +0200 Subject: [PATCH 2/7] Remove tls config --- tests/integration/helpers.py | 7 ++----- tests/integration/test_backups_aws.py | 2 -- tests/integration/test_backups_ceph.py | 1 - tests/integration/test_backups_gcp.py | 2 -- tests/integration/test_backups_pitr_aws.py | 2 -- tests/integration/test_backups_pitr_gcp.py | 2 -- tests/integration/test_charm.py | 2 +- tests/integration/test_tls.py | 5 +---- tests/spread/test_spaced_async_replication.py/task.yaml | 8 ++++++++ 9 files changed, 12 insertions(+), 19 deletions(-) create mode 100644 tests/spread/test_spaced_async_replication.py/task.yaml diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index f08dc3d944e..e47431a33cb 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -1207,7 +1207,6 @@ async def backup_operations( ops_test: OpsTest, s3_integrator_app_name: str, tls_certificates_app_name: str, - tls_config, tls_channel, credentials, cloud, @@ -1215,13 +1214,11 @@ async def backup_operations( charm, ) -> None: """Basic set of operations for backup testing in different cloud providers.""" - use_tls = all([tls_certificates_app_name, tls_config, tls_channel]) + use_tls = all([tls_certificates_app_name, tls_channel]) # Deploy S3 Integrator and TLS Certificates Operator. await ops_test.model.deploy(s3_integrator_app_name) if use_tls: - await ops_test.model.deploy( - tls_certificates_app_name, config=tls_config, channel=tls_channel - ) + await ops_test.model.deploy(tls_certificates_app_name, channel=tls_channel) # Deploy and relate PostgreSQL to S3 integrator (one database app for each cloud for now # as archive_mode is disabled after restoring the backup) and to TLS Certificates Operator diff --git a/tests/integration/test_backups_aws.py b/tests/integration/test_backups_aws.py index 14e61951751..c4fb9f2f7cc 100644 --- a/tests/integration/test_backups_aws.py +++ b/tests/integration/test_backups_aws.py @@ -26,7 +26,6 @@ S3_INTEGRATOR_APP_NAME = "s3-integrator" tls_certificates_app_name = "self-signed-certificates" tls_channel = "1/stable" -tls_config = {"ca-common-name": "Test CA"} logger = logging.getLogger(__name__) @@ -41,7 +40,6 @@ async def test_backup_aws(ops_test: OpsTest, aws_cloud_configs: tuple[dict, dict ops_test, S3_INTEGRATOR_APP_NAME, tls_certificates_app_name, - tls_config, tls_channel, credentials, AWS, diff --git a/tests/integration/test_backups_ceph.py b/tests/integration/test_backups_ceph.py index 3b9f508da77..8958bdaf8d2 100644 --- a/tests/integration/test_backups_ceph.py +++ b/tests/integration/test_backups_ceph.py @@ -166,7 +166,6 @@ async def test_backup_ceph(ops_test: OpsTest, cloud_configs, cloud_credentials, S3_INTEGRATOR_APP_NAME, None, None, - None, cloud_credentials, "ceph", cloud_configs, diff --git a/tests/integration/test_backups_gcp.py b/tests/integration/test_backups_gcp.py index ed19d3ff43f..7431343911d 100644 --- a/tests/integration/test_backups_gcp.py +++ b/tests/integration/test_backups_gcp.py @@ -27,7 +27,6 @@ S3_INTEGRATOR_APP_NAME = "s3-integrator" tls_certificates_app_name = "self-signed-certificates" tls_channel = "1/stable" -tls_config = {"ca-common-name": "Test CA"} logger = logging.getLogger(__name__) @@ -42,7 +41,6 @@ async def test_backup_gcp(ops_test: OpsTest, gcp_cloud_configs: tuple[dict, dict ops_test, S3_INTEGRATOR_APP_NAME, tls_certificates_app_name, - tls_config, tls_channel, credentials, GCP, diff --git a/tests/integration/test_backups_pitr_aws.py b/tests/integration/test_backups_pitr_aws.py index 1bdf2c475a7..17ccc3b2010 100644 --- a/tests/integration/test_backups_pitr_aws.py +++ b/tests/integration/test_backups_pitr_aws.py @@ -21,7 +21,6 @@ S3_INTEGRATOR_APP_NAME = "s3-integrator" TLS_CERTIFICATES_APP_NAME = "self-signed-certificates" TLS_CHANNEL = "1/stable" -TLS_CONFIG = {"ca-common-name": "Test CA"} logger = logging.getLogger(__name__) @@ -333,7 +332,6 @@ async def test_pitr_backup_aws( ops_test, S3_INTEGRATOR_APP_NAME, TLS_CERTIFICATES_APP_NAME, - TLS_CONFIG, TLS_CHANNEL, credentials, AWS, diff --git a/tests/integration/test_backups_pitr_gcp.py b/tests/integration/test_backups_pitr_gcp.py index f1ce4b1213a..186cef80220 100644 --- a/tests/integration/test_backups_pitr_gcp.py +++ b/tests/integration/test_backups_pitr_gcp.py @@ -21,7 +21,6 @@ S3_INTEGRATOR_APP_NAME = "s3-integrator" TLS_CERTIFICATES_APP_NAME = "self-signed-certificates" TLS_CHANNEL = "1/stable" -TLS_CONFIG = {"ca-common-name": "Test CA"} logger = logging.getLogger(__name__) @@ -333,7 +332,6 @@ async def test_pitr_backup_gcp( ops_test, S3_INTEGRATOR_APP_NAME, TLS_CERTIFICATES_APP_NAME, - TLS_CONFIG, TLS_CHANNEL, credentials, GCP, diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index d659da1fb8f..1136923c113 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -22,7 +22,7 @@ convert_records_to_dict, db_connect, ) -from .high_availability_helpers_new import ( +from .high_availability.high_availability_helpers_new import ( get_unit_ip, get_user_password, wait_for_apps_status, diff --git a/tests/integration/test_tls.py b/tests/integration/test_tls.py index 123f03297b8..dc48f81c35e 100644 --- a/tests/integration/test_tls.py +++ b/tests/integration/test_tls.py @@ -31,7 +31,6 @@ APP_NAME = METADATA["name"] tls_certificates_app_name = "self-signed-certificates" tls_channel = "1/stable" -tls_config = {"ca-common-name": "Test CA"} @pytest.mark.abort_on_fail @@ -55,9 +54,7 @@ async def test_tls_enabled(ops_test: OpsTest) -> None: """Test that TLS is enabled when relating to the TLS Certificates Operator.""" async with ops_test.fast_forward(): # Deploy TLS Certificates operator. - await ops_test.model.deploy( - tls_certificates_app_name, config=tls_config, channel=tls_channel, base=CHARM_BASE - ) + await ops_test.model.deploy(tls_certificates_app_name, channel=tls_channel) # Relate it to the PostgreSQL to enable TLS. await ops_test.model.relate( diff --git a/tests/spread/test_spaced_async_replication.py/task.yaml b/tests/spread/test_spaced_async_replication.py/task.yaml new file mode 100644 index 00000000000..58b14d9ba74 --- /dev/null +++ b/tests/spread/test_spaced_async_replication.py/task.yaml @@ -0,0 +1,8 @@ +summary: test_spaced_async_replication.py +environment: + TEST_MODULE: spaces/test_spaced_async_replication.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results + From 3b185a05adb9ddb6896fdb8b3913ab2fe8993608 Mon Sep 17 00:00:00 2001 From: Dragomir Penev Date: Fri, 7 Nov 2025 20:11:43 +0200 Subject: [PATCH 3/7] Remove base --- tests/integration/test_backups_pitr_aws.py | 3 +-- tests/integration/test_backups_pitr_gcp.py | 3 +-- tests/integration/test_charm.py | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_backups_pitr_aws.py b/tests/integration/test_backups_pitr_aws.py index 17ccc3b2010..125974f55f8 100644 --- a/tests/integration/test_backups_pitr_aws.py +++ b/tests/integration/test_backups_pitr_aws.py @@ -29,7 +29,6 @@ async def pitr_backup_operations( ops_test: OpsTest, s3_integrator_app_name: str, tls_certificates_app_name: str, - tls_config, tls_channel, credentials, cloud, @@ -49,7 +48,7 @@ async def pitr_backup_operations( logger.info("deploying the next charms: s3-integrator, self-signed-certificates, postgresql") await ops_test.model.deploy(s3_integrator_app_name) - await ops_test.model.deploy(tls_certificates_app_name, config=tls_config, channel=tls_channel) + await ops_test.model.deploy(tls_certificates_app_name, channel=tls_channel) await ops_test.model.deploy( charm, application_name=database_app_name, diff --git a/tests/integration/test_backups_pitr_gcp.py b/tests/integration/test_backups_pitr_gcp.py index 186cef80220..55a204507e8 100644 --- a/tests/integration/test_backups_pitr_gcp.py +++ b/tests/integration/test_backups_pitr_gcp.py @@ -29,7 +29,6 @@ async def pitr_backup_operations( ops_test: OpsTest, s3_integrator_app_name: str, tls_certificates_app_name: str, - tls_config, tls_channel, credentials, cloud, @@ -49,7 +48,7 @@ async def pitr_backup_operations( logger.info("deploying the next charms: s3-integrator, self-signed-certificates, postgresql") await ops_test.model.deploy(s3_integrator_app_name) - await ops_test.model.deploy(tls_certificates_app_name, config=tls_config, channel=tls_channel) + await ops_test.model.deploy(tls_certificates_app_name, channel=tls_channel) await ops_test.model.deploy( charm, application_name=database_app_name, diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 1136923c113..9078dc3e43e 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -16,7 +16,6 @@ from locales import SNAP_LOCALES from .helpers import ( - CHARM_BASE, DATABASE_APP_NAME, STORAGE_PATH, convert_records_to_dict, @@ -39,7 +38,7 @@ def test_deploy(juju: Juju, charm) -> None: juju.deploy( charm=charm, app=DB_APP_NAME, - base=CHARM_BASE, + base="ubuntu@24.04", config={"profile": "testing"}, num_units=3, ) From 74ee645e89802129d86ba966af39c4ca388bea1c Mon Sep 17 00:00:00 2001 From: Dragomir Penev Date: Fri, 7 Nov 2025 20:43:18 +0200 Subject: [PATCH 4/7] Add back landscape --- tests/integration/spaces/test_spaced_async_replication.py | 8 ++++---- tests/integration/test_charm.py | 2 +- tests/integration/test_subordinates.py | 7 ++++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/integration/spaces/test_spaced_async_replication.py b/tests/integration/spaces/test_spaced_async_replication.py index 72e0846e9af..21d975019a5 100644 --- a/tests/integration/spaces/test_spaced_async_replication.py +++ b/tests/integration/spaces/test_spaced_async_replication.py @@ -11,7 +11,7 @@ from jubilant import Juju from tenacity import Retrying, stop_after_attempt -from .. import architecture +from ..architecture import architecture from ..high_availability.high_availability_helpers_new import ( get_app_leader, get_app_units, @@ -88,7 +88,7 @@ def first_model_continuous_writes(first_model: str) -> Generator: def test_deploy(first_model: str, second_model: str, lxd_spaces, charm) -> None: """Simple test to ensure that the database application charms get deployed.""" configuration = {"profile": "testing"} - constraints = {"arch": architecture.architecture, "spaces": "client,peers"} + constraints = {"arch": architecture, "spaces": "client,peers"} bind = {"database-peers": "peers", "database": "client"} logging.info("Deploying postgresql clusters") @@ -115,7 +115,7 @@ def test_deploy(first_model: str, second_model: str, lxd_spaces, charm) -> None: ) logging.info("Deploying tls operators") - constraints = {"arch": architecture.architecture} + constraints = {"arch": architecture} model_1.deploy( charm="self-signed-certificates", channel="1/stable", @@ -132,7 +132,7 @@ def test_deploy(first_model: str, second_model: str, lxd_spaces, charm) -> None: model_2.integrate(f"{DB_APP_2}:peer-certificates", "certificates-offer") logging.info("Deploying test application") - constraints = {"arch": architecture.architecture, "spaces": "client"} + constraints = {"arch": architecture, "spaces": "client"} bind = {"database": "client"} model_1.deploy( charm=DB_TEST_APP_NAME, diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 9078dc3e43e..0882d32fe1d 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -149,7 +149,7 @@ def test_settings_are_correct(juju: Juju, unit_id: int): assert settings["maximum_lag_on_failover"] == 1048576 logging.warning("Asserting port ranges") - unit = juju.status.apps[DATABASE_APP_NAME].units[f"{DB_APP_NAME}/{unit_id}"] + unit = juju.status().apps[DATABASE_APP_NAME].units[f"{DB_APP_NAME}/{unit_id}"] assert unit.open_ports == ["5432/tcp"] diff --git a/tests/integration/test_subordinates.py b/tests/integration/test_subordinates.py index ff6d301e276..c3caeb2d0fe 100644 --- a/tests/integration/test_subordinates.py +++ b/tests/integration/test_subordinates.py @@ -63,11 +63,12 @@ async def test_deploy(ops_test: OpsTest, charm: str, check_subordinate_env_vars) ) await ops_test.model.wait_for_idle(apps=[DATABASE_APP_NAME], status="active", timeout=2000) + await ops_test.model.relate(f"{DATABASE_APP_NAME}:juju-info", f"{LS_CLIENT}:container") await ops_test.model.relate( f"{DATABASE_APP_NAME}:juju-info", f"{UBUNTU_PRO_APP_NAME}:juju-info" ) await ops_test.model.wait_for_idle( - apps=[UBUNTU_PRO_APP_NAME, DATABASE_APP_NAME], status="active" + apps=[LS_CLIENT, UBUNTU_PRO_APP_NAME, DATABASE_APP_NAME], status="active" ) @@ -75,7 +76,7 @@ async def test_scale_up(ops_test: OpsTest, check_subordinate_env_vars): await scale_application(ops_test, DATABASE_APP_NAME, 4) await ops_test.model.wait_for_idle( - apps=[UBUNTU_PRO_APP_NAME, DATABASE_APP_NAME], status="active", timeout=1500 + apps=[LS_CLIENT, UBUNTU_PRO_APP_NAME, DATABASE_APP_NAME], status="active", timeout=1500 ) @@ -83,5 +84,5 @@ async def test_scale_down(ops_test: OpsTest, check_subordinate_env_vars): await scale_application(ops_test, DATABASE_APP_NAME, 3) await ops_test.model.wait_for_idle( - apps=[UBUNTU_PRO_APP_NAME, DATABASE_APP_NAME], status="active", timeout=1500 + apps=[LS_CLIENT, UBUNTU_PRO_APP_NAME, DATABASE_APP_NAME], status="active", timeout=1500 ) From ee16c2eba389e70df1bb9872da87834a151099a2 Mon Sep 17 00:00:00 2001 From: Dragomir Penev Date: Fri, 7 Nov 2025 21:39:12 +0200 Subject: [PATCH 5/7] Spaces package --- tests/integration/spaces/__init__.py | 0 tests/integration/test_charm.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 tests/integration/spaces/__init__.py diff --git a/tests/integration/spaces/__init__.py b/tests/integration/spaces/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 0882d32fe1d..3df9b195195 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -177,7 +177,7 @@ def test_postgresql_parameters_change(juju: Juju) -> None: "experimental_max_connections": "200", }, ) - juju.wait(ready=wait_for_apps_status(jubilant.all_active, DB_APP_NAME)) + juju.wait(ready=wait_for_apps_status(jubilant.all_active, DB_APP_NAME), successes=6) password = get_user_password(juju, DB_APP_NAME, "operator") # Connect to PostgreSQL. From 641ef9e57e1fc083540f8bcb02b394760d9f5a76 Mon Sep 17 00:00:00 2001 From: Dragomir Penev Date: Fri, 7 Nov 2025 22:23:21 +0200 Subject: [PATCH 6/7] Sleep for config event --- tests/integration/test_charm.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 3df9b195195..70bcc0d7f59 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -4,6 +4,7 @@ import logging +from time import sleep from typing import get_args import jubilant @@ -177,7 +178,8 @@ def test_postgresql_parameters_change(juju: Juju) -> None: "experimental_max_connections": "200", }, ) - juju.wait(ready=wait_for_apps_status(jubilant.all_active, DB_APP_NAME), successes=6) + sleep(5) + juju.wait(ready=wait_for_apps_status(jubilant.all_active, DB_APP_NAME)) password = get_user_password(juju, DB_APP_NAME, "operator") # Connect to PostgreSQL. From 10596703a39ae405c1ff807f1dfe3e87ceb89bf0 Mon Sep 17 00:00:00 2001 From: Dragomir Penev Date: Sun, 15 Feb 2026 00:59:13 +0200 Subject: [PATCH 7/7] Restore old part of the test --- tests/integration/test_charm.py | 152 ++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 70bcc0d7f59..c5cf6b39099 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -13,14 +13,24 @@ import requests from jubilant import Juju from psycopg2 import sql +from pytest_operator.plugin import OpsTest +from tenacity import Retrying, stop_after_attempt, wait_exponential, wait_fixed from locales import SNAP_LOCALES +from .ha_tests.helpers import get_cluster_roles from .helpers import ( DATABASE_APP_NAME, STORAGE_PATH, + check_cluster_members, convert_records_to_dict, db_connect, + find_unit, + get_password, + get_primary, + get_unit_address, + scale_application, + switchover, ) from .high_availability.high_availability_helpers_new import ( get_unit_ip, @@ -215,3 +225,145 @@ def test_postgresql_parameters_change(juju: Juju) -> None: assert settings["max_connections"] == "200" finally: connection.close() + + +async def test_scale_down_and_up(ops_test: OpsTest): + """Test data is replicated to new units after a scale up.""" + # Ensure the initial number of units in the application. + initial_scale = len(UNIT_IDS) + await scale_application(ops_test, DATABASE_APP_NAME, initial_scale) + + # Scale down the application. + await scale_application(ops_test, DATABASE_APP_NAME, initial_scale - 1) + + # Ensure the member was correctly removed from the cluster + # (by comparing the cluster members and the current units). + await check_cluster_members(ops_test, DATABASE_APP_NAME) + + # Scale up the application (2 more units than the current scale). + await scale_application(ops_test, DATABASE_APP_NAME, initial_scale + 1) + + # Assert the correct members are part of the cluster. + await check_cluster_members(ops_test, DATABASE_APP_NAME) + + # Test the deletion of the unit that is both the leader and the primary. + any_unit_name = ops_test.model.applications[DATABASE_APP_NAME].units[0].name + primary = await get_primary(ops_test, any_unit_name) + leader_unit = await find_unit(ops_test, leader=True, application=DATABASE_APP_NAME) + + # Trigger a switchover if the primary and the leader are not the same unit. + patroni_password = await get_password(ops_test, "patroni") + + if primary != leader_unit.name: + switchover(ops_test, primary, patroni_password, leader_unit.name) + + # Get the new primary unit. + primary = await get_primary(ops_test, any_unit_name) + # Check that the primary changed. + for attempt in Retrying( + stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=30) + ): + with attempt: + assert primary == leader_unit.name + + await ops_test.model.applications[DATABASE_APP_NAME].destroy_units(leader_unit.name) + await ops_test.model.wait_for_idle( + apps=[DATABASE_APP_NAME], status="active", timeout=1000, wait_for_exact_units=initial_scale + ) + + # Assert the correct members are part of the cluster. + await check_cluster_members(ops_test, DATABASE_APP_NAME) + + # Scale up the application (2 more units than the current scale). + await scale_application(ops_test, DATABASE_APP_NAME, initial_scale + 2) + + # Test the deletion of both the unit that is the leader and the unit that is the primary. + any_unit_name = ops_test.model.applications[DATABASE_APP_NAME].units[0].name + primary = await get_primary(ops_test, any_unit_name) + leader_unit = await find_unit(ops_test, DATABASE_APP_NAME, True) + + # Trigger a switchover if the primary and the leader are the same unit. + if primary == leader_unit.name: + switchover(ops_test, primary, patroni_password) + + # Get the new primary unit. + primary = await get_primary(ops_test, any_unit_name) + # Check that the primary changed. + for attempt in Retrying( + stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=30) + ): + with attempt: + assert primary != leader_unit.name + + await ops_test.model.applications[DATABASE_APP_NAME].destroy_units(primary, leader_unit.name) + await ops_test.model.wait_for_idle( + apps=[DATABASE_APP_NAME], + status="active", + timeout=2000, + idle_period=30, + wait_for_exact_units=initial_scale, + ) + + # Wait some time to elect a new primary. + sleep(30) + + # Assert the correct members are part of the cluster. + await check_cluster_members(ops_test, DATABASE_APP_NAME) + + # End with the cluster having the initial number of units. + await scale_application(ops_test, DATABASE_APP_NAME, initial_scale) + + +async def test_switchover_sync_standby(ops_test: OpsTest): + original_roles = await get_cluster_roles( + ops_test, ops_test.model.applications[DATABASE_APP_NAME].units[0].name + ) + run_action = await ops_test.model.units[original_roles["sync_standbys"][0]].run_action( + "promote-to-primary", scope="unit" + ) + await run_action.wait() + + await ops_test.model.wait_for_idle(status="active", timeout=200) + + new_roles = await get_cluster_roles( + ops_test, ops_test.model.applications[DATABASE_APP_NAME].units[0].name + ) + assert new_roles["primaries"][0] == original_roles["sync_standbys"][0] + + +async def test_persist_data_through_primary_deletion(ops_test: OpsTest): + """Test data persists through a primary deletion.""" + # Set a composite application name in order to test in more than one series at the same time. + any_unit_name = ops_test.model.applications[DATABASE_APP_NAME].units[0].name + for attempt in Retrying(stop=stop_after_attempt(3), wait=wait_fixed(5), reraise=True): + with attempt: + primary = await get_primary(ops_test, any_unit_name) + password = await get_password(ops_test) + + # Write data to primary IP. + host = get_unit_address(ops_test, primary) + logging.info(f"connecting to primary {primary} on {host}") + with db_connect(host, password) as connection: + connection.autocommit = True + with connection.cursor() as cursor: + cursor.execute("CREATE TABLE primarydeletiontest (testcol INT);") + connection.close() + + # Remove one unit. + await ops_test.model.destroy_units( + primary, + ) + await ops_test.model.wait_for_idle(apps=[DATABASE_APP_NAME], status="active", timeout=1500) + + # Add the unit again. + await ops_test.model.applications[DATABASE_APP_NAME].add_unit(count=1) + await ops_test.model.wait_for_idle(apps=[DATABASE_APP_NAME], status="active", timeout=2000) + + # Testing write occurred to every postgres instance by reading from them + for unit in ops_test.model.applications[DATABASE_APP_NAME].units: + host = unit.public_address + logging.info("connecting to the database host: %s", host) + with db_connect(host, password) as connection, connection.cursor() as cursor: + # Ensure we can read from "primarydeletiontest" table + cursor.execute("SELECT * FROM primarydeletiontest;") + connection.close()