diff --git a/scripts/seed_nft_tables.py b/scripts/seed_nft_tables.py index e74b94d1b..5053fa7e0 100644 --- a/scripts/seed_nft_tables.py +++ b/scripts/seed_nft_tables.py @@ -11,18 +11,20 @@ # import json import numpy as np -from nrlf.consumer.fhir.r4.model import DocumentReference from nrlf.core.constants import ( CATEGORY_ATTRIBUTES, SNOMED_SYSTEM_URL, TYPE_ATTRIBUTES, TYPE_CATEGORIES, - Categories, - PointerTypes, ) from nrlf.core.dynamodb.model import DocumentPointer from nrlf.core.logger import logger from nrlf.tests.data import load_document_reference +from tests.performance.seed_data_constants import ( # DEFAULT_COUNT_DISTRIBUTIONS, + CHECKSUM_WEIGHTS, + DEFAULT_CUSTODIAN_DISTRIBUTIONS, + DEFAULT_TYPE_DISTRIBUTIONS, +) dynamodb = boto3.client("dynamodb") resource = boto3.resource("dynamodb") @@ -31,69 +33,6 @@ DOC_REF_TEMPLATE = load_document_reference("NFT-template") -CHECKSUM_WEIGHTS = [i for i in range(10, 1, -1)] - -# These are based on the Nov 7th 2025 pointer stats report -DEFAULT_TYPE_DISTRIBUTIONS = { - "736253002": 65, # mental health crisis plan - "1382601000000107": 5, # respect form - "887701000000100": 15, # emergency healthcare plan - "861421000000109": 5, # eol care coordination summary - "735324008": 5, # treatment escalation plan - "824321000000109": 5, # summary record -} - -DEFAULT_CUSTODIAN_DISTRIBUTIONS = { - "736253002": { - "TRPG": 9, - "TRHA": 1, - "TRRE": 20, - "TRAT": 10, - "TWR4": 4, - "TRKL": 9, - "TRW1": 5, - "TRH5": 1, - "TRP7": 13, - "TRWK": 8, - "TRQY": 3, - "TRV5": 3, - "TRJ8": 2, - "TRXA": 4, - "T11X": 1, - "TG6V": 2, - }, - "1382601000000107": {"T8GX8": 3, "TQUY": 2}, # respect form - "887701000000100": { - "TV1": 1, - "TV2": 2, - "TV3": 1, - "TV4": 1, - "TV5": 3, - "TV6": 1, - }, # emergency healthcare plan - "861421000000109": { - "TV1": 2, - "TV2": 2, - "TV3": 1, - "TV4": 1, - "TV5": 3, - "TV6": 1, - }, # eol care coordination summary - "735324008": { - "TV1": 1, - "TV2": 1, - "TV3": 1, - "TV4": 2, - "TV5": 2, - "TV6": 1, - }, # treatment escalation plan - "824321000000109": { - "TRXT": 1, - }, # summary record currently has only one supplier -} - -DEFAULT_COUNT_DISTRIBUTIONS = {"1": 91, "2": 8, "3": 1} - class TestNhsNumbersIterator: def __iter__(self): @@ -262,7 +201,7 @@ def _get_pointer_count_poisson_distributions( def _set_up_custodian_iterators( - custodian_dists: dict[str, dict[str, int]] + custodian_dists: dict[str, dict[str, int]], ) -> dict[str, Iterator[str]]: custodian_iters: dict[str, Iterator[str]] = {} for pointer_type in custodian_dists: diff --git a/terraform/account-wide-infrastructure/modules/pointers-table/vars.tf b/terraform/account-wide-infrastructure/modules/pointers-table/vars.tf index 29d04b60e..d0f2d36e3 100644 --- a/terraform/account-wide-infrastructure/modules/pointers-table/vars.tf +++ b/terraform/account-wide-infrastructure/modules/pointers-table/vars.tf @@ -23,6 +23,6 @@ variable "kms_deletion_window_in_days" { variable "enable_backups" { type = bool - description = "Enable AwS cloud backup" + description = "Enable AWS cloud backup" default = false } diff --git a/terraform/account-wide-infrastructure/test/dynamodb__pointers-table.tf b/terraform/account-wide-infrastructure/test/dynamodb__pointers-table.tf index 98c7bae25..cdbe7dbc7 100644 --- a/terraform/account-wide-infrastructure/test/dynamodb__pointers-table.tf +++ b/terraform/account-wide-infrastructure/test/dynamodb__pointers-table.tf @@ -33,5 +33,5 @@ module "ref-pointers-table" { module "perftest-pointers-table" { source = "../modules/pointers-table" - name_prefix = "nhsd-nrlf--perftest" + name_prefix = "nhsd-nrlf--perftest-baseline" } diff --git a/terraform/infrastructure/README.md b/terraform/infrastructure/README.md index 6b9eec9b4..65c3a4437 100644 --- a/terraform/infrastructure/README.md +++ b/terraform/infrastructure/README.md @@ -116,6 +116,8 @@ replacing `{ENV_NAME}` with the environment name (e.g. `dev`, `qa`, `qa-sandbox` To tear down the infrastructure, you need to use Terraform to destroy the resources in your Terraform workspace. +First `make build-artifacts`. Then assume management and run `make get-s3-perms ENV={ENV_NAME}` in the project root. + To teardown the infrastructure, do the following: ``` diff --git a/terraform/infrastructure/etc/perftest.tfvars b/terraform/infrastructure/etc/perftest.tfvars index 2ddb7ecc5..56478f9fe 100644 --- a/terraform/infrastructure/etc/perftest.tfvars +++ b/terraform/infrastructure/etc/perftest.tfvars @@ -1,7 +1,7 @@ account_name = "perftest" aws_account_name = "test" -dynamodb_pointers_table_prefix = "nhsd-nrlf--perftest" +dynamodb_pointers_table_prefix = "nhsd-nrlf--perftest-baseline" domain = "perftest.record-locator.national.nhs.uk" public_domain = "perftest.api.service.nhs.uk" diff --git a/tests/performance/README.md b/tests/performance/README.md index 6c01cc1a5..10d3fa5d6 100644 --- a/tests/performance/README.md +++ b/tests/performance/README.md @@ -1,20 +1,44 @@ # Performance Testing -some high level context short +We have performance tests which give us a benchmark of how NRLF performs under load for consumers and producers. -## Run perf tests +## Run performance tests ### Prep the environment Perf tests are generally conducted in the perftest env. There's a selection of tables in the perftest env representing different pointer volume scenarios e.g. perftest-baseline vs perftest-1million (todo: update with real names!). -To reset this table to the expected state for perftests, restore the table from a backup. +#### Point perftest at a different pointers table -In the steps below, make sure the table name is the table your environment is pointing at. You might need to redeploy NRLF lambdas to point at the desired table. +We (will) have multiple tables representing different states of NRLF in the future e.g. all patients receiving an IPS (International Patient Summary), onboarding particular high-volume suppliers. + +In order to run performance tests to get figures for these different states, we can point the perftest environment at one of these tables. + +Currently, this requires tearing down the existing environment and restoring from scratch: + +1. Follow instructions in terraform/infrastructure/readme.md to tear down the perf test environment. + - Do **not** tear down shared account-wide infrastructure +2. Update `perftest-pointers-table.name_prefix` in `terraform/account-wide-infrastructure/test/dynamodb__pointers-table.tf` to be the table name you want, minus "-pointers-table" + - e.g. to use the baseline table `nhsd-nrlf--perftest-baseline-pointers-table`, set `name_prefix = "nhsd-nrlf--perftest-baseline"` +3. Update `dynamodb_pointers_table_prefix` in `terraform/infrastructure/etc/perftest.tfvars` same as above. + - e.g. to use the baseline table `dynamodb_pointers_table_prefix = "nhsd-nrlf--perftest-baseline"` +4. Commit changes to a branch & push +5. Run the [Deploy Account-wide infrastructure](https://github.com/NHSDigital/NRLF/actions/workflows/deploy-account-wide-infra.yml) workflow against your branch & `account-test`. + - If you get a terraform failure like "tried to create table but it already exists", you will need to do some fanangaling: + 1. make sure there is a backup of your chosen table or create one if not. In the AWS console: dynamodb > tables > your perftest table > backups > create backup > Create on-demand backup > leave all settings as defaults > create backup. This might take up to an hour to complete. + 2. once backed up, delete your table. In the AWS console: dynamodb > tables > your perftest table > actions > delete table + 3. Rerun the Deploy Account-wide infrastructure action. + 4. Terraform will create an empty table with the correct name & (most importantly!) read/write IAM policies. + 5. Delete the empty table created by terraform and restore from the backup, specifying the same table name you've defined in code. +6. Run the [Persistent Environment Deploy](https://github.com/NHSDigital/NRLF/actions/workflows/persistent-environment.yml) workflow against your branch & `perftest` to restore the environment with lambdas pointed at your chosen table. +7. You can check this has been successful by checking the table name in the lambdas. + - In the AWS console: Lambda > functions > pick any perftest-1 lambda > Configuration > Environment variables > `TABLE_NAME` should be your desired pointer table e.g. `nhsd-nrlf--perftest-baseline-pointers-table` + +If you've followed these steps, you will also need to [generate permissions](#generate-permissions) as the organisation permissions will have been lost when the environment was torn down. ### Prepare to run tests -#### Pull certs for env +#### Pull certs for perftest ```sh assume management @@ -26,19 +50,21 @@ make truststore-pull-all ENV=perftest You will need to generate pointer permissions the first time performance tests are run in an environment e.g. if the perftest environment is destroyed & recreated. ```sh -make generate permissions # makes a bunch of json permission files +# In project root +make generate permissions # makes a bunch of json permission files for test organisations make build # will take all permissions & create nrlf_permissions.zip file # apply this new permissions zip file to your environment cd ./terraform/infrastructure -assume test # needed? +assume nhsd-nrlf-test make init TF_WORKSPACE_NAME=perftest-1 ENV=perftest -tf apply +make ENV=perftest USE_SHARED_RESOURCES=true apply ``` #### Generate input files ```sh +assume nhsd-nrlf-test # creates 2 csv files and a json file make perftest-prepare PERFTEST_TABLE_NAME=perftest-baseline ``` @@ -49,3 +75,13 @@ make perftest-prepare PERFTEST_TABLE_NAME=perftest-baseline make perftest-consumer ENV_TYPE=perftest PERFTEST_HOST=perftest-1.perftest.record-locator.national.nhs.uk make perftest-producer ENV_TYPE=perftest PERFTEST_HOST=perftest-1.perftest.record-locator.national.nhs.uk ``` + +## Assumptions / Caveats + +- Run performance tests in the perftest environment only\* +- Both producer & consumer tests are repeatable +- These tests work on the assumption that all nhs numbers in the test data are serial and lie within a fixed range i.e. picking any number between NHS_NUMBER_MINIMUM & NHS_NUMBER_MAXIMUM will yield a patient with pointer(s). +- Configure scenarios in the `consumer/perftest.config.json` & `producer/perftest.config.json` files. This does not alter the number of stages per scenario, that's fixed in `perftest.js`. +- Consider running these tests multiple times to get figures for a warm environment - perftest, unlike prod, is not well-used so you will get cold-start figures on your first run + +\*These performance tests are tightly coupled to the seed scripts that populate test data. This means these tests can only be run in an environment containing solely test data created by the seed data scripts. `perftest` is a dedicated environment to do this in, but in theory any environment could be populated with the seed data and used. diff --git a/tests/performance/perftest_environment.py b/tests/performance/perftest_environment.py index f96a60371..2ae81d764 100644 --- a/tests/performance/perftest_environment.py +++ b/tests/performance/perftest_environment.py @@ -5,6 +5,7 @@ import re import boto3 +from seed_data_constants import CHECKSUM_WEIGHTS DYNAMODB = boto3.resource("dynamodb", region_name="eu-west-2") @@ -55,10 +56,6 @@ def extract_consumer_data(output_dir="."): print(f"Consumer data written to {out}") # noqa: T201 -# Semi-deterministic NHS number generator (duplicated from seed_nft_tables.py) -CHECKSUM_WEIGHTS = [i for i in range(10, 1, -1)] - - class TestNhsNumbersIterator: def __iter__(self): self.first9 = 900000000 diff --git a/tests/performance/producer/delete_permissions.py b/tests/performance/producer/delete_permissions.py index b9d0d7d60..4922b8a0f 100644 --- a/tests/performance/producer/delete_permissions.py +++ b/tests/performance/producer/delete_permissions.py @@ -1,7 +1,8 @@ from pathlib import Path import fire -from seed_nft_tables import DEFAULT_CUSTODIAN_DISTRIBUTIONS + +from tests.performance.seed_data_constants import DEFAULT_CUSTODIAN_DISTRIBUTIONS def main(permissions_dir="../../dist/nrlf_permissions/K6PerformanceTest"): diff --git a/tests/performance/producer/generate_distributions.py b/tests/performance/producer/generate_distributions.py index 1ea674dce..c81c9fbc9 100644 --- a/tests/performance/producer/generate_distributions.py +++ b/tests/performance/producer/generate_distributions.py @@ -1,8 +1,10 @@ import json from pathlib import Path -# Import the constants from your seed_nft_tables.py -from seed_nft_tables import DEFAULT_CUSTODIAN_DISTRIBUTIONS, DEFAULT_TYPE_DISTRIBUTIONS +from tests.performance.seed_data_constants import ( + DEFAULT_CUSTODIAN_DISTRIBUTIONS, + DEFAULT_TYPE_DISTRIBUTIONS, +) def expand_distribution(dist): diff --git a/tests/performance/producer/generate_permissions.py b/tests/performance/producer/generate_permissions.py index 29b289e6c..5713eedf8 100644 --- a/tests/performance/producer/generate_permissions.py +++ b/tests/performance/producer/generate_permissions.py @@ -2,7 +2,8 @@ from pathlib import Path import fire -from seed_nft_tables import DEFAULT_CUSTODIAN_DISTRIBUTIONS + +from tests.performance.seed_data_constants import DEFAULT_CUSTODIAN_DISTRIBUTIONS def main(output_dir="../../dist/nrlf_permissions/K6PerformanceTest"): diff --git a/tests/performance/producer/seed_nft_tables.py b/tests/performance/seed_data_constants.py similarity index 90% rename from tests/performance/producer/seed_nft_tables.py rename to tests/performance/seed_data_constants.py index f59a85af1..27dec83cd 100644 --- a/tests/performance/producer/seed_nft_tables.py +++ b/tests/performance/seed_data_constants.py @@ -1,11 +1,3 @@ -import boto3 - -dynamodb = boto3.client("dynamodb") -resource = boto3.resource("dynamodb") - - -# DOC_REF_TEMPLATE = load_document_reference("NFT-template") - CHECKSUM_WEIGHTS = [i for i in range(10, 1, -1)] # These are based on the Nov 7th 2025 pointer stats report @@ -66,3 +58,5 @@ "TRXT": 1, }, # summary record currently has only one supplier } + +DEFAULT_COUNT_DISTRIBUTIONS = {"1": 91, "2": 8, "3": 1}