Skip to content

NRL-1875 Add profiles for 15m and 55m perf tests #1117

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sandyforresternhs merged 3 commits into from
Jan 8, 2026
Merged

NRL-1875 Add profiles for 15m and 55m perf tests #1117

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5a4af5e
NRL-1875 Define dynamodb pointers table as deploy parameter
mattdean3-nhs Jan 8, 2026
12dd046
NRL-1875 Add 15m and 55m volume distributions for perfs tests
mattdean3-nhs Jan 8, 2026
0fc48d0
NRL-1875 Remove unused IAM policies from API lambdas
mattdean3-nhs Jan 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions scripts/seed_nft_tables.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
from nrlf.tests.data import load_document_reference
from tests.performance.seed_data_constants import ( # DEFAULT_COUNT_DISTRIBUTIONS,
CHECKSUM_WEIGHTS,
DEFAULT_CUSTODIAN_DISTRIBUTIONS,
DEFAULT_TYPE_DISTRIBUTIONS,
CUSTODIAN_DISTRIBUTION_PROFILES,
TYPE_DISTRIBUTION_PROFILES,
)

dynamodb = boto3.client("dynamodb")
Expand Down Expand Up @@ -87,22 +87,24 @@ def _populate_seed_table(
table_name: str,
px_with_pointers: int,
pointers_per_px: float = 1.0,
type_dists: dict[str, int] = DEFAULT_TYPE_DISTRIBUTIONS,
custodian_dists: dict[str, dict[str, int]] = DEFAULT_CUSTODIAN_DISTRIBUTIONS,
type_dist_profile: str = "default",
custodian_dist_profile: str = "default",
):
"""
Seeds a table with example data for non-functional testing.
"""
if pointers_per_px < 1.0:
raise ValueError("Cannot populate table with patients with zero pointers")

type_dists = TYPE_DISTRIBUTION_PROFILES[type_dist_profile]
custodian_dists = CUSTODIAN_DISTRIBUTION_PROFILES[custodian_dist_profile]

# set up iterations
type_iter = _set_up_cyclical_iterator(type_dists)
custodian_iters = _set_up_custodian_iterators(custodian_dists)
# count_iter = _set_up_cyclical_iterator(DEFAULT_COUNT_DISTRIBUTIONS)
count_iter = _get_pointer_count_poisson_distributions(
px_with_pointers, pointers_per_px
)
# count_iter = _get_pointer_count_negbinom_distributions(px_with_pointers, pointers_per_px)
testnum_cls = TestNhsNumbersIterator()
testnum_iter = iter(testnum_cls)

Expand Down
18 changes: 4 additions & 14 deletions terraform/infrastructure/data.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,22 +17,12 @@ data "aws_iam_policy" "auth-store-read-policy" {

data "aws_dynamodb_table" "pointers-table" {
count = var.use_shared_resources ? 1 : 0
name = "${local.pointers_table_prefix}-pointers-table"
name = local.shared_pointers_table_name
}

data "aws_iam_policy" "pointers-table-read" {
count = var.use_shared_resources ? 1 : 0
name = "${local.pointers_table_prefix}-pointers-table-read"
}

data "aws_iam_policy" "pointers-table-write" {
count = var.use_shared_resources ? 1 : 0
name = "${local.pointers_table_prefix}-pointers-table-write"
}

data "aws_iam_policy" "pointers-kms-read-write" {
count = var.use_shared_resources ? 1 : 0
name = "${local.pointers_table_prefix}-pointers-kms-read-write"
data "aws_kms_key" "pointers-table-key" {
count = var.use_shared_resources ? 1 : 0
key_id = "alias/${local.shared_pointers_table_name}-key"
}

data "external" "current-info" {
Expand Down
4 changes: 2 additions & 2 deletions terraform/infrastructure/etc/dev.tfvars
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
account_name = "dev"
aws_account_name = "dev"

dynamodb_pointers_table_prefix = "nhsd-nrlf--dev"
dynamodb_sandbox_pointers_table_prefix = "nhsd-nrlf--dev-sandbox"
dynamodb_pointers_table_name = "nhsd-nrlf--dev-pointers-table"
dynamodb_sandbox_pointers_table_name = "nhsd-nrlf--dev-sandbox-pointers-table"

domain = "api.record-locator.dev.national.nhs.uk"
public_domain = "internal-dev.api.service.nhs.uk"
Expand Down
6 changes: 3 additions & 3 deletions terraform/infrastructure/etc/int.tfvars
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
account_name = "int"
aws_account_name = "test"

dynamodb_pointers_table_prefix = "nhsd-nrlf--int"
dynamodb_sandbox_pointers_table_prefix = "nhsd-nrlf--int-sandbox"
deletion_protection = true
dynamodb_pointers_table_name = "nhsd-nrlf--int-pointers-table"
dynamodb_sandbox_pointers_table_name = "nhsd-nrlf--int-sandbox-pointers-table"
deletion_protection = true

domain = "api.record-locator.int.national.nhs.uk"
public_domain = "int.api.service.nhs.uk"
Expand Down
2 changes: 1 addition & 1 deletion terraform/infrastructure/etc/perftest.tfvars
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
account_name = "perftest"
aws_account_name = "test"

dynamodb_pointers_table_prefix = "nhsd-nrlf--perftest-baseline"
dynamodb_pointers_table_name = "nhsd-nrlf--perftest-baseline-pointers-table"

domain = "perftest.record-locator.national.nhs.uk"
public_domain = "perftest.api.service.nhs.uk"
Expand Down
4 changes: 2 additions & 2 deletions terraform/infrastructure/etc/prod.tfvars
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
account_name = "prod"
aws_account_name = "prod"

dynamodb_pointers_table_prefix = "nhsd-nrlf--prod"
deletion_protection = true
dynamodb_pointers_table_name = "nhsd-nrlf--prod-pointers-table"
deletion_protection = true

domain = "api.record-locator.national.nhs.uk"
public_domain = "api.service.nhs.uk"
Expand Down
4 changes: 2 additions & 2 deletions terraform/infrastructure/etc/qa.tfvars
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
account_name = "qa"
aws_account_name = "test"

dynamodb_pointers_table_prefix = "nhsd-nrlf--qa"
dynamodb_sandbox_pointers_table_prefix = "nhsd-nrlf--qa-sandbox"
dynamodb_pointers_table_name = "nhsd-nrlf--qa-pointers-table"
dynamodb_sandbox_pointers_table_name = "nhsd-nrlf--qa-sandbox-pointers-table"

domain = "qa.record-locator.national.nhs.uk"
public_domain = "internal-qa.api.service.nhs.uk"
Expand Down
2 changes: 1 addition & 1 deletion terraform/infrastructure/etc/ref.tfvars
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
account_name = "ref"
aws_account_name = "test"

dynamodb_pointers_table_prefix = "nhsd-nrlf--ref"
dynamodb_pointers_table_name = "nhsd-nrlf--ref-pointers-table"

domain = "api.record-locator.ref.national.nhs.uk"
public_domain = "ref.api.service.nhs.uk"
Expand Down
86 changes: 86 additions & 0 deletions terraform/infrastructure/iam.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
resource "aws_iam_policy" "pointers-table-read" {
count = var.use_shared_resources ? 1 : 0
name = "${local.prefix}-allow-pointers-table-read"
description = "Read the pointers-table"
policy = jsonencode({
Version = "2012-10-17"
Statement = [
{
Action = [
"kms:Decrypt",
"kms:DescribeKey"
]
Effect = "Allow"
Resource = [
data.aws_kms_key.pointers-table-key[0].arn
]
},
{
Effect = "Allow"
Action = [
"dynamodb:Query",
"dynamodb:Scan",
"dynamodb:GetItem",
],
Resource = [
"${data.aws_dynamodb_table.pointers-table[0].arn}*"
]
}
]
})
}

resource "aws_iam_policy" "pointers-table-write" {
count = var.use_shared_resources ? 1 : 0
name = "${local.prefix}-allow-pointers-table-write"
description = "Write to the pointers-table"
policy = jsonencode({
Version = "2012-10-17"
Statement = [
{
Action = [
"kms:Encrypt",
"kms:GenerateDataKey"
]
Effect = "Allow"
Resource = [
data.aws_kms_key.pointers-table-key[0].arn
]
},
{
Effect = "Allow"
Action = [
"dynamodb:PutItem",
"dynamodb:UpdateItem",
"dynamodb:DeleteItem",
],
Resource = [
"${data.aws_dynamodb_table.pointers-table[0].arn}*"
]
}
]
})
}

resource "aws_iam_policy" "pointers-kms-read-write" {
count = var.use_shared_resources ? 1 : 0
name = "${local.prefix}-allow-pointers-kms-read-write"
description = "Encrypt and decrypt with the pointers table kms key"
policy = jsonencode({
Version = "2012-10-17"
Statement = [
{
Action = [
"kms:Decrypt",
"kms:DescribeKey",
"kms:Encrypt",
"kms:GenerateDataKey"
]
Effect = "Allow"
Resource = [
data.aws_kms_key.pointers-table-key[0].arn
]
}
]
})
}
12 changes: 0 additions & 12 deletions terraform/infrastructure/lambda.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ module "consumer__readDocumentReference" {
}
additional_policies = [
local.pointers_table_read_policy_arn,
local.pointers_kms_read_write_arn,
local.auth_store_read_policy_arn
]
firehose_subscriptions = local.firehose_lambda_subscriptions
Expand All @@ -44,7 +43,6 @@ module "consumer__searchDocumentReference" {
}
additional_policies = [
local.pointers_table_read_policy_arn,
local.pointers_kms_read_write_arn,
local.auth_store_read_policy_arn
]
firehose_subscriptions = local.firehose_lambda_subscriptions
Expand All @@ -71,7 +69,6 @@ module "consumer__searchPostDocumentReference" {
}
additional_policies = [
local.pointers_table_read_policy_arn,
local.pointers_kms_read_write_arn,
local.auth_store_read_policy_arn
]
firehose_subscriptions = local.firehose_lambda_subscriptions
Expand Down Expand Up @@ -99,7 +96,6 @@ module "producer__createDocumentReference" {
additional_policies = [
local.pointers_table_write_policy_arn,
local.pointers_table_read_policy_arn,
local.pointers_kms_read_write_arn,
local.auth_store_read_policy_arn
]
firehose_subscriptions = local.firehose_lambda_subscriptions
Expand Down Expand Up @@ -127,7 +123,6 @@ module "producer__deleteDocumentReference" {
additional_policies = [
local.pointers_table_write_policy_arn,
local.pointers_table_read_policy_arn,
local.pointers_kms_read_write_arn,
local.auth_store_read_policy_arn
]
firehose_subscriptions = local.firehose_lambda_subscriptions
Expand All @@ -154,7 +149,6 @@ module "producer__readDocumentReference" {
}
additional_policies = [
local.pointers_table_read_policy_arn,
local.pointers_kms_read_write_arn,
local.auth_store_read_policy_arn
]
firehose_subscriptions = local.firehose_lambda_subscriptions
Expand All @@ -181,7 +175,6 @@ module "producer__searchDocumentReference" {
}
additional_policies = [
local.pointers_table_read_policy_arn,
local.pointers_kms_read_write_arn,
local.auth_store_read_policy_arn
]
firehose_subscriptions = local.firehose_lambda_subscriptions
Expand All @@ -208,7 +201,6 @@ module "producer__searchPostDocumentReference" {
}
additional_policies = [
local.pointers_table_read_policy_arn,
local.pointers_kms_read_write_arn,
local.auth_store_read_policy_arn
]
firehose_subscriptions = local.firehose_lambda_subscriptions
Expand Down Expand Up @@ -236,7 +228,6 @@ module "producer__updateDocumentReference" {
additional_policies = [
local.pointers_table_read_policy_arn,
local.pointers_table_write_policy_arn,
local.pointers_kms_read_write_arn,
local.auth_store_read_policy_arn
]
firehose_subscriptions = local.firehose_lambda_subscriptions
Expand Down Expand Up @@ -264,7 +255,6 @@ module "producer__upsertDocumentReference" {
additional_policies = [
local.pointers_table_write_policy_arn,
local.pointers_table_read_policy_arn,
local.pointers_kms_read_write_arn,
local.auth_store_read_policy_arn
]
firehose_subscriptions = local.firehose_lambda_subscriptions
Expand Down Expand Up @@ -292,7 +282,6 @@ module "consumer__status" {
}
additional_policies = [
local.pointers_table_read_policy_arn,
local.pointers_kms_read_write_arn,
local.auth_store_read_policy_arn
]
firehose_subscriptions = local.firehost_lambda_splunk_only_subscription
Expand Down Expand Up @@ -321,7 +310,6 @@ module "producer__status" {
}
additional_policies = [
local.pointers_table_read_policy_arn,
local.pointers_kms_read_write_arn,
local.auth_store_read_policy_arn
]
firehose_subscriptions = local.firehost_lambda_splunk_only_subscription
Expand Down
9 changes: 4 additions & 5 deletions terraform/infrastructure/locals.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,9 @@ locals {
auth_store_id = var.use_shared_resources ? data.aws_s3_bucket.authorization-store[0].id : module.ephemeral-s3-permission-store[0].bucket_id
auth_store_read_policy_arn = var.use_shared_resources ? data.aws_iam_policy.auth-store-read-policy[0].arn : module.ephemeral-s3-permission-store[0].bucket_read_policy_arn

pointers_table_prefix = local.is_sandbox_env ? "${var.dynamodb_sandbox_pointers_table_prefix}" : "${var.dynamodb_pointers_table_prefix}"

shared_pointers_table_name = local.is_sandbox_env ? var.dynamodb_sandbox_pointers_table_name : var.dynamodb_pointers_table_name
pointers_table_name = var.use_shared_resources ? data.aws_dynamodb_table.pointers-table[0].name : module.ephemeral-pointers-table[0].table_name
pointers_table_read_policy_arn = var.use_shared_resources ? data.aws_iam_policy.pointers-table-read[0].arn : module.ephemeral-pointers-table[0].read_policy_arn
pointers_table_write_policy_arn = var.use_shared_resources ? data.aws_iam_policy.pointers-table-write[0].arn : module.ephemeral-pointers-table[0].write_policy_arn
pointers_kms_read_write_arn = var.use_shared_resources ? data.aws_iam_policy.pointers-kms-read-write[0].arn : module.ephemeral-pointers-table[0].kms_read_write_policy_arn
pointers_table_key_arn = var.use_shared_resources ? data.aws_kms_key.pointers-table-key[0].arn : module.ephemeral-pointers-table[0].kms_key_arn
pointers_table_read_policy_arn = var.use_shared_resources ? aws_iam_policy.pointers-table-read[0].arn : module.ephemeral-pointers-table[0].read_policy_arn
pointers_table_write_policy_arn = var.use_shared_resources ? aws_iam_policy.pointers-table-write[0].arn : module.ephemeral-pointers-table[0].write_policy_arn
}
5 changes: 5 additions & 0 deletions terraform/infrastructure/modules/pointers-table/output.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,8 @@ output "kms_read_write_policy_arn" {
description = "Policy to encrypt and decrypt the pointers table with the kms key"
value = aws_iam_policy.pointers-kms-read-write.arn
}

output "kms_key_arn" {
description = "KMS key arn for the pointers table"
value = aws_kms_key.pointers-table-key.arn
}
8 changes: 4 additions & 4 deletions terraform/infrastructure/vars.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,13 +68,13 @@ variable "disable_firehose_lambda_subscriptions" {
default = false
}

variable "dynamodb_pointers_table_prefix" {
variable "dynamodb_pointers_table_name" {
type = string
description = "The prefix of the DynamoDB pointers table to use when using shared resources"
description = "The name of the DynamoDB pointers table to use when using shared resources"
}

variable "dynamodb_sandbox_pointers_table_prefix" {
variable "dynamodb_sandbox_pointers_table_name" {
type = string
description = "The prefix of the DynamoDB pointers table to use when using shared resources in a sandbox environment"
description = "The name of the DynamoDB pointers table to use when using shared resources in a sandbox environment"
default = null
}
20 changes: 20 additions & 0 deletions tests/performance/seed_data_constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,4 +59,24 @@
}, # summary record currently has only one supplier
}

VOL_15M_POINTERS_TYPE_DISTRIBUTIONS = {"824321000000109": 1}

VOL_15M_POINTERS_CUSTODIAN_DISTRIBUTIONS = {"16521000000101": {"TD2L9A": 1}}

VOL_55M_POINTERS_TYPE_DISTRIBUTIONS = {"16521000000101": 1}

VOL_55M_POINTERS_CUSTODIAN_DISTRIBUTIONS = {"16521000000101": {"TX26": 1}}

DEFAULT_COUNT_DISTRIBUTIONS = {"1": 91, "2": 8, "3": 1}

TYPE_DISTRIBUTION_PROFILES = {
"default": DEFAULT_TYPE_DISTRIBUTIONS,
"15m_pointers": VOL_15M_POINTERS_TYPE_DISTRIBUTIONS,
"55m_pointers": VOL_55M_POINTERS_TYPE_DISTRIBUTIONS,
}

CUSTODIAN_DISTRIBUTION_PROFILES = {
"default": DEFAULT_CUSTODIAN_DISTRIBUTIONS,
"15m_pointers": VOL_15M_POINTERS_CUSTODIAN_DISTRIBUTIONS,
"55m_pointers": VOL_55M_POINTERS_CUSTODIAN_DISTRIBUTIONS,
}
Loading