Skip to content

DB Optimizations #36

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mjiori wants to merge 11 commits into
base: qa
Choose a base branch
from
Open

DB Optimizations #36

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
dc0a5da
Added migration for removing and converting columns
mjiori Sep 5, 2025
d10c579
Making code fixes alongside the optimization migrations
mjiori Sep 5, 2025
24b14eb
Changing eventType to int
mjiori Sep 5, 2025
8d261f5
Added note about what to do if an unknown event type is found during …
mjiori Sep 5, 2025
f19554f
Making code changes in line with different data types, more to do
mjiori Sep 8, 2025
54f37f9
Fixing various errors
mjiori Sep 17, 2025
06639b1
Finish optimization migration
mjiori Dec 9, 2025
8762c8e
Checkpointing
mjiori Jan 28, 2026
97078f8
Checkpoint
mjiori Feb 19, 2026
1d31431
New release for patching update on containers.
Caesonia Mar 30, 2026
28d56b4
Merge branch 'master' into shrink-db-size
mjiori Apr 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cfn/cfn-registry-cluster.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ Resources:
- Condition: SUCCESS
ContainerName: Nsqadmin_ResolvConf_InitContainer
Essential: true
Image: 997427182289.dkr.ecr.us-east-1.amazonaws.com/docker-hub/aptrust/registry:2159d11-master
Image: 997427182289.dkr.ecr.us-east-1.amazonaws.com/docker-hub/aptrust/registry:1cefc5d-master
Secrets:
- Name: APT_ENV
ValueFrom: !Sub 'arn:aws:ssm:us-east-1:997427182289:parameter/${Envn}/REGISTRY/APT_ENV'
Expand Down
292 changes: 161 additions & 131 deletions constants/constants.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,136 +5,154 @@ import (
)

const (
AccessConsortia = "consortia"
AccessInstitution = "institution"
AccessRestricted = "restricted"
ActionApproveDelete = "ApproveDelete"
ActionCreate = "Create"
ActionDelete = "Delete"
ActionFinishBulkDelete = "FinishBulkDelete"
ActionFixityCheck = "Fixity Check"
ActionRestoreFile = "Restore File"
ActionGlacierRestore = "Glacier Restore"
ActionIngest = "Ingest"
ActionRead = "Read"
ActionRequestDelete = "RequestDelete"
ActionRestoreObject = "Restore Object"
ActionUpdate = "Update"
AlertDeletionCancelled = "Deletion Cancelled"
AlertDeletionCompleted = "Deletion Completed"
AlertDeletionConfirmed = "Deletion Confirmed"
AlertDeletionRequested = "Deletion Requested"
AlertFailedFixity = "Failed Fixity Check"
AlertPasswordChanged = "Password Changed"
AlertPasswordReset = "Password Reset"
AlertRestorationCompleted = "Restoration Completed"
AlertStalledItems = "Stalled Work Items"
AlertWelcome = "Welcome New User"
AlgMd5 = "md5"
AlgSha1 = "sha1"
AlgSha256 = "sha256"
AlgSha512 = "sha512"
APIUserHeader = "X-Pharos-API-User"
APIKeyHeader = "X-Pharos-API-Key"
APIPrefixAdmin = "/admin-api/"
APIPrefixMember = "/member-api/"
APTrustOpsEmail = "ops@aptrust.org"
BTRProfileIdentifier = "https://github.com/dpscollaborative/btr_bagit_profile/releases/download/1.0/btr-bagit-profile.json"
CSRFCookieName = "csrf_token"
CSRFHeaderName = "X-CSRF-Token"
CSRFTokenName = "csrf_token"
DefaultProfileIdentifier = "https://raw.githubusercontent.com/APTrust/preservation-services/master/profiles/aptrust-v2.2.json"
EmailServiceSES = "SES"
EmailServiceSMTP = "SMTP"
EventAccessAssignment = "access assignment"
EventCapture = "capture"
EventCompression = "compression"
EventCreation = "creation"
EventDeaccession = "deaccession"
EventDecompression = "decompression"
EventDecryption = "decryption"
EventDeletion = "deletion"
EventDigestCalculation = "message digest calculation"
EventFixityCheck = "fixity check"
EventIdentifierAssignment = "identifier assignment"
EventIngestion = "ingestion"
EventMigration = "migration"
EventNormalization = "normalization"
EventReplication = "replication"
EventSignatureValidation = "digital signature validation"
EventValidation = "validation"
EventVirusCheck = "virus check"
IngestPreFetch = "ingest01_prefetch"
IngestValidation = "ingest02_bag_validation"
IngestReingestCheck = "ingest03_reingest_check"
IngestStaging = "ingest04_staging"
IngestFormatIdentification = "ingest05_format_identification"
IngestStorage = "ingest06_storage"
IngestStorageValidation = "ingest07_storage_validation"
IngestRecord = "ingest08_record"
IngestCleanup = "ingest09_cleanup"
InstTypeMember = "MemberInstitution"
InstTypeSubscriber = "SubscriptionInstitution"
MetaFixityAlertsLastRun = "fixity alerts last run"
MetaSpotTestsRunning = "spot restore is running"
MetaSpotTestsLastRun = "spot restore last run"
OutcomeFailure = "Failure"
OutcomeSuccess = "Success"
RoleInstAdmin = "institutional_admin"
RoleInstUser = "institutional_user"
RoleNone = "none"
RoleSysAdmin = "admin"
SecondFactorAuthy = "Authy"
SecondFactorBackupCode = "Backup Code"
SecondFactorSMS = "SMS"
StageAvailableInS3 = "Available in S3"
StageCleanup = "Cleanup"
StageCopyToStaging = "Copy To Staging"
StageFetch = "Fetch"
StageFormatIdentification = "Format Identification"
StagePackage = "Package"
StageReceive = "Receive"
StageRecord = "Record"
StageReingestCheck = "Reingest Check"
StageRequested = "Requested"
StageResolve = "Resolve"
StageRestoring = "Restoring"
StageStorageValidation = "Storage Validation"
StageStore = "Store"
StageUnpack = "Unpack"
StageValidate = "Validate"
StateActive = "A"
StateDeleted = "D"
StatusCancelled = "Cancelled"
StatusFailed = "Failed"
StatusPending = "Pending"
StatusStarted = "Started"
StatusSuccess = "Success"
StatusSuspended = "Suspended"
StorageOptionGlacierDeepOH = "Glacier-Deep-OH"
StorageOptionGlacierDeepOR = "Glacier-Deep-OR"
StorageOptionGlacierDeepVA = "Glacier-Deep-VA"
StorageOptionGlacierOH = "Glacier-OH"
StorageOptionGlacierOR = "Glacier-OR"
StorageOptionGlacierVA = "Glacier-VA"
StorageOptionStandard = "Standard"
StorageOptionWasabiOR = "Wasabi-OR"
StorageOptionWasabiTX = "Wasabi-TX"
StorageOptionWasabiVA = "Wasabi-VA"
SystemUser = "system@aptrust.org"
TopicDelete = "delete_item"
TopicE2EDelete = "e2e_deletion_post_test"
TopicE2EFixity = "e2e_fixity_post_test"
TopicE2EIngest = "e2e_ingest_post_test"
TopicE2EReingest = "e2e_reingest_post_test"
TopicE2ERestore = "e2e_restoration_post_test"
TopicFileRestore = "restore_file"
TopicFixity = "fixity_check"
TopicGlacierRestore = "restore_glacier"
TopicObjectRestore = "restore_object"
TwoFactorAuthy = "onetouch"
TwoFactorNone = "none"
TwoFactorSMS = "sms"
AccessConsortia = "consortia"
AccessInstitution = "institution"
AccessRestricted = "restricted"
ActionApproveDelete = "ApproveDelete"
ActionCreate = "Create"
ActionDelete = "Delete"
ActionFinishBulkDelete = "FinishBulkDelete"
ActionFixityCheck = "Fixity Check"
ActionRestoreFile = "Restore File"
ActionGlacierRestore = "Glacier Restore"
ActionIngest = "Ingest"
ActionRead = "Read"
ActionRequestDelete = "RequestDelete"
ActionRestoreObject = "Restore Object"
ActionUpdate = "Update"
AlertDeletionCancelled = "Deletion Cancelled"
AlertDeletionCompleted = "Deletion Completed"
AlertDeletionConfirmed = "Deletion Confirmed"
AlertDeletionRequested = "Deletion Requested"
AlertFailedFixity = "Failed Fixity Check"
AlertPasswordChanged = "Password Changed"
AlertPasswordReset = "Password Reset"
AlertRestorationCompleted = "Restoration Completed"
AlertStalledItems = "Stalled Work Items"
AlertWelcome = "Welcome New User"
AlgMd5 = "md5"
AlgSha1 = "sha1"
AlgSha256 = "sha256"
AlgSha512 = "sha512"
APIUserHeader = "X-Pharos-API-User"
APIKeyHeader = "X-Pharos-API-Key"
APIPrefixAdmin = "/admin-api/"
APIPrefixMember = "/member-api/"
APTrustOpsEmail = "ops@aptrust.org"
BTRProfileIdentifier = "https://github.com/dpscollaborative/btr_bagit_profile/releases/download/1.0/btr-bagit-profile.json"
CSRFCookieName = "csrf_token"
CSRFHeaderName = "X-CSRF-Token"
CSRFTokenName = "csrf_token"
DefaultProfileIdentifier = "https://raw.githubusercontent.com/APTrust/preservation-services/master/profiles/aptrust-v2.2.json"
EmailServiceSES = "SES"
EmailServiceSMTP = "SMTP"
EventAccessAssignment = 1 // "access assignment"
EventCapture = 4 // "capture"
EventCompression = 6 //"compression"
EventCreation = 7 //"creation"
EventDeaccession = 8 //"deaccession"
EventDecompression = 9 //"decompression"
EventDecryption = 10 //"decryption"
EventDeletion = 11 //"deletion"
EventDigestCalculation = 33 //"message digest calculation"
EventFixityCheck = 21 //"fixity check"
EventIdentifierAssignment = 24 //"identifier assignment"
EventIngestion = 29 //"ingestion"
EventMigration = 36 //"migration"
EventNormalization = 38 //"normalization"
EventReplication = 47 //"replication"
EventSignatureValidation = 13 //"digital signature validation"
EventValidation = 51 //"validation"
EventVirusCheck = 52 //"virus check"
EventAccessAssignmentString = "access assignment"
EventCaptureString = "capture"
EventCompressionString = "compression"
EventCreationString = "creation"
EventDeaccessionString = "deaccession"
EventDecompressionString = "decompression"
EventDecryptionString = "decryption"
EventDeletionString = "deletion"
EventDigestCalculationString = "message digest calculation"
EventFixityCheckString = "fixity check"
EventIdentifierAssignmentString = "identifier assignment"
EventIngestionString = "ingestion"
EventMigrationString = "migration"
EventNormalizationString = "normalization"
EventReplicationString = "replication"
EventSignatureValidationString = "digital signature validation"
EventValidationString = "validation"
EventVirusCheckString = "virus check"
IngestPreFetch = "ingest01_prefetch"
IngestValidation = "ingest02_bag_validation"
IngestReingestCheck = "ingest03_reingest_check"
IngestStaging = "ingest04_staging"
IngestFormatIdentification = "ingest05_format_identification"
IngestStorage = "ingest06_storage"
IngestStorageValidation = "ingest07_storage_validation"
IngestRecord = "ingest08_record"
IngestCleanup = "ingest09_cleanup"
InstTypeMember = "MemberInstitution"
InstTypeSubscriber = "SubscriptionInstitution"
MetaFixityAlertsLastRun = "fixity alerts last run"
MetaSpotTestsRunning = "spot restore is running"
MetaSpotTestsLastRun = "spot restore last run"
OutcomeFailure = "Failure"
OutcomeSuccess = "Success"
RoleInstAdmin = "institutional_admin"
RoleInstUser = "institutional_user"
RoleNone = "none"
RoleSysAdmin = "admin"
SecondFactorAuthy = "Authy"
SecondFactorBackupCode = "Backup Code"
SecondFactorSMS = "SMS"
StageAvailableInS3 = "Available in S3"
StageCleanup = "Cleanup"
StageCopyToStaging = "Copy To Staging"
StageFetch = "Fetch"
StageFormatIdentification = "Format Identification"
StagePackage = "Package"
StageReceive = "Receive"
StageRecord = "Record"
StageReingestCheck = "Reingest Check"
StageRequested = "Requested"
StageResolve = "Resolve"
StageRestoring = "Restoring"
StageStorageValidation = "Storage Validation"
StageStore = "Store"
StageUnpack = "Unpack"
StageValidate = "Validate"
StateActive = "A"
StateDeleted = "D"
StatusCancelled = "Cancelled"
StatusFailed = "Failed"
StatusPending = "Pending"
StatusStarted = "Started"
StatusSuccess = "Success"
StatusSuspended = "Suspended"
StorageOptionGlacierDeepOH = "Glacier-Deep-OH"
StorageOptionGlacierDeepOR = "Glacier-Deep-OR"
StorageOptionGlacierDeepVA = "Glacier-Deep-VA"
StorageOptionGlacierOH = "Glacier-OH"
StorageOptionGlacierOR = "Glacier-OR"
StorageOptionGlacierVA = "Glacier-VA"
StorageOptionStandard = "Standard"
StorageOptionWasabiOR = "Wasabi-OR"
StorageOptionWasabiTX = "Wasabi-TX"
StorageOptionWasabiVA = "Wasabi-VA"
SystemUser = "system@aptrust.org"
TopicDelete = "delete_item"
TopicE2EDelete = "e2e_deletion_post_test"
TopicE2EFixity = "e2e_fixity_post_test"
TopicE2EIngest = "e2e_ingest_post_test"
TopicE2EReingest = "e2e_reingest_post_test"
TopicE2ERestore = "e2e_restoration_post_test"
TopicFileRestore = "restore_file"
TopicFixity = "fixity_check"
TopicGlacierRestore = "restore_glacier"
TopicObjectRestore = "restore_object"
TwoFactorAuthy = "onetouch"
TwoFactorNone = "none"
TwoFactorSMS = "sms"
)

var AccessSettings = []string{
Expand Down Expand Up @@ -179,7 +197,7 @@ var EventOutcomes = []string{
OutcomeSuccess,
}

var EventTypes = []string{
var EventTypes = []int{
EventAccessAssignment,
EventCreation,
EventDeletion,
Expand All @@ -191,6 +209,18 @@ var EventTypes = []string{
EventValidation,
}

var EventTypesString = []string{
EventAccessAssignmentString,
EventCreationString,
EventDeletionString,
EventDigestCalculationString,
EventFixityCheckString,
EventIdentifierAssignmentString,
EventIngestionString,
EventReplicationString,
EventValidationString,
}

var GlacierOnlyOptions = []string{
StorageOptionGlacierDeepOH,
StorageOptionGlacierDeepOR,
Expand Down
Loading