From 5637f56f9b9b71ba8ae688bd688e27f55d74277e Mon Sep 17 00:00:00 2001 From: Qingyang Hu Date: Thu, 13 Nov 2025 10:19:40 -0500 Subject: [PATCH 1/3] update unified integration --- internal/integration/mtest/mongotest.go | 6 -- internal/integration/unified/client_entity.go | 82 +++++++++++++++++-- internal/integration/unified/entity.go | 1 + internal/spectest/skip.go | 13 --- 4 files changed, 78 insertions(+), 24 deletions(-) diff --git a/internal/integration/mtest/mongotest.go b/internal/integration/mtest/mongotest.go index 3924b58604..dbd69da3b9 100644 --- a/internal/integration/mtest/mongotest.go +++ b/internal/integration/mtest/mongotest.go @@ -797,12 +797,6 @@ func verifyRunOnBlockConstraint(rob RunOnBlock) error { return err } - // TODO(GODRIVER-3486): Once auto encryption is supported by the unified test - // format,this check should be removed. - if rob.CSFLEEnabled() && rob.CSFLE.Options != nil { - return fmt.Errorf("Auto encryption required (GODRIVER-3486)") - } - if rob.CSFLEEnabled() && !IsCSFLEEnabled() { return fmt.Errorf("runOnBlock requires CSFLE to be enabled. Build with the cse tag to enable") } else if !rob.CSFLEEnabled() && IsCSFLEEnabled() { diff --git a/internal/integration/unified/client_entity.go b/internal/integration/unified/client_entity.go index bc981793df..6c08974359 100644 --- a/internal/integration/unified/client_entity.go +++ b/internal/integration/unified/client_entity.go @@ -9,6 +9,7 @@ package unified import ( "context" "fmt" + "os" "strings" "sync" "sync/atomic" @@ -32,11 +33,16 @@ import ( // exceed the default truncation length. const defaultMaxDocumentLen = 10_000 -// Security-sensitive commands that should be ignored in command monitoring by default. -var securitySensitiveCommands = []string{ - "authenticate", "saslStart", "saslContinue", "getnonce", - "createUser", "updateUser", "copydbgetnonce", "copydbsaslstart", "copydb", -} +var ( + // Security-sensitive commands that should be ignored in command monitoring by default. + securitySensitiveCommands = []string{ + "authenticate", "saslStart", "saslContinue", "getnonce", + "createUser", "updateUser", "copydbgetnonce", "copydbsaslstart", "copydb", + } + + awsAccessKeyID = os.Getenv("FLE_AWS_KEY") + awsSecretAccessKey = os.Getenv("FLE_AWS_SECRET") +) // clientEntity is a wrapper for a mongo.Client object that also holds additional information required during test // execution. @@ -217,6 +223,13 @@ func newClientEntity(ctx context.Context, em *EntityMap, entityOptions *entityOp } else { integtest.AddTestServerAPIVersion(clientOpts) } + if entityOptions.AutoEncryptOpts != nil { + aeo, err := createAutoEncryptionOptions(entityOptions.AutoEncryptOpts) + if err != nil { + return nil, fmt.Errorf("error parsing auto encryption options: %w", err) + } + clientOpts.SetAutoEncryptionOptions(aeo) + } for _, cmd := range entityOptions.IgnoredCommands { entity.ignoredCommands[cmd] = struct{}{} } @@ -251,6 +264,65 @@ func getURIForClient(opts *entityOptions) string { } } +func createAutoEncryptionOptions(opts bson.Raw) (*options.AutoEncryptionOptions, error) { + aeo := options.AutoEncryption() + var kvnsFound bool + elems, err := opts.Elements() + if err != nil { + return nil, err + } + + for _, elem := range elems { + name := elem.Key() + opt := elem.Value() + + switch name { + case "kmsProviders": + providers := make(map[string]map[string]any) + elems, err := opt.Document().Elements() + if err != nil { + return nil, err + } + for _, elem := range elems { + provider := elem.Key() + providerOpt := elem.Value() + switch provider { + case "aws": + providers["aws"] = map[string]any{ + "accessKeyId": awsAccessKeyID, + "secretAccessKey": awsSecretAccessKey, + } + case "local": + _, key := providerOpt.Document().Lookup("key").Binary() + providers["local"] = map[string]any{ + "key": key, + } + default: + return nil, fmt.Errorf("unrecognized KMS provider: %v", provider) + } + } + aeo.SetKmsProviders(providers) + case "schemaMap": + var schemaMap map[string]any + err := bson.Unmarshal(opt.Document(), &schemaMap) + if err != nil { + return nil, err + } + aeo.SetSchemaMap(schemaMap) + case "keyVaultNamespace": + kvnsFound = true + aeo.SetKeyVaultNamespace(opt.StringValue()) + default: + return nil, fmt.Errorf("unrecognized option: %v", name) + } + } + if !kvnsFound { + aeo.SetKeyVaultNamespace("keyvault.datakeys") + } + + return aeo, nil +} + // disconnect disconnects the client associated with this entity. It is an // idempotent operation, unlike the mongo client's disconnect method. This // property will help avoid unnecessary errors when calling disconnect on a diff --git a/internal/integration/unified/entity.go b/internal/integration/unified/entity.go index b1b827a124..57772b7236 100644 --- a/internal/integration/unified/entity.go +++ b/internal/integration/unified/entity.go @@ -52,6 +52,7 @@ type entityOptions struct { ID string `bson:"id"` // Options for client entities. + AutoEncryptOpts bson.Raw `bson:"autoEncryptOpts"` URIOptions bson.M `bson:"uriOptions"` UseMultipleMongoses *bool `bson:"useMultipleMongoses"` ObserveEvents []string `bson:"observeEvents"` diff --git a/internal/spectest/skip.go b/internal/spectest/skip.go index 189b38b447..0a1cf6e430 100644 --- a/internal/spectest/skip.go +++ b/internal/spectest/skip.go @@ -392,19 +392,6 @@ var skipTests = map[string][]string{ "TestClientSideEncryptionSpec/timeoutMS.json/timeoutMS_applied_to_listCollections_to_get_collection_schema", }, - // TODO(GODRIVER-3486): Support auto encryption in unified tests. - "Support auto encryption in unified tests (GODRIVER-3486)": { - "TestUnifiedSpec/unified-test-format/tests/valid-pass/poc-queryable-encryption.json/insert,_replace,_and_find_with_queryable_encryption", - }, - - // TODO(DRIVERS-3106): Support auto encryption in unified tests. - "Support auto encryption in unified tests (DRIVERS-3106)": { - "TestUnifiedSpec/client-side-encryption/tests/unified/localSchema.json/A_local_schema_should_override", - "TestUnifiedSpec/client-side-encryption/tests/unified/localSchema.json/A_local_schema_with_no_encryption_is_an_error", - "TestUnifiedSpec/client-side-encryption/tests/unified/fle2v2-BypassQueryAnalysis.json/BypassQueryAnalysis_decrypts", - "TestUnifiedSpec/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json/encryptedFieldsMap_is_preferred_over_remote_encryptedFields", - }, - // TODO(GODRIVER-3076): CSFLE/QE Support for more than 1 KMS provider per // type. "Support multiple KMS providers per type (GODRIVER-3076)": { From 95e8fedea084ece5bf259d3e5d7a545df9c842c0 Mon Sep 17 00:00:00 2001 From: Qingyang Hu Date: Thu, 13 Nov 2025 17:46:23 -0500 Subject: [PATCH 2/3] updates --- internal/integration/unified/client_entity.go | 9 +++++++++ internal/integration/unified/operation.go | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/internal/integration/unified/client_entity.go b/internal/integration/unified/client_entity.go index 6c08974359..27d6740f91 100644 --- a/internal/integration/unified/client_entity.go +++ b/internal/integration/unified/client_entity.go @@ -42,6 +42,9 @@ var ( awsAccessKeyID = os.Getenv("FLE_AWS_KEY") awsSecretAccessKey = os.Getenv("FLE_AWS_SECRET") + azureTenantID = os.Getenv("FLE_AZURE_TENANTID") + azureClientID = os.Getenv("FLE_AZURE_CLIENTID") + azureClientSecret = os.Getenv("FLE_AZURE_CLIENTSECRET") ) // clientEntity is a wrapper for a mongo.Client object that also holds additional information required during test @@ -292,6 +295,12 @@ func createAutoEncryptionOptions(opts bson.Raw) (*options.AutoEncryptionOptions, "accessKeyId": awsAccessKeyID, "secretAccessKey": awsSecretAccessKey, } + case "azure": + providers["azure"] = map[string]any{ + "tenantId": azureTenantID, + "clientId": azureClientID, + "clientSecret": azureClientSecret, + } case "local": _, key := providerOpt.Document().Lookup("key").Binary() providers["local"] = map[string]any{ diff --git a/internal/integration/unified/operation.go b/internal/integration/unified/operation.go index 1b591d66af..195c2a33fc 100644 --- a/internal/integration/unified/operation.go +++ b/internal/integration/unified/operation.go @@ -276,7 +276,7 @@ func (op *operation) run(ctx context.Context, loopDone <-chan struct{}) (*operat return executeDecrypt(ctx, op) // Unsupported operations - case "count", "listIndexNames": + case "count", "listIndexNames", "mapReduce": return nil, newSkipTestError(fmt.Sprintf("the %q operation is not supported", op.Name)) default: return nil, fmt.Errorf("unrecognized entity operation %q", op.Name) From 6bf84805c482cc0c1630a66d0b4191c3eb318f7b Mon Sep 17 00:00:00 2001 From: Qingyang Hu Date: Fri, 14 Nov 2025 21:55:30 -0500 Subject: [PATCH 3/3] updates --- internal/integration/unified/client_entity.go | 9 ++++++- .../integration/unified/collection_data.go | 24 +++++++++++-------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/internal/integration/unified/client_entity.go b/internal/integration/unified/client_entity.go index 27d6740f91..d9ea091672 100644 --- a/internal/integration/unified/client_entity.go +++ b/internal/integration/unified/client_entity.go @@ -8,6 +8,7 @@ package unified import ( "context" + "encoding/base64" "fmt" "os" "strings" @@ -302,7 +303,11 @@ func createAutoEncryptionOptions(opts bson.Raw) (*options.AutoEncryptionOptions, "clientSecret": azureClientSecret, } case "local": - _, key := providerOpt.Document().Lookup("key").Binary() + str := providerOpt.Document().Lookup("key").StringValue() + key, err := base64.StdEncoding.DecodeString(str) + if err != nil { + return nil, err + } providers["local"] = map[string]any{ "key": key, } @@ -321,6 +326,8 @@ func createAutoEncryptionOptions(opts bson.Raw) (*options.AutoEncryptionOptions, case "keyVaultNamespace": kvnsFound = true aeo.SetKeyVaultNamespace(opt.StringValue()) + case "bypassQueryAnalysis": + aeo.SetBypassQueryAnalysis(opt.Boolean()) default: return nil, fmt.Errorf("unrecognized option: %v", name) } diff --git a/internal/integration/unified/collection_data.go b/internal/integration/unified/collection_data.go index 02ceb3f147..3a62945e96 100644 --- a/internal/integration/unified/collection_data.go +++ b/internal/integration/unified/collection_data.go @@ -27,8 +27,9 @@ type collectionData struct { } type createOptions struct { - Capped *bool `bson:"capped"` - SizeInBytes *int64 `bson:"size"` + Capped *bool `bson:"capped"` + SizeInBytes *int64 `bson:"size"` + EncryptedFields any `bson:"encryptedFields"` } // createCollection configures the collection represented by the receiver using the internal client. This function @@ -49,14 +50,15 @@ func (c *collectionData) createCollection(ctx context.Context) error { if c.Options.SizeInBytes != nil { createOpts = createOpts.SetSizeInBytes(*c.Options.SizeInBytes) } + if c.Options.EncryptedFields != nil { + createOpts = createOpts.SetEncryptedFields(c.Options.EncryptedFields) + } if err := db.CreateCollection(ctx, c.CollectionName, createOpts); err != nil { return fmt.Errorf("error creating collection: %w", err) } - } - - // If neither documents nor options are provided, still create the collection with write concern "majority". - if len(c.Documents) == 0 && c.Options == nil { + } else { + // If options are provided, still create the collection with write concern "majority". // The write concern has to be manually specified in the command document because RunCommand does not honor // the database's write concern. create := bson.D{ @@ -68,13 +70,15 @@ func (c *collectionData) createCollection(ctx context.Context) error { if err := db.RunCommand(ctx, create).Err(); err != nil { return fmt.Errorf("error creating collection: %w", err) } - return nil } - docs := bsonutil.RawToInterfaces(c.Documents...) - if _, err := coll.InsertMany(ctx, docs); err != nil { - return fmt.Errorf("error inserting data: %w", err) + if len(c.Documents) != 0 { + docs := bsonutil.RawToInterfaces(c.Documents...) + if _, err := coll.InsertMany(ctx, docs); err != nil { + return fmt.Errorf("error inserting data: %w", err) + } } + return nil }