GODRIVER-3540 Support Prefix/Suffix/Substring Indexes.

.evergreen/config.yml

@@ -26,11 +26,6 @@ timeout:
       args: [ls, -la]
 
 functions:
-  assume-test-secrets-ec2-role: 
-    - command: ec2.assume_role
-      params:
-        role_arn: ${aws_test_secrets_role}
-
   setup-system:
     # Executes clone and applies the submitted patch, if any
     - command: git.get_project
@@ -598,6 +593,22 @@ functions:
           KMS_FAILPOINT_SERVER_RUNNING: "true"
         args: [*task-runner, evg-test-retry-kms-requests]
 
+  run-client-side-encryption-test:
+    - command: subprocess.exec
+      params:
+        binary: "bash"
+        env: 
+          GO_BUILD_TAGS: cse
+        include_expansions_in_env: [AUTH, SSL, MONGODB_URI, TOPOLOGY, 
+          MONGO_GO_DRIVER_COMPRESSOR]
+        args: [*task-runner, setup-test]
+    - command: subprocess.exec
+      type: test
+      retry_on_failure: true
+      params:
+        binary: "bash"
+        args: [*task-runner, evg-test-client-side-encryption]
+
   run-fuzz-tests:
     - command: subprocess.exec
       type: test
@@ -1572,6 +1583,21 @@ tasks:
           AUTH: "noauth"
           SSL: "nossl"
 
+  - name: "test-client-side-encryption"
+    tags: ["client-side-encryption-test"]
+    commands:
+      - func: bootstrap-mongo-orchestration
+        vars:
+          TOPOLOGY: "server"
+          AUTH: "noauth"
+          SSL: "nossl"
+      - func: start-cse-servers
+      - func: run-client-side-encryption-test
+        vars:
+          TOPOLOGY: "server"
+          AUTH: "noauth"
+          SSL: "nossl"
+
   - name: "test-retry-kms-requests"
     tags: ["kms-test"]
     commands:
@@ -2253,6 +2279,12 @@ buildvariants:
     tasks:
       - name: ".kms-test"
 
+  - matrix_name: "client-side-encryption-test"
+    matrix_spec: { version: ["latest"], os-ssl-40: ["rhel87-64"] }
+    display_name: "Client Side Encryption Tests ${os-ssl-40}"
+    tasks:
+      - name: ".client-side-encryption-test"
+
   - matrix_name: "load-balancer-test"
     tags: ["pullrequest"]
     matrix_spec: { version: ["5.0", "6.0", "7.0", "8.0"], os-ssl-40: ["rhel87-64"] }

Taskfile.yml

@@ -150,6 +150,9 @@ tasks:
   evg-test-retry-kms-requests:
     - go test -exec "env PKG_CONFIG_PATH=${PKG_CONFIG_PATH} LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" ${BUILD_TAGS} -v -timeout {{.TEST_TIMEOUT}}s ./internal/integration -run TestClientSideEncryptionProse/kms_retry_tests >> test.suite
 
+  evg-test-client-side-encryption:
+    - go test -exec "env PKG_CONFIG_PATH=${PKG_CONFIG_PATH} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} DYLD_LIBRARY_PATH=${MACOS_LIBRARY_PATH}" ${BUILD_TAGS} -v -timeout {{.TEST_TIMEOUT}}s ./internal/integration -run TestClientSideEncryptionProse >> test.suite
+
   evg-test-load-balancers:
     # Load balancer should be tested with all unified tests as well as tests in the following
     # components: retryable reads, retryable writes, change streams, initial DNS seedlist discovery.

etc/install-libmongocrypt.sh

@@ -3,7 +3,7 @@
 # This script installs libmongocrypt into an "install" directory.
 set -eux
 
-LIBMONGOCRYPT_TAG="1.12.0"
+LIBMONGOCRYPT_TAG="1.15.1"
 
 # Install libmongocrypt based on OS.
 if [ "Windows_NT" = "${OS:-}" ]; then

internal/integration/client_side_encryption_prose_test.go

@@ -3144,6 +3144,127 @@ func TestClientSideEncryptionProse(t *testing.T) {
 			})
 		}
 	})
+
+	mt.RunOpts("27. text Explicit Encryption", qeRunOpts.MinServerVersion("8.2"), func(mt *mtest.T) {
+		encryptedFields := readJSONFile(mt, "encryptedFields-prefix-suffix.json")
+		key1Document := readJSONFile(mt, "key1-document.json")
+		subtype, data := key1Document.Lookup("_id").Binary()
+		key1ID := bson.Binary{Subtype: subtype, Data: data}
+
+		testSetup := func() (*mongo.Client, *mongo.ClientEncryption) {
+			for _, collName := range []string{"prefix-suffix", "substring"} {
+				mtest.DropEncryptedCollection(mt, mt.Client.Database("db").Collection(collName), encryptedFields)
+				cco := options.CreateCollection().SetEncryptedFields(encryptedFields)
+				err := mt.Client.Database("db").CreateCollection(context.Background(), collName, cco)
+				require.NoError(mt, err, "error on CreateCollection: %v", err)
+			}
+			err := mt.Client.Database("keyvault").Collection("datakeys").Drop(context.Background())
+			require.NoError(mt, err, "error on Drop: %v", err)
+			opts := options.Client().ApplyURI(mtest.ClusterURI())
+			integtest.AddTestServerAPIVersion(opts)
+			keyVaultClient, err := mongo.Connect(opts)
+			require.NoError(mt, err, "error on Connect: %v", err)
+			datakeysColl := keyVaultClient.Database("keyvault").Collection("datakeys", options.Collection().SetWriteConcern(mtest.MajorityWc))
+			_, err = datakeysColl.InsertOne(context.Background(), key1Document)
+			require.NoError(mt, err, "error on InsertOne: %v", err)
+			kmsProvidersMap := map[string]map[string]any{
+				"local": {"key": localMasterKey},
+			}
+			// Create a ClientEncryption.
+			ceo := options.ClientEncryption().
+				SetKeyVaultNamespace("keyvault.datakeys").
+				SetKmsProviders(kmsProvidersMap)
+			clientEncryption, err := mongo.NewClientEncryption(keyVaultClient, ceo)
+			require.NoError(mt, err, "error on NewClientEncryption: %v", err)
+
+			// Create a MongoClient with AutoEncryptionOpts and bypassQueryAnalysis=true.
+			aeo := options.AutoEncryption().
+				SetKeyVaultNamespace("keyvault.datakeys").
+				SetKmsProviders(kmsProvidersMap).
+				SetBypassQueryAnalysis(true)
+			co := options.Client().SetAutoEncryptionOptions(aeo).ApplyURI(mtest.ClusterURI())
+			integtest.AddTestServerAPIVersion(co)
+			encryptedClient, err := mongo.Connect(co)
+			require.NoError(mt, err, "error on Connect: %v", err)
+
+			foobarbaz := bson.RawValue{Type: bson.TypeString, Value: bsoncore.AppendString(nil, "foobarbaz")}
+			for _, c := range []struct {
+				collection string
+				textOpts   *options.TextOptionsBuilder
+			}{
+				{
+					collection: "prefix-suffix",
+					textOpts: options.Text().
+						SetPrefix(options.PrefixOptions{
+							StrMaxQueryLength: 10,
+							StrMinQueryLength: 2,
+						}).
+						SetSuffix(options.SuffixOptions{
+							StrMaxQueryLength: 10,
+							StrMinQueryLength: 2,
+						}).
+						SetCaseSensitive(true).
+						SetDiacriticSensitive(true),
+				},
+				{
+					collection: "substring",
+					textOpts: options.Text().
+						SetSubstring(options.SubstringOptions{
+							StrMaxLength:      10,
+							StrMaxQueryLength: 10,
+							StrMinQueryLength: 2,
+						}).
+						SetCaseSensitive(true).
+						SetDiacriticSensitive(true),
+				},
+			} {
+				coll := encryptedClient.Database("db").Collection(c.collection, options.Collection().SetWriteConcern(mtest.MajorityWc))
+				eo := options.Encrypt().
+					SetKeyID(key1ID).
+					SetAlgorithm("TextPreview").
+					SetContentionFactor(0).
+					SetTextOptions(c.textOpts)
+				insertPayload, err := clientEncryption.Encrypt(context.Background(), foobarbaz, eo)
+				require.NoError(mt, err, "error in Encrypt: %v", err)
+				_, err = coll.InsertOne(context.Background(), bson.D{{"_id", 0}, {"encryptedText", insertPayload}})
+				require.NoError(mt, err, "error in InsertOne: %v", err)
+			}
+
+			return encryptedClient, clientEncryption
+		}
+
+		mt.Run("Case 1: can decrypt a payload", func(mt *mtest.T) {
+			encryptedClient, clientEncryption := testSetup()
+			defer clientEncryption.Close(context.Background())
+			defer encryptedClient.Disconnect(context.Background())
+
+			foo := bson.RawValue{Type: bson.TypeString, Value: bsoncore.AppendString(nil, "foo")}
+			eo := options.Encrypt().
+				SetAlgorithm("TextPreview").
+				SetKeyID(key1ID).
+				SetContentionFactor(0).
+				SetTextOptions(options.Text().
+					SetPrefix(options.PrefixOptions{
+						StrMaxQueryLength: 10,
+						StrMinQueryLength: 2,
+					}).
+					SetCaseSensitive(true).
+					SetDiacriticSensitive(true))
+			payload, err := clientEncryption.Encrypt(context.Background(), foo, eo)
+			require.NoError(mt, err, "error in Encrypt: %v", err)
+			coll := encryptedClient.Database("db").Collection("prefix-suffix")
+			got, err := coll.FindOne(context.Background(), bson.D{
+				{"$expr", bson.D{
+					{"$encStrStartsWith", bson.D{
+						{"input", "$encryptedText"},
+						{"prefix", payload},
+					}},
+				}},
+			}).Raw()
+			require.NoError(mt, err, "error in FindOne: %v", err)
+			assert.FailNow(mt, "got: %v", got)
+		})
+	})
 }
 
 func getWatcher(mt *mtest.T, streamType mongo.StreamType, cpt *cseProseTest) watcher {

internal/integration/mtest/mongotest.go

@@ -797,12 +797,6 @@ func verifyRunOnBlockConstraint(rob RunOnBlock) error {
 		return err
 	}
 
-	// TODO(GODRIVER-3486): Once auto encryption is supported by the unified test
-	// format,this check should be removed.
-	if rob.CSFLEEnabled() && rob.CSFLE.Options != nil {
-		return fmt.Errorf("auto encryption required (GODRIVER-3486)")
-	}
-
 	if rob.CSFLEEnabled() && !IsCSFLEEnabled() {
 		return fmt.Errorf("runOnBlock requires CSFLE to be enabled. Build with the cse tag to enable")
 	} else if !rob.CSFLEEnabled() && IsCSFLEEnabled() {

internal/integration/unified/client_entity.go

@@ -8,6 +8,7 @@ package unified
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"strings"
 	"sync"
@@ -217,6 +218,13 @@ func newClientEntity(ctx context.Context, em *EntityMap, entityOptions *entityOp
 	} else {
 		integtest.AddTestServerAPIVersion(clientOpts)
 	}
+	if entityOptions.AutoEncryptOpts != nil {
+		aeo, err := createAutoEncryptionOptions(entityOptions.AutoEncryptOpts)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing auto encryption options: %w", err)
+		}
+		clientOpts.SetAutoEncryptionOptions(aeo)
+	}
 	for _, cmd := range entityOptions.IgnoredCommands {
 		entity.ignoredCommands[cmd] = struct{}{}
 	}
@@ -251,6 +259,82 @@ func getURIForClient(opts *entityOptions) string {
 	}
 }
 
+func createAutoEncryptionOptions(opts bson.Raw) (*options.AutoEncryptionOptions, error) {
+	aeo := options.AutoEncryption()
+	var kvnsFound bool
+	elems, err := opts.Elements()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, elem := range elems {
+		name := elem.Key()
+		opt := elem.Value()
+
+		switch name {
+		case "kmsProviders":
+			providers := make(map[string]map[string]any)
+			elems, err := opt.Document().Elements()
+			if err != nil {
+				return nil, err
+			}
+			for _, elem := range elems {
+				key := elem.Key()
+				opt := elem.Value().Document()
+				provider, err := getKmsProvider(key, opt)
+				if err != nil {
+					return nil, err
+				}
+				if provider == nil {
+					continue
+				}
+				providers[key] = provider
+				if key == "kmip" && tlsClientCertificateKeyFile != "" && tlsCAFile != "" {
+					cfg, err := options.BuildTLSConfig(map[string]any{
+						"tlsCertificateKeyFile": tlsClientCertificateKeyFile,
+						"tlsCAFile":             tlsCAFile,
+					})
+					if err != nil {
+						return nil, fmt.Errorf("error constructing tls config: %w", err)
+					}
+					aeo.SetTLSConfig(map[string]*tls.Config{
+						"kmip": cfg,
+					})
+				}
+			}
+			aeo.SetKmsProviders(providers)
+		case "schemaMap":
+			var schemaMap map[string]any
+			err := bson.Unmarshal(opt.Document(), &schemaMap)
+			if err != nil {
+				return nil, fmt.Errorf("error creating schema map: %v", err)
+			}
+			aeo.SetSchemaMap(schemaMap)
+		case "keyVaultNamespace":
+			kvnsFound = true
+			aeo.SetKeyVaultNamespace(opt.StringValue())
+		case "bypassAutoEncryption":
+			aeo.SetBypassAutoEncryption(opt.Boolean())
+		case "encryptedFieldsMap":
+			var encryptedFieldsMap map[string]any
+			err := bson.Unmarshal(opt.Document(), &encryptedFieldsMap)
+			if err != nil {
+				return nil, fmt.Errorf("error creating encryptedFieldsMap: %v", err)
+			}
+			aeo.SetEncryptedFieldsMap(encryptedFieldsMap)
+		case "bypassQueryAnalysis":
+			aeo.SetBypassQueryAnalysis(opt.Boolean())
+		default:
+			return nil, fmt.Errorf("unrecognized option: %v", name)
+		}
+	}
+	if !kvnsFound {
+		aeo.SetKeyVaultNamespace("keyvault.datakeys")
+	}
+
+	return aeo, nil
+}
+
 // disconnect disconnects the client associated with this entity. It is an
 // idempotent operation, unlike the mongo client's disconnect method. This
 // property will help avoid unnecessary errors when calling disconnect on a

internal/integration/unified/collection_data.go

@@ -27,8 +27,10 @@ type collectionData struct {
 }
 
 type createOptions struct {
-	Capped      *bool  `bson:"capped"`
-	SizeInBytes *int64 `bson:"size"`
+	Capped          *bool    `bson:"capped"`
+	SizeInBytes     *int64   `bson:"size"`
+	EncryptedFields bson.Raw `bson:"encryptedFields"`
+	Validator       bson.Raw `bson:"validator"`
 }
 
 // createCollection configures the collection represented by the receiver using the internal client. This function
@@ -49,14 +51,18 @@ func (c *collectionData) createCollection(ctx context.Context) error {
 		if c.Options.SizeInBytes != nil {
 			createOpts = createOpts.SetSizeInBytes(*c.Options.SizeInBytes)
 		}
+		if c.Options.EncryptedFields != nil {
+			createOpts = createOpts.SetEncryptedFields(c.Options.EncryptedFields)
+		}
+		if c.Options.Validator != nil {
+			createOpts = createOpts.SetValidator(c.Options.Validator)
+		}
 
 		if err := db.CreateCollection(ctx, c.CollectionName, createOpts); err != nil {
 			return fmt.Errorf("error creating collection: %w", err)
 		}
-	}
-
-	// If neither documents nor options are provided, still create the collection with write concern "majority".
-	if len(c.Documents) == 0 && c.Options == nil {
+	} else {
+		// If options are provided, still create the collection with write concern "majority".
 		// The write concern has to be manually specified in the command document because RunCommand does not honor
 		// the database's write concern.
 		create := bson.D{
@@ -68,13 +74,15 @@ func (c *collectionData) createCollection(ctx context.Context) error {
 		if err := db.RunCommand(ctx, create).Err(); err != nil {
 			return fmt.Errorf("error creating collection: %w", err)
 		}
-		return nil
 	}
 
-	docs := bsonutil.RawToInterfaces(c.Documents...)
-	if _, err := coll.InsertMany(ctx, docs); err != nil {
-		return fmt.Errorf("error inserting data: %w", err)
+	if len(c.Documents) != 0 {
+		docs := bsonutil.RawToInterfaces(c.Documents...)
+		if _, err := coll.InsertMany(ctx, docs); err != nil {
+			return fmt.Errorf("error inserting data: %w", err)
+		}
 	}
+
 	return nil
 }