Start Always Encrypted feature branch

.github/workflows/pr-validation.yml

@@ -10,8 +10,8 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: ['1.16','1.17', '1.18']
-        sqlImage: ['2017-latest','2019-latest']
+        go: ['1.19','1.20']
+        sqlImage: ['2019-latest','2022-latest']
     steps:
     - uses: actions/checkout@v2
     - name: Setup go

.github/workflows/reviewdog.yml

@@ -6,9 +6,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: golangci-lint
-        uses: reviewdog/action-golangci-lint@v1
+        uses: reviewdog/action-golangci-lint@v2
         with:
           level: warning
           reporter: github-pr-review

.pipelines/TestSql2017.yml

@@ -8,7 +8,7 @@ variables:
 steps: 
 - task: GoTool@0
   inputs:
-    version: '1.16.5'
+    version: '1.19'
 - task: Go@0
   displayName: 'Go: get sources'
   inputs:

README.md

@@ -7,7 +7,7 @@
 
 ## Install
 
-Requires Go 1.10 or above.
+Requires Go 1.16 or above.
 
 Install with `go install github.com/microsoft/go-mssqldb@latest`.
 
@@ -63,6 +63,7 @@ Other supported formats are listed below.
 * `Workstation ID` - The workstation name (default is the host name)
 * `ApplicationIntent` - Can be given the value `ReadOnly` to initiate a read-only connection to an Availability Group listener. The `database` must be specified when connecting with `Application Intent` set to `ReadOnly`.
 * `protocol` - forces use of a protocol. Make sure the corresponding package is imported.
+* `columnencryption` or `column encryption setting` - a boolean value indicating whether Always Encrypted should be enabled on the connection.
 
 ### Connection parameters for namedpipe package
 * `pipe`  - If set, no Browser query is made and named pipe used will be `\\<host>\pipe\<pipe>`
@@ -377,8 +378,56 @@ db.QueryContext(ctx, `select * from t2 where user_name = @p1;`, mssql.VarChar(na
 // Note: Mismatched data types on table and parameter may cause long running queries
 ```
 
+## Using Always Encrypted
+
+The protocol and cryptography details for AE are [detailed elsewhere](https://learn.microsoft.com/sql/relational-databases/security/encryption/always-encrypted-database-engine?view=sql-server-ver16).
+
+### Enablement
+
+To enable AE on a connection, set the `ColumnEncryption` value to true on a config or pass `columnencryption=true` in the connection string.
+
+Decryption and encryption won't succeed, however, without also including a decryption key provider. To avoid code size impacts on non-AE applications, key providers are not included by default.
+
+Include the local certificate providers:
+
+```go
+ import (
+  "github.com/microsoft/go-mssqldb/aecmk/localcert"
+ )
+ ```
+
+You can also instantiate a key provider directly in code and hand it to a `Connector` instance.
+
+```go
+c := mssql.NewConnectorConfig(myconfig)
+c.RegisterCekProvider(providerName, MyProviderType{})
+```
+
+### Decryption
+
+If the correct key provider is included in your application, decryption of encrypted cells happens automatically with no extra server round trips.
+
+### Encryption
+
+Encryption of parameters passed to `Exec` and `Query` variants requires an extra round trip per query to fetch the encryption metadata. If the error returned by a query attempt indicates a type mismatch between the parameter and the destination table, most likely your input type is not a strict match for the SQL Server data type of the destination. You may be using a Go `string` when you need to use one of the driver-specific aliases like `VarChar` or `NVarCharMax`.
+
+*** NOTE *** - Currently `char` and `varchar` types do not include a collation parameter component so can't be used for inserting encrypted values. Also, using a nullable sql package type like `sql.NullableInt32` to pass a `NULL` value for an encrypted column will not work unless the encrypted column type is `nvarchar`. 
+https://github.com/microsoft/go-mssqldb/issues/129
+https://github.com/microsoft/go-mssqldb/issues/130
+
+
+### Local certificate AE key provider
+
+Key provider configuration is managed separately without any properties in the connection string.
+The `pfx` provider exposes its instance as the variable `PfxKeyProvider`. You can give it passwords for certificates using `SetCertificatePassword(pathToCertificate, path)`. Use an empty string or `"*"` as the path to use the same password for all certificates.
+
+The `MSSQL_CERTIFICATE_STORE` provider exposes its instance as the variable `WindowsCertificateStoreKeyProvider`.
+
+Both providers can be constrained to an allowed list of encryption key paths by appending paths to `provider.AllowedLocations`.
+
 ## Important Notes
 
+
 * [LastInsertId](https://golang.org/pkg/database/sql/#Result.LastInsertId) should
     not be used with this driver (or SQL Server) due to how the TDS protocol
  works. Please use the [OUTPUT Clause](https://docs.microsoft.com/en-us/sql/t-sql/queries/output-clause-transact-sql)
@@ -409,7 +458,9 @@ db.QueryContext(ctx, `select * from t2 where user_name = @p1;`, mssql.VarChar(na
 * A `namedpipe` package to support connections using named pipes (np:) on Windows
 * A `sharedmemory` package to support connections using shared memory (lpc:) on Windows
 * Dedicated Administrator Connection (DAC) is supported using `admin` protocol
-
+* Always Encrypted
+  - `MSSQL_CERTIFICATE_STORE` provider on Windows
+  - `pfx` provider on Linux and Windows
 ## Tests
 
 `go test` is used for testing. A running instance of MSSQL server is required.
@@ -449,6 +500,7 @@ To fix SQL Server 2008 R2 issue, install SQL Server 2008 R2 Service Pack 2.
 To fix SQL Server 2008 issue, install Microsoft SQL Server 2008 Service Pack 3 and Cumulative update package 3 for SQL Server 2008 SP3.
 More information: <http://support.microsoft.com/kb/2653857>
 
+* Bulk copy does not yet support encrypting column values using Always Encrypted. Tracked in [#127](https://github.com/microsoft/go-mssqldb/issues/127)
 
 # Contributing
 This project is a fork of [https://github.com/denisenkom/go-mssqldb](https://github.com/denisenkom/go-mssqldb) and welcomes new and previous contributors. For more informaton on contributing to this project, please see [Contributing](./CONTRIBUTING.md).

aecmk/keyprovider.go

@@ -0,0 +1,112 @@
+package aecmk
+
+import (
+	"fmt"
+	"sync"
+	"time"
+)
+
+const (
+	CertificateStoreKeyProvider = "MSSQL_CERTIFICATE_STORE"
+	CspKeyProvider              = "MSSQL_CSP_PROVIDER"
+	CngKeyProvider              = "MSSQL_CNG_STORE"
+	AzureKeyVaultKeyProvider    = "AZURE_KEY_VAULT"
+	JavaKeyProvider             = "MSSQL_JAVA_KEYSTORE"
+	KeyEncryptionAlgorithm      = "RSA_OAEP"
+)
+
+// ColumnEncryptionKeyLifetime is the default lifetime of decrypted Column Encryption Keys in the global cache.
+// The default is 2 hours
+var ColumnEncryptionKeyLifetime time.Duration = 2 * time.Hour
+
+type cekCacheEntry struct {
+	Expiry time.Time
+	Key    []byte
+}
+
+type cekCache map[string]cekCacheEntry
+
+type CekProvider struct {
+	Provider      ColumnEncryptionKeyProvider
+	decryptedKeys cekCache
+	mutex         sync.Mutex
+}
+
+func NewCekProvider(provider ColumnEncryptionKeyProvider) *CekProvider {
+	return &CekProvider{Provider: provider, decryptedKeys: make(cekCache), mutex: sync.Mutex{}}
+}
+
+func (cp *CekProvider) GetDecryptedKey(keyPath string, encryptedBytes []byte) (decryptedKey []byte, err error) {
+	cp.mutex.Lock()
+	ev, cachedKey := cp.decryptedKeys[keyPath]
+	if cachedKey {
+		if ev.Expiry.Before(time.Now()) {
+			delete(cp.decryptedKeys, keyPath)
+			cachedKey = false
+		} else {
+			decryptedKey = ev.Key
+		}
+	}
+	// decrypting a key can take a while, so let multiple callers race
+	// Key providers can choose to optimize their own concurrency.
+	// For example - there's probably minimal value in serializing access to a local certificate,
+	// but there'd be high value in having a queue of waiters for decrypting a key stored in the cloud.
+	cp.mutex.Unlock()
+	if !cachedKey {
+		decryptedKey = cp.Provider.DecryptColumnEncryptionKey(keyPath, KeyEncryptionAlgorithm, encryptedBytes)
+	}
+	if !cachedKey {
+		duration := cp.Provider.KeyLifetime()
+		if duration == nil {
+			duration = &ColumnEncryptionKeyLifetime
+		}
+		expiry := time.Now().Add(*duration)
+		cp.mutex.Lock()
+		cp.decryptedKeys[keyPath] = cekCacheEntry{Expiry: expiry, Key: decryptedKey}
+		cp.mutex.Unlock()
+	}
+	return
+}
+
+// no synchronization on this map. Providers register during init.
+type ColumnEncryptionKeyProviderMap map[string]*CekProvider
+
+var globalCekProviderFactoryMap = ColumnEncryptionKeyProviderMap{}
+
+// ColumnEncryptionKeyProvider is the interface for decrypting and encrypting column encryption keys.
+// It is similar to .Net https://learn.microsoft.com/dotnet/api/microsoft.data.sqlclient.sqlcolumnencryptionkeystoreprovider.
+type ColumnEncryptionKeyProvider interface {
+	// DecryptColumnEncryptionKey decrypts the specified encrypted value of a column encryption key.
+	// The encrypted value is expected to be encrypted using the column master key with the specified key path and using the specified algorithm.
+	DecryptColumnEncryptionKey(masterKeyPath string, encryptionAlgorithm string, encryptedCek []byte) []byte
+	// EncryptColumnEncryptionKey encrypts a column encryption key using the column master key with the specified key path and using the specified algorithm.
+	EncryptColumnEncryptionKey(masterKeyPath string, encryptionAlgorithm string, cek []byte) []byte
+	// SignColumnMasterKeyMetadata digitally signs the column master key metadata with the column master key
+	// referenced by the masterKeyPath parameter. The input values used to generate the signature should be the
+	// specified values of the masterKeyPath and allowEnclaveComputations parameters. May return an empty slice if not supported.
+	SignColumnMasterKeyMetadata(masterKeyPath string, allowEnclaveComputations bool) []byte
+	// VerifyColumnMasterKeyMetadata verifies the specified signature is valid for the column master key
+	// with the specified key path and the specified enclave behavior. Return nil if not supported.
+	VerifyColumnMasterKeyMetadata(masterKeyPath string, allowEnclaveComputations bool) *bool
+	// KeyLifetime is an optional Duration. Keys fetched by this provider will be discarded after their lifetime expires.
+	// If it returns nil, the keys will expire based on the value of ColumnEncryptionKeyLifetime.
+	// If it returns zero, the keys will not be cached.
+	KeyLifetime() *time.Duration
+}
+
+func RegisterCekProvider(name string, provider ColumnEncryptionKeyProvider) error {
+	_, ok := globalCekProviderFactoryMap[name]
+	if ok {
+		return fmt.Errorf("CEK provider %s is already registered", name)
+	}
+	globalCekProviderFactoryMap[name] = &CekProvider{Provider: provider, decryptedKeys: cekCache{}, mutex: sync.Mutex{}}
+	return nil
+}
+
+func GetGlobalCekProviders() (providers ColumnEncryptionKeyProviderMap) {
+	providers = make(ColumnEncryptionKeyProviderMap)
+	for i, p := range globalCekProviderFactoryMap {
+		providers[i] = p
+	}
+	return
+}