From 34fd7cc7f9396df6338cb1a30f562c7d9ad037cb Mon Sep 17 00:00:00 2001 From: Neil MacDougall Date: Thu, 20 Feb 2020 15:51:56 +0000 Subject: [PATCH 1/5] Add support for TLS with MySQL --- .../console/templates/deployment.yaml | 2 + deploy/kubernetes/console/values.yaml | 5 +- .../datastore/20190522121200_LocalUsers.go | 62 +++---------------- .../20190930092500_LocalUsersTriggerFix.go | 5 +- src/jetstream/datastore/datastore.go | 31 +++++----- src/jetstream/go.mod | 2 +- src/jetstream/go.sum | 4 ++ 7 files changed, 37 insertions(+), 74 deletions(-) diff --git a/deploy/kubernetes/console/templates/deployment.yaml b/deploy/kubernetes/console/templates/deployment.yaml index c4632e97ed..abe58b20f0 100644 --- a/deploy/kubernetes/console/templates/deployment.yaml +++ b/deploy/kubernetes/console/templates/deployment.yaml @@ -96,6 +96,8 @@ spec: value: {{ default "3306" .Values.mariadb.port | quote }} - name: DATABASE_PROVIDER value: {{ default "mysql" .Values.mariadb.type | quote }} + - name: DB_SSL_MODE + value: {{ default "false" .Values.mariadb.tls | quote }} - name: HTTP_CONNECTION_TIMEOUT_IN_SECS value: "10" - name: HTTP_CLIENT_TIMEOUT_IN_SECS diff --git a/deploy/kubernetes/console/values.yaml b/deploy/kubernetes/console/values.yaml index ef1036bce1..6fcbcf0c73 100644 --- a/deploy/kubernetes/console/values.yaml +++ b/deploy/kubernetes/console/values.yaml @@ -120,13 +120,12 @@ mariadb: userPassword: # Leave password blank to auto-generate (not needed for external database) rootPassword: - # DB Host (when using an external database) host: - # Override port (default 3306) (when using an external database) port: - + # TLS Mode - can be true, false, skip-verify or preferred + tls: # Override DB type - default is mysql type: diff --git a/src/jetstream/datastore/20190522121200_LocalUsers.go b/src/jetstream/datastore/20190522121200_LocalUsers.go index 27b34e56b0..e2e964c92f 100644 --- a/src/jetstream/datastore/20190522121200_LocalUsers.go +++ b/src/jetstream/datastore/20190522121200_LocalUsers.go @@ -7,6 +7,11 @@ import ( "bitbucket.org/liamstask/goose/lib/goose" ) +// NOTE: This migration script has been modified +// We originally had a triiger, which is removed in a later migration script +// This requires a certain level of privilege to create, so it has been removed in this scipt for new installs. +// Upgrades will still remove the trigger if it exists + func init() { RegisterMigration(20190522121200, "LocalUsers", func(txn *sql.Tx, conf *goose.DBConf) error { binaryDataType := "BYTEA" @@ -31,34 +36,10 @@ func init() { createLocalUsers += "last_updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, " createLocalUsers += "PRIMARY KEY (user_guid) )" - //Trigger to update last_updated timestamp - createUpdateModifiedTrigger := "CREATE TRIGGER update_last_updated " - createUpdateModifiedTrigger += "AFTER UPDATE ON local_users " - createUpdateModifiedTrigger += "BEGIN UPDATE local_users SET last_updated = DATETIME('now') WHERE _rowid_ = new._rowid_; " - createUpdateModifiedTrigger += "END;" - - //Configure Postgres migration options + //Configure Postgres migration options if strings.Contains(conf.Driver.Name, "postgres") { createLocalUsers += " WITH (OIDS=FALSE);" - - //Postgres requires a trigger function - //Create trigger function and generate trigger statement - postgresTrigger, err := setupPostgresTrigger(txn) - if err != nil { - return err - } - createUpdateModifiedTrigger = postgresTrigger - - } else if strings.Contains(conf.Driver.Name, "mysql") { - // MySQL - createUpdateModifiedTrigger = "CREATE TRIGGER update_last_updated " - createUpdateModifiedTrigger += "AFTER UPDATE ON local_users " - createUpdateModifiedTrigger += "FOR EACH ROW BEGIN " - createUpdateModifiedTrigger += "UPDATE local_users SET last_updated = NOW() WHERE user_guid = NEW.user_guid; " - createUpdateModifiedTrigger += "END;" - - createLocalUsers += ";" - } else { + } else { createLocalUsers += ";" } @@ -67,11 +48,6 @@ func init() { return err } - _, err = txn.Exec(createUpdateModifiedTrigger) - if err != nil { - return err - } - createIndex := "CREATE INDEX local_users_user_guid ON local_users (user_guid);" _, err = txn.Exec(createIndex) if err != nil { @@ -86,27 +62,3 @@ func init() { return nil }) } - -func setupPostgresTrigger(txn *sql.Tx) (string, error) { - - createPostgresUpdateModifiedTrigger := "CREATE TRIGGER update_trigger " - createPostgresUpdateModifiedTrigger += "AFTER UPDATE ON local_users FOR EACH ROW " - createPostgresUpdateModifiedTrigger += "EXECUTE PROCEDURE update_last_modified_time(); " - - postgresTriggerFunction := "CREATE OR REPLACE FUNCTION update_last_modified_time() " - postgresTriggerFunction += "RETURNS trigger AS " - postgresTriggerFunction += "$BODY$ " - postgresTriggerFunction += "BEGIN " - postgresTriggerFunction += "UPDATE local_users " - postgresTriggerFunction += "SET last_updated = CURRENT_TIMESTAMP WHERE new.user_guid = old.user_guid; " - postgresTriggerFunction += "RETURN NEW; END; $BODY$ " - postgresTriggerFunction += "LANGUAGE plpgsql VOLATILE COST 100;" - - _, err := txn.Exec(postgresTriggerFunction) - if err != nil { - return "", err - } - - return createPostgresUpdateModifiedTrigger, err -} - diff --git a/src/jetstream/datastore/20190930092500_LocalUsersTriggerFix.go b/src/jetstream/datastore/20190930092500_LocalUsersTriggerFix.go index e1d0c209e9..6903ee015b 100644 --- a/src/jetstream/datastore/20190930092500_LocalUsersTriggerFix.go +++ b/src/jetstream/datastore/20190930092500_LocalUsersTriggerFix.go @@ -21,7 +21,10 @@ func init() { dropTrigger = "DROP TRIGGER update_trigger ON local_users;" } else if strings.Contains(conf.Driver.Name, "mysql") { // MYSQL - dropTrigger = "DROP TRIGGER update_last_updated;" + dropTrigger = "DROP TRIGGER IF EXISTS update_last_updated;" + // Ignore error - most likely permission bug issue on Mysql + txn.Exec(dropTrigger) + return nil } if len(dropTrigger) > 0 { diff --git a/src/jetstream/datastore/datastore.go b/src/jetstream/datastore/datastore.go index e4876e4b11..6905116ba4 100644 --- a/src/jetstream/datastore/datastore.go +++ b/src/jetstream/datastore/datastore.go @@ -54,6 +54,7 @@ type DatabaseConfig struct { type SSLValidationMode string const ( + // PGSQL SSL Modes // SSLDisabled means no checking of SSL SSLDisabled SSLValidationMode = "disable" // SSLRequired requires SSL without validation @@ -105,12 +106,15 @@ func NewDatabaseConnectionParametersFromConfig(dc DatabaseConfig) (DatabaseConfi if dc.SSLMode == string(SSLDisabled) || dc.SSLMode == string(SSLRequired) || dc.SSLMode == string(SSLVerifyCA) || dc.SSLMode == string(SSLVerifyFull) { return dc, nil - } else { - // Invalid SSL mode - return dc, fmt.Errorf("Invalid SSL mode: %s", dc.SSLMode) } + // Invalid SSL mode + return dc, fmt.Errorf("Invalid SSL mode: %s", dc.SSLMode) } else if dc.DatabaseProvider == MYSQL { - return dc, nil + if dc.SSLMode == "true" || dc.SSLMode == "false" || dc.SSLMode == "skip-verify" || dc.SSLMode == "preferred" { + return dc, nil + } + // Invalid SSL mode + return dc, fmt.Errorf("Invalid SSL mode: %s", dc.SSLMode) } return dc, fmt.Errorf("Invalid provider %v", dc) } @@ -268,25 +272,23 @@ func buildConnectionString(dc DatabaseConfig) string { } func buildConnectionStringForMysql(dc DatabaseConfig) string { - log.Debug("buildConnectionString") + log.Debug("buildConnectionStringForMysql") escapeStr := func(in string) string { return strings.Replace(in, `'`, `\'`, -1) } - connStr := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", + connStr := fmt.Sprintf("%s:%%s@tcp(%s:%d)/%s?parseTime=true", escapeStr(dc.Username), - escapeStr(dc.Password), dc.Host, dc.Port, escapeStr(dc.Database)) - log.Printf("DB Connection string: %s:*********@tcp(%s:%d)/%s?parseTime=true", - escapeStr(dc.Username), - dc.Host, - dc.Port, - escapeStr(dc.Database)) - - return connStr + if len(dc.SSLMode) > 0 { + log.Infof("Setting SSL Mode for mysql: %s", dc.SSLMode) + connStr = fmt.Sprintf("%s&tls=%s", connStr, dc.SSLMode) + } + log.Infof(connStr, "*********") + return fmt.Sprintf(connStr, escapeStr(dc.Password)) } // Ping - ping the database to ensure the connection/pool works. @@ -294,6 +296,7 @@ func Ping(db *sql.DB) error { log.Debug("Ping database") err := db.Ping() if err != nil { + log.Warn(db.Ping().Error()) return fmt.Errorf("Unable to ping the database: %+v", err) } diff --git a/src/jetstream/go.mod b/src/jetstream/go.mod index 570329df91..06b7801e69 100644 --- a/src/jetstream/go.mod +++ b/src/jetstream/go.mod @@ -31,7 +31,7 @@ require ( github.com/dsnet/compress v0.0.0-20171208185109-cc9eb1d7ad76 // indirect github.com/elazarl/goproxy/ext v0.0.0-20191011121108-aa519ddbe484 // indirect github.com/fatih/color v1.7.0 // indirect - github.com/go-sql-driver/mysql v1.4.1 + github.com/go-sql-driver/mysql v1.5.0 github.com/google/go-querystring v1.0.0 // indirect github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e // indirect github.com/gorilla/context v1.1.1 diff --git a/src/jetstream/go.sum b/src/jetstream/go.sum index 9e9247c616..c060d65455 100644 --- a/src/jetstream/go.sum +++ b/src/jetstream/go.sum @@ -260,6 +260,8 @@ github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+ github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= @@ -775,6 +777,8 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= helm.sh/helm/v3 v3.0.0 h1:or/9cs1GgfcTQeEnR2CVJNw893/rmqIG1KsNHmUiSFw= From 9ac57dfdc2095ce28d4e033a3ec5867685256a1c Mon Sep 17 00:00:00 2001 From: Neil MacDougall Date: Thu, 20 Feb 2020 17:09:27 +0000 Subject: [PATCH 2/5] Fix for other dbs --- .../datastore/20190930092500_LocalUsersTriggerFix.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jetstream/datastore/20190930092500_LocalUsersTriggerFix.go b/src/jetstream/datastore/20190930092500_LocalUsersTriggerFix.go index 6903ee015b..d8c2337a17 100644 --- a/src/jetstream/datastore/20190930092500_LocalUsersTriggerFix.go +++ b/src/jetstream/datastore/20190930092500_LocalUsersTriggerFix.go @@ -14,11 +14,11 @@ func init() { if strings.Contains(conf.Driver.Name, "sqlite") { //SQLITE - dropTrigger = "DROP TRIGGER update_last_updated;" + dropTrigger = "DROP TRIGGER IF EXISTS update_last_updated;" } if strings.Contains(conf.Driver.Name, "postgres") { // POSTGRESQL - dropTrigger = "DROP TRIGGER update_trigger ON local_users;" + dropTrigger = "DROP TRIGGER IF EXISTS update_trigger ON local_users;" } else if strings.Contains(conf.Driver.Name, "mysql") { // MYSQL dropTrigger = "DROP TRIGGER IF EXISTS update_last_updated;" From 94ec502adb7ae9dc2a204a15c9b8773daffe7a3e Mon Sep 17 00:00:00 2001 From: Neil MacDougall Date: Thu, 20 Feb 2020 22:33:16 +0000 Subject: [PATCH 3/5] Backend unit test fix --- src/jetstream/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jetstream/main_test.go b/src/jetstream/main_test.go index 34e989f2af..1d0ff4e15f 100644 --- a/src/jetstream/main_test.go +++ b/src/jetstream/main_test.go @@ -130,7 +130,7 @@ func TestLoadDatabaseConfig(t *testing.T) { "DB_HOST": "localhost", "DB_PORT": "5432", "DB_CONNECT_TIMEOUT_IN_SECS": "5", - "DB_SSL_MODE": "disable", + "DB_SSL_MODE": "false", }))) if err != nil { From 657ee827755833f70f59ef23c1e6c14fca2d910d Mon Sep 17 00:00:00 2001 From: Neil MacDougall Date: Fri, 21 Feb 2020 08:12:05 +0000 Subject: [PATCH 4/5] Remove debug log stmt --- src/jetstream/datastore/datastore.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/jetstream/datastore/datastore.go b/src/jetstream/datastore/datastore.go index 6905116ba4..d0561b0fe9 100644 --- a/src/jetstream/datastore/datastore.go +++ b/src/jetstream/datastore/datastore.go @@ -296,7 +296,6 @@ func Ping(db *sql.DB) error { log.Debug("Ping database") err := db.Ping() if err != nil { - log.Warn(db.Ping().Error()) return fmt.Errorf("Unable to ping the database: %+v", err) } From d1148ab7d067bdca8dbfa1d4a685ca395c1a212e Mon Sep 17 00:00:00 2001 From: Neil MacDougall Date: Fri, 21 Feb 2020 08:13:12 +0000 Subject: [PATCH 5/5] Remove duplicate yaml --- src/jetstream/go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/jetstream/go.sum b/src/jetstream/go.sum index f91e3e009f..1cab656957 100644 --- a/src/jetstream/go.sum +++ b/src/jetstream/go.sum @@ -768,8 +768,6 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= helm.sh/helm/v3 v3.0.0 h1:or/9cs1GgfcTQeEnR2CVJNw893/rmqIG1KsNHmUiSFw=