Enable binding of postgres CF db service to CF hosted console

.cfignore

@@ -3,3 +3,7 @@ node_modules/
 bower_components/
 dist/
 components/*/backend/vendor
+dev-certs/
+out/
+outputs/
+tmp/

.gitignore

@@ -40,3 +40,4 @@ deploy/development.rc
 deploy/ci/secrets.yml
 deploy/kubernetes/values.yaml
 outputs/
+deploy/cloud-foundry/db-migration/goose

components/app-core/backend/auth_test.go

@@ -49,7 +49,7 @@ func TestLoginToUAA(t *testing.T) {
 			WillReturnRows(expectNoRows())
 
 		mock.ExpectExec(insertIntoTokens).
-		// WithArgs(mockUserGUID, "uaa", mockTokenRecord.AuthToken, mockTokenRecord.RefreshToken, newExpiry).
+			// WithArgs(mockUserGUID, "uaa", mockTokenRecord.AuthToken, mockTokenRecord.RefreshToken, newExpiry).
 			WillReturnResult(sqlmock.NewResult(1, 1))
 
 		Convey("Should not fail to login", func() {
@@ -88,7 +88,6 @@ func TestLoginToUAAWithBadCreds(t *testing.T) {
 		pp.Config.ConsoleConfig.UAAEndpoint = uaaUrl
 		pp.Config.ConsoleConfig.SkipSSLValidation = true
 
-
 		err := pp.loginToUAA(ctx)
 		Convey("Login to UAA should fail", func() {
 			So(err, ShouldNotBeNil)
@@ -129,9 +128,8 @@ func TestLoginToUAAButCantSaveToken(t *testing.T) {
 		pp.Config.ConsoleConfig.UAAEndpoint = uaaUrl
 		pp.Config.ConsoleConfig.SkipSSLValidation = true
 
-
 		mock.ExpectQuery(selectAnyFromTokens).
-		// WithArgs(mockUserGUID).
+			// WithArgs(mockUserGUID).
 			WillReturnRows(sqlmock.NewRows([]string{"COUNT(*)"}).AddRow("0"))
 
 		// --- set up the database expectation for pp.saveUAAToken
@@ -207,7 +205,7 @@ func TestLoginToCNSI(t *testing.T) {
 		// Setup expectation that the CNSI token will get saved
 		//encryptedUAAToken, _ := tokens.EncryptToken(pp.Config.EncryptionKeyInBytes, mockUAAToken)
 		mock.ExpectExec(insertIntoTokens).
-		//WithArgs(mockCNSIGUID, mockUserGUID, "cnsi", encryptedUAAToken, encryptedUAAToken, sessionValues["exp"]).
+			//WithArgs(mockCNSIGUID, mockUserGUID, "cnsi", encryptedUAAToken, encryptedUAAToken, sessionValues["exp"]).
 			WillReturnResult(sqlmock.NewResult(1, 1))
 
 		// do the call
@@ -660,7 +658,7 @@ func TestVerifySessionExpired(t *testing.T) {
 
 		mock.ExpectQuery(selectAnyFromTokens).
 			WillReturnRows(sqlmock.NewRows([]string{"auth_token", "refresh_token", "token_expiry"}).
-			AddRow(mockUAAToken, mockUAAToken, sessionValues["exp"]))
+				AddRow(mockUAAToken, mockUAAToken, sessionValues["exp"]))
 		err := pp.verifySession(ctx)
 
 		Convey("Should fail to verify session", func() {

components/app-core/backend/cnsi.go

@@ -262,19 +262,31 @@ func (p *portalProxy) GetCNSIRecord(guid string) (interfaces.CNSIRecord, error)
 	return rec, nil
 }
 
-func (p *portalProxy) cnsiRecordExists(endpoint string) bool {
-	log.Debug("cnsiRecordExists")
+func (p *portalProxy) GetCNSIRecordByEndpoint(endpoint string) (interfaces.CNSIRecord, error) {
+	log.Debug("GetCNSIRecordByEndpoint")
+	var rec interfaces.CNSIRecord
+
 	cnsiRepo, err := cnsis.NewPostgresCNSIRepository(p.DatabaseConnectionPool)
 	if err != nil {
-		return false
+		return rec, err
 	}
 
-	_, err = cnsiRepo.FindByAPIEndpoint(endpoint)
+	rec, err = cnsiRepo.FindByAPIEndpoint(endpoint)
 	if err != nil {
-		return false
+		return rec, err
 	}
 
-	return true
+	// Ensure that trailing slash is removed from the API Endpoint
+	rec.APIEndpoint.Path = strings.TrimRight(rec.APIEndpoint.Path, "/")
+
+	return rec, nil
+}
+
+func (p *portalProxy) cnsiRecordExists(endpoint string) bool {
+	log.Debug("cnsiRecordExists")
+
+	_, err := p.GetCNSIRecordByEndpoint(endpoint);
+	return err == nil
 }
 
 func (p *portalProxy) setCNSIRecord(guid string, c interfaces.CNSIRecord) error {

components/app-core/backend/datastore/database_cf_config.go

@@ -0,0 +1,68 @@
+package datastore
+
+import (
+	"encoding/json"
+	"github.com/SUSE/stratos-ui/components/app-core/backend/config"
+	log "github.com/Sirupsen/logrus"
+	"strconv"
+	"strings"
+)
+
+const (
+	SERVICES_ENV = "VCAP_SERVICES"
+)
+
+type VCAPService struct {
+	Credentials VCAPCredential `json:"credentials"`
+	Tags        []string       `json:"tags"`
+}
+
+type VCAPCredential struct {
+	Username string `json:"username"`
+	Password string `json:"password"`
+	Dbname   string `json:"dbname"`
+	Hostname string `json:"hostname"`
+	Port     string `json:"port"`
+	Uri      string `json:"uri"`
+}
+
+// Discover cf db services via their 'uri' env var and apply settings to the DatabaseConfig objects
+func ParseCFEnvs(db *DatabaseConfig) bool {
+	if config.IsSet(SERVICES_ENV) == false {
+		return false
+	}
+
+	// Extract struts from VCAP_SERVICES env
+	vcapServicesStr := config.GetString(SERVICES_ENV)
+	var vcapServices map[string][]VCAPService
+	err := json.Unmarshal([]byte(vcapServicesStr), &vcapServices)
+	if err != nil {
+		log.Warnf("Unable to convert %s env var into JSON", SERVICES_ENV)
+		return false
+	}
+
+	for _, services := range vcapServices {
+		if len(services) == 0 {
+			continue
+		}
+		service := services[0]
+
+		for _, tag := range service.Tags {
+			if strings.HasPrefix(tag, "stratos_postgresql") {
+				dbCredentials := service.Credentials
+				// At the moment we only handle Postgres
+				db.DatabaseProvider = "pgsql"
+				db.Username = dbCredentials.Username
+				db.Password = dbCredentials.Password
+				db.Database = dbCredentials.Dbname
+				db.Host = dbCredentials.Hostname
+				db.Port, err = strconv.Atoi(dbCredentials.Port)
+				db.SSLMode = "disable"
+				log.Info("Discovered Cloud Foundry postgres service and applied config")
+				return true
+			}
+		}
+	}
+
+	return false
+}

components/app-core/backend/main.go

@@ -312,7 +312,10 @@ func loadPortalConfig(pc interfaces.PortalConfig) (interfaces.PortalConfig, erro
 
 func loadDatabaseConfig(dc datastore.DatabaseConfig) (datastore.DatabaseConfig, error) {
 	log.Debug("loadDatabaseConfig")
-	if err := config.Load(&dc); err != nil {
+
+	if datastore.ParseCFEnvs(&dc) == true {
+		log.Info("Using Cloud Foundry DB service")
+	} else if err := config.Load(&dc); err != nil {
 		return dc, fmt.Errorf("Unable to load database configuration. %v", err)
 	}
 

components/app-core/backend/main_test.go

@@ -88,7 +88,6 @@ func TestLoadPortalConfig(t *testing.T) {
 		t.Error("Unable to get TLSAddress from config")
 	}
 
-
 	if result.CFClient != "portal-proxy" {
 		t.Error("Unable to get CFClient from config")
 	}

components/app-core/backend/mock_server_test.go

@@ -131,12 +131,11 @@ func setupPortalProxy(db *sql.DB) *portalProxy {
 	urlP, _ := url.Parse("https://login.52.38.188.107.nip.io:50450")
 	pc := interfaces.PortalConfig{
 		ConsoleConfig: &interfaces.ConsoleConfig{
-			ConsoleClient:        "console",
-			ConsoleClientSecret:  "",
-			UAAEndpoint:          urlP,
-			SkipSSLValidation:    true,
+			ConsoleClient:       "console",
+			ConsoleClientSecret: "",
+			UAAEndpoint:         urlP,
+			SkipSSLValidation:   true,
 			ConsoleAdminScope:   UAAAdminIdentifier,
-
 		},
 		SessionStoreSecret:   "hiddenraisinsohno!",
 		EncryptionKeyInBytes: mockEncryptionKey,
@@ -263,21 +262,21 @@ var mockUAAResponse = UAAResponse{
 }
 
 const (
-	mockAPIEndpoint = "https://api.127.0.0.1"
-	mockAuthEndpoint = "https://login.127.0.0.1"
-	mockTokenEndpoint = "https://uaa.127.0.0.1"
+	mockAPIEndpoint     = "https://api.127.0.0.1"
+	mockAuthEndpoint    = "https://login.127.0.0.1"
+	mockTokenEndpoint   = "https://uaa.127.0.0.1"
 	mockDopplerEndpoint = "https://doppler.127.0.0.1"
-	mockProxyVersion = 20161117141922
+	mockProxyVersion    = 20161117141922
 
 	stringCFType = "cf"
 	stringCEType = "hce"
 
 	selectAnyFromTokens = `SELECT .+ FROM tokens WHERE .+`
-	insertIntoTokens = `INSERT INTO tokens`
-	updateTokens = `UPDATE tokens`
-	selectAnyFromCNSIs = `SELECT (.+) FROM cnsis WHERE (.+)`
-	insertIntoCNSIs = `INSERT INTO cnsis`
-	getDbVersion = `SELECT version_id FROM goose_db_version WHERE is_applied = 't' ORDER BY id DESC LIMIT 1`
+	insertIntoTokens    = `INSERT INTO tokens`
+	updateTokens        = `UPDATE tokens`
+	selectAnyFromCNSIs  = `SELECT (.+) FROM cnsis WHERE (.+)`
+	insertIntoCNSIs     = `INSERT INTO cnsis`
+	getDbVersion        = `SELECT version_id FROM goose_db_version WHERE is_applied = 't' ORDER BY id DESC LIMIT 1`
 )
 
 var rowFieldsForCNSI = []string{"guid", "name", "cnsi_type", "api_endpoint", "auth_endpoint", "token_endpoint", "doppler_logging_endpoint", "skip_ssl_validation"}

components/app-core/backend/repository/interfaces/portal_proxy.go

@@ -28,6 +28,7 @@ type PortalProxy interface {
 	DoLoginToCNSI(c echo.Context, cnsiGUID string) (*LoginRes, error)
 	// Expose internal portal proxy records to extensions
 	GetCNSIRecord(guid string) (CNSIRecord, error)
+	GetCNSIRecordByEndpoint(endpoint string) (CNSIRecord, error)
 	GetCNSITokenRecord(cnsiGUID string, userGUID string) (TokenRecord, bool)
 	GetCNSIUser(cnsiGUID string, userGUID string) (*ConnectedUser, bool)
 	GetConfig() *PortalConfig

components/cloud-foundry-hosting/backend/main.go

@@ -155,15 +155,28 @@ func (ch *CFHosting) Init() error {
 			return fmt.Errorf("Failed to save console configuration due to %s", err)
 		}
 
-		// Auto-register the Cloud Foundry
-		cfCnsi, regErr := ch.portalProxy.DoRegisterEndpoint("Cloud Foundry", appData.API, true, cfEndpointSpec.Info)
-		if regErr != nil {
-			log.Fatal("Could not auto-register the Cloud Foundry endpoint", err)
-			ch.portalProxy.GetConfig().CloudFoundryInfo = &interfaces.CFInfo{
-				SpaceGUID: appData.SpaceID,
-				AppGUID:   appData.ApplicationID,
+		var cfCnsi interfaces.CNSIRecord
+
+		cfCnsi, err = ch.portalProxy.GetCNSIRecordByEndpoint(appData.API)
+		if err != nil {
+			return fmt.Errorf("Failed to discover if an endpoint for hosting cf exists due to %s", err)
+		} else if cfCnsi.CNSIType != "" {
+			log.Info("Found existing endpoint matching Cloud Foundry API. Will not auto-register or auto-connect")
+		} else {
+			log.Info("Auto-registering endpoint for Cloud Foundry API")
+			var regErr error
+			// Auto-register the Cloud Foundry
+			cfCnsi, regErr = ch.portalProxy.DoRegisterEndpoint("Cloud Foundry", appData.API, true, cfEndpointSpec.Info)
+			if regErr != nil {
+				log.Fatal("Could not auto-register the Cloud Foundry endpoint", err)
+				ch.portalProxy.GetConfig().CloudFoundryInfo = &interfaces.CFInfo{
+					SpaceGUID: appData.SpaceID,
+					AppGUID:   appData.ApplicationID,
+				}
+				return nil
 			}
-			return nil
+			// Add login hook to automatically connect to the Cloud Foundry when the user logs in
+			ch.portalProxy.GetConfig().LoginHook = ch.cfLoginHook
 		}
 
 		// Store the space and id of the ConsocfLoginHookle application - we can use these to prevent stop/delete in the front-end
@@ -173,9 +186,6 @@ func (ch *CFHosting) Init() error {
 			EndpointGUID: cfCnsi.GUID,
 		}
 
-		// Add login hook to automatically conneect to the Cloud Foundry when the user logs in
-		ch.portalProxy.GetConfig().LoginHook = ch.cfLoginHook
-
 		log.Info("All done for Cloud Foundry deployment")
 	}
 	return nil

deploy/cloud-foundry/README.md

@@ -140,9 +140,9 @@ applications:
 
 ### Enable Endpoints Dashboard to register additional Cloud Foundry endpoints
 
->**NOTE** This method is meant to demonstrate the capabilities of the console with multiple endpoints and is not meant for production environments
+>**NOTE** This feature, on it's own, is meant to demonstrate the capabilities of the console with multiple endpoints and is not meant for production environments.
 
-This method comes with two caveats.
+This method comes with two caveats. To remove these caveats see [here](#Associate-Cloud-Foundry-database-service).
 
 1. The console will lose stored data when a cf app instance is restarted
 2. Multiple instances of the app will contain multiple separate stored data instances. This will mean the user may connect to a different one with a different storage when revisiting the console.
@@ -162,3 +162,6 @@ applications:
   env:
     FORCE_ENDPOINT_DASHBOARD: true
 ```
+
+### Associate Cloud Foundry database service
+Follow instructions [here](db-migration/README.md).

deploy/cloud-foundry/db-migration/README.md

@@ -0,0 +1,35 @@
+# Associated a Cloud Foundry database service
+
+As mentioned in the standard cf push instructions [here]("../README.md") the console as deployed via cf push
+ does not contain any way to persist date over application restarts and db entries such as registered endpoints
+ and user tokens are lost. To resolve this a Cloud Foundry db service can be binded to the console. Run through 
+ the steps below to implement.
+
+> **NOTE** The console supports postgresql. Your Cloud Foundry deployment should contain a service for
+ the desired db tagged with 'postgresql'.
+
+1. Enable the endpoint dashboard
+    * This is not strictly required, but at the moment is the only motivator to follow the next steps
+    * Add the following to the manifest
+    ```
+    env:
+        FORCE_ENDPOINT_DASHBOARD: true
+    ```
+1. Create the console app and associated a postgres service instance
+    * Use the instructions [here]("../README.md")
+    * Log into the console and then navigate to the console application
+    * In the `Services` tab associate a new postgres service instance to the application
+    * Validate in the `Variables` tab that `VCAP_SERVICES` is populated with new postgres credentials
+    * Remember to update the manifest with the services section
+1. Set up the associated db instance. Run the following from the root of the console
+    ```
+    cf push -c "deploy/cloud-foundry/db-migration/db-migrate.sh" -u "process"
+    ```
+    > **NOTE** All subsequent pushes, restarts, restaging will use this migration command.
+    It's therefore very important to execute the next step in order for the console to start
+1. Restart the app via cf push
+    ```
+    cf push -c "null"
+    ```
+
+

deploy/cloud-foundry/db-migration/db-migrate.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -e
+
+echo "Attempting to migrating database"
+
+DB_MIGRATE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+DEPLOY_DIR=${DB_MIGRATE_DIR}/../../
+
+export STRATOS_TEMP=$(mktemp -d)
+
+export GOPATH=${DB_MIGRATE_DIR}/goose
+export GOBIN=$GOPATH/bin
+go get bitbucket.org/liamstask/goose/cmd/goose
+
+export STRATOS_DB_ENV="$STRATOS_TEMP/db.env"
+node ${DB_MIGRATE_DIR}/parse_db_environment.js $STRATOS_DB_ENV
+source $STRATOS_DB_ENV
+
+cd $DEPLOY_DIR
+case $DB_TYPE in
+"postgresql")
+    echo "Migrating postgresql instance on $DB_HOST"
+    $GOBIN/goose -env cf_postgres up
+    if [ $? -eq 0 ]; then
+        while
+            echo "Database successfully migrated. Please restart the application via 'cf push -c \"null\"'";
+            sleep 60
+        do
+            :
+        done
+    else
+        echo Database migration failed
+    fi
+    ;;
+*)
+    echo Unknown DB type '$DB_TYPE'
+    ;;
+esac