Backend changes to support long-running requests

deploy/all-in-one/config.all-in-one.properties

@@ -3,6 +3,7 @@ DATABASE_PROVIDER=sqlite
 HTTP_CONNECTION_TIMEOUT_IN_SECS=10
 HTTP_CLIENT_TIMEOUT_IN_SECS=30
 HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS=120
+HTTP_CLIENT_TIMEOUT_LONGRUNNING_IN_SECS=600
 CONSOLE_PROXY_TLS_ADDRESS=:443
 CF_ADMIN_ROLE=cloud_controller.admin
 CF_CLIENT=cf

deploy/ci/travis/config.properties

@@ -3,6 +3,7 @@ DATABASE_PROVIDER=sqlite
 HTTP_CONNECTION_TIMEOUT_IN_SECS=10
 HTTP_CLIENT_TIMEOUT_IN_SECS=30
 HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS=120
+HTTP_CLIENT_TIMEOUT_LONGRUNNING_IN_SECS=600
 SKIP_SSL_VALIDATION=true
 CONSOLE_PROXY_TLS_ADDRESS=:5443
 CONSOLE_CLIENT=console

deploy/cloud-foundry/config.properties

@@ -3,6 +3,7 @@ DATABASE_PROVIDER=sqlite
 HTTP_CONNECTION_TIMEOUT_IN_SECS=10
 HTTP_CLIENT_TIMEOUT_IN_SECS=30
 HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS=120
+HTTP_CLIENT_TIMEOUT_LONGRUNNING_IN_SECS=600
 SKIP_SSL_VALIDATION=true
 CONSOLE_PROXY_TLS_ADDRESS=:443
 CONSOLE_CLIENT=console

deploy/kubernetes/console/templates/deployment.yaml

@@ -123,6 +123,8 @@ spec:
           value: "30"
         - name: HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS
           value: "120"
+        - name: HTTP_CLIENT_TIMEOUT_LONGRUNNING_IN_SECS
+          value: "600"
         - name: SKIP_TLS_VERIFICATION
           value: "false"
         - name: CONSOLE_PROXY_TLS_ADDRESS

deploy/proxy.env

@@ -7,6 +7,7 @@ DB_PORT=3306
 HTTP_CONNECTION_TIMEOUT_IN_SECS=10
 HTTP_CLIENT_TIMEOUT_IN_SECS=30
 HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS=120
+HTTP_CLIENT_TIMEOUT_LONGRUNNING_IN_SECS=600
 SKIP_SSL_VALIDATION=true
 CONSOLE_PROXY_TLS_ADDRESS=:3003
 CONSOLE_CLIENT=console

deploy/stratos-ui-release/jobs/backend/templates/config.properties.erb

@@ -1,6 +1,7 @@
 HTTP_CONNECTION_TIMEOUT_IN_SECS=10
 HTTP_CLIENT_TIMEOUT_IN_SECS=30
 HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS=120
+HTTP_CLIENT_TIMEOUT_LONGRUNNING_IN_SECS=600
 SKIP_SSL_VALIDATION=<%= p('stratos_ui.backend.skip_ssl_validation') %>
 CONSOLE_PROXY_TLS_ADDRESS=<%= p('stratos_ui.backend.address') %>:<%= p('stratos_ui.backend.port') %>
 CF_CLIENT=cf

src/frontend/packages/store/src/actions/service-instances.actions.ts

@@ -110,6 +110,8 @@ export class CreateServiceInstance extends CFStartAction implements ICFAction {
     this.options.params = new URLSearchParams();
     this.options.params.set('accepts_incomplete', 'true');
     this.options.method = 'post';
+    this.options.headers = new Headers();
+    this.options.headers.set('x-cap-long-running', 'true');
     this.options.body = {
       name,
       space_guid: spaceGuid,

src/jetstream/default.config.properties

@@ -3,6 +3,7 @@ DATABASE_PROVIDER=sqlite
 HTTP_CONNECTION_TIMEOUT_IN_SECS=10
 HTTP_CLIENT_TIMEOUT_IN_SECS=30
 HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS=120
+HTTP_CLIENT_TIMEOUT_LONGRUNNING_IN_SECS=600
 SKIP_SSL_VALIDATION=true
 CONSOLE_PROXY_TLS_ADDRESS=:5443
 CONSOLE_CLIENT=console

src/jetstream/go.sum

@@ -222,6 +222,7 @@ github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
 github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
@@ -253,6 +254,7 @@ github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/technosophos/moniker v0.0.0-20180509230615-a5dbd03a2245/go.mod h1:O1c8HleITsZqzNZDjSNzirUGsMT0oGu9LhHKoJrqO+A=
 github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8=

src/jetstream/main.go

@@ -731,9 +731,20 @@ func (p *portalProxy) GetHttpClient(skipSSLValidation bool) http.Client {
 	return p.getHttpClient(skipSSLValidation, false)
 }
 
+// GetHttpClientForRequest returns an Http Client for the giving request
 func (p *portalProxy) GetHttpClientForRequest(req *http.Request, skipSSLValidation bool) http.Client {
 	isMutating := req.Method != "GET" && req.Method != "HEAD"
-	return p.getHttpClient(skipSSLValidation, isMutating)
+	client := p.getHttpClient(skipSSLValidation, isMutating)
+
+	// Is this is a long-running request, then use a different timeout
+	if req.Header.Get(longRunningTimeoutHeader) == "true" {
+		longRunningClient := http.Client{}
+		longRunningClient.Transport = client.Transport
+		longRunningClient.Timeout = time.Duration(p.GetConfig().HTTPClientTimeoutLongRunningInSecs) * time.Second
+		return longRunningClient
+	}
+
+	return client
 }
 
 func (p *portalProxy) getHttpClient(skipSSLValidation bool, mutating bool) http.Client {

src/jetstream/main_test.go

@@ -63,17 +63,18 @@ func TestLoadPortalConfig(t *testing.T) {
 	var pc interfaces.PortalConfig
 
 	result, err := loadPortalConfig(pc, env.NewVarSet(env.WithMapLookup(map[string]string{
-		"HTTP_CLIENT_TIMEOUT_IN_SECS":          "10",
-		"HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS": "35",
-		"SKIP_SSL_VALIDATION":                  "true",
-		"CONSOLE_PROXY_TLS_ADDRESS":            ":8080",
-		"CONSOLE_CLIENT":                       "portal-proxy",
-		"CONSOLE_CLIENT_SECRET":                "ohsosecret!",
-		"CF_CLIENT":                            "portal-proxy",
-		"CF_CLIENT_SECRET":                     "ohsosecret!",
-		"UAA_ENDPOINT":                         "https://login.cf.org.com:443",
-		"ALLOWED_ORIGINS":                      "https://localhost,https://127.0.0.1",
-		"SESSION_STORE_SECRET":                 "cookiesecret",
+		"HTTP_CLIENT_TIMEOUT_IN_SECS":             "10",
+		"HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS":    "35",
+		"HTTP_CLIENT_TIMEOUT_LONGRUNNING_IN_SECS": "123",
+		"SKIP_SSL_VALIDATION":                     "true",
+		"CONSOLE_PROXY_TLS_ADDRESS":               ":8080",
+		"CONSOLE_CLIENT":                          "portal-proxy",
+		"CONSOLE_CLIENT_SECRET":                   "ohsosecret!",
+		"CF_CLIENT":                               "portal-proxy",
+		"CF_CLIENT_SECRET":                        "ohsosecret!",
+		"UAA_ENDPOINT":                            "https://login.cf.org.com:443",
+		"ALLOWED_ORIGINS":                         "https://localhost,https://127.0.0.1",
+		"SESSION_STORE_SECRET":                    "cookiesecret",
 	})))
 
 	if err != nil {

src/jetstream/passthrough.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/labstack/echo"
 	log "github.com/sirupsen/logrus"
@@ -20,6 +21,12 @@ import (
 // API Host Prefix to replace if the custom header is supplied
 const apiPrefix = "api."
 
+const longRunningTimeoutHeader = "x-cap-long-running"
+
+// Timeout for long-running requests, after which we will return indicating request it still active
+// to prevent hitting the 2 minute browser timeout
+const longRunningRequestTimeout = 30
+
 type PassthroughErrorStatus struct {
 	StatusCode int    `json:"statusCode"`
 	Status     string `json:"status"`
@@ -215,6 +222,7 @@ func (p *portalProxy) ProxyRequest(c echo.Context, uri *url.URL) (map[string]*in
 	log.Debug("proxy")
 	cnsiList := strings.Split(c.Request().Header.Get("x-cap-cnsi-list"), ",")
 	shouldPassthrough := "true" == c.Request().Header.Get("x-cap-passthrough")
+	longRunning := "true" == c.Request().Header.Get(longRunningTimeoutHeader)
 
 	if err := p.validateCNSIList(cnsiList); err != nil {
 		return nil, echo.NewHTTPError(http.StatusBadRequest, err.Error())
@@ -240,13 +248,22 @@ func (p *portalProxy) ProxyRequest(c echo.Context, uri *url.URL) (map[string]*in
 		}
 	}
 
+	// Only support one endpoint for long running operation (due to way we do timeout with the response channel)
+	if longRunning {
+		if len(cnsiList) > 1 {
+			err := errors.New("Requested long-running proxy to multiple CNSIs. Only single CNSI is supported for long running passthrough")
+			return nil, echo.NewHTTPError(http.StatusBadRequest, err.Error())
+		}
+	}
+
 	// send the request to each CNSI
 	done := make(chan *interfaces.CNSIRequest)
 	for _, cnsi := range cnsiList {
 		cnsiRequest, buildErr := p.buildCNSIRequest(cnsi, portalUserGUID, req.Method, uri, body, header)
 		if buildErr != nil {
 			return nil, echo.NewHTTPError(http.StatusBadRequest, buildErr.Error())
 		}
+		cnsiRequest.LongRunning = longRunning
 		// Allow the host part of the API URL to be overridden
 		apiHost := c.Request().Header.Get("x-cap-api-host")
 		// Don't allow any '.' chars in the api name
@@ -265,15 +282,62 @@ func (p *portalProxy) ProxyRequest(c echo.Context, uri *url.URL) (map[string]*in
 		go p.doRequest(&cnsiRequest, done)
 	}
 
+	// Wait for all responses
 	responses := make(map[string]*interfaces.CNSIRequest)
-	for range cnsiList {
-		res := <-done
-		responses[res.GUID] = res
+
+	if !longRunning {
+		for range cnsiList {
+			res := <-done
+			responses[res.GUID] = res
+		}
+	} else {
+		// Long running has a timeout
+		for range cnsiList {
+			select {
+			case res := <-done:
+				responses[res.GUID] = res
+			case <-time.After(longRunningRequestTimeout * time.Second):
+				// For all those that have not completed, add a timeout response
+				for _, id := range cnsiList {
+					if _, ok := responses[id]; !ok {
+						// Did not get a response for the endpoint
+						responses[id] = &interfaces.CNSIRequest{
+							GUID:         id,
+							UserGUID:     portalUserGUID,
+							Method:       req.Method,
+							StatusCode:   http.StatusAccepted,
+							Status:       "Long Running Operation still active",
+							Response:     makeLongRunningTimeoutError(),
+							Error:        nil,
+							ResponseGUID: id,
+						}
+					}
+				}
+				break
+			}
+		}
 	}
 
 	return responses, nil
 }
 
+func makeLongRunningTimeoutError() []byte {
+	var errorStatus = &PassthroughErrorStatus{
+		StatusCode: http.StatusAccepted,
+		Status:     "Long Running Operation still active",
+	}
+	errorResponse := []byte(fmt.Sprint("{\"longRunningTimeout\": true}"))
+	passthroughError := &PassthroughError{}
+	passthroughError.Error = errorStatus
+	passthroughError.ErrorResponse = (*json.RawMessage)(&errorResponse)
+	res, e := json.Marshal(passthroughError)
+	if e != nil {
+		log.Errorf("makeLongRunningTimeoutError: could not marshal JSON: %+v", e)
+	}
+	return res
+}
+
+// TODO: This should be used by the function above
 func (p *portalProxy) DoProxyRequest(requests []interfaces.ProxyRequestInfo) (map[string]*interfaces.CNSIRequest, error) {
 	log.Debug("DoProxyRequest")
 
@@ -401,6 +465,11 @@ func (p *portalProxy) doRequest(cnsiRequest *interfaces.CNSIRequest, done chan<-
 	// Copy original headers through, except custom portal-proxy Headers
 	fwdCNSIStandardHeaders(cnsiRequest, req)
 
+	// If this is a long running request, add a header which we can use at request time to change the timeout
+	if cnsiRequest.LongRunning {
+		req.Header.Set(longRunningTimeoutHeader, "true")
+	}
+
 	// Find the auth provider for the auth type - default ot oauthflow
 	authHandler := p.GetAuthProvider(tokenRec.AuthType)
 	if authHandler.Handler != nil {

src/jetstream/repository/interfaces/structs.go

@@ -309,46 +309,48 @@ type CNSIRequest struct {
 	StatusCode  int         `json:"statusCode"`
 	Status      string      `json:"status"`
 	PassThrough bool        `json:"-"`
+	LongRunning bool        `json:"-"`
 
 	Response     []byte `json:"-"`
 	Error        error  `json:"-"`
 	ResponseGUID string `json:"-"`
 }
 
 type PortalConfig struct {
-	HTTPClientTimeoutInSecs         int64    `configName:"HTTP_CLIENT_TIMEOUT_IN_SECS"`
-	HTTPClientTimeoutMutatingInSecs int64    `configName:"HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS"`
-	HTTPConnectionTimeoutInSecs     int64    `configName:"HTTP_CONNECTION_TIMEOUT_IN_SECS"`
-	TLSAddress                      string   `configName:"CONSOLE_PROXY_TLS_ADDRESS"`
-	TLSCert                         string   `configName:"CONSOLE_PROXY_CERT"`
-	TLSCertKey                      string   `configName:"CONSOLE_PROXY_CERT_KEY"`
-	TLSCertPath                     string   `configName:"CONSOLE_PROXY_CERT_PATH"`
-	TLSCertKeyPath                  string   `configName:"CONSOLE_PROXY_CERT_KEY_PATH"`
-	CFClient                        string   `configName:"CF_CLIENT"`
-	CFClientSecret                  string   `configName:"CF_CLIENT_SECRET"`
-	AllowedOrigins                  []string `configName:"ALLOWED_ORIGINS"`
-	SessionStoreSecret              string   `configName:"SESSION_STORE_SECRET"`
-	EncryptionKeyVolume             string   `configName:"ENCRYPTION_KEY_VOLUME"`
-	EncryptionKeyFilename           string   `configName:"ENCRYPTION_KEY_FILENAME"`
-	EncryptionKey                   string   `configName:"ENCRYPTION_KEY"`
-	AutoRegisterCFUrl               string   `configName:"AUTO_REG_CF_URL"`
-	AutoRegisterCFName              string   `configName:"AUTO_REG_CF_NAME"`
-	SSOLogin                        bool     `configName:"SSO_LOGIN"`
-	SSOOptions                      string   `configName:"SSO_OPTIONS"`
-	SSOWhiteList                    string   `configName:"SSO_WHITELIST"`
-	AuthEndpointType                string   `configName:"AUTH_ENDPOINT_TYPE"`
-	CookieDomain                    string   `configName:"COOKIE_DOMAIN"`
-	LogLevel                        string   `configName:"LOG_LEVEL"`
-	CFAdminIdentifier               string
-	CloudFoundryInfo                *CFInfo
-	HTTPS                           bool
-	EncryptionKeyInBytes            []byte
-	ConsoleVersion                  string
-	IsCloudFoundry                  bool
-	LoginHooks                      []LoginHook
-	SessionStore                    SessionStorer
-	ConsoleConfig                   *ConsoleConfig
-	PluginConfig                    map[string]string
-	DatabaseProviderName            string
-	EnableTechPreview               bool `configName:"ENABLE_TECH_PREVIEW"`
+	HTTPClientTimeoutInSecs            int64    `configName:"HTTP_CLIENT_TIMEOUT_IN_SECS"`
+	HTTPClientTimeoutMutatingInSecs    int64    `configName:"HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS"`
+	HTTPClientTimeoutLongRunningInSecs int64    `configName:"HTTP_CLIENT_TIMEOUT_LONGRUNNING_IN_SECS"`
+	HTTPConnectionTimeoutInSecs        int64    `configName:"HTTP_CONNECTION_TIMEOUT_IN_SECS"`
+	TLSAddress                         string   `configName:"CONSOLE_PROXY_TLS_ADDRESS"`
+	TLSCert                            string   `configName:"CONSOLE_PROXY_CERT"`
+	TLSCertKey                         string   `configName:"CONSOLE_PROXY_CERT_KEY"`
+	TLSCertPath                        string   `configName:"CONSOLE_PROXY_CERT_PATH"`
+	TLSCertKeyPath                     string   `configName:"CONSOLE_PROXY_CERT_KEY_PATH"`
+	CFClient                           string   `configName:"CF_CLIENT"`
+	CFClientSecret                     string   `configName:"CF_CLIENT_SECRET"`
+	AllowedOrigins                     []string `configName:"ALLOWED_ORIGINS"`
+	SessionStoreSecret                 string   `configName:"SESSION_STORE_SECRET"`
+	EncryptionKeyVolume                string   `configName:"ENCRYPTION_KEY_VOLUME"`
+	EncryptionKeyFilename              string   `configName:"ENCRYPTION_KEY_FILENAME"`
+	EncryptionKey                      string   `configName:"ENCRYPTION_KEY"`
+	AutoRegisterCFUrl                  string   `configName:"AUTO_REG_CF_URL"`
+	AutoRegisterCFName                 string   `configName:"AUTO_REG_CF_NAME"`
+	SSOLogin                           bool     `configName:"SSO_LOGIN"`
+	SSOOptions                         string   `configName:"SSO_OPTIONS"`
+	SSOWhiteList                       string   `configName:"SSO_WHITELIST"`
+	AuthEndpointType                   string   `configName:"AUTH_ENDPOINT_TYPE"`
+	CookieDomain                       string   `configName:"COOKIE_DOMAIN"`
+	LogLevel                           string   `configName:"LOG_LEVEL"`
+	CFAdminIdentifier                  string
+	CloudFoundryInfo                   *CFInfo
+	HTTPS                              bool
+	EncryptionKeyInBytes               []byte
+	ConsoleVersion                     string
+	IsCloudFoundry                     bool
+	LoginHooks                         []LoginHook
+	SessionStore                       SessionStorer
+	ConsoleConfig                      *ConsoleConfig
+	PluginConfig                       map[string]string
+	DatabaseProviderName               string
+	EnableTechPreview                  bool `configName:"ENABLE_TECH_PREVIEW"`
 }