From 707a547609c7b35d0b09837e7ca9de14e57d9022 Mon Sep 17 00:00:00 2001 From: Neil MacDougall Date: Wed, 4 Dec 2019 12:41:50 +0000 Subject: [PATCH] Backend changs to support long-running requests --- .../all-in-one/config.all-in-one.properties | 1 + deploy/ci/travis/config.properties | 1 + deploy/cloud-foundry/config.properties | 1 + .../console/templates/deployment.yaml | 2 + deploy/proxy.env | 1 + .../backend/templates/config.properties.erb | 1 + .../src/actions/service-instances.actions.ts | 2 + src/jetstream/default.config.properties | 1 + src/jetstream/go.sum | 2 + src/jetstream/main.go | 13 +++- src/jetstream/main_test.go | 23 +++--- src/jetstream/passthrough.go | 75 ++++++++++++++++++- .../repository/interfaces/structs.go | 72 +++++++++--------- 13 files changed, 145 insertions(+), 50 deletions(-) diff --git a/deploy/all-in-one/config.all-in-one.properties b/deploy/all-in-one/config.all-in-one.properties index 0befba7923..632f685de3 100644 --- a/deploy/all-in-one/config.all-in-one.properties +++ b/deploy/all-in-one/config.all-in-one.properties @@ -3,6 +3,7 @@ DATABASE_PROVIDER=sqlite HTTP_CONNECTION_TIMEOUT_IN_SECS=10 HTTP_CLIENT_TIMEOUT_IN_SECS=30 HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS=120 +HTTP_CLIENT_TIMEOUT_LONGRUNNING_IN_SECS=600 CONSOLE_PROXY_TLS_ADDRESS=:443 CF_ADMIN_ROLE=cloud_controller.admin CF_CLIENT=cf diff --git a/deploy/ci/travis/config.properties b/deploy/ci/travis/config.properties index 9bc67178ce..e7a11953c2 100644 --- a/deploy/ci/travis/config.properties +++ b/deploy/ci/travis/config.properties @@ -3,6 +3,7 @@ DATABASE_PROVIDER=sqlite HTTP_CONNECTION_TIMEOUT_IN_SECS=10 HTTP_CLIENT_TIMEOUT_IN_SECS=30 HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS=120 +HTTP_CLIENT_TIMEOUT_LONGRUNNING_IN_SECS=600 SKIP_SSL_VALIDATION=true CONSOLE_PROXY_TLS_ADDRESS=:5443 CONSOLE_CLIENT=console diff --git a/deploy/cloud-foundry/config.properties b/deploy/cloud-foundry/config.properties index c3c95ab593..a5cb9bd20a 100644 --- a/deploy/cloud-foundry/config.properties +++ b/deploy/cloud-foundry/config.properties @@ -3,6 +3,7 @@ DATABASE_PROVIDER=sqlite HTTP_CONNECTION_TIMEOUT_IN_SECS=10 HTTP_CLIENT_TIMEOUT_IN_SECS=30 HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS=120 +HTTP_CLIENT_TIMEOUT_LONGRUNNING_IN_SECS=600 SKIP_SSL_VALIDATION=true CONSOLE_PROXY_TLS_ADDRESS=:443 CONSOLE_CLIENT=console diff --git a/deploy/kubernetes/console/templates/deployment.yaml b/deploy/kubernetes/console/templates/deployment.yaml index 7ed9f7bb50..287fd253fc 100644 --- a/deploy/kubernetes/console/templates/deployment.yaml +++ b/deploy/kubernetes/console/templates/deployment.yaml @@ -123,6 +123,8 @@ spec: value: "30" - name: HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS value: "120" + - name: HTTP_CLIENT_TIMEOUT_LONGRUNNING_IN_SECS + value: "600" - name: SKIP_TLS_VERIFICATION value: "false" - name: CONSOLE_PROXY_TLS_ADDRESS diff --git a/deploy/proxy.env b/deploy/proxy.env index c02f412d9d..be65e004fb 100644 --- a/deploy/proxy.env +++ b/deploy/proxy.env @@ -7,6 +7,7 @@ DB_PORT=3306 HTTP_CONNECTION_TIMEOUT_IN_SECS=10 HTTP_CLIENT_TIMEOUT_IN_SECS=30 HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS=120 +HTTP_CLIENT_TIMEOUT_LONGRUNNING_IN_SECS=600 SKIP_SSL_VALIDATION=true CONSOLE_PROXY_TLS_ADDRESS=:3003 CONSOLE_CLIENT=console diff --git a/deploy/stratos-ui-release/jobs/backend/templates/config.properties.erb b/deploy/stratos-ui-release/jobs/backend/templates/config.properties.erb index 882edde652..1dd838a55f 100644 --- a/deploy/stratos-ui-release/jobs/backend/templates/config.properties.erb +++ b/deploy/stratos-ui-release/jobs/backend/templates/config.properties.erb @@ -1,6 +1,7 @@ HTTP_CONNECTION_TIMEOUT_IN_SECS=10 HTTP_CLIENT_TIMEOUT_IN_SECS=30 HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS=120 +HTTP_CLIENT_TIMEOUT_LONGRUNNING_IN_SECS=600 SKIP_SSL_VALIDATION=<%= p('stratos_ui.backend.skip_ssl_validation') %> CONSOLE_PROXY_TLS_ADDRESS=<%= p('stratos_ui.backend.address') %>:<%= p('stratos_ui.backend.port') %> CF_CLIENT=cf diff --git a/src/frontend/packages/store/src/actions/service-instances.actions.ts b/src/frontend/packages/store/src/actions/service-instances.actions.ts index 75d5fa16ec..80ab3b74fa 100644 --- a/src/frontend/packages/store/src/actions/service-instances.actions.ts +++ b/src/frontend/packages/store/src/actions/service-instances.actions.ts @@ -110,6 +110,8 @@ export class CreateServiceInstance extends CFStartAction implements ICFAction { this.options.params = new URLSearchParams(); this.options.params.set('accepts_incomplete', 'true'); this.options.method = 'post'; + this.options.headers = new Headers(); + this.options.headers.set('x-cap-long-running', 'true'); this.options.body = { name, space_guid: spaceGuid, diff --git a/src/jetstream/default.config.properties b/src/jetstream/default.config.properties index 8d6a68ddc0..f831caab2d 100644 --- a/src/jetstream/default.config.properties +++ b/src/jetstream/default.config.properties @@ -3,6 +3,7 @@ DATABASE_PROVIDER=sqlite HTTP_CONNECTION_TIMEOUT_IN_SECS=10 HTTP_CLIENT_TIMEOUT_IN_SECS=30 HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS=120 +HTTP_CLIENT_TIMEOUT_LONGRUNNING_IN_SECS=600 SKIP_SSL_VALIDATION=true CONSOLE_PROXY_TLS_ADDRESS=:5443 CONSOLE_CLIENT=console diff --git a/src/jetstream/go.sum b/src/jetstream/go.sum index 8f5f40696b..469a93e006 100644 --- a/src/jetstream/go.sum +++ b/src/jetstream/go.sum @@ -222,6 +222,7 @@ github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= @@ -253,6 +254,7 @@ github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/technosophos/moniker v0.0.0-20180509230615-a5dbd03a2245/go.mod h1:O1c8HleITsZqzNZDjSNzirUGsMT0oGu9LhHKoJrqO+A= github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8= diff --git a/src/jetstream/main.go b/src/jetstream/main.go index 5aa1128777..5c8321080d 100644 --- a/src/jetstream/main.go +++ b/src/jetstream/main.go @@ -731,9 +731,20 @@ func (p *portalProxy) GetHttpClient(skipSSLValidation bool) http.Client { return p.getHttpClient(skipSSLValidation, false) } +// GetHttpClientForRequest returns an Http Client for the giving request func (p *portalProxy) GetHttpClientForRequest(req *http.Request, skipSSLValidation bool) http.Client { isMutating := req.Method != "GET" && req.Method != "HEAD" - return p.getHttpClient(skipSSLValidation, isMutating) + client := p.getHttpClient(skipSSLValidation, isMutating) + + // Is this is a long-running request, then use a different timeout + if req.Header.Get(longRunningTimeoutHeader) == "true" { + longRunningClient := http.Client{} + longRunningClient.Transport = client.Transport + longRunningClient.Timeout = time.Duration(p.GetConfig().HTTPClientTimeoutLongRunningInSecs) * time.Second + return longRunningClient + } + + return client } func (p *portalProxy) getHttpClient(skipSSLValidation bool, mutating bool) http.Client { diff --git a/src/jetstream/main_test.go b/src/jetstream/main_test.go index 3e5f0d85f7..34e989f2af 100644 --- a/src/jetstream/main_test.go +++ b/src/jetstream/main_test.go @@ -63,17 +63,18 @@ func TestLoadPortalConfig(t *testing.T) { var pc interfaces.PortalConfig result, err := loadPortalConfig(pc, env.NewVarSet(env.WithMapLookup(map[string]string{ - "HTTP_CLIENT_TIMEOUT_IN_SECS": "10", - "HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS": "35", - "SKIP_SSL_VALIDATION": "true", - "CONSOLE_PROXY_TLS_ADDRESS": ":8080", - "CONSOLE_CLIENT": "portal-proxy", - "CONSOLE_CLIENT_SECRET": "ohsosecret!", - "CF_CLIENT": "portal-proxy", - "CF_CLIENT_SECRET": "ohsosecret!", - "UAA_ENDPOINT": "https://login.cf.org.com:443", - "ALLOWED_ORIGINS": "https://localhost,https://127.0.0.1", - "SESSION_STORE_SECRET": "cookiesecret", + "HTTP_CLIENT_TIMEOUT_IN_SECS": "10", + "HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS": "35", + "HTTP_CLIENT_TIMEOUT_LONGRUNNING_IN_SECS": "123", + "SKIP_SSL_VALIDATION": "true", + "CONSOLE_PROXY_TLS_ADDRESS": ":8080", + "CONSOLE_CLIENT": "portal-proxy", + "CONSOLE_CLIENT_SECRET": "ohsosecret!", + "CF_CLIENT": "portal-proxy", + "CF_CLIENT_SECRET": "ohsosecret!", + "UAA_ENDPOINT": "https://login.cf.org.com:443", + "ALLOWED_ORIGINS": "https://localhost,https://127.0.0.1", + "SESSION_STORE_SECRET": "cookiesecret", }))) if err != nil { diff --git a/src/jetstream/passthrough.go b/src/jetstream/passthrough.go index 53f7c24489..5b3cc0ef10 100644 --- a/src/jetstream/passthrough.go +++ b/src/jetstream/passthrough.go @@ -10,6 +10,7 @@ import ( "net/http" "net/url" "strings" + "time" "github.com/labstack/echo" log "github.com/sirupsen/logrus" @@ -20,6 +21,12 @@ import ( // API Host Prefix to replace if the custom header is supplied const apiPrefix = "api." +const longRunningTimeoutHeader = "x-cap-long-running" + +// Timeout for long-running requests, after which we will return indicating request it still active +// to prevent hitting the 2 minute browser timeout +const longRunningRequestTimeout = 30 + type PassthroughErrorStatus struct { StatusCode int `json:"statusCode"` Status string `json:"status"` @@ -215,6 +222,7 @@ func (p *portalProxy) ProxyRequest(c echo.Context, uri *url.URL) (map[string]*in log.Debug("proxy") cnsiList := strings.Split(c.Request().Header.Get("x-cap-cnsi-list"), ",") shouldPassthrough := "true" == c.Request().Header.Get("x-cap-passthrough") + longRunning := "true" == c.Request().Header.Get(longRunningTimeoutHeader) if err := p.validateCNSIList(cnsiList); err != nil { return nil, echo.NewHTTPError(http.StatusBadRequest, err.Error()) @@ -240,6 +248,14 @@ func (p *portalProxy) ProxyRequest(c echo.Context, uri *url.URL) (map[string]*in } } + // Only support one endpoint for long running operation (due to way we do timeout with the response channel) + if longRunning { + if len(cnsiList) > 1 { + err := errors.New("Requested long-running proxy to multiple CNSIs. Only single CNSI is supported for long running passthrough") + return nil, echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } + } + // send the request to each CNSI done := make(chan *interfaces.CNSIRequest) for _, cnsi := range cnsiList { @@ -247,6 +263,7 @@ func (p *portalProxy) ProxyRequest(c echo.Context, uri *url.URL) (map[string]*in if buildErr != nil { return nil, echo.NewHTTPError(http.StatusBadRequest, buildErr.Error()) } + cnsiRequest.LongRunning = longRunning // Allow the host part of the API URL to be overridden apiHost := c.Request().Header.Get("x-cap-api-host") // Don't allow any '.' chars in the api name @@ -265,15 +282,62 @@ func (p *portalProxy) ProxyRequest(c echo.Context, uri *url.URL) (map[string]*in go p.doRequest(&cnsiRequest, done) } + // Wait for all responses responses := make(map[string]*interfaces.CNSIRequest) - for range cnsiList { - res := <-done - responses[res.GUID] = res + + if !longRunning { + for range cnsiList { + res := <-done + responses[res.GUID] = res + } + } else { + // Long running has a timeout + for range cnsiList { + select { + case res := <-done: + responses[res.GUID] = res + case <-time.After(longRunningRequestTimeout * time.Second): + // For all those that have not completed, add a timeout response + for _, id := range cnsiList { + if _, ok := responses[id]; !ok { + // Did not get a response for the endpoint + responses[id] = &interfaces.CNSIRequest{ + GUID: id, + UserGUID: portalUserGUID, + Method: req.Method, + StatusCode: http.StatusAccepted, + Status: "Long Running Operation still active", + Response: makeLongRunningTimeoutError(), + Error: nil, + ResponseGUID: id, + } + } + } + break + } + } } return responses, nil } +func makeLongRunningTimeoutError() []byte { + var errorStatus = &PassthroughErrorStatus{ + StatusCode: http.StatusAccepted, + Status: "Long Running Operation still active", + } + errorResponse := []byte(fmt.Sprint("{\"longRunningTimeout\": true}")) + passthroughError := &PassthroughError{} + passthroughError.Error = errorStatus + passthroughError.ErrorResponse = (*json.RawMessage)(&errorResponse) + res, e := json.Marshal(passthroughError) + if e != nil { + log.Errorf("makeLongRunningTimeoutError: could not marshal JSON: %+v", e) + } + return res +} + +// TODO: This should be used by the function above func (p *portalProxy) DoProxyRequest(requests []interfaces.ProxyRequestInfo) (map[string]*interfaces.CNSIRequest, error) { log.Debug("DoProxyRequest") @@ -401,6 +465,11 @@ func (p *portalProxy) doRequest(cnsiRequest *interfaces.CNSIRequest, done chan<- // Copy original headers through, except custom portal-proxy Headers fwdCNSIStandardHeaders(cnsiRequest, req) + // If this is a long running request, add a header which we can use at request time to change the timeout + if cnsiRequest.LongRunning { + req.Header.Set(longRunningTimeoutHeader, "true") + } + // Find the auth provider for the auth type - default ot oauthflow authHandler := p.GetAuthProvider(tokenRec.AuthType) if authHandler.Handler != nil { diff --git a/src/jetstream/repository/interfaces/structs.go b/src/jetstream/repository/interfaces/structs.go index b7ed395771..7cc8358d62 100644 --- a/src/jetstream/repository/interfaces/structs.go +++ b/src/jetstream/repository/interfaces/structs.go @@ -309,6 +309,7 @@ type CNSIRequest struct { StatusCode int `json:"statusCode"` Status string `json:"status"` PassThrough bool `json:"-"` + LongRunning bool `json:"-"` Response []byte `json:"-"` Error error `json:"-"` @@ -316,39 +317,40 @@ type CNSIRequest struct { } type PortalConfig struct { - HTTPClientTimeoutInSecs int64 `configName:"HTTP_CLIENT_TIMEOUT_IN_SECS"` - HTTPClientTimeoutMutatingInSecs int64 `configName:"HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS"` - HTTPConnectionTimeoutInSecs int64 `configName:"HTTP_CONNECTION_TIMEOUT_IN_SECS"` - TLSAddress string `configName:"CONSOLE_PROXY_TLS_ADDRESS"` - TLSCert string `configName:"CONSOLE_PROXY_CERT"` - TLSCertKey string `configName:"CONSOLE_PROXY_CERT_KEY"` - TLSCertPath string `configName:"CONSOLE_PROXY_CERT_PATH"` - TLSCertKeyPath string `configName:"CONSOLE_PROXY_CERT_KEY_PATH"` - CFClient string `configName:"CF_CLIENT"` - CFClientSecret string `configName:"CF_CLIENT_SECRET"` - AllowedOrigins []string `configName:"ALLOWED_ORIGINS"` - SessionStoreSecret string `configName:"SESSION_STORE_SECRET"` - EncryptionKeyVolume string `configName:"ENCRYPTION_KEY_VOLUME"` - EncryptionKeyFilename string `configName:"ENCRYPTION_KEY_FILENAME"` - EncryptionKey string `configName:"ENCRYPTION_KEY"` - AutoRegisterCFUrl string `configName:"AUTO_REG_CF_URL"` - AutoRegisterCFName string `configName:"AUTO_REG_CF_NAME"` - SSOLogin bool `configName:"SSO_LOGIN"` - SSOOptions string `configName:"SSO_OPTIONS"` - SSOWhiteList string `configName:"SSO_WHITELIST"` - AuthEndpointType string `configName:"AUTH_ENDPOINT_TYPE"` - CookieDomain string `configName:"COOKIE_DOMAIN"` - LogLevel string `configName:"LOG_LEVEL"` - CFAdminIdentifier string - CloudFoundryInfo *CFInfo - HTTPS bool - EncryptionKeyInBytes []byte - ConsoleVersion string - IsCloudFoundry bool - LoginHooks []LoginHook - SessionStore SessionStorer - ConsoleConfig *ConsoleConfig - PluginConfig map[string]string - DatabaseProviderName string - EnableTechPreview bool `configName:"ENABLE_TECH_PREVIEW"` + HTTPClientTimeoutInSecs int64 `configName:"HTTP_CLIENT_TIMEOUT_IN_SECS"` + HTTPClientTimeoutMutatingInSecs int64 `configName:"HTTP_CLIENT_TIMEOUT_MUTATING_IN_SECS"` + HTTPClientTimeoutLongRunningInSecs int64 `configName:"HTTP_CLIENT_TIMEOUT_LONGRUNNING_IN_SECS"` + HTTPConnectionTimeoutInSecs int64 `configName:"HTTP_CONNECTION_TIMEOUT_IN_SECS"` + TLSAddress string `configName:"CONSOLE_PROXY_TLS_ADDRESS"` + TLSCert string `configName:"CONSOLE_PROXY_CERT"` + TLSCertKey string `configName:"CONSOLE_PROXY_CERT_KEY"` + TLSCertPath string `configName:"CONSOLE_PROXY_CERT_PATH"` + TLSCertKeyPath string `configName:"CONSOLE_PROXY_CERT_KEY_PATH"` + CFClient string `configName:"CF_CLIENT"` + CFClientSecret string `configName:"CF_CLIENT_SECRET"` + AllowedOrigins []string `configName:"ALLOWED_ORIGINS"` + SessionStoreSecret string `configName:"SESSION_STORE_SECRET"` + EncryptionKeyVolume string `configName:"ENCRYPTION_KEY_VOLUME"` + EncryptionKeyFilename string `configName:"ENCRYPTION_KEY_FILENAME"` + EncryptionKey string `configName:"ENCRYPTION_KEY"` + AutoRegisterCFUrl string `configName:"AUTO_REG_CF_URL"` + AutoRegisterCFName string `configName:"AUTO_REG_CF_NAME"` + SSOLogin bool `configName:"SSO_LOGIN"` + SSOOptions string `configName:"SSO_OPTIONS"` + SSOWhiteList string `configName:"SSO_WHITELIST"` + AuthEndpointType string `configName:"AUTH_ENDPOINT_TYPE"` + CookieDomain string `configName:"COOKIE_DOMAIN"` + LogLevel string `configName:"LOG_LEVEL"` + CFAdminIdentifier string + CloudFoundryInfo *CFInfo + HTTPS bool + EncryptionKeyInBytes []byte + ConsoleVersion string + IsCloudFoundry bool + LoginHooks []LoginHook + SessionStore SessionStorer + ConsoleConfig *ConsoleConfig + PluginConfig map[string]string + DatabaseProviderName string + EnableTechPreview bool `configName:"ENABLE_TECH_PREVIEW"` }