Add skipped, broken and focus statuses for tests definitions & export statuses in allure report's

README-ru.md

@@ -76,6 +76,7 @@ func TestFuncCases(t *testing.T) {
 ```yaml
 - name: КОГДА запрашивается список заказов ДОЛЖЕН успешно возвращаться
   method: GET
+  status: ""
   path: /jsonrpc/v2/order.getBriefList
   query: ?id=550e8400-e29b-41d4-a716-446655440000&jsonrpc=2.0&user_id=00001
 
@@ -186,6 +187,14 @@ responseHeaders:
           }
 ```
 
+### Статус теста
+
+`status` - параметр, для того чтобы помечать тесты, может иметь следующие значения:
+  - `broken` - такой тест не будет запущен, в отчете будет отмечен как `broken`
+  - `skipped` - такой тест не будет запущен, в отчете будет отмечен как `skipped`
+  - `focus` - если у теста выставлен такой статус, все остальные тесты в suite у которых не проставлен статус, будут отмечены как `skipped` и будут запущены только тесты с статусом `focus`
+
+
 ### HTTP-запрос
 
 `method` - параметр для передачи типа HTTP запроса, формат передачи указан в примере выше

README.md

@@ -78,6 +78,7 @@ The tests can be now ran with `go test`, for example: `go test ./...`.
 ```yaml
 - name: WHEN the list of orders is requested MUST successfully response
   method: GET
+  status: ""
   path: /jsonrpc/v2/order.getBriefList
   query: ?id=550e8400-e29b-41d4-a716-446655440000&jsonrpc=2.0&user_id=00001
 
@@ -189,6 +190,13 @@ or for elements of map/array (if it's JSON):
           }
 ```
 
+### Test status
+
+`status` - a parameter, for specially mark tests, can have following values:
+  - `broken` - do not run test, only mark it as broken
+  - `skipped` - do not run test, skip it
+  - `focus` - run only this specific test, and mark all other tests with unset status as `skipped`
+
 ### HTTP-request
 
 `method` - a parameter for HTTP request type, the format is in the example above.

examples/with-db-example/cases/focused-tests.yaml

@@ -0,0 +1,24 @@
+- name: Get random number
+  method: GET
+  path: /randint/
+  response:
+    200: '{ "num": {"generated": "$matchRegexp(\\d)" } }'
+  variables_to_set:
+    200:
+      info_id: num.generated
+
+- name: Get info with database
+  status: focus
+  method: GET
+  variables:
+    info_id: 10
+  path: "/info/{{ $info_id }}"
+  variables_to_set:
+    200:
+      golang_id: query_result.0.0
+  response:
+    200: '{"result_id": "{{ $info_id }}", "query_result": [[ {{ $golang_id }}, "golang"], [2, "gonkey"]]}'
+  dbQuery: >
+    SELECT id, name FROM testing WHERE id={{ $golang_id }}
+  dbResponse:
+    - '{"id": {{ $golang_id }}, "name": "golang"}'

examples/with-db-example/cases/skipped-and-broken-tests.yaml

@@ -0,0 +1,23 @@
+- name: Get random number - will not be run
+  status: broken # mark test as broken
+  method: GET
+  path: /bad-path/
+  response:
+    200: '{ "num": {"generated": "$matchRegexp(\\d)" } }'
+  variables_to_set:
+    200:
+      info_id: num.generated
+
+- name: Get info with database
+  status: skipped
+  method: GET
+  path: "/info/{{ $info_id }}"
+  variables_to_set:
+    200:
+      golang_id: query_result.0.0
+  response:
+    200: '{"result_id": "{{ $info_id }}", "query_result": [[ {{ $golang_id }}, "golang"], [2, "gonkey"]]}'
+  dbQuery: >
+    SELECT id, name FROM testing WHERE id={{ $golang_id }}
+  dbResponse:
+    - '{"id": {{ $golang_id }}, "name": "golang"}'

models/result.go

@@ -1,5 +1,7 @@
 package models
 
+import "errors"
+
 // Result of test execution
 type Result struct {
 	Path                string // TODO: remove
@@ -16,6 +18,51 @@ type Result struct {
 	Test                TestInterface
 }
 
+func allureStatus(status string) bool {
+	switch status {
+	case "passed", "failed", "broken", "skipped":
+		return true
+	default:
+		return false
+	}
+}
+
+func notRunnedStatus(status string) bool {
+	switch status {
+	case "broken", "skipped":
+		return true
+	default:
+		return false
+	}
+}
+
+func (r *Result) AllureStatus() (string, error) {
+	testStatus := r.Test.GetStatus()
+	if testStatus != "" && allureStatus(testStatus) && notRunnedStatus(testStatus) {
+		return testStatus, nil
+	}
+
+	var (
+		status             = "passed"
+		testErrors []error = nil
+	)
+
+	if len(r.Errors) != 0 {
+		status = "failed"
+		testErrors = r.Errors
+	}
+
+	if len(testErrors) != 0 {
+		errText := ""
+		for _, err := range testErrors {
+			errText = errText + err.Error() + "\n"
+		}
+		return status, errors.New(errText)
+	}
+
+	return status, nil
+}
+
 // Passed returns true if test passed (false otherwise)
 func (r *Result) Passed() bool {
 	return len(r.Errors) == 0

models/test.go

@@ -11,6 +11,8 @@ type TestInterface interface {
 	GetResponse(code int) (string, bool)
 	GetResponseHeaders(code int) (map[string]string, bool)
 	GetName() string
+	GetStatus() string
+	SetStatus(string)
 	Fixtures() []string
 	ServiceMocks() map[string]interface{}
 	Pause() int
@@ -57,5 +59,7 @@ type Form struct {
 type Summary struct {
 	Success bool
 	Failed  int
+	Skipped int
+	Broken  int
 	Total   int
 }

output/allure_report/allure_report.go

@@ -2,7 +2,6 @@ package allure_report
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -56,15 +55,10 @@ func (o *AllureReportOutput) Process(t models.TestInterface, result *models.Resu
 			*bytes.NewBufferString(fmt.Sprintf(`Respone: %s`, result.DbResponse)),
 			"txt")
 	}
-	if !result.Passed() {
-		ers := ""
-		for _, e := range result.Errors {
-			ers = ers + e.Error() + "\n"
-		}
-		o.allure.EndCase("failed", errors.New(ers), time.Now())
-	} else {
-		o.allure.EndCase("passed", nil, time.Now())
-	}
+
+	status, err := result.AllureStatus()
+	o.allure.EndCase(status, err, time.Now())
+
 	return nil
 }
 

output/console_colored/console_colored.go

@@ -112,5 +112,12 @@ func templateFuncMap() template.FuncMap {
 }
 
 func (o *ConsoleColoredOutput) ShowSummary(summary *models.Summary) {
-	o.coloredPrintf("\nFailed tests: %d/%d\n", summary.Failed, summary.Total)
+	o.coloredPrintf(
+		"\nsuccess %d, failed %d, skipped %d, broken %d, total %d\n",
+		summary.Total-summary.Broken-summary.Failed-summary.Skipped,
+		summary.Failed,
+		summary.Skipped,
+		summary.Broken,
+		summary.Total,
+	)
 }

runner/runner.go

@@ -1,6 +1,7 @@
 package runner
 
 import (
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -63,20 +64,48 @@ func (r *Runner) Run() (*models.Summary, error) {
 		return nil, err
 	}
 
+	tests := []models.TestInterface{}
+	hasFocused := false
+	for test := range loader {
+		tests = append(tests, test)
+		if test.GetStatus() == "focus" {
+			hasFocused = true
+		}
+	}
+
 	client, err := newClient()
 	if err != nil {
 		return nil, err
 	}
 
 	totalTests := 0
 	failedTests := 0
+	skippedTests := 0
+	brokenTests := 0
+
+	for _, v := range tests {
+		if hasFocused {
+			switch v.GetStatus() {
+			case "focus":
+				v.SetStatus("")
+			case "broken":
+				// do nothing
+			default:
+				v.SetStatus("skipped")
+			}
+		}
 
-	for v := range loader {
 		testResult, err := r.executeTest(v, client)
-		if err != nil {
+		switch {
+		case err != nil && errors.Is(err, errTestSkipped):
+			skippedTests++
+		case err != nil && errors.Is(err, errTestBroken):
+			brokenTests++
+		case err != nil:
 			// todo: populate error with test name. Currently it is not possible here to get test name.
 			return nil, err
 		}
+
 		totalTests++
 		if len(testResult.Errors) > 0 {
 			failedTests++
@@ -90,15 +119,32 @@ func (r *Runner) Run() (*models.Summary, error) {
 
 	s := &models.Summary{
 		Success: failedTests == 0,
+		Skipped: skippedTests,
+		Broken:  brokenTests,
 		Failed:  failedTests,
 		Total:   totalTests,
 	}
 
 	return s, nil
 }
 
+var (
+	errTestSkipped = errors.New("test was skipped")
+	errTestBroken  = errors.New("test was broken")
+)
+
 func (r *Runner) executeTest(v models.TestInterface, client *http.Client) (*models.Result, error) {
 
+	if v.GetStatus() != "" {
+		if v.GetStatus() == "broken" {
+			return &models.Result{Test: v}, errTestBroken
+		}
+
+		if v.GetStatus() == "skipped" {
+			return &models.Result{Test: v}, errTestSkipped
+		}
+	}
+
 	r.config.Variables.Load(v.GetVariables())
 	v = r.config.Variables.Apply(v)
 

testloader/yaml_file/test.go

@@ -10,7 +10,7 @@ type Test struct {
 	TestDefinition
 
 	Filename string
-  
+
 	Request            string
 	Responses          map[int]string
 	ResponseHeaders    map[int]map[string]string
@@ -62,6 +62,10 @@ func (t *Test) GetName() string {
 	return t.Name
 }
 
+func (t *Test) GetStatus() string {
+	return t.Status
+}
+
 func (t *Test) IgnoreArraysOrdering() bool {
 	return t.ComparisonParams.IgnoreArraysOrdering
 }
@@ -175,3 +179,7 @@ func (t *Test) SetDbQueryString(query string) {
 func (t *Test) SetDbResponseJson(responses []string) {
 	t.DbResponse = responses
 }
+
+func (t *Test) SetStatus(status string) {
+	t.Status = status
+}

testloader/yaml_file/test_definition.go

@@ -4,6 +4,7 @@ import "github.com/lamoda/gonkey/models"
 
 type TestDefinition struct {
 	Name                     string                    `json:"name" yaml:"name"`
+	Status                   string                    `json:"status" yaml:"status"`
 	Variables                map[string]string         `json:"variables" yaml:"variables"`
 	VariablesToSet           VariablesToSet            `json:"variables_to_set" yaml:"variables_to_set"`
 	Form                     *models.Form              `json:"form" yaml:"form"`