From d7a858576c02514313c6928a3d05f1fcdfaf941a Mon Sep 17 00:00:00 2001 From: rafaela-soares Date: Fri, 25 Mar 2022 14:44:52 +0000 Subject: [PATCH 1/2] add scanned lines + parsed lines --- e2e/utils/helper.go | 2 +- internal/console/helpers/helpers.go | 2 ++ internal/tracker/ci.go | 12 ++++++++++++ pkg/kics/resolver_sink.go | 5 +++++ pkg/kics/service.go | 28 ++++++++++++++++++++++++---- pkg/kics/sink.go | 8 +++++++- pkg/model/summary.go | 2 ++ pkg/parser/parser.go | 3 +++ pkg/scan/post_scan.go | 2 ++ 9 files changed, 58 insertions(+), 6 deletions(-) diff --git a/e2e/utils/helper.go b/e2e/utils/helper.go index d566fa15285..40ed363b908 100644 --- a/e2e/utils/helper.go +++ b/e2e/utils/helper.go @@ -28,7 +28,7 @@ func RunCommand(kicsDockerImage string, kicsArgs []string, useMock bool) (*CmdOu baseDir := filepath.Dir(cwd) dockerArgs := []string{"run", "-e", descriptionServer, "--add-host=host.docker.internal:host-gateway", "-v", baseDir + ":/path", kicsDockerImage} - completeArgs := append(dockerArgs, kicsArgs...) + completeArgs := append(dockerArgs, kicsArgs...) //nolint cmd := exec.Command("docker", completeArgs...) //nolint cmd.Env = append(os.Environ(), descriptionServer) diff --git a/internal/console/helpers/helpers.go b/internal/console/helpers/helpers.go index 78e9ff03011..947cb6e315a 100644 --- a/internal/console/helpers/helpers.go +++ b/internal/console/helpers/helpers.go @@ -124,7 +124,9 @@ func PrintResult(summary *model.Summary, failedQueries map[string]error, printer fmt.Printf("TOTAL: %d\n\n", summary.SeveritySummary.TotalCounter) log.Info().Msgf("Files scanned: %d", summary.ScannedFiles) + log.Info().Msgf("Lines scanned: %d", summary.ScannedFilesLines) log.Info().Msgf("Parsed files: %d", summary.ParsedFiles) + log.Info().Msgf("Lines parsed: %d", summary.ParsedFilesLines) log.Info().Msgf("Queries loaded: %d", summary.TotalQueries) log.Info().Msgf("Queries failed to execute: %d", summary.FailedToExecuteQueries) log.Info().Msg("Inspector stopped") diff --git a/internal/tracker/ci.go b/internal/tracker/ci.go index 96410d43e94..2e1e91c1ef4 100644 --- a/internal/tracker/ci.go +++ b/internal/tracker/ci.go @@ -25,6 +25,8 @@ type CITracker struct { ScanSecrets int ScanPaths int lines int + FoundCountLines int + ParsedCountLines int Version model.Version } @@ -96,3 +98,13 @@ func (c *CITracker) TrackScanPath() { func (c *CITracker) TrackVersion(retrievedVersion model.Version) { c.Version = retrievedVersion } + +// TrackFileFoundCountLines - information about the lines of the scanned files +func (c *CITracker) TrackFileFoundCountLines(countLines int) { + c.FoundCountLines += countLines +} + +// TrackFileParserCountLines - information about the lines of the parsed files +func (c *CITracker) TrackFileParseCountLines(countLines int) { + c.ParsedCountLines += countLines +} diff --git a/pkg/kics/resolver_sink.go b/pkg/kics/resolver_sink.go index 2dfd48eccec..1d2190e3a63 100644 --- a/pkg/kics/resolver_sink.go +++ b/pkg/kics/resolver_sink.go @@ -1,6 +1,7 @@ package kics import ( + "bytes" "context" "encoding/json" "fmt" @@ -25,6 +26,9 @@ func (s *Service) resolverSink(ctx context.Context, filename, scanID string) ([] for _, rfile := range resFiles.File { s.Tracker.TrackFileFound() + countLines := bytes.Count(rfile.Content, []byte{'\n'}) + 1 + s.Tracker.TrackFileFoundCountLines(countLines) + documents, err := s.Parser.Parse(rfile.FileName, rfile.Content) if err != nil { if documents.Kind == "break" { @@ -66,6 +70,7 @@ func (s *Service) resolverSink(ctx context.Context, filename, scanID string) ([] s.saveToFile(ctx, &file) } s.Tracker.TrackFileParse() + s.Tracker.TrackFileParseCountLines(documents.CountLines) } return resFiles.Excluded, nil } diff --git a/pkg/kics/service.go b/pkg/kics/service.go index ba59f5b4f97..fd868bf7406 100644 --- a/pkg/kics/service.go +++ b/pkg/kics/service.go @@ -1,6 +1,7 @@ package kics import ( + "bytes" "context" "encoding/json" "io" @@ -39,6 +40,8 @@ type Storage interface { type Tracker interface { TrackFileFound() TrackFileParse() + TrackFileFoundCountLines(countLines int) + TrackFileParseCountLines(countLines int) } // Service is a struct that contains a SourceProvider to receive sources, a storage to save and retrieve scanning informations @@ -111,17 +114,30 @@ func (s *Service) StartScan( } } +// Content keeps the content of the file and the number of lines +type Content struct { + Content *[]byte + CountLines int +} + /* getContent will read the passed file 1MB at a time to prevent resource exhaustion and return its content */ -func getContent(rc io.Reader) (*[]byte, error) { +func getContent(rc io.Reader) (*Content, error) { maxSizeMB := 5 // Max size of file in MBs var content []byte + countLines := 0 data := make([]byte, mbConst) + + c := &Content{ + Content: &[]byte{}, + CountLines: 0, + } + for { if maxSizeMB < 0 { - return &[]byte{}, errors.New("file size limit exceeded") + return c, errors.New("file size limit exceeded") } data = data[:cap(data)] n, err := rc.Read(data) @@ -129,12 +145,16 @@ func getContent(rc io.Reader) (*[]byte, error) { if err == io.EOF { break } - return &[]byte{}, err + return c, err } + countLines += bytes.Count(data[:n], []byte{'\n'}) + 1 content = append(content, data[:n]...) maxSizeMB-- } - return &content, nil + c.Content = &content + c.CountLines = countLines + + return c, nil } // GetVulnerabilities returns a list of scan detected vulnerabilities diff --git a/pkg/kics/sink.go b/pkg/kics/sink.go index 105fed9da95..13120b1931b 100644 --- a/pkg/kics/sink.go +++ b/pkg/kics/sink.go @@ -27,7 +27,12 @@ var ( func (s *Service) sink(ctx context.Context, filename, scanID string, rc io.Reader) error { s.Tracker.TrackFileFound() - content, err := getContent(rc) + c, err := getContent(rc) + + content := c.Content + + s.Tracker.TrackFileFoundCountLines(c.CountLines) + if err != nil { return errors.Wrapf(err, "failed to get file content: %s", filename) } @@ -71,6 +76,7 @@ func (s *Service) sink(ctx context.Context, filename, scanID string, rc io.Reade s.saveToFile(ctx, &file) } s.Tracker.TrackFileParse() + s.Tracker.TrackFileParseCountLines(documents.CountLines) return errors.Wrap(err, "failed to save file content") } diff --git a/pkg/model/summary.go b/pkg/model/summary.go index 98212726cdc..c2cb3862475 100644 --- a/pkg/model/summary.go +++ b/pkg/model/summary.go @@ -63,7 +63,9 @@ type QueryResultSlice []QueryResult // and how many queries failed to execute type Counters struct { ScannedFiles int `json:"files_scanned"` + ScannedFilesLines int `json:"lines_scanned"` ParsedFiles int `json:"files_parsed"` + ParsedFilesLines int `json:"lines_parsed"` FailedToScanFiles int `json:"files_failed_to_scan"` TotalQueries int `json:"queries_total"` FailedToExecuteQueries int `json:"queries_failed_to_execute"` diff --git a/pkg/parser/parser.go b/pkg/parser/parser.go index 0f97b92f368..d49a67b5be9 100644 --- a/pkg/parser/parser.go +++ b/pkg/parser/parser.go @@ -1,6 +1,7 @@ package parser import ( + "bytes" "errors" "os" "path/filepath" @@ -79,6 +80,7 @@ type ParsedDocument struct { Kind model.FileKind Content string IgnoreLines []int + CountLines int } // CommentsCommands gets commands on comments in the file beginning, before the code starts @@ -138,6 +140,7 @@ func (c *Parser) Parse(filePath string, fileContent []byte) (ParsedDocument, err Kind: c.parsers.GetKind(), Content: cont, IgnoreLines: igLines, + CountLines: bytes.Count(*resolved, []byte{'\n'}) + 1, }, nil } return ParsedDocument{ diff --git a/pkg/scan/post_scan.go b/pkg/scan/post_scan.go index 28a21e49194..46171275f87 100644 --- a/pkg/scan/post_scan.go +++ b/pkg/scan/post_scan.go @@ -20,6 +20,8 @@ import ( func (c *Client) getSummary(results []model.Vulnerability, end time.Time, pathParameters model.PathParameters) model.Summary { counters := model.Counters{ ScannedFiles: c.Tracker.FoundFiles, + ScannedFilesLines: c.Tracker.FoundCountLines, + ParsedFilesLines: c.Tracker.ParsedCountLines, ParsedFiles: c.Tracker.ParsedFiles, TotalQueries: c.Tracker.LoadedQueries, FailedToExecuteQueries: c.Tracker.ExecutingQueries - c.Tracker.ExecutedQueries, From 75ccdd4acd7ba53367e189b47008a828dae1ac73 Mon Sep 17 00:00:00 2001 From: rafaela-soares Date: Fri, 25 Mar 2022 15:14:10 +0000 Subject: [PATCH 2/2] correcting test errors --- e2e/fixtures/E2E_CLI_032_RESULT.json | 2 ++ e2e/fixtures/E2E_CLI_033_RESULT.json | 2 ++ e2e/fixtures/E2E_CLI_036_RESULT.json | 2 ++ e2e/fixtures/E2E_CLI_036_RESULT_2.json | 2 ++ e2e/fixtures/schemas/result.json | 10 ++++++++++ internal/tracker/ci.go | 2 +- 6 files changed, 19 insertions(+), 1 deletion(-) diff --git a/e2e/fixtures/E2E_CLI_032_RESULT.json b/e2e/fixtures/E2E_CLI_032_RESULT.json index f63caceb305..133e56fc202 100644 --- a/e2e/fixtures/E2E_CLI_032_RESULT.json +++ b/e2e/fixtures/E2E_CLI_032_RESULT.json @@ -1,7 +1,9 @@ { "kics_version": "development", "files_scanned": 1, + "lines_scanned": 278, "files_parsed": 1, + "lines_parsed": 278, "files_failed_to_scan": 0, "queries_total": 963, "queries_failed_to_execute": 0, diff --git a/e2e/fixtures/E2E_CLI_033_RESULT.json b/e2e/fixtures/E2E_CLI_033_RESULT.json index 4c95c78e7ac..3bdb2a8d649 100644 --- a/e2e/fixtures/E2E_CLI_033_RESULT.json +++ b/e2e/fixtures/E2E_CLI_033_RESULT.json @@ -1,7 +1,9 @@ { "kics_version": "development", "files_scanned": 1, + "lines_scanned": 5, "files_parsed": 1, + "lines_parsed": 5, "files_failed_to_scan": 0, "queries_total": 855, "queries_failed_to_execute": 0, diff --git a/e2e/fixtures/E2E_CLI_036_RESULT.json b/e2e/fixtures/E2E_CLI_036_RESULT.json index 9a5572384a6..d814f517e60 100644 --- a/e2e/fixtures/E2E_CLI_036_RESULT.json +++ b/e2e/fixtures/E2E_CLI_036_RESULT.json @@ -1,7 +1,9 @@ { "kics_version": "development", "files_scanned": 1, + "lines_scanned": 278, "files_parsed": 1, + "lines_parsed": 278, "files_failed_to_scan": 0, "queries_total": 13, "queries_failed_to_execute": 0, diff --git a/e2e/fixtures/E2E_CLI_036_RESULT_2.json b/e2e/fixtures/E2E_CLI_036_RESULT_2.json index b2e1219bb38..9ac5744417e 100644 --- a/e2e/fixtures/E2E_CLI_036_RESULT_2.json +++ b/e2e/fixtures/E2E_CLI_036_RESULT_2.json @@ -1,7 +1,9 @@ { "kics_version": "development", "files_scanned": 1, + "lines_scanned": 278, "files_parsed": 1, + "lines_parsed": 278, "files_failed_to_scan": 0, "queries_total": 1, "queries_failed_to_execute": 0, diff --git a/e2e/fixtures/schemas/result.json b/e2e/fixtures/schemas/result.json index 43453d72f83..a21dbdab85f 100644 --- a/e2e/fixtures/schemas/result.json +++ b/e2e/fixtures/schemas/result.json @@ -3,7 +3,9 @@ "required": [ "kics_version", "files_scanned", + "lines_scanned", "files_parsed", + "lines_parsed", "files_failed_to_scan", "queries_total", "queries_failed_to_execute", @@ -24,10 +26,18 @@ "type": "integer", "minimum": 0 }, + "lines_scanned": { + "type": "integer", + "minimum": 0 + }, "files_parsed": { "type": "integer", "minimum": 0 }, + "lines_parsed": { + "type": "integer", + "minimum": 0 + }, "files_failed_to_scan": { "type": "integer", "const": 0 diff --git a/internal/tracker/ci.go b/internal/tracker/ci.go index 2e1e91c1ef4..2fd2524ea11 100644 --- a/internal/tracker/ci.go +++ b/internal/tracker/ci.go @@ -104,7 +104,7 @@ func (c *CITracker) TrackFileFoundCountLines(countLines int) { c.FoundCountLines += countLines } -// TrackFileParserCountLines - information about the lines of the parsed files +// TrackFileParseCountLines - information about the lines of the parsed files func (c *CITracker) TrackFileParseCountLines(countLines int) { c.ParsedCountLines += countLines }