🐛 modify alternative isText to accept carriage returns (#2421)

* modify IsText from golang.org/x/tools/godoc/util to accept carriage returns. Signed-off-by: Spencer Schrock <[email protected]> * add TODO reminder to cleanup after release tests Signed-off-by: Spencer Schrock <[email protected]> Signed-off-by: Spencer Schrock <[email protected]>
ossf · Nov 3, 2022 · 0e3f81c · 0e3f81c
1 parent 043a720
commit 0e3f81c
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 6 deletions.
diff --git a/attestor/go.mod b/attestor/go.mod
@@ -57,7 +57,6 @@ require (
 	golang.org/x/sync v0.1.0 // indirect
 	golang.org/x/term v0.1.0 // indirect
 	golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect
-	golang.org/x/tools v0.2.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/api v0.18.8 // indirect

diff --git a/attestor/go.sum b/attestor/go.sum
@@ -1206,8 +1206,6 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE=
-golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
 golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

diff --git a/checks/raw/binary_artifact.go b/checks/raw/binary_artifact.go
@@ -22,12 +22,12 @@ import (
 	"regexp"
 	"strings"
 	"unicode"
+	"unicode/utf8"
 
 	semver "github.com/Masterminds/semver/v3"
 	"github.com/h2non/filetype"
 	"github.com/h2non/filetype/types"
 	"github.com/rhysd/actionlint"
-	"golang.org/x/tools/godoc/util"
 
 	"github.com/ossf/scorecard/v4/checker"
 	"github.com/ossf/scorecard/v4/checks/fileparser"
@@ -158,9 +158,10 @@ var checkBinaryFileContent fileparser.DoWhileTrueOnFileContent = func(path strin
 	}
 
 	exists2 := binaryFileTypes[strings.ReplaceAll(filepath.Ext(path), ".", "")]
+	// TODO remove the comparison after testing in release-test worker
 	isTextFile := isText(content)
-	if _, enabled := os.LookupEnv("SCORECARD_COMPARE_ISTEXT"); enabled && isTextFile != util.IsText(content) {
-		log.Printf("isText implementations differ for file: %s", path)
+	if _, enabled := os.LookupEnv("SCORECARD_COMPARE_ISTEXT"); enabled && isTextFile != isText2(content) {
+		log.Printf("isText implementations differ (raw.isText: %t) for file: %s", isTextFile, path)
 	}
 	if !isTextFile && exists2 {
 		*pfiles = append(*pfiles, checker.File{
@@ -186,6 +187,26 @@ func isText(content []byte) bool {
 	return true
 }
 
+// TODO decine between isText and isText2 after testing in release-test worker
+// modified version of golang.org/x/tools/godoc/util.
+func isText2(s []byte) bool {
+	const max = 1024 // at least utf8.UTFMax
+	if len(s) > max {
+		s = s[0:max]
+	}
+	for i, c := range string(s) {
+		if i+utf8.UTFMax > len(s) {
+			// last char may be incomplete - ignore
+			break
+		}
+		if c == 0xFFFD || c < ' ' && c != '\n' && c != '\t' && c != '\r' {
+			// decoding error or control character - not a text file
+			return false
+		}
+	}
+	return true
+}
+
 // gradleWrapperValidated checks for the gradle-wrapper-verify action being
 // used in a non-failing workflow on the latest commit.
 func gradleWrapperValidated(c clients.RepoClient) (bool, error) {