diff --git a/pyproject.toml b/pyproject.toml index 36e1a133b6e..c3609385cc6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ module = [ "freezegun", "hdrh", "hdrh.histogram", + "ijson", "pex.*", "psutil", "pystache", diff --git a/src/python/pants/backend/experimental/go/register.py b/src/python/pants/backend/experimental/go/register.py index ad175912fc5..910a536b16b 100644 --- a/src/python/pants/backend/experimental/go/register.py +++ b/src/python/pants/backend/experimental/go/register.py @@ -1,7 +1,7 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from pants.backend.go import build, distribution, import_analysis, target_type_rules +from pants.backend.go import build, distribution, import_analysis, module, target_type_rules from pants.backend.go import target_types as go_target_types from pants.backend.go.target_types import GoBinary, GoExternalModule, GoModule, GoPackage @@ -17,4 +17,5 @@ def rules(): *import_analysis.rules(), *go_target_types.rules(), *target_type_rules.rules(), + *module.rules(), ] diff --git a/src/python/pants/backend/go/import_analysis.py b/src/python/pants/backend/go/import_analysis.py index 1507795bc64..789ded30a65 100644 --- a/src/python/pants/backend/go/import_analysis.py +++ b/src/python/pants/backend/go/import_analysis.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from typing import Dict -import ijson # type: ignore +import ijson from pants.backend.go.distribution import GoLangDistribution from pants.core.util_rules.external_tool import DownloadedExternalTool, ExternalToolRequest @@ -69,6 +69,8 @@ async def analyze_imports_for_golang_distribution( goroot.get_request(platform), ) + # Note: The `go` tool requires GOPATH to be an absolute path which can only be resolved from within the + # execution sandbox. Thus, this code uses a bash script to be able to resolve that path. analyze_script_digest = await Get( Digest, CreateDigest( diff --git a/src/python/pants/backend/go/module.py b/src/python/pants/backend/go/module.py new file mode 100644 index 00000000000..5fdf260b967 --- /dev/null +++ b/src/python/pants/backend/go/module.py @@ -0,0 +1,202 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +import logging +import textwrap +from dataclasses import dataclass +from typing import List, Optional, Tuple + +import ijson + +from pants.backend.go.distribution import GoLangDistribution +from pants.backend.go.target_types import GoModuleSources +from pants.build_graph.address import Address +from pants.core.util_rules.external_tool import DownloadedExternalTool, ExternalToolRequest +from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest +from pants.engine.addresses import Addresses +from pants.engine.fs import ( + CreateDigest, + Digest, + DigestContents, + FileContent, + MergeDigests, + RemovePrefix, + Snapshot, + Workspace, +) +from pants.engine.goal import Goal, GoalSubsystem +from pants.engine.internals.selectors import Get +from pants.engine.platform import Platform +from pants.engine.process import BashBinary, Process, ProcessResult +from pants.engine.rules import collect_rules, goal_rule, rule +from pants.engine.target import UnexpandedTargets +from pants.util.logging import LogLevel +from pants.util.ordered_set import FrozenOrderedSet + +_logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class ModuleDescriptor: + import_path: str + module_path: str + module_version: str + + +@dataclass(frozen=True) +class ResolvedGoModule: + import_path: str + minimum_go_version: Optional[str] + modules: FrozenOrderedSet[ModuleDescriptor] + digest: Digest + + +@dataclass(frozen=True) +class ResolveGoModuleRequest: + address: Address + + +# Perform a minimal parsing of go.mod for the `module` and `go` directives. Full resolution of go.mod is left to +# the go toolchain. This could also probably be replaced by a go shim to make use of: +# https://pkg.go.dev/golang.org/x/mod/modfile +# TODO: Add full path to expections for applicable go.mod. +def basic_parse_go_mod(raw_text: bytes) -> Tuple[Optional[str], Optional[str]]: + module_path = None + minimum_go_version = None + for line in raw_text.decode("utf-8").splitlines(): + parts = line.strip().split() + if len(parts) >= 2: + if parts[0] == "module": + if module_path is not None: + raise ValueError("Multiple `module` directives found in go.mod file.") + module_path = parts[1] + elif parts[0] == "go": + if minimum_go_version is not None: + raise ValueError("Multiple `go` directives found in go.mod file.") + minimum_go_version = parts[1] + return module_path, minimum_go_version + + +# Parse the output of `go mod download` into a list of module descriptors. +def parse_module_descriptors(raw_json: bytes) -> List[ModuleDescriptor]: + module_descriptors = [] + for raw_module_descriptor in ijson.items(raw_json, "", multiple_values=True): + module_descriptor = ModuleDescriptor( + import_path=raw_module_descriptor["Path"], + module_path=raw_module_descriptor["Path"], + module_version=raw_module_descriptor["Version"], + ) + module_descriptors.append(module_descriptor) + return module_descriptors + + +@rule +async def resolve_go_module( + request: ResolveGoModuleRequest, + goroot: GoLangDistribution, + platform: Platform, + bash: BashBinary, +) -> ResolvedGoModule: + downloaded_goroot = await Get( + DownloadedExternalTool, + ExternalToolRequest, + goroot.get_request(platform), + ) + + targets = await Get(UnexpandedTargets, Addresses([request.address])) + if not targets: + raise AssertionError(f"Address `{request.address}` did not resolve to any targets.") + elif len(targets) > 1: + raise AssertionError(f"Address `{request.address}` resolved to multiple targets.") + target = targets[0] + + sources = await Get(SourceFiles, SourceFilesRequest([target.get(GoModuleSources)])) + flattened_sources_snapshot = await Get( + Snapshot, RemovePrefix(sources.snapshot.digest, request.address.spec_path) + ) + + # Note: The `go` tool requires GOPATH to be an absolute path which can only be resolved from within the + # execution sandbox. Thus, this code uses a bash script to be able to resolve that path. + analyze_script_digest = await Get( + Digest, + CreateDigest( + [ + FileContent( + "analyze.sh", + textwrap.dedent( + """\ + export GOROOT="./go" + export GOPATH="$(/bin/pwd)/gopath" + export GOCACHE="$(/bin/pwd)/cache" + mkdir -p "$GOPATH" "$GOCACHE" + exec ./go/bin/go mod download -json all + """ + ).encode("utf-8"), + ) + ] + ), + ) + + input_root_digest = await Get( + Digest, + MergeDigests( + [flattened_sources_snapshot.digest, downloaded_goroot.digest, analyze_script_digest] + ), + ) + + process = Process( + argv=[bash.path, "./analyze.sh"], + input_digest=input_root_digest, + description="Resolve go_module metadata.", + output_files=["go.mod", "go.sum"], + level=LogLevel.DEBUG, + ) + + result = await Get(ProcessResult, Process, process) + + # Parse the go.mod for the module path and minimum Go version. + module_path = None + minimum_go_version = None + digest_contents = await Get(DigestContents, Digest, flattened_sources_snapshot.digest) + for entry in digest_contents: + if entry.path == "go.mod": + module_path, minimum_go_version = basic_parse_go_mod(entry.content) + + if module_path is None: + raise ValueError("No `module` directive found in go.mod.") + + return ResolvedGoModule( + import_path=module_path, + minimum_go_version=minimum_go_version, + modules=FrozenOrderedSet(parse_module_descriptors(result.stdout)), + digest=result.output_digest, + ) + + +# TODO: Add integration tests for the `go-resolve` goal once we figure out its final form. For now, it is a debug +# tool to help update go.sum while developing the Go plugin and will probably change. +class GoResolveSubsystem(GoalSubsystem): + name = "go-resolve" + help = "Resolve a Go module's go.mod and update go.sum accordingly." + + +class GoResolveGoal(Goal): + subsystem_cls = GoResolveSubsystem + + +@goal_rule +async def run_go_resolve(targets: UnexpandedTargets, workspace: Workspace) -> GoResolveGoal: + # TODO: Use MultiGet to resolve the go_module targets. + # TODO: Combine all of the go.sum's into a single Digest to write. + for target in targets: + if target.has_field(GoModuleSources) and not target.address.is_file_target: + resolved_go_module = await Get(ResolvedGoModule, ResolveGoModuleRequest(target.address)) + # TODO: Only update the files if they actually changed. + workspace.write_digest(resolved_go_module.digest, path_prefix=target.address.spec_path) + _logger.info(f"{target.address}: Updated go.mod and go.sum.\n") + else: + _logger.info(f"{target.address}: Skipping because target is not a `go_module`.\n") + return GoResolveGoal(exit_code=0) + + +def rules(): + return collect_rules() diff --git a/src/python/pants/backend/go/module_integration_test.py b/src/python/pants/backend/go/module_integration_test.py new file mode 100644 index 00000000000..b89446a18b4 --- /dev/null +++ b/src/python/pants/backend/go/module_integration_test.py @@ -0,0 +1,56 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +import pytest + +from pants.backend.go import module +from pants.backend.go.module import ResolvedGoModule, ResolveGoModuleRequest +from pants.backend.go.target_types import GoExternalModule, GoModule, GoPackage +from pants.build_graph.address import Address +from pants.core.util_rules import external_tool, source_files +from pants.engine.rules import QueryRule +from pants.testutil.rule_runner import RuleRunner + + +@pytest.fixture +def rule_runner() -> RuleRunner: + rule_runner = RuleRunner( + rules=[ + *external_tool.rules(), + *source_files.rules(), + *module.rules(), + QueryRule(ResolvedGoModule, [ResolveGoModuleRequest]), + ], + target_types=[GoPackage, GoModule, GoExternalModule], + ) + rule_runner.set_options(["--backend-packages=pants.backend.experimental.go"]) + return rule_runner + + +def test_basic_parse_go_mod() -> None: + content = b"module go.example.com/foo\ngo 1.16\nrequire github.com/golang/protobuf v1.4.2\n" + module_path, minimum_go_version = module.basic_parse_go_mod(content) + assert module_path == "go.example.com/foo" + assert minimum_go_version == "1.16" + + +def test_resolve_go_module(rule_runner: RuleRunner) -> None: + rule_runner.add_to_build_file("foo", "go_module(name='mod')\ngo_package(name='pkg')\n") + rule_runner.write_files( + { + "foo/pkg/foo.go": "package pkg\n", + "foo/go.mod": "module go.example.com/foo\ngo 1.16\nrequire github.com/golang/protobuf v1.4.2\n", + "foo/go.sum": "", + "foo/main.go": "package main\nfunc main() { }\n", + } + ) + resolved_go_module = rule_runner.request( + ResolvedGoModule, [ResolveGoModuleRequest(Address("foo", target_name="mod"))] + ) + assert resolved_go_module.import_path == "go.example.com/foo" + assert resolved_go_module.minimum_go_version == "1.16" + assert len(resolved_go_module.modules) > 0 + found_protobuf_module = False + for module_descriptor in resolved_go_module.modules: + if module_descriptor.module_path == "github.com/golang/protobuf": + found_protobuf_module = True + assert found_protobuf_module diff --git a/src/python/pants/backend/go/target_types.py b/src/python/pants/backend/go/target_types.py index ba2b5146323..06f5ea4bdbd 100644 --- a/src/python/pants/backend/go/target_types.py +++ b/src/python/pants/backend/go/target_types.py @@ -1,7 +1,17 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +import os +from typing import Sequence + from pants.engine.rules import collect_rules -from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies, Sources, StringField, Target +from pants.engine.target import ( + COMMON_TARGET_FIELDS, + Dependencies, + InvalidFieldException, + Sources, + StringField, + Target, +) class GoSources(Sources): @@ -34,6 +44,12 @@ class GoPackage(Target): class GoModuleSources(Sources): alias = "_sources" default = ("go.mod", "go.sum") + expected_num_files = range(1, 3) + + def validate_resolved_files(self, files: Sequence[str]) -> None: + super().validate_resolved_files(files) + if "go.mod" not in [os.path.basename(f) for f in files]: + raise InvalidFieldException(f"""No go.mod file was found for target {self.address}.""") class GoModule(Target): diff --git a/testprojects/src/go/BUILD b/testprojects/src/go/BUILD new file mode 100644 index 00000000000..26749bef630 --- /dev/null +++ b/testprojects/src/go/BUILD @@ -0,0 +1,4 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +go_module() + diff --git a/testprojects/src/go/go.mod b/testprojects/src/go/go.mod new file mode 100644 index 00000000000..00f6a28cda8 --- /dev/null +++ b/testprojects/src/go/go.mod @@ -0,0 +1,7 @@ +module github.com/pantsbuild/pants/testprojects/src/go + +go 1.16 + +require ( + github.com/google/uuid v1.2.0 +) diff --git a/testprojects/src/go/go.sum b/testprojects/src/go/go.sum new file mode 100644 index 00000000000..a4d2b5ecb01 --- /dev/null +++ b/testprojects/src/go/go.sum @@ -0,0 +1,2 @@ +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=