go: resolve go_module target into dependent modules (#12394)

Go plugin updates:

- Adds a `ResolvedGoModule` type that represents a `go_module` with all relevant module and external module metadata  hydrated.
- Adds experimental `go-resolve` goal that updates `go.sum`.

Next steps will be to use this support for dependency inference of `go_package` targets to `go_external_module` targets.

Author: Tom Dyas

pyproject.toml

@@ -55,6 +55,7 @@ module = [
     "freezegun",
     "hdrh",
     "hdrh.histogram",
+    "ijson",
     "pex.*",
     "psutil",
     "pystache",

src/python/pants/backend/experimental/go/register.py

@@ -1,7 +1,7 @@
 # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-from pants.backend.go import build, distribution, import_analysis, target_type_rules
+from pants.backend.go import build, distribution, import_analysis, module, target_type_rules
 from pants.backend.go import target_types as go_target_types
 from pants.backend.go.target_types import GoBinary, GoExternalModule, GoModule, GoPackage
 
@@ -17,4 +17,5 @@ def rules():
         *import_analysis.rules(),
         *go_target_types.rules(),
         *target_type_rules.rules(),
+        *module.rules(),
     ]

src/python/pants/backend/go/import_analysis.py

@@ -6,7 +6,7 @@
 from dataclasses import dataclass
 from typing import Dict
 
-import ijson  # type: ignore
+import ijson
 
 from pants.backend.go.distribution import GoLangDistribution
 from pants.core.util_rules.external_tool import DownloadedExternalTool, ExternalToolRequest
@@ -69,6 +69,8 @@ async def analyze_imports_for_golang_distribution(
         goroot.get_request(platform),
     )
 
+    # Note: The `go` tool requires GOPATH to be an absolute path which can only be resolved from within the
+    # execution sandbox. Thus, this code uses a bash script to be able to resolve that path.
     analyze_script_digest = await Get(
         Digest,
         CreateDigest(

src/python/pants/backend/go/module.py

@@ -0,0 +1,202 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+import logging
+import textwrap
+from dataclasses import dataclass
+from typing import List, Optional, Tuple
+
+import ijson
+
+from pants.backend.go.distribution import GoLangDistribution
+from pants.backend.go.target_types import GoModuleSources
+from pants.build_graph.address import Address
+from pants.core.util_rules.external_tool import DownloadedExternalTool, ExternalToolRequest
+from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
+from pants.engine.addresses import Addresses
+from pants.engine.fs import (
+    CreateDigest,
+    Digest,
+    DigestContents,
+    FileContent,
+    MergeDigests,
+    RemovePrefix,
+    Snapshot,
+    Workspace,
+)
+from pants.engine.goal import Goal, GoalSubsystem
+from pants.engine.internals.selectors import Get
+from pants.engine.platform import Platform
+from pants.engine.process import BashBinary, Process, ProcessResult
+from pants.engine.rules import collect_rules, goal_rule, rule
+from pants.engine.target import UnexpandedTargets
+from pants.util.logging import LogLevel
+from pants.util.ordered_set import FrozenOrderedSet
+
+_logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class ModuleDescriptor:
+    import_path: str
+    module_path: str
+    module_version: str
+
+
+@dataclass(frozen=True)
+class ResolvedGoModule:
+    import_path: str
+    minimum_go_version: Optional[str]
+    modules: FrozenOrderedSet[ModuleDescriptor]
+    digest: Digest
+
+
+@dataclass(frozen=True)
+class ResolveGoModuleRequest:
+    address: Address
+
+
+# Perform a minimal parsing of go.mod for the `module` and `go` directives. Full resolution of go.mod is left to
+# the go toolchain. This could also probably be replaced by a go shim to make use of:
+# https://pkg.go.dev/golang.org/x/mod/modfile
+# TODO: Add full path to expections for applicable go.mod.
+def basic_parse_go_mod(raw_text: bytes) -> Tuple[Optional[str], Optional[str]]:
+    module_path = None
+    minimum_go_version = None
+    for line in raw_text.decode("utf-8").splitlines():
+        parts = line.strip().split()
+        if len(parts) >= 2:
+            if parts[0] == "module":
+                if module_path is not None:
+                    raise ValueError("Multiple `module` directives found in go.mod file.")
+                module_path = parts[1]
+            elif parts[0] == "go":
+                if minimum_go_version is not None:
+                    raise ValueError("Multiple `go` directives found in go.mod file.")
+                minimum_go_version = parts[1]
+    return module_path, minimum_go_version
+
+
+# Parse the output of `go mod download` into a list of module descriptors.
+def parse_module_descriptors(raw_json: bytes) -> List[ModuleDescriptor]:
+    module_descriptors = []
+    for raw_module_descriptor in ijson.items(raw_json, "", multiple_values=True):
+        module_descriptor = ModuleDescriptor(
+            import_path=raw_module_descriptor["Path"],
+            module_path=raw_module_descriptor["Path"],
+            module_version=raw_module_descriptor["Version"],
+        )
+        module_descriptors.append(module_descriptor)
+    return module_descriptors
+
+
+@rule
+async def resolve_go_module(
+    request: ResolveGoModuleRequest,
+    goroot: GoLangDistribution,
+    platform: Platform,
+    bash: BashBinary,
+) -> ResolvedGoModule:
+    downloaded_goroot = await Get(
+        DownloadedExternalTool,
+        ExternalToolRequest,
+        goroot.get_request(platform),
+    )
+
+    targets = await Get(UnexpandedTargets, Addresses([request.address]))
+    if not targets:
+        raise AssertionError(f"Address `{request.address}` did not resolve to any targets.")
+    elif len(targets) > 1:
+        raise AssertionError(f"Address `{request.address}` resolved to multiple targets.")
+    target = targets[0]
+
+    sources = await Get(SourceFiles, SourceFilesRequest([target.get(GoModuleSources)]))
+    flattened_sources_snapshot = await Get(
+        Snapshot, RemovePrefix(sources.snapshot.digest, request.address.spec_path)
+    )
+
+    # Note: The `go` tool requires GOPATH to be an absolute path which can only be resolved from within the
+    # execution sandbox. Thus, this code uses a bash script to be able to resolve that path.
+    analyze_script_digest = await Get(
+        Digest,
+        CreateDigest(
+            [
+                FileContent(
+                    "analyze.sh",
+                    textwrap.dedent(
+                        """\
+                export GOROOT="./go"
+                export GOPATH="$(/bin/pwd)/gopath"
+                export GOCACHE="$(/bin/pwd)/cache"
+                mkdir -p "$GOPATH" "$GOCACHE"
+                exec ./go/bin/go mod download -json all
+                """
+                    ).encode("utf-8"),
+                )
+            ]
+        ),
+    )
+
+    input_root_digest = await Get(
+        Digest,
+        MergeDigests(
+            [flattened_sources_snapshot.digest, downloaded_goroot.digest, analyze_script_digest]
+        ),
+    )
+
+    process = Process(
+        argv=[bash.path, "./analyze.sh"],
+        input_digest=input_root_digest,
+        description="Resolve go_module metadata.",
+        output_files=["go.mod", "go.sum"],
+        level=LogLevel.DEBUG,
+    )
+
+    result = await Get(ProcessResult, Process, process)
+
+    # Parse the go.mod for the module path and minimum Go version.
+    module_path = None
+    minimum_go_version = None
+    digest_contents = await Get(DigestContents, Digest, flattened_sources_snapshot.digest)
+    for entry in digest_contents:
+        if entry.path == "go.mod":
+            module_path, minimum_go_version = basic_parse_go_mod(entry.content)
+
+    if module_path is None:
+        raise ValueError("No `module` directive found in go.mod.")
+
+    return ResolvedGoModule(
+        import_path=module_path,
+        minimum_go_version=minimum_go_version,
+        modules=FrozenOrderedSet(parse_module_descriptors(result.stdout)),
+        digest=result.output_digest,
+    )
+
+
+# TODO: Add integration tests for the `go-resolve` goal once we figure out its final form. For now, it is a debug
+# tool to help update go.sum while developing the Go plugin and will probably change.
+class GoResolveSubsystem(GoalSubsystem):
+    name = "go-resolve"
+    help = "Resolve a Go module's go.mod and update go.sum accordingly."
+
+
+class GoResolveGoal(Goal):
+    subsystem_cls = GoResolveSubsystem
+
+
+@goal_rule
+async def run_go_resolve(targets: UnexpandedTargets, workspace: Workspace) -> GoResolveGoal:
+    # TODO: Use MultiGet to resolve the go_module targets.
+    # TODO: Combine all of the go.sum's into a single Digest to write.
+    for target in targets:
+        if target.has_field(GoModuleSources) and not target.address.is_file_target:
+            resolved_go_module = await Get(ResolvedGoModule, ResolveGoModuleRequest(target.address))
+            # TODO: Only update the files if they actually changed.
+            workspace.write_digest(resolved_go_module.digest, path_prefix=target.address.spec_path)
+            _logger.info(f"{target.address}: Updated go.mod and go.sum.\n")
+        else:
+            _logger.info(f"{target.address}: Skipping because target is not a `go_module`.\n")
+    return GoResolveGoal(exit_code=0)
+
+
+def rules():
+    return collect_rules()

src/python/pants/backend/go/module_integration_test.py

@@ -0,0 +1,56 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+import pytest
+
+from pants.backend.go import module
+from pants.backend.go.module import ResolvedGoModule, ResolveGoModuleRequest
+from pants.backend.go.target_types import GoExternalModule, GoModule, GoPackage
+from pants.build_graph.address import Address
+from pants.core.util_rules import external_tool, source_files
+from pants.engine.rules import QueryRule
+from pants.testutil.rule_runner import RuleRunner
+
+
+@pytest.fixture
+def rule_runner() -> RuleRunner:
+    rule_runner = RuleRunner(
+        rules=[
+            *external_tool.rules(),
+            *source_files.rules(),
+            *module.rules(),
+            QueryRule(ResolvedGoModule, [ResolveGoModuleRequest]),
+        ],
+        target_types=[GoPackage, GoModule, GoExternalModule],
+    )
+    rule_runner.set_options(["--backend-packages=pants.backend.experimental.go"])
+    return rule_runner
+
+
+def test_basic_parse_go_mod() -> None:
+    content = b"module go.example.com/foo\ngo 1.16\nrequire github.com/golang/protobuf v1.4.2\n"
+    module_path, minimum_go_version = module.basic_parse_go_mod(content)
+    assert module_path == "go.example.com/foo"
+    assert minimum_go_version == "1.16"
+
+
+def test_resolve_go_module(rule_runner: RuleRunner) -> None:
+    rule_runner.add_to_build_file("foo", "go_module(name='mod')\ngo_package(name='pkg')\n")
+    rule_runner.write_files(
+        {
+            "foo/pkg/foo.go": "package pkg\n",
+            "foo/go.mod": "module go.example.com/foo\ngo 1.16\nrequire github.com/golang/protobuf v1.4.2\n",
+            "foo/go.sum": "",
+            "foo/main.go": "package main\nfunc main() { }\n",
+        }
+    )
+    resolved_go_module = rule_runner.request(
+        ResolvedGoModule, [ResolveGoModuleRequest(Address("foo", target_name="mod"))]
+    )
+    assert resolved_go_module.import_path == "go.example.com/foo"
+    assert resolved_go_module.minimum_go_version == "1.16"
+    assert len(resolved_go_module.modules) > 0
+    found_protobuf_module = False
+    for module_descriptor in resolved_go_module.modules:
+        if module_descriptor.module_path == "github.com/golang/protobuf":
+            found_protobuf_module = True
+    assert found_protobuf_module

src/python/pants/backend/go/target_types.py

@@ -1,7 +1,17 @@
 # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
+import os
+from typing import Sequence
+
 from pants.engine.rules import collect_rules
-from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies, Sources, StringField, Target
+from pants.engine.target import (
+    COMMON_TARGET_FIELDS,
+    Dependencies,
+    InvalidFieldException,
+    Sources,
+    StringField,
+    Target,
+)
 
 
 class GoSources(Sources):
@@ -34,6 +44,12 @@ class GoPackage(Target):
 class GoModuleSources(Sources):
     alias = "_sources"
     default = ("go.mod", "go.sum")
+    expected_num_files = range(1, 3)
+
+    def validate_resolved_files(self, files: Sequence[str]) -> None:
+        super().validate_resolved_files(files)
+        if "go.mod" not in [os.path.basename(f) for f in files]:
+            raise InvalidFieldException(f"""No go.mod file was found for target {self.address}.""")
 
 
 class GoModule(Target):

testprojects/src/go/BUILD

@@ -0,0 +1,4 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+go_module()
+

testprojects/src/go/go.mod

@@ -0,0 +1,7 @@
+module github.com/pantsbuild/pants/testprojects/src/go
+
+go 1.16
+
+require (
+	github.com/google/uuid v1.2.0
+)

testprojects/src/go/go.sum

@@ -0,0 +1,2 @@
+github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
+github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=