Implement ignoring directories based on wildcards. (#24203)

This is accomplished by a new directive in REPO.bazel,
"ignore_directories()". It takes a single argument, a list of
directories to ignore and it allows the same wildcards as glob().

This is done separately from .bazelignore to provide a migration path
off of that weird single-purpose configuration file.

Implementing this requires splitting RepoFileFunction into two: a part
that parses the repository file and one that creates a PackageArgs
instance. This was necessary to avoid a Skyframe dependency cycle: when
a WORKSPACE file is present and it loads a .bzl file from a repository
with a REPO.bazel file, the repo mapping for the main repository depends
on the WORKSPACE file, which depends on the .bzl file, which depends on
the IgnoredPackagePrefixesValue of its repository, which then depends on
the repo mapping of the main repository and the one the .bzl file is in,
which then depend on the WORKSPACE file.

Fixes #7093.

RELNOTES[NEW]: REPO.bazel now allows another directive,
"ignore_directories()". It takes a list of directories to ignore just
like .bazelignore does, but with glob semantics.

Closes #24032.

PiperOrigin-RevId: 693227896
Change-Id: Ia3e02a2bfe9caf999fc641f75261b528b19c1d03

Author: lberki

src/main/java/com/google/devtools/build/lib/analysis/starlark/StarlarkGlobalsImpl.java

@@ -23,7 +23,7 @@
 import com.google.devtools.build.lib.collect.nestedset.Depset;
 import com.google.devtools.build.lib.packages.BuildGlobals;
 import com.google.devtools.build.lib.packages.Proto;
-import com.google.devtools.build.lib.packages.RepoCallable;
+import com.google.devtools.build.lib.packages.RepoFileGlobals;
 import com.google.devtools.build.lib.packages.SelectorList;
 import com.google.devtools.build.lib.packages.StarlarkGlobals;
 import com.google.devtools.build.lib.packages.StarlarkNativeModule;
@@ -133,7 +133,7 @@ public ImmutableMap<String, Object> getModuleToplevels() {
   @Override
   public ImmutableMap<String, Object> getRepoToplevels() {
     ImmutableMap.Builder<String, Object> env = ImmutableMap.builder();
-    Starlark.addMethods(env, RepoCallable.INSTANCE);
+    Starlark.addMethods(env, RepoFileGlobals.INSTANCE);
     return env.buildOrThrow();
   }
 

src/main/java/com/google/devtools/build/lib/cmdline/BUILD

@@ -51,6 +51,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code",
         "//src/main/java/com/google/devtools/build/lib/util:hash_codes",
         "//src/main/java/com/google/devtools/build/lib/util:string",
+        "//src/main/java/com/google/devtools/build/lib/vfs",
         "//src/main/java/com/google/devtools/build/lib/vfs:ospathpolicy",
         "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
         "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",

src/main/java/com/google/devtools/build/lib/cmdline/IgnoredSubdirectories.java

@@ -14,57 +14,132 @@
 
 package com.google.devtools.build.lib.cmdline;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.skyframe.serialization.DeserializationContext;
+import com.google.devtools.build.lib.skyframe.serialization.ObjectCodec;
+import com.google.devtools.build.lib.skyframe.serialization.SerializationContext;
+import com.google.devtools.build.lib.skyframe.serialization.SerializationException;
 import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.UnixGlob;
+import com.google.protobuf.CodedInputStream;
+import com.google.protobuf.CodedOutputStream;
+import java.io.IOException;
 import java.util.Objects;
 import javax.annotation.Nullable;
 
 /**
  * A set of subdirectories to ignore during target pattern matching or globbing.
- *
- * <p>This is currently just a prefix, but will eventually support glob-style wildcards.
  */
 public final class IgnoredSubdirectories {
-  public static final IgnoredSubdirectories EMPTY = new IgnoredSubdirectories(ImmutableSet.of());
+  public static final IgnoredSubdirectories EMPTY =
+      new IgnoredSubdirectories(ImmutableSet.of(), ImmutableList.of());
+
+  private static final Splitter SLASH_SPLITTER = Splitter.on("/");
 
   private final ImmutableSet<PathFragment> prefixes;
 
-  private IgnoredSubdirectories(ImmutableSet<PathFragment> prefixes) {
-    for (PathFragment prefix : prefixes) {
-      Preconditions.checkArgument(!prefix.isAbsolute());
+  // String[] is mutable; we keep the split version because that's faster to match and the non-split
+  // one because that allows for simpler equality checking and then matchingEntry() doesn't need to
+  // allocate new objects.
+  private final ImmutableList<String> patterns;
+  private final ImmutableList<String[]> splitPatterns;
+
+  private static class Codec implements ObjectCodec<IgnoredSubdirectories> {
+    private static final Codec INSTANCE = new Codec();
+
+    @Override
+    public Class<? extends IgnoredSubdirectories> getEncodedClass() {
+      return IgnoredSubdirectories.class;
+    }
+
+    @Override
+    public void serialize(
+        SerializationContext context, IgnoredSubdirectories obj, CodedOutputStream codedOut)
+        throws SerializationException, IOException {
+      context.serialize(obj.prefixes, codedOut);
+      context.serialize(obj.patterns, codedOut);
     }
+
+    @Override
+    public IgnoredSubdirectories deserialize(
+        DeserializationContext context, CodedInputStream codedIn)
+        throws SerializationException, IOException {
+      ImmutableSet<PathFragment> prefixes = context.deserialize(codedIn);
+      ImmutableList<String> patterns = context.deserialize(codedIn);
+
+      return new IgnoredSubdirectories(prefixes, patterns);
+    }
+  }
+
+  private IgnoredSubdirectories(
+      ImmutableSet<PathFragment> prefixes, ImmutableList<String> patterns) {
     this.prefixes = prefixes;
+    this.patterns = patterns;
+    this.splitPatterns =
+        patterns.stream()
+            .map(p -> Iterables.toArray(SLASH_SPLITTER.split(p), String.class))
+            .collect(toImmutableList());
   }
 
   public static IgnoredSubdirectories of(ImmutableSet<PathFragment> prefixes) {
-    if (prefixes.isEmpty()) {
+    return of(prefixes, ImmutableList.of());
+  }
+
+  public static IgnoredSubdirectories of(
+      ImmutableSet<PathFragment> prefixes, ImmutableList<String> patterns) {
+    if (prefixes.isEmpty() && patterns.isEmpty()) {
       return EMPTY;
-    } else {
-      return new IgnoredSubdirectories(prefixes);
     }
+
+    for (PathFragment prefix : prefixes) {
+      Preconditions.checkArgument(!prefix.isAbsolute());
+    }
+
+    return new IgnoredSubdirectories(prefixes, patterns);
   }
 
   public IgnoredSubdirectories withPrefix(PathFragment prefix) {
-    ImmutableSet<PathFragment> prefixed =
+    Preconditions.checkArgument(!prefix.isAbsolute());
+
+    ImmutableSet<PathFragment> prefixedPrefixes =
         prefixes.stream().map(prefix::getRelative).collect(toImmutableSet());
-    return new IgnoredSubdirectories(prefixed);
+
+    ImmutableList<String> prefixedPatterns =
+        patterns.stream().map(p -> prefix + "/" + p).collect(toImmutableList());
+
+    return new IgnoredSubdirectories(prefixedPrefixes, prefixedPatterns);
   }
 
   public IgnoredSubdirectories union(IgnoredSubdirectories other) {
     return new IgnoredSubdirectories(
-        ImmutableSet.<PathFragment>builder().addAll(prefixes).addAll(other.prefixes).build());
+        ImmutableSet.<PathFragment>builder().addAll(prefixes).addAll(other.prefixes).build(),
+        ImmutableList.copyOf(
+            ImmutableSet.<String>builder().addAll(patterns).addAll(other.patterns).build()));
   }
 
   /** Filters out entries that cannot match anything under {@code directory}. */
   public IgnoredSubdirectories filterForDirectory(PathFragment directory) {
     ImmutableSet<PathFragment> filteredPrefixes =
         prefixes.stream().filter(p -> p.startsWith(directory)).collect(toImmutableSet());
 
-    return new IgnoredSubdirectories(filteredPrefixes);
+    String[] splitDirectory =
+        Iterables.toArray(SLASH_SPLITTER.split(directory.getPathString()), String.class);
+    ImmutableList.Builder<String> filteredPatterns = ImmutableList.builder();
+    for (int i = 0; i < patterns.size(); i++) {
+      if (UnixGlob.canMatchChild(splitPatterns.get(i), splitDirectory)) {
+        filteredPatterns.add(patterns.get(i));
+      }
+    }
+
+    return new IgnoredSubdirectories(filteredPrefixes, filteredPatterns.build());
   }
 
   public ImmutableSet<PathFragment> prefixes() {
@@ -95,10 +170,17 @@ public boolean allPathsAreUnder(PathFragment directory) {
 
   /** Returns the entry that matches a given directory or {@code null} if none. */
   @Nullable
-  public PathFragment matchingEntry(PathFragment directory) {
+  public String matchingEntry(PathFragment directory) {
     for (PathFragment prefix : prefixes) {
       if (directory.startsWith(prefix)) {
-        return prefix;
+        return prefix.getPathString();
+      }
+    }
+
+    String[] segmentArray = Iterables.toArray(directory.segments(), String.class);
+    for (int i = 0; i < patterns.size(); i++) {
+      if (UnixGlob.matchesPrefix(splitPatterns.get(i), segmentArray)) {
+        return patterns.get(i);
       }
     }
 
@@ -111,17 +193,22 @@ public boolean equals(Object other) {
       return false;
     }
 
+    // splitPatterns is a function of patterns so it's enough to check if patterns is equal
     IgnoredSubdirectories that = (IgnoredSubdirectories) other;
-    return Objects.equals(this.prefixes, that.prefixes);
+    return Objects.equals(this.prefixes, that.prefixes)
+        && Objects.equals(this.patterns, that.patterns);
   }
 
   @Override
   public int hashCode() {
-    return prefixes.hashCode();
+    return Objects.hash(prefixes, patterns);
   }
 
   @Override
   public String toString() {
-    return MoreObjects.toStringHelper("IgnoredSubdirectories").add("prefixes", prefixes).toString();
+    return MoreObjects.toStringHelper("IgnoredSubdirectories")
+        .add("prefixes", prefixes)
+        .add("patterns", patterns)
+        .toString();
   }
 }

src/main/java/com/google/devtools/build/lib/cmdline/TargetPattern.java

@@ -595,8 +595,7 @@ public <T, E extends Exception & QueryExceptionMarkerInterface> void eval(
           this,
           excludedSubdirectories);
       IgnoredSubdirectories ignoredSubdirectories = ignoredSubdirectoriesSupplier.get();
-      PathFragment matchingEntry =
-          ignoredSubdirectories.matchingEntry(directory.getPackageFragment());
+      String matchingEntry = ignoredSubdirectories.matchingEntry(directory.getPackageFragment());
       if (warnIfFiltered(matchingEntry, resolver)) {
         return;
       }
@@ -632,8 +631,7 @@ ListenableFuture<Void> evalAsync(
       IgnoredSubdirectories filteredIgnoredSubdirectories;
       try {
         IgnoredSubdirectories ignoredSubdirectories = ignoredSubdirectoriesSupplier.get();
-        PathFragment matchingEntry =
-            ignoredSubdirectories.matchingEntry(directory.getPackageFragment());
+        String matchingEntry = ignoredSubdirectories.matchingEntry(directory.getPackageFragment());
         if (warnIfFiltered(matchingEntry, resolver)) {
           return immediateVoidFuture();
         }
@@ -654,7 +652,7 @@ ListenableFuture<Void> evalAsync(
           executor);
     }
 
-    private boolean warnIfFiltered(PathFragment matchingEntry, TargetPatternResolver<?> resolver) {
+    private boolean warnIfFiltered(String matchingEntry, TargetPatternResolver<?> resolver) {
       if (matchingEntry != null) {
         resolver.warn(
             "Pattern '"

src/main/java/com/google/devtools/build/lib/packages/RepoCallable.java

@@ -0,0 +1,81 @@
+// Copyright 2023 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.packages;
+
+import java.util.Map;
+import net.starlark.java.annot.Param;
+import net.starlark.java.annot.ParamType;
+import net.starlark.java.annot.StarlarkMethod;
+import net.starlark.java.eval.EvalException;
+import net.starlark.java.eval.Sequence;
+import net.starlark.java.eval.Starlark;
+import net.starlark.java.eval.StarlarkThread;
+
+/** Definition of the {@code repo()} function used in REPO.bazel files. */
+public final class RepoFileGlobals {
+  private RepoFileGlobals() {}
+
+  public static final RepoFileGlobals INSTANCE = new RepoFileGlobals();
+
+  @StarlarkMethod(
+      name = "ignore_directories",
+      doc =
+          "The list of directories to ignore in this repository. <p>This function takes a list"
+              + " of strings and a directory is ignored if any of the given strings matches its"
+              + " repository-relative path according to the semantics of the <code>glob()</code>"
+              + " function. This function can be used to ignore directories that are implementation"
+              + " details of source control systems, output files of other build systems, etc.",
+      useStarlarkThread = true,
+      parameters = {
+        @Param(
+            name = "dirs",
+            allowedTypes = {
+              @ParamType(type = Sequence.class, generic1 = String.class),
+            })
+      })
+  public void ignoreDirectories(Iterable<?> dirsUnchecked, StarlarkThread thread)
+      throws EvalException {
+    Sequence<String> dirs = Sequence.cast(dirsUnchecked, String.class, "dirs");
+    RepoThreadContext context = RepoThreadContext.fromOrFail(thread, "repo()");
+
+    if (context.isIgnoredDirectoriesSet()) {
+      throw new EvalException("'ignored_directories()' can only be called once");
+    }
+
+    context.setIgnoredDirectories(dirs);
+  }
+
+  @StarlarkMethod(
+      name = "repo",
+      documented = false, // documented separately
+      extraKeywords = @Param(name = "kwargs"),
+      useStarlarkThread = true)
+  public void repoCallable(Map<String, Object> kwargs, StarlarkThread thread) throws EvalException {
+    RepoThreadContext context = RepoThreadContext.fromOrFail(thread, "repo()");
+    if (context.isRepoFunctionCalled()) {
+      throw Starlark.errorf("'repo' can only be called once in the REPO.bazel file");
+    }
+
+    if (context.isIgnoredDirectoriesSet()) {
+      throw Starlark.errorf("if repo() is called, it must be called before any other functions");
+    }
+
+    if (kwargs.isEmpty()) {
+      throw Starlark.errorf("at least one argument must be given to the 'repo' function");
+    }
+
+    context.setPackageArgsMap(kwargs);
+  }
+}