Merge branch 'main' into issue-1092

benchmarks/jmh/src/jmh/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryBenchmark.java

@@ -75,7 +75,7 @@ public void commit(Blackhole bh) throws Exception {
 
     private Revision addCommit() {
         final Revision revision =
-                repo.commit(new Revision(currentRevision), currentRevision * 1000, AUTHOR,
+                repo.commit(new Revision(currentRevision), currentRevision * 1000L, AUTHOR,
                             "Summary", "Detail", Markup.PLAINTEXT,
                             Change.ofTextUpsert("/file_" + rnd() + ".txt",
                                                 String.valueOf(currentRevision))).join().revision();

benchmarks/jmh/src/jmh/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryHistoryBenchmark.java

@@ -0,0 +1,96 @@
+/*
+ * Copyright 2025 LINE Corporation
+ *
+ * LINE Corporation licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.linecorp.centraldogma.server.internal.storage.repository.git;
+
+import static com.linecorp.centraldogma.server.CentralDogmaBuilder.DEFAULT_REPOSITORY_CACHE_SPEC;
+import static org.mockito.Mockito.mock;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.util.concurrent.ForkJoinPool;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.infra.Blackhole;
+
+import com.linecorp.armeria.common.metric.NoopMeterRegistry;
+import com.linecorp.centraldogma.common.Author;
+import com.linecorp.centraldogma.common.Change;
+import com.linecorp.centraldogma.common.Markup;
+import com.linecorp.centraldogma.common.Revision;
+import com.linecorp.centraldogma.internal.Util;
+import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache;
+import com.linecorp.centraldogma.server.storage.project.Project;
+
+@State(Scope.Benchmark)
+public class GitRepositoryHistoryBenchmark {
+
+    private static final Author AUTHOR = Author.ofEmail("[email protected]");
+
+    @Param({ "100", "1000"})
+    private int noCommits;
+
+    @Param({ "1", "3", "5", "10", "30" })
+    private int noFiles;
+
+    private File repoDir;
+    private GitRepository repo;
+    private int currentRevision;
+    private RepositoryCache cache;
+
+    @Setup
+    public void init() throws Exception {
+        repoDir = Files.createTempDirectory("jmh-gitrepository.").toFile();
+        cache = new RepositoryCache(DEFAULT_REPOSITORY_CACHE_SPEC, NoopMeterRegistry.get());
+        repo = new GitRepository(mock(Project.class), repoDir, ForkJoinPool.commonPool(),
+                                 System.currentTimeMillis(), AUTHOR,
+                                 cache);
+        currentRevision = 1;
+
+        // 1000 is the maximum number of allowed commits for a single history query.
+        for (int i = 0; i < noCommits; i++) {
+            addCommit(i);
+        }
+    }
+
+    @TearDown
+    public void destroy() throws Exception {
+        repo.internalClose();
+        Util.deleteFileTree(repoDir);
+    }
+
+    @Benchmark
+    public void history(Blackhole bh) throws Exception {
+        cache.clear();
+        for (int i = 0; i < noFiles; i++) {
+            bh.consume(repo.blockingHistory(new Revision(noCommits), Revision.INIT,
+                                            "/dir/file_" + i + ".txt", 1));
+        }
+    }
+
+    private void addCommit(int index) {
+        repo.commit(new Revision(currentRevision), currentRevision * 1000L, AUTHOR,
+                    "Summary", "Detail", Markup.PLAINTEXT,
+                    Change.ofTextUpsert("/dir/file_" + index + ".txt",
+                                        String.valueOf(currentRevision))).join();
+        currentRevision++;
+    }
+}

it/server/src/test/java/com/linecorp/centraldogma/it/CacheTest.java

@@ -67,9 +67,12 @@ void getFile(ClientType clientType, TestInfo testInfo) {
         final Map<String, Double> meters2 = metersSupplier.get();
         // Metadata needs to access to check a write quota (one cache miss).
         if (clientType == ClientType.LEGACY) {
-            // NB: A push operation involves a history() operation to retrieve the last commit.
-            //     Therefore we should observe one cache miss. (Thrift only)
-            assertThat(missCount(meters2)).isEqualTo(missCount(meters1) + 2);
+            // NB: A push operation involves a history() operation to retrieve the last commit when pushing.
+            //     Therefore we should observe five cache miss. (Thrift only)
+            assertThat(missCount(meters2)).isEqualTo(
+                    missCount(meters1) + 1 +
+                    1 + // (CacheableHistoryCall)
+                    4); // (CacheableObjectLoaderCall: 2 for revision2 and 2 for revision1)
         } else {
             assertThat(missCount(meters2)).isEqualTo(missCount(meters1) + 1);
         }
@@ -108,16 +111,40 @@ void history(ClientType clientType, TestInfo testInfo) {
         client.createProject(project).join();
         client.createRepository(project, REPO_FOO).join();
 
+        double prevMissCount = missCount(metersSupplier.get());
         final PushResult res1 = client.forRepo(project, REPO_FOO)
                                       .commit("Add a file", Change.ofTextUpsert("/foo.txt", "bar"))
                                       .push()
                                       .join();
-
-        final Map<String, Double> meters1 = metersSupplier.get();
+        double currentMissCount = missCount(metersSupplier.get());
+        // Metadata needs to access to check a write quota (one cache miss).
+        if (clientType == ClientType.LEGACY) {
+            // NB: A push operation involves a history() operation to retrieve the last commit when pushing.
+            //     Therefore we should observe five cache miss. (Thrift only)
+            assertThat(currentMissCount).isEqualTo(
+                    prevMissCount + 1 +
+                    1 + // (CacheableHistoryCall)
+                    4); // (CacheableObjectLoaderCall: 2 for revision2 and 2 for revision1)
+        } else {
+            assertThat(currentMissCount).isEqualTo(prevMissCount + 1);
+        }
+        prevMissCount = currentMissCount;
 
         // Get the history in various combination of from/to revisions.
         final List<Commit> history1 =
                 client.getHistory(project, REPO_FOO, HEAD, new Revision(-2), PathPattern.all(), 0).join();
+
+        currentMissCount = missCount(metersSupplier.get());
+        if (clientType == ClientType.LEGACY) {
+            assertThat(currentMissCount).isEqualTo(prevMissCount + 1); // (CacheableHistoryCall)
+        } else {
+            assertThat(currentMissCount).isEqualTo(
+                    prevMissCount +
+                    1 + // (CacheableHistoryCall)
+                    4); // (CacheableObjectLoaderCall: 2 for revision2 and 2 for revision1)
+        }
+        prevMissCount = currentMissCount;
+
         final List<Commit> history2 =
                 client.getHistory(project, REPO_FOO, HEAD, INIT, PathPattern.all(), 0).join();
         final List<Commit> history3 =
@@ -130,12 +157,9 @@ void history(ClientType clientType, TestInfo testInfo) {
         assertThat(history1).isEqualTo(history2);
         assertThat(history1).isEqualTo(history3);
         assertThat(history1).isEqualTo(history4);
-
-        final Map<String, Double> meters2 = metersSupplier.get();
-
-        // Should miss once and hit 3 times.
-        assertThat(missCount(meters2)).isEqualTo(missCount(meters1) + 1);
-        assertThat(hitCount(meters2)).isEqualTo(hitCount(meters1) + 3);
+        currentMissCount = missCount(metersSupplier.get());
+        // All cached.
+        assertThat(currentMissCount).isEqualTo(prevMissCount);
     }
 
     @ParameterizedTest(name = "getDiffs [{index}: {0}]")

server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java

@@ -75,7 +75,7 @@ public final class CentralDogmaBuilder {
     static final int DEFAULT_NUM_REPOSITORY_WORKERS = 16;
     static final long DEFAULT_MAX_REMOVED_REPOSITORY_AGE_MILLIS = 604_800_000;  // 7 days
 
-    static final String DEFAULT_REPOSITORY_CACHE_SPEC =
+    public static final String DEFAULT_REPOSITORY_CACHE_SPEC =
             "maximumWeight=134217728," + // Cache up to apx. 128-megachars.
             "expireAfterAccess=5m";      // Expire on 5 minutes of inactivity.
 

server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryCache.java

@@ -19,6 +19,8 @@
 import static java.util.Objects.requireNonNull;
 
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.locks.Lock;
+import java.util.function.Supplier;
 
 import javax.annotation.Nullable;
 
@@ -96,6 +98,40 @@ public <T> void put(CacheableCall<T> call, T value) {
         cache.put(call, CompletableFuture.completedFuture(value));
     }
 
+    public <T> T load(CacheableCall<T> key, Supplier<T> supplier, boolean logIfMiss) {
+        CompletableFuture<T> existingFuture = getIfPresent(key);
+        if (existingFuture != null) {
+            final T existingValue = existingFuture.getNow(null);
+            if (existingValue != null) {
+                // Cached already.
+                return existingValue;
+            }
+        }
+
+        // Not cached yet.
+        final Lock lock = key.coarseGrainedLock();
+        lock.lock();
+        try {
+            existingFuture = getIfPresent(key);
+            if (existingFuture != null) {
+                final T existingValue = existingFuture.getNow(null);
+                if (existingValue != null) {
+                    // Other thread already put the entries to the cache before we acquire the lock.
+                    return existingValue;
+                }
+            }
+
+            final T value = supplier.get();
+            put(key, value);
+            if (logIfMiss) {
+                logger.debug("Cache miss: {}", key);
+            }
+            return value;
+        } finally {
+            lock.unlock();
+        }
+    }
+
     public CacheStats stats() {
         return cache.synchronous().stats();
     }

server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CacheableObjectLoaderCall.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2025 LINE Corporation
+ *
+ * LINE Corporation licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package com.linecorp.centraldogma.server.internal.storage.repository.git;
+
+import java.util.concurrent.CompletableFuture;
+
+import org.eclipse.jgit.lib.AnyObjectId;
+import org.eclipse.jgit.lib.ObjectLoader;
+
+import com.google.common.base.MoreObjects.ToStringHelper;
+import com.google.common.primitives.Ints;
+
+import com.linecorp.centraldogma.server.internal.storage.repository.CacheableCall;
+import com.linecorp.centraldogma.server.storage.repository.Repository;
+
+final class CacheableObjectLoaderCall extends CacheableCall<ObjectLoader> {
+
+    private final AnyObjectId objectId;
+    private final int hashCode;
+
+    CacheableObjectLoaderCall(Repository repo, AnyObjectId objectId) {
+        super(repo);
+        this.objectId = objectId;
+        hashCode = objectId.hashCode() * 31 + System.identityHashCode(repo);
+    }
+
+    @Override
+    protected int weigh(ObjectLoader value) {
+        return Ints.saturatedCast(value.getSize());
+    }
+
+    /**
+     * Never invoked because {@link GitRepository} produces the value of this call.
+     */
+    @Override
+    public CompletableFuture<ObjectLoader> execute() {
+        throw new IllegalStateException();
+    }
+
+    @Override
+    public int hashCode() {
+        return hashCode;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (!super.equals(o)) {
+            return false;
+        }
+
+        final CacheableObjectLoaderCall that = (CacheableObjectLoaderCall) o;
+        return objectId.equals(that.objectId);
+    }
+
+    @Override
+    protected void toString(ToStringHelper helper) {
+        helper.add("objectId", objectId);
+    }
+}

server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CachingTreeObjectReader.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2025 LINE Corporation
+ *
+ * LINE Corporation licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.linecorp.centraldogma.server.internal.storage.repository.git;
+
+import static org.eclipse.jgit.lib.Constants.OBJ_TREE;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+
+import javax.annotation.Nullable;
+
+import org.eclipse.jgit.lib.AnyObjectId;
+import org.eclipse.jgit.lib.ObjectLoader;
+import org.eclipse.jgit.lib.ObjectReader;
+import org.eclipse.jgit.lib.ObjectReader.Filter;
+
+import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache;
+import com.linecorp.centraldogma.server.storage.repository.Repository;
+
+final class CachingTreeObjectReader extends Filter {
+
+    private final Repository repository;
+
+    private final ObjectReader delegate;
+
+    @Nullable
+    private final RepositoryCache cache;
+
+    CachingTreeObjectReader(Repository repository, ObjectReader delegate, @Nullable RepositoryCache cache) {
+        this.repository = repository;
+        this.delegate = delegate;
+        this.cache = cache;
+    }
+
+    @Override
+    protected ObjectReader delegate() {
+        return delegate;
+    }
+
+    @Override
+    public ObjectLoader open(AnyObjectId objectId, int typeHint)
+            throws IOException {
+        // Only cache tree objects.
+        if (OBJ_TREE != typeHint || cache == null) {
+            return delegate.open(objectId, typeHint);
+        }
+
+        // Need to convert to objectId from MutableObjectId
+        final AnyObjectId objectId0 = objectId.toObjectId();
+
+        final CacheableObjectLoaderCall key = new CacheableObjectLoaderCall(repository, objectId0);
+        return cache.load(key, () -> {
+            try {
+                return delegate.open(objectId0, typeHint);
+            } catch (IOException e) {
+                throw new UncheckedIOException("failed to open an object: " + objectId0, e);
+            }
+        }, false);
+    }
+}