Add benchmark infratructure without actual tests

.gitignore

@@ -1,5 +1,6 @@
 .DS_Store
 /.build
+/Benchmarks/.build
 /Packages
 /*.xcodeproj
 xcuserdata/

Benchmarks/Benchmarks/SwiftKafkaConsumerBenchmarks/KafkaConsumerBenchmark.swift

@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the swift-kafka-client open source project
+//
+// Copyright (c) 2023 Apple Inc. and the swift-kafka-client project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of swift-kafka-client project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Benchmark
+import Kafka
+
+let benchmarks = {
+    Benchmark.defaultConfiguration = .init(
+        metrics: [.wallClock, .cpuTotal, .allocatedResidentMemory, .contextSwitches, .throughput] + .arc,
+        warmupIterations: 0,
+        scalingFactor: .one,
+        maxDuration: .seconds(5),
+        maxIterations: 100
+    )
+
+    Benchmark.setup = {}
+
+    Benchmark.teardown = {}
+}

Benchmarks/Benchmarks/SwiftKafkaProducerBenchmarks/KafkaProducerBenchmark.swift

@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the swift-kafka-client open source project
+//
+// Copyright (c) 2023 Apple Inc. and the swift-kafka-client project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of swift-kafka-client project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Benchmark
+import Crdkafka
+import Kafka
+
+let benchmarks = {
+    Benchmark.defaultConfiguration = .init(
+        metrics: [.wallClock, .cpuTotal, .allocatedResidentMemory, .contextSwitches, .throughput] + .arc,
+        warmupIterations: 0,
+        scalingFactor: .one,
+        maxDuration: .seconds(5),
+        maxIterations: 100
+    )
+
+    Benchmark.setup = {}
+
+    Benchmark.teardown = {}
+}

Benchmarks/Package.swift

@@ -0,0 +1,51 @@
+// swift-tools-version: 5.7
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the swift-kafka-client open source project
+//
+// Copyright (c) 2023 Apple Inc. and the swift-kafka-client project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of swift-kafka-client project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import PackageDescription
+
+let package = Package(
+    name: "benchmarks",
+    platforms: [
+        .macOS(.v13),
+    ],
+    dependencies: [
+        .package(path: "../"),
+        .package(url: "https://github.com/ordo-one/package-benchmark.git", from: "1.11.1"),
+    ],
+    targets: [
+        .executableTarget(
+            name: "SwiftKafkaConsumerBenchmarks",
+            dependencies: [
+                .product(name: "Benchmark", package: "package-benchmark"),
+                .product(name: "Kafka", package: "swift-kafka-client"),
+            ],
+            path: "Benchmarks/SwiftKafkaConsumerBenchmarks",
+            plugins: [
+                .plugin(name: "BenchmarkPlugin", package: "package-benchmark"),
+            ]
+        ),
+        .executableTarget(
+            name: "SwiftKafkaProducerBenchmarks",
+            dependencies: [
+                .product(name: "Benchmark", package: "package-benchmark"),
+                .product(name: "Kafka", package: "swift-kafka-client"),
+            ],
+            path: "Benchmarks/SwiftKafkaProducerBenchmarks",
+            plugins: [
+                .plugin(name: "BenchmarkPlugin", package: "package-benchmark"),
+            ]
+        ),
+    ]
+)

dev/update-benchmark-thresholds.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+##===----------------------------------------------------------------------===##
+##
+## This source file is part of the swift-kafka-client open source project
+##
+## Copyright (c) 2023 Apple Inc. and the swift-kafka-client project authors
+## Licensed under Apache License v2.0
+##
+## See LICENSE.txt for license information
+## See CONTRIBUTORS.txt for the list of swift-kafka-client project authors
+##
+## SPDX-License-Identifier: Apache-2.0
+##
+##===----------------------------------------------------------------------===##
+
+set -eu
+set -o pipefail
+
+here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+target_repo=${2-"$here/.."}
+
+for f in 57 58 59 510 -nightly; do
+    echo "swift$f"
+
+    docker_file=$(if [[ "$f" == "-nightly" ]]; then f=main; fi && ls "$target_repo/docker/docker-compose."*"$f"*".yaml")
+
+    docker-compose -f docker/docker-compose.yaml -f $docker_file run update-benchmark-baseline
+done

docker/Dockerfile

@@ -15,6 +15,8 @@ ENV LANGUAGE en_US.UTF-8
 # Dependencies
 RUN apt-get update
 RUN apt-get install libsasl2-dev -y
+RUN apt-get install libjemalloc-dev -y
+
 
 # tools
 RUN mkdir -p $HOME/.tools

docker/docker-compose.2204.510.yaml

@@ -19,5 +19,10 @@ services:
       - STRICT_CONCURRENCY_ARG=-Xswiftc -strict-concurrency=complete
       # - SANITIZER_ARG=--sanitize=thread # TSan broken still
 
+  update-benchmark-baseline:
+    image: swift-kafka-client:22.04-5.10
+    environment:
+      - SWIFT_VERSION=5.10
+
   shell:
     image: swift-kafka-client:22.04-5.10

docker/docker-compose.2204.57.yaml

@@ -19,5 +19,10 @@ services:
       - STRICT_CONCURRENCY_ARG=-Xswiftc -strict-concurrency=complete
       # - SANITIZER_ARG=--sanitize=thread # TSan broken still
 
+  update-benchmark-baseline:
+    image: swift-kafka-client:22.04-5.7
+    environment:
+      - SWIFT_VERSION=5.7
+
   shell:
     image: swift-kafka-client:22.04-5.7

docker/docker-compose.2204.58.yaml

@@ -20,5 +20,10 @@ services:
       - STRICT_CONCURRENCY_ARG=-Xswiftc -strict-concurrency=complete
       # - SANITIZER_ARG=--sanitize=thread # TSan broken still
 
+  update-benchmark-baseline:
+    image: swift-kafka-client:22.04-5.8
+    environment:
+      - SWIFT_VERSION=5.8
+
   shell:
     image: swift-kafka-client:22.04-5.8

docker/docker-compose.2204.59.yaml

@@ -20,5 +20,10 @@ services:
       - STRICT_CONCURRENCY_ARG=-Xswiftc -strict-concurrency=complete
       # - SANITIZER_ARG=--sanitize=thread # TSan broken still
 
+  update-benchmark-baseline:
+    image: swift-kafka-client:22.04-5.9
+    environment:
+      - SWIFT_VERSION=5.9
+
   shell:
     image: swift-kafka-client:22.04-5.9

docker/docker-compose.2204.main.yaml

@@ -16,5 +16,10 @@ services:
       - STRICT_CONCURRENCY_ARG=-Xswiftc -strict-concurrency=complete
       # - SANITIZER_ARG=--sanitize=thread # TSan broken still
 
+  update-benchmark-baseline:
+    image: swift-kafka-client:22.04-main
+    environment:
+      - SWIFT_VERSION=main
+
   shell:
     image: swift-kafka-client:22.04-main

docker/docker-compose.yaml

@@ -34,8 +34,8 @@ services:
     depends_on: [runtime-setup]
     volumes:
       - ~/.ssh:/root/.ssh
-      - ..:/code:z
-    working_dir: /code
+      - ..:/swift-kafka-client:z
+    working_dir: /swift-kafka-client
 
   soundness:
     <<: *common
@@ -54,9 +54,27 @@ services:
     command: >
       /bin/bash -xcl "
         swift build --build-tests $${SANITIZER_ARG-} && \
-        swift $${SWIFT_TEST_VERB-test} $${WARN_AS_ERROR_ARG-} $${SANITIZER_ARG-} $${IMPORT_CHECK_ARG-} $${STRICT_CONCURRENCY_ARG-}
+        swift $${SWIFT_TEST_VERB-test} $${WARN_AS_ERROR_ARG-} $${SANITIZER_ARG-} $${IMPORT_CHECK_ARG-} $${STRICT_CONCURRENCY_ARG-} && \
+        cd Benchmarks && swift package --disable-sandbox benchmark baseline check --check-absolute-path Thresholds/$${SWIFT_VERSION-}/
       "
 
+  benchmark:
+    <<: *common
+    depends_on: [kafka, runtime-setup]
+    environment:
+      KAFKA_HOST: kafka
+    command: >
+      /bin/bash -xcl "
+        cd Benchmarks && swift package --disable-sandbox benchmark
+      "
+
+  update-benchmark-baseline:
+    <<: *common
+    depends_on: [kafka, runtime-setup]
+    environment:
+      KAFKA_HOST: kafka
+    command: /bin/bash -xcl "cd Benchmarks && swift package --disable-sandbox --scratch-path .build/$${SWIFT_VERSION-}/ --allow-writing-to-package-directory benchmark --format metricP90AbsoluteThresholds --path Thresholds/$${SWIFT_VERSION-}/"
+
   # util
 
   shell: