feat(sql): add SQL bootstrapper

DEPENDENCIES

@@ -1,4 +1,4 @@
 maven/mavencentral/com.apicatalog/carbon-did/0.3.0, Apache-2.0, approved, clearlydefined
 maven/mavencentral/com.apicatalog/copper-multibase/0.5.0, Apache-2.0, approved, #14501
 maven/mavencentral/com.apicatalog/copper-multicodec/0.1.1, Apache-2.0, approved, #14500
 maven/mavencentral/com.apicatalog/iron-ed25519-cryptosuite-2020/0.14.0, Apache-2.0, approved, #14503
@@ -218,7 +218,7 @@
 maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.13, Apache-2.0, approved, CQ23528
 maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.14, Apache-2.0, approved, CQ23528
 maven/mavencentral/org.apache.httpcomponents/httpmime/4.5.13, Apache-2.0, approved, CQ11718
-maven/mavencentral/org.apache.kafka/kafka-clients/3.7.1, Apache-2.0 AND (Apache-2.0 AND MIT) AND (Apache-2.0 AND BSD-3-Clause), approved, #13623
+maven/mavencentral/org.apache.kafka/kafka-clients/3.8.0, Apache-2.0, restricted, clearlydefined
 maven/mavencentral/org.apache.maven.doxia/doxia-core/1.12.0, Apache-2.0, approved, clearlydefined
 maven/mavencentral/org.apache.maven.doxia/doxia-logging-api/1.12.0, Apache-2.0, approved, clearlydefined
 maven/mavencentral/org.apache.maven.doxia/doxia-module-xdoc/1.12.0, Apache-2.0, approved, clearlydefined
@@ -291,10 +291,10 @@
 maven/mavencentral/org.hamcrest/hamcrest/2.1, BSD-3-Clause, approved, clearlydefined
 maven/mavencentral/org.hamcrest/hamcrest/2.2, BSD-3-Clause, approved, clearlydefined
 maven/mavencentral/org.hdrhistogram/HdrHistogram/2.2.2, BSD-2-Clause AND CC0-1.0 AND CC0-1.0, approved, #14828
-maven/mavencentral/org.jacoco/org.jacoco.agent/0.8.9, EPL-2.0, approved, CQ23285
-maven/mavencentral/org.jacoco/org.jacoco.ant/0.8.9, EPL-2.0, approved, #1068
-maven/mavencentral/org.jacoco/org.jacoco.core/0.8.9, EPL-2.0, approved, CQ23283
-maven/mavencentral/org.jacoco/org.jacoco.report/0.8.9, EPL-2.0 AND Apache-2.0, approved, CQ23284
+maven/mavencentral/org.jacoco/org.jacoco.agent/0.8.11, EPL-2.0, approved, CQ23285
+maven/mavencentral/org.jacoco/org.jacoco.ant/0.8.11, EPL-2.0, approved, #1068
+maven/mavencentral/org.jacoco/org.jacoco.core/0.8.11, EPL-2.0, approved, CQ23283
+maven/mavencentral/org.jacoco/org.jacoco.report/0.8.11, EPL-2.0 AND Apache-2.0, approved, CQ23284
 maven/mavencentral/org.javassist/javassist/3.28.0-GA, Apache-2.0 OR LGPL-2.1-or-later OR MPL-1.1, approved, #327
 maven/mavencentral/org.javassist/javassist/3.30.2-GA, Apache-2.0 AND LGPL-2.1-or-later AND MPL-1.1, approved, #12108
 maven/mavencentral/org.jetbrains.kotlin/kotlin-stdlib-common/1.9.10, Apache-2.0, approved, #14186
@@ -326,12 +326,12 @@
 maven/mavencentral/org.mozilla/rhino/1.7.7.2, MPL-2.0 AND BSD-3-Clause AND ISC, approved, CQ16320
 maven/mavencentral/org.objenesis/objenesis/3.3, Apache-2.0, approved, clearlydefined
 maven/mavencentral/org.opentest4j/opentest4j/1.3.0, Apache-2.0, approved, #9713
-maven/mavencentral/org.ow2.asm/asm-commons/9.5, BSD-3-Clause, approved, #7553
+maven/mavencentral/org.ow2.asm/asm-commons/9.6, BSD-3-Clause, approved, #10775
 maven/mavencentral/org.ow2.asm/asm-commons/9.7, BSD-3-Clause, approved, #14075
-maven/mavencentral/org.ow2.asm/asm-tree/9.5, BSD-3-Clause, approved, #7555
+maven/mavencentral/org.ow2.asm/asm-tree/9.6, BSD-3-Clause, approved, #10773
 maven/mavencentral/org.ow2.asm/asm-tree/9.7, BSD-3-Clause, approved, #14073
 maven/mavencentral/org.ow2.asm/asm/9.1, BSD-3-Clause, approved, CQ23029
-maven/mavencentral/org.ow2.asm/asm/9.5, BSD-3-Clause, approved, #7554
+maven/mavencentral/org.ow2.asm/asm/9.6, BSD-3-Clause, approved, #10776
 maven/mavencentral/org.ow2.asm/asm/9.7, BSD-3-Clause, approved, #14076
 maven/mavencentral/org.postgresql/postgresql/42.7.3, BSD-2-Clause AND Apache-2.0, approved, #11681
 maven/mavencentral/org.reflections/reflections/0.10.2, Apache-2.0 AND WTFPL, approved, clearlydefined
@@ -349,7 +349,7 @@
 maven/mavencentral/org.testcontainers/junit-jupiter/1.20.0, MIT, approved, clearlydefined
 maven/mavencentral/org.testcontainers/kafka/1.20.0, MIT, approved, clearlydefined
 maven/mavencentral/org.testcontainers/postgresql/1.20.0, MIT, approved, clearlydefined
-maven/mavencentral/org.testcontainers/testcontainers/1.20.0, None, restricted, #15747
+maven/mavencentral/org.testcontainers/testcontainers/1.20.0, MIT, approved, #15747
 maven/mavencentral/org.testcontainers/vault/1.20.0, MIT, approved, clearlydefined
 maven/mavencentral/org.xerial.snappy/snappy-java/1.1.10.5, Apache-2.0 AND (Apache-2.0 AND BSD-3-Clause), approved, #9098
 maven/mavencentral/org.xmlresolver/xmlresolver/5.2.2, Apache-2.0, approved, clearlydefined

extensions/common/sql/sql-bootstrapper/build.gradle.kts

@@ -0,0 +1,31 @@
+/*
+ *  Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation
+ *
+ */
+
+plugins {
+    `java-library`
+    `java-test-fixtures`
+    `maven-publish`
+}
+
+dependencies {
+    api(project(":spi:common:core-spi"))
+    api(project(":spi:common:transaction-spi"))
+    implementation(project(":spi:common:transaction-datasource-spi"))
+    implementation(project(":extensions:common:sql:sql-core")) // SqlQueryExecutor
+//
+    testImplementation(project(":core:common:junit"))
+    testImplementation(libs.assertj)
+}
+
+

extensions/common/sql/sql-bootstrapper/src/main/java/org/eclipse/edc/sql/bootstrapper/QueuedStatementRecord.java

@@ -0,0 +1,18 @@
+/*
+ *  Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation
+ *
+ */
+
+package org.eclipse.edc.sql.bootstrapper;
+
+record QueuedStatementRecord(String name, String datasourceName, String sql) {
+}

extensions/common/sql/sql-bootstrapper/src/main/java/org/eclipse/edc/sql/bootstrapper/SqlDmlStatementRunner.java

@@ -0,0 +1,83 @@
+/*
+ *  Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation
+ *
+ */
+
+package org.eclipse.edc.sql.bootstrapper;
+
+import org.eclipse.edc.spi.monitor.Monitor;
+import org.eclipse.edc.spi.persistence.EdcPersistenceException;
+import org.eclipse.edc.spi.result.Result;
+import org.eclipse.edc.sql.QueryExecutor;
+import org.eclipse.edc.transaction.datasource.spi.DataSourceRegistry;
+import org.eclipse.edc.transaction.spi.TransactionContext;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Map;
+
+import static org.eclipse.edc.spi.result.Result.failure;
+import static org.eclipse.edc.spi.result.Result.success;
+
+/**
+ * Internal class to the SQL Bootstrapper Extension module with the intended purpose to execute a series of DML statements against
+ * the database.
+ */
+class SqlDmlStatementRunner {
+
+    private final TransactionContext transactionContext;
+    private final QueryExecutor queryExecutor;
+    private final Monitor monitor;
+    private final DataSourceRegistry dataSourceRegistry;
+
+    SqlDmlStatementRunner(TransactionContext transactionContext, QueryExecutor queryExecutor, Monitor monitor, DataSourceRegistry dataSourceRegistry) {
+        this.transactionContext = transactionContext;
+        this.queryExecutor = queryExecutor;
+        this.monitor = monitor;
+        this.dataSourceRegistry = dataSourceRegistry;
+    }
+
+    /**
+     * Executes the queued DML statements one after the other. This method is intended to be called only from the {@link SqlSchemaBootstrapperExtension}.
+     *
+     * @param statements A map containing the datasource name as key and the SQL statements as value
+     * @return A summary result of all the statements.
+     */
+    public Result<Void> executeSql(Map<String, List<String>> statements) {
+        monitor.debug("Running DML statements: [%s]".formatted(String.join(", ", statements.keySet())));
+        return transactionContext.execute(() -> statements.entrySet().stream()
+                .map(statement -> {
+                    var connectionResult = getConnection(statement.getKey());
+                    return connectionResult.compose(connection -> {
+                        try {
+                            queryExecutor.execute(connection, String.join("", statement.getValue()));
+                        } catch (EdcPersistenceException sqlException) {
+                            return failure(sqlException.getMessage());
+                        }
+                        return success();
+                    });
+                })
+                .reduce(Result::merge)
+                .orElse(Result.success()));
+    }
+
+    public Result<Connection> getConnection(String datasourceName) {
+        try {
+            var resolve = dataSourceRegistry.resolve(datasourceName);
+            return resolve != null ? success(resolve.getConnection()) :
+                    failure("No datasource found with name '%s'".formatted(datasourceName));
+        } catch (SQLException e) {
+            return failure(e.getMessage());
+        }
+    }
+}

extensions/common/sql/sql-bootstrapper/src/main/java/org/eclipse/edc/sql/bootstrapper/SqlSchemaBootstrapper.java

@@ -0,0 +1,59 @@
+/*
+ *  Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation
+ *
+ */
+
+package org.eclipse.edc.sql.bootstrapper;
+
+import org.eclipse.edc.spi.system.ServiceExtension;
+import org.eclipse.edc.spi.system.ServiceExtensionContext;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Provides a convenient way to create database structures in an SQL database. DML statements can be added in the {@code initialize()}
+ * phase of extensions and the bootstrapper takes care of executing them against the database.
+ */
+public interface SqlSchemaBootstrapper {
+    /**
+     * Extensions that operate a store based on an SQL database and thus require a certain database structure to be present,
+     * can use this class to have their schema auto-generated. The entire DDL has to be in a file that is available from the resources.
+     * <p>
+     * Note that all DDL statements <strong>must</strong> be queued during the {@link ServiceExtension#initialize(ServiceExtensionContext)} phase and
+     * individual statements <strong>must not</strong> rely on ordering, since that depends on extension ordering.
+     *
+     * @param datasourceName The name of the datasource against which the statements are to be run
+     * @param resourceName   An SQL DDL statement. Cannot contain prepared statements. Do not add DML statements here!
+     */
+    default void addStatementFromResource(String datasourceName, String resourceName) {
+        addStatementFromResource(datasourceName, resourceName, getClass().getClassLoader());
+    }
+
+    /**
+     * Extensions that operate a store based on an SQL database and thus require a certain database structure to be present,
+     * can use this class to have their schema auto-generated. The entire DDL has to be in a file that is available from the resources.
+     * <p>
+     * Note that all DDL statements <strong>must</strong> be queued during the {@link ServiceExtension#initialize(ServiceExtensionContext)} phase and
+     * individual statements <strong>must not</strong> rely on ordering, since that depends on extension ordering.
+     *
+     * @param datasourceName The name of the datasource against which the statements are to be run
+     * @param resourceName   An SQL DDL statement. Cannot contain prepared statements. Do not add DML statements here!
+     * @param classLoader    A classloader which is used to resolve the resource
+     */
+    void addStatementFromResource(String datasourceName, String resourceName, ClassLoader classLoader);
+
+    /**
+     * Gets all registered DML statements as a map where the datasource name is the key, and the SQL statement(s) is the value.
+     */
+    Map<String, List<String>> getStatements();
+}

extensions/common/sql/sql-bootstrapper/src/main/java/org/eclipse/edc/sql/bootstrapper/SqlSchemaBootstrapperExtension.java

@@ -0,0 +1,72 @@
+/*
+ *  Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation
+ *
+ */
+
+package org.eclipse.edc.sql.bootstrapper;
+
+import org.eclipse.edc.runtime.metamodel.annotation.Extension;
+import org.eclipse.edc.runtime.metamodel.annotation.Inject;
+import org.eclipse.edc.runtime.metamodel.annotation.Provider;
+import org.eclipse.edc.spi.monitor.Monitor;
+import org.eclipse.edc.spi.persistence.EdcPersistenceException;
+import org.eclipse.edc.spi.system.ServiceExtension;
+import org.eclipse.edc.spi.system.ServiceExtensionContext;
+import org.eclipse.edc.sql.QueryExecutor;
+import org.eclipse.edc.transaction.datasource.spi.DataSourceRegistry;
+import org.eclipse.edc.transaction.spi.TransactionContext;
+
+import static org.eclipse.edc.sql.bootstrapper.SqlSchemaBootstrapperExtension.NAME;
+
+@Extension(value = NAME, categories = { "sql", "persistence", "storage" })
+public class SqlSchemaBootstrapperExtension implements ServiceExtension {
+    public static final String NAME = "SQL Schema Bootstrapper Extension";
+    public static final String SCHEMA_AUTOCREATE_PROPERTY = "edc.sql.schema.autocreate";
+    public static final boolean SCHEMA_AUTOCREATE_DEFAULT = false;
+
+    @Inject
+    private TransactionContext transactionContext;
+    @Inject
+    private QueryExecutor queryExecutor;
+    @Inject
+    private DataSourceRegistry datasourceRegistry;
+    @Inject
+    private Monitor monitor;
+
+    private SqlSchemaBootstrapperImpl bootstrapper;
+    private Boolean shouldAutoCreate;
+
+    @Override
+    public void initialize(ServiceExtensionContext context) {
+        shouldAutoCreate = context.getConfig().getBoolean(SCHEMA_AUTOCREATE_PROPERTY, SCHEMA_AUTOCREATE_DEFAULT);
+    }
+
+    @Override
+    public void prepare() {
+        if (shouldAutoCreate) {
+            var statements = getBootstrapper().getStatements();
+            new SqlDmlStatementRunner(transactionContext, queryExecutor, monitor, datasourceRegistry).executeSql(statements)
+                    .orElseThrow(f -> new EdcPersistenceException("Failed to bootstrap SQL schema, error '%s'".formatted(f.getFailureDetail())));
+
+        } else {
+            monitor.debug("Automatic SQL schema creation is disabled. To enable it, set '%s' = true".formatted(SCHEMA_AUTOCREATE_PROPERTY));
+        }
+    }
+
+    @Provider
+    public SqlSchemaBootstrapper getBootstrapper() {
+        if (bootstrapper == null) {
+            bootstrapper = new SqlSchemaBootstrapperImpl();
+        }
+        return bootstrapper;
+    }
+}

extensions/common/sql/sql-bootstrapper/src/main/java/org/eclipse/edc/sql/bootstrapper/SqlSchemaBootstrapperImpl.java

@@ -0,0 +1,48 @@
+/*
+ *  Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation
+ *
+ */
+
+package org.eclipse.edc.sql.bootstrapper;
+
+import org.eclipse.edc.spi.EdcException;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Scanner;
+
+import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.mapping;
+import static java.util.stream.Collectors.toList;
+
+public class SqlSchemaBootstrapperImpl implements SqlSchemaBootstrapper {
+
+    private final List<QueuedStatementRecord> statements = new ArrayList<>();
+
+    @Override
+    public void addStatementFromResource(String datasourceName, String resourceName, ClassLoader classLoader) {
+        try (var sqlStream = classLoader.getResourceAsStream(resourceName); var scanner = new Scanner(Objects.requireNonNull(sqlStream)).useDelimiter("\\A")) {
+            var sql = scanner.next();
+            statements.add(new QueuedStatementRecord(resourceName, datasourceName, sql));
+        } catch (IOException e) {
+            throw new EdcException(e);
+        }
+    }
+
+    @Override
+    public Map<String, List<String>> getStatements() {
+        return statements.stream().collect(groupingBy(QueuedStatementRecord::datasourceName, mapping(QueuedStatementRecord::sql, toList())));
+    }
+}

extensions/common/sql/sql-bootstrapper/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension

@@ -0,0 +1,15 @@
+#
+#  Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
+#
+#  This program and the accompanying materials are made available under the
+#  terms of the Apache License, Version 2.0 which is available at
+#  https://www.apache.org/licenses/LICENSE-2.0
+#
+#  SPDX-License-Identifier: Apache-2.0
+#
+#  Contributors:
+#       Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation
+#
+#
+
+org.eclipse.edc.sql.bootstrapper.SqlSchemaBootstrapperExtension