test: enable integration tests against real S3 (fixes #1899) (#1951)

* test: enable integration tests against real S3 (fixes #1899)

* avoid overwriting task settins of root project.

* refactored method checking MinIO availability.

* added documentation about running integration tests against S3.

* split method checking backend availability for readability.

* fixed invalid endpoint URL in example cmdline.

Author: iwasakims

extensions/common/aws/aws-test/README.md

@@ -1,5 +1,7 @@
 # AWS Test
 
+## Local testing using MinIO
+
 To run AWS integration tests you will need a MinIO instance running:
 ```
 docker run -d -p 9000:9000 -e MINIO_ROOT_USER=root -e MINIO_ROOT_PASSWORD=password bitnami/minio:latest
@@ -10,3 +12,23 @@ Then set the two environment variables:
 S3_ACCESS_KEY_ID=root
 S3_SECRET_ACCESS_KEY=password
 ```
+
+## Test using your AWS credential
+
+`IT_AWS_ENDPOINT` can be used to override [endpoint](https://docs.aws.amazon.com/general/latest/gr/s3.html) URI
+for running integration tests against AWS S3 by environment variable:
+
+```
+$ IT_AWS_ENDPOINT=https://s3.us-east-1.amazonaws.com/ \
+  IT_AWS_REGION=us-east-1 \
+  IT_AWS_PROFILE=myprofie \
+  ./gradlew clean test -DincludeTags="AwsS3IntegrationTest" --tests '*S3StatusCheckerIntegrationTest'
+```
+
+`IT_AWS_REGION` must be set to your region code in order to avoid
+["A conflicting conditional operation is currently in progress against this resource." error](http://stackoverflow.com/questions/13898057/aws-error-message-a-conflicting-conditional-operation-is-currently-in-progress).
+
+`IT_AWS_PROFILE` can be used to specify
+[named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html)
+referring your own credential.
+You can also use access key and secret access key by `S3_ACCESS_KEY_ID` and `S3_SECRET_ACCESS_KEY`.

extensions/common/aws/aws-test/src/testFixtures/java/org/eclipse/dataspaceconnector/aws/testfixtures/AbstractS3Test.java

@@ -9,6 +9,7 @@
  *
  *  Contributors:
  *       Microsoft Corporation - initial API and implementation
+ *       NTT DATA - added endpoint override
  *
  */
 
@@ -59,13 +60,14 @@
  */
 public abstract class AbstractS3Test {
 
-    protected static final String REGION = System.getProperty("it.aws.region", Region.US_EAST_1.id());
+    protected static final String REGION = propOrEnv("it.aws.region", Region.US_EAST_1.id());
     // Adding REGION to bucket prevents errors of
     //      "A conflicting conditional operation is currently in progress against this resource."
     // when bucket is rapidly added/deleted and consistency propagation causes this error.
     // (Should not be necessary if REGION remains static, but added to prevent future frustration.)
     // [see http://stackoverflow.com/questions/13898057/aws-error-message-a-conflicting-conditional-operation-is-currently-in-progress]
-    protected static final URI S3_ENDPOINT = URI.create("http://localhost:9000");
+    protected static final String MINIO_ENDPOINT = "http://localhost:9000";
+    protected static final URI S3_ENDPOINT = URI.create(propOrEnv("it.aws.endpoint", MINIO_ENDPOINT));
     protected final UUID processId = UUID.randomUUID();
     protected String bucketName = createBucketName();
     protected S3AsyncClient s3AsyncClient;
@@ -83,15 +85,27 @@ static void prepareAll() {
                 .pollInterval(Duration.ofSeconds(2))
                 .ignoreException(IOException.class) // thrown by pingMinio
                 .ignoreException(ConnectException.class)
-                .until(AbstractS3Test::pingMinio);
+                .until(AbstractS3Test::isBackendAvailable);
+    }
+
+    private static boolean isBackendAvailable() throws IOException {
+        if (isMinio()) {
+            return isMinioAvailable();
+        } else {
+            return true;
+        }
+    }
+
+    private static boolean isMinio() {
+        return MINIO_ENDPOINT.equals(S3_ENDPOINT.toString());
     }
 
     /**
      * pings <a href="https://docs.min.io/minio/baremetal/monitoring/healthcheck-probe.html">MinIO's health endpoint</a>
      *
      * @return true if HTTP status [200..300[
      */
-    private static boolean pingMinio() throws IOException {
+    private static boolean isMinioAvailable() throws IOException {
         var httpClient = new OkHttpClient();
         var healthRq = new Request.Builder().url(S3_ENDPOINT + "/minio/health/live").get().build();
         try (var response = httpClient.newCall(healthRq).execute()) {
@@ -158,7 +172,7 @@ protected void putStringOnBucket(String bucketName, String key, String content)
     }
 
     protected @NotNull AwsCredentials getCredentials() {
-        String profile = propOrEnv("AWS_PROFILE", null);
+        var profile = propOrEnv("it.aws.profile", null);
         if (profile != null) {
             return ProfileCredentialsProvider.create(profile).resolveCredentials();
         }