Expose AWS_REQUEST_PAYER environment variable

CHANGELOG.md

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 - Add RasterSourceRDD.tiledLayerRDD within the geometry intersection [#3474](https://github.com/locationtech/geotrellis/pull/3474)
+- Expose AWS_REQUEST_PAYER environment variable [#3479](https://github.com/locationtech/geotrellis/pull/3479)
 
 ### Changed
 - Migration to CE3 and other major dependencies upgrade [#3389](https://github.com/locationtech/geotrellis/pull/3389)

s3-spark/src/main/resources/reference.conf → s3-spark/src/test/resources/reference.conf

@@ -1,4 +1,4 @@
-# Copyright 2016 Azavea
+# Copyright 2022 Azavea
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-geotrellis.s3.rdd.read.window-size = 1024
+geotrellis.s3.request-payer = ${?AWS_REQUEST_PAYER}

s3-spark/src/test/scala/geotrellis/spark/store/s3/MockS3Client.scala

@@ -16,7 +16,11 @@
 
 package geotrellis.spark.store.s3
 
+import geotrellis.store.s3._
+import geotrellis.store.s3.conf.S3Config
+
 import software.amazon.awssdk.auth.credentials.{AwsBasicCredentials, StaticCredentialsProvider}
+import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration
 import software.amazon.awssdk.services.s3.S3Client
 import software.amazon.awssdk.regions.Region
 
@@ -26,7 +30,10 @@ object MockS3Client {
   def apply(): S3Client = {
     val cred = AwsBasicCredentials.create("minio", "password")
     val credProvider = StaticCredentialsProvider.create(cred)
+    val overrideConfig = ClientOverrideConfiguration.builder().requestPayer(S3Config.requestPayer).build()
+
     S3Client.builder()
+      .overrideConfiguration(overrideConfig)
       .endpointOverride(new URI("http://localhost:9091"))
       .credentialsProvider(credProvider)
       .region(Region.US_EAST_1)

s3/src/main/resources/reference.conf

@@ -0,0 +1,15 @@
+# Copyright 2022 Azavea
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+geotrellis.s3.request-payer = ${?AWS_REQUEST_PAYER}

s3/src/main/scala/geotrellis/store/s3/S3ClientProducer.scala

@@ -16,6 +16,7 @@
 
 package geotrellis.store.s3
 
+import geotrellis.store.s3.conf.S3Config
 import software.amazon.awssdk.services.s3.S3Client
 import software.amazon.awssdk.core.retry.backoff.FullJitterBackoffStrategy
 import software.amazon.awssdk.core.retry.conditions.{OrRetryCondition, RetryCondition}
@@ -47,13 +48,14 @@ object S3ClientProducer {
         .build()
     val retryPolicy =
       RetryPolicy.defaultRetryPolicy()
-        .toBuilder()
+        .toBuilder
         .retryCondition(retryCondition)
         .backoffStrategy(backoffStrategy)
         .build()
     val overrideConfig =
       ClientOverrideConfiguration.builder()
         .retryPolicy(retryPolicy)
+        .requestPayer(S3Config.requestPayer)
         .build()
 
     S3Client.builder()

s3/src/main/scala/geotrellis/store/s3/conf/S3Config.scala

@@ -16,12 +16,15 @@
 
 package geotrellis.store.s3.conf
 
-import pureconfig.ConfigSource
+import pureconfig.{ConfigReader, ConfigSource}
 import pureconfig.generic.auto._
+import software.amazon.awssdk.services.s3.model.RequestPayer
 
-case class S3Config(windowsSize: Int = 1024)
+case class S3Config(requestPayer: Option[RequestPayer] = None)
 
 object S3Config {
-  lazy val conf: S3Config = ConfigSource.default.at("geotrellis.s3.rdd.read").loadOrThrow[S3Config]
+  implicit val requestPayerReader: ConfigReader[RequestPayer] = ConfigReader[String].map(RequestPayer.fromValue)
+
+  lazy val conf: S3Config = ConfigSource.default.at("geotrellis.s3").loadOrThrow[S3Config]
   implicit def s3ConfigToClass(obj: S3Config.type): S3Config = conf
 }

s3/src/main/scala/geotrellis/store/s3/package.scala

@@ -19,12 +19,18 @@ package geotrellis.store
 import geotrellis.raster.CellGrid
 import geotrellis.raster.io.geotiff.GeoTiff
 import geotrellis.raster.render.{Jpg, Png}
+
+import cats.syntax.either._
+import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration
 import software.amazon.awssdk.services.s3.S3Client
-import software.amazon.awssdk.services.s3.model.{NoSuchKeyException, HeadObjectRequest}
+import software.amazon.awssdk.services.s3.model.{HeadObjectRequest, NoSuchKeyException, RequestPayer}
 
-import scala.util.{Try, Success, Failure}
+import scala.util.Try
 
 package object s3 {
+  // https://github.com/aws/aws-sdk-java/blob/1.12.267/aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/Headers.java#L201
+  val REQUESTER_PAYS_HEADER: String = "x-amz-request-payer"
+
   private[geotrellis] def makePath(chunks: String*): String =
     chunks
       .collect { case str if str.nonEmpty => if(str.endsWith("/")) str.dropRight(1) else str }
@@ -36,14 +42,12 @@ package object s3 {
         .bucket(bucket)
         .key(key)
         .build()
-      val objectExists =
-        Try(client.headObject(request))
-          .map(_ => true)
-          .recoverWith({ case nske: NoSuchKeyException => Success(false) })
-      objectExists match {
-        case Success(bool) => bool
-        case Failure(throwable) => throw throwable
-      }
+
+      Try(client.headObject(request))
+        .map(_ => true)
+        .recover { case _: NoSuchKeyException => false }
+        .toEither
+        .valueOr(throw _)
     }
 
     def objectExists(path: String): Boolean = {
@@ -54,6 +58,11 @@ package object s3 {
     }
   }
 
+  implicit class ClientOverrideConfigurationBuilderOps(val self: ClientOverrideConfiguration.Builder) extends AnyVal {
+    def requestPayer(requestPayer: Option[RequestPayer]): ClientOverrideConfiguration.Builder =
+      requestPayer.map(_.toString).fold(self)(self.putHeader(REQUESTER_PAYS_HEADER, _))
+  }
+
   implicit class withJpgS3WriteMethods(val self: Jpg) extends JpgS3WriteMethods(self)
 
   implicit class withPngS3WriteMethods(val self: Png) extends PngS3WriteMethods(self)