From c18fc7b2f109609f0fe2fe03b57d1ce74bca2b5d Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Thu, 7 May 2026 09:19:41 +0100 Subject: [PATCH 01/46] Disable lower environment. --- image-loader/app/model/Uploader.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala index 6b3f722d71..e50089c98d 100644 --- a/image-loader/app/model/Uploader.scala +++ b/image-loader/app/model/Uploader.scala @@ -475,6 +475,7 @@ class Uploader( )) // TODO: centralise where all these URLs are constructed } yield { + /* config.maybeLowerEnvironmentQueueBucketToSampleInto.foreach { lowerEnvironmentQueueBucket => if (math.random() < config.lowerEnvironmentSamplingPercentageAsDecimal) { val mediaId = imageUpload.image.id @@ -494,7 +495,7 @@ class Uploader( } } } - + */ UploadStatusUri(s"${config.rootUri(instance)}/uploadStatus/${uploadRequest.imageId}") } From 0a5609a79ff05ebc8a1cf5f14ceace1a75ae2bf1 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Sat, 8 Aug 2026 23:19:43 +0100 Subject: [PATCH 02/46] S3 supplies deleteObject. --- common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index c8c264a2e5..19e9740e07 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -226,6 +226,9 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with } } + def deleteObject(bucket: Bucket, key: String): Unit = + client.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(key).build()) + } object S3Ops { From fcd6bde28e8f1fe64503c6e964a6f36159a9ef08 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Sun, 9 Aug 2026 12:15:15 +0100 Subject: [PATCH 03/46] S3 provides deleteVersion --- common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 19e9740e07..08ab80f306 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -229,6 +229,9 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with def deleteObject(bucket: Bucket, key: String): Unit = client.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(key).build()) + def deleteVersion(bucket: Bucket, key: String, objectVersion: String): Unit = + client.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(key).versionId(objectVersion).build()) + } object S3Ops { From 2d22f0208587256856a90db6c86f71631e781688 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Sat, 8 Aug 2026 23:17:40 +0100 Subject: [PATCH 04/46] S3ImageStorage uses deleteObject on S3 class. Removes reference to the underlying S3 client. --- .../scala/com/gu/mediaservice/lib/S3ImageStorage.scala | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala index acaf5503db..8e66090ab3 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala @@ -28,9 +28,8 @@ class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage } def deleteImage(bucket: String, key: String)(implicit logMarker: LogMarker) = Future { + deleteObject(bucket, key) logger.info(logMarker, s"Deleted image $key from bucket $bucket") - client.deleteObject( - DeleteObjectRequest.builder().bucket(bucket).key(key).build()) } def deleteVersionedImage(bucket: String, id: String)(implicit logMarker: LogMarker) = Future { @@ -44,10 +43,8 @@ class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage ListObjectsV2Request.builder().bucket(bucket).prefix(id).build() ).contents().asScala.toList logger.info(s"Found ${files.size} files to delete in folder $id") - files.foreach(file => client.deleteObject( - DeleteObjectRequest.builder().bucket(bucket).key(file.key()).build() - )) - logger.info(logMarker, s"Deleting images in folder $id from bucket $bucket") + files.foreach(file => deleteObject(bucket, file.key())) + logger.info(logMarker, s"Deleting images in folder $id from bucket $bucket") } } From 0bbea22fe0ec44e491ba326be186a7f8c2b64123 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Sun, 9 Aug 2026 12:31:58 +0100 Subject: [PATCH 05/46] S3Metadata includes the objectVersion field so that getMetadata can be used to read the object version. --- .../src/main/scala/com/gu/mediaservice/lib/aws/S3.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 08ab80f306..f3b6955349 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -48,7 +48,7 @@ object S3Object { } } -case class S3Metadata(userMetadata: Map[String, String], objectMetadata: S3ObjectMetadata) +case class S3Metadata(userMetadata: Map[String, String], objectMetadata: S3ObjectMetadata, objectVersion: Option[String] = None) object S3Metadata { def apply(meta: HeadObjectResponse): S3Metadata = { @@ -58,7 +58,8 @@ object S3Metadata { contentType = Option(meta.contentType()).filterNot(_.toLowerCase == "application/octet-stream").map(MimeType.apply), cacheControl = Option(meta.cacheControl()), lastModified = Option(meta.lastModified()).map(l => new DateTime(l.toEpochMilli).withZone(DateTimeZone.UTC)) - ) + ), + objectVersion = Option(meta.versionId()) ) } } From a0c0eb862f025ab1381d4b5c3555b2681a87c66f Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Sun, 9 Aug 2026 12:32:29 +0100 Subject: [PATCH 06/46] S3ImageStorage uses S3 getMetadata to read object version. Removes reference to the underlying S3 client. --- .../scala/com/gu/mediaservice/lib/S3ImageStorage.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala index 8e66090ab3..10f3a8bb71 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala @@ -4,7 +4,7 @@ import com.gu.mediaservice.lib.aws.S3 import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} import com.gu.mediaservice.model.MimeType -import software.amazon.awssdk.services.s3.model.{DeleteObjectRequest, HeadObjectRequest, ListObjectsV2Request} +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request import java.io.File import scala.concurrent.Future @@ -33,8 +33,10 @@ class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage } def deleteVersionedImage(bucket: String, id: String)(implicit logMarker: LogMarker) = Future { - val objectVersion = client.headObject(HeadObjectRequest.builder().bucket(bucket).key(id).build()).versionId() - client.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(id).versionId(objectVersion).build()) + val objectVersion = getMetadata(bucket, id).objectVersion.getOrElse( + throw new IllegalStateException(s"No version id found for $id in bucket $bucket") + ) + deleteVersion(bucket, id, objectVersion) logger.info(logMarker, s"Deleted image $id from bucket $bucket (version: $objectVersion)") } From 1870876193b2aff582859cce3ca4fded6d8082cb Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Sun, 9 Aug 2026 12:57:40 +0100 Subject: [PATCH 07/46] S3ImageStorage deleteFolder uses S3 list to get folder files to delete. --- .../scala/com/gu/mediaservice/lib/S3ImageStorage.scala | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala index 10f3a8bb71..bacd17b615 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala @@ -40,13 +40,10 @@ class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage logger.info(logMarker, s"Deleted image $id from bucket $bucket (version: $objectVersion)") } - def deleteFolder(bucket: String, id: String)(implicit logMarker: LogMarker) = Future { - val files = client.listObjectsV2( - ListObjectsV2Request.builder().bucket(bucket).prefix(id).build() - ).contents().asScala.toList + def deleteFolder(bucket: String, id: String)(implicit logMarker: LogMarker): Future[Unit] = list(bucket, id).map { files => logger.info(s"Found ${files.size} files to delete in folder $id") - files.foreach(file => deleteObject(bucket, file.key())) + files.foreach(file => deleteObject(bucket, file.uri.getPath.stripPrefix("/"))) logger.info(logMarker, s"Deleting images in folder $id from bucket $bucket") - } + } } From bc7ffd2c273c92bda6434060ae9c8386cda01fa5 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Mon, 10 Aug 2026 19:58:38 +0100 Subject: [PATCH 08/46] S3 provides deleteObjects so that ImageIngestOperations.bulkDeleteV2 can release it's client usage. --- .../lib/ImageIngestOperations.scala | 18 +---------------- .../com/gu/mediaservice/lib/aws/S3.scala | 20 ++++++++++++++++++- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala index ef96e09cf1..1f65d3f441 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala @@ -7,11 +7,9 @@ import com.gu.mediaservice.lib.logging.LogMarker import com.gu.mediaservice.model.{Instance, MimeType} import com.typesafe.scalalogging.StrictLogging import org.joda.time.DateTime -import software.amazon.awssdk.services.s3.model.{Delete, DeleteObjectsRequest, ObjectIdentifier} import java.io.File import scala.concurrent.Future -import scala.jdk.CollectionConverters._ object ImageIngestOperations { def fileKeyFromId(id: String)(implicit instance: Instance): String = instance.id + "/" + snippetForId(id) @@ -57,24 +55,10 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config overwrite = true) } - private def bulkDelete(bucket: String, keys: List[String]): Future[Map[String, Boolean]] = keys match { case Nil => Future.successful(Map.empty) case _ => Future { - val objects = keys.map { key => - ObjectIdentifier.builder() - .key(key) - .build() - }.asJava - val response = client.deleteObjects( - DeleteObjectsRequest.builder().bucket(bucket) - .delete(Delete.builder().objects(objects).build()) - .build() - ) - val errorKeys = response.errors().asScala.toList.map(_.key()) - keys.map { key => - key -> !errorKeys.contains(key) - }.toMap + deleteObjects(bucket, keys) } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index f3b6955349..297e52ae6b 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -7,15 +7,16 @@ import org.joda.time.{DateTime, DateTimeZone} import software.amazon.awssdk.core.ResponseInputStream import software.amazon.awssdk.core.sync.RequestBody import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.s3.{S3Client, S3Configuration} import software.amazon.awssdk.services.s3.model._ import software.amazon.awssdk.services.s3.presigner.S3Presigner import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest -import software.amazon.awssdk.services.s3.{S3Client, S3Configuration} import java.io.File import java.net.{URI, URL} import java.nio.charset.StandardCharsets import java.time.Duration +import java.util import scala.concurrent.{ExecutionContext, Future} import scala.jdk.CollectionConverters._ @@ -230,6 +231,23 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with def deleteObject(bucket: Bucket, key: String): Unit = client.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(key).build()) + def deleteObjects(bucket: Bucket, keys: List[String]): Map[String, Boolean] = { + val objects: util.List[ObjectIdentifier] = keys.map { key => + ObjectIdentifier.builder() + .key(key) + .build() + }.asJava + val response = client.deleteObjects( + DeleteObjectsRequest.builder().bucket(bucket) + .delete(Delete.builder().objects(objects).build()) + .build() + ) + val errorKeys = response.errors().asScala.toList.map(_.key()) + keys.map { key => + key -> !errorKeys.contains(key) + }.toMap + } + def deleteVersion(bucket: Bucket, key: String, objectVersion: String): Unit = client.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(key).versionId(objectVersion).build()) From bf06d1860ae88b3be49e95cf17df9e5412a556fe Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Mon, 10 Aug 2026 19:59:19 +0100 Subject: [PATCH 09/46] ImageLoaderStore use S3.deleteObject to release client usage. --- image-loader/app/lib/ImageLoaderStore.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/image-loader/app/lib/ImageLoaderStore.scala b/image-loader/app/lib/ImageLoaderStore.scala index 455a833716..db2e2f5792 100644 --- a/image-loader/app/lib/ImageLoaderStore.scala +++ b/image-loader/app/lib/ImageLoaderStore.scala @@ -77,10 +77,8 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati logger.warn(logMarker, s"Attempted to copy $key from ingest bucket to fail bucket, but it does not exist.") } - def deleteObjectFromIngestBucket(key: String)(implicit logMarker: LogMarker) = handleNotFound(key) { - client.deleteObject( - DeleteObjectRequest.builder().bucket(config.maybeIngestBucket.get).key(key).build()) - () + def deleteObjectFromIngestBucket(key: String)(implicit logMarker: LogMarker): Unit = handleNotFound(key) { + deleteObject(config.maybeIngestBucket.get, key) } { logger.warn(logMarker, s"Attempted to delete $key from ingest bucket, but it does not exist.") } From 51384d1cc9f2a8cab6a35af2c560f571d4bc4b28 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Mon, 10 Aug 2026 20:08:06 +0100 Subject: [PATCH 10/46] ReaperController uses S3.list --- thrall/app/controllers/ReaperController.scala | 38 ++++++++----------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/thrall/app/controllers/ReaperController.scala b/thrall/app/controllers/ReaperController.scala index c4ae8367e1..2ed3246039 100644 --- a/thrall/app/controllers/ReaperController.scala +++ b/thrall/app/controllers/ReaperController.scala @@ -19,11 +19,9 @@ import play.api.libs.json.{JsValue, Json, OWrites} import play.api.libs.ws.WSClient import play.api.mvc.{Action, AnyContent, ControllerComponents} import scalaz.NonEmptyList -import software.amazon.awssdk.services.s3.model.ListObjectsV2Request import scala.concurrent.duration.DurationInt import scala.concurrent.{ExecutionContext, Future} -import scala.jdk.CollectionConverters._ import scala.language.postfixOps import scala.util.control.NonFatal import scala.util.{Failure, Success} @@ -200,31 +198,27 @@ class ReaperController( }.toMap }).map(Json.toJson(_)) } - def index = withLoginRedirect { + def index: Action[AnyContent] = withLoginRedirectAsync { val now = DateTime.now(DateTimeZone.UTC) (config.maybeReaperBucket, config.maybeReaperCountPerRun) match { - case (None, _) => NotImplemented("'s3.reaper.bucket' not configured in thrall.conf") - case (_, None) => NotImplemented("'reaper.countPerRun' not configured in thrall.conf") + case (None, _) => Future.successful(NotImplemented("'s3.reaper.bucket' not configured in thrall.conf")) + case (_, None) => Future.successful(NotImplemented("'reaper.countPerRun' not configured in thrall.conf")) case (Some(reaperBucket), Some(countOfImagesToReap)) => - val recentRecords = List(now, now.minusDays(1), now.minusDays(2)).flatMap { day => + Future.sequence(List(now, now.minusDays(1), now.minusDays(2)).map { day => val s3DirName = s3DirNameFromDate(day) - val softDeletes = store.client.listObjectsV2( - ListObjectsV2Request.builder().bucket(reaperBucket).prefix(s"soft/$s3DirName/").build() - ).contents().asScala.toList - - val hardDeletes = store.client.listObjectsV2( - ListObjectsV2Request.builder().bucket(reaperBucket).prefix(s"hard/$s3DirName/").build() - ).contents().asScala.toList - - softDeletes ++ hardDeletes + for { + softDeletes <- store.list(reaperBucket, s"soft/$s3DirName") + hardDeletes <- store.list(reaperBucket, s"hard/$s3DirName") + } yield softDeletes ++ hardDeletes + }).map { recentRecords => + val recentRecordKeys = recentRecords.flatten + .filter(_.metadata.objectMetadata.lastModified.exists(_ isAfter now.minusHours(48))) + .sortBy(_.metadata.objectMetadata.lastModified.map(_.getMillis)) + .reverse + .map(_.uri.getPath.stripPrefix("/")) + + Ok(views.html.reaper(isPaused, INTERVAL.toString(), countOfImagesToReap, recentRecordKeys)) } - val recentRecordKeys = recentRecords - .filter(_.lastModified() isAfter now.minusHours(48).toDate.toInstant) - .sortBy(_.lastModified()) - .reverse - .map(_.key()) - - Ok(views.html.reaper(isPaused, INTERVAL.toString(), countOfImagesToReap, recentRecordKeys)) }} def reaperRecord(key: String) = auth { config.maybeReaperBucket match { From 0c73f9dea544a161b66de7c75285f3ec07aa9322 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Mon, 10 Aug 2026 20:19:44 +0100 Subject: [PATCH 11/46] KeyStore uses S3.list --- .../scala/com/gu/mediaservice/lib/auth/KeyStore.scala | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala index 2bd89378de..49f48bcbec 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala @@ -3,10 +3,9 @@ package com.gu.mediaservice.lib.auth import com.gu.mediaservice.lib.BaseStore import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.model.Instance -import software.amazon.awssdk.services.s3.model.ListObjectsV2Request -import scala.jdk.CollectionConverters._ -import scala.concurrent.ExecutionContext +import scala.concurrent.duration._ +import scala.concurrent.{Await, ExecutionContext} class KeyStore(bucket: String, config: CommonConfig)(implicit ec: ExecutionContext) extends BaseStore[String, ApiAccessor](bucket, config)(ec) { @@ -20,9 +19,8 @@ class KeyStore(bucket: String, config: CommonConfig)(implicit ec: ExecutionConte } private def fetchAll: Map[String, ApiAccessor] = { - val contents = s3.client.listObjectsV2(ListObjectsV2Request.builder().bucket(bucket).build()) - .contents().asScala.toList - val keys = contents.map(_.key()) + val objects = Await.result(s3.list(bucket, ""), 10.seconds) + val keys = objects.map(_.uri.getPath.stripPrefix("/")) keys.flatMap(k => getS3Object(k).map(k -> ApiAccessor(_))).toMap } } From 294228ffbfee24f9b6fcd280626e754371c70abc Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Mon, 10 Aug 2026 20:22:12 +0100 Subject: [PATCH 12/46] Unused getLatestS3Stream. --- .../com/gu/mediaservice/lib/BaseStore.scala | 24 +------------------ 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala index 9c6aa257b5..f72d2a1400 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala @@ -1,15 +1,12 @@ package com.gu.mediaservice.lib -import org.apache.pekko.actor.{Cancellable, Scheduler} import com.gu.mediaservice.lib.aws.S3 import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.GridLogging +import org.apache.pekko.actor.{Cancellable, Scheduler} import org.joda.time.DateTime -import software.amazon.awssdk.services.s3.model.{GetObjectRequest, ListObjectsV2Request} import java.util.concurrent.atomic.AtomicReference -import java.io.InputStream -import scala.jdk.CollectionConverters._ import scala.concurrent.ExecutionContext import scala.concurrent.duration._ import scala.util.control.NonFatal @@ -25,25 +22,6 @@ abstract class BaseStore[TStoreKey, TStoreVal](bucket: String, config: CommonCon protected def getS3Object(key: String): Option[String] = s3.getObjectAsString(bucket, key) - protected def getLatestS3Stream: Option[InputStream] = { - val objects = s3.client.listObjectsV2(ListObjectsV2Request.builder().bucket(bucket).build()) - .contents().asScala.toList - .filterNot(_.key() == "AMAZON_SES_SETUP_NOTIFICATION") - - if (objects.nonEmpty) { - val obj = objects.maxBy(_.lastModified()) - logger.info(s"Latest key ${obj.key} in bucket $bucket") - - val stream = s3.client.getObject( - GetObjectRequest.builder().key(obj.key()).bucket(bucket).build() - ) - Some(stream) - } else { - logger.error(s"Bucket $bucket is empty") - None - } - } - private var cancellable: Option[Cancellable] = None def scheduleUpdates(scheduler: Scheduler): Unit = { From a7b999437fe6f1529418101992a860ab4e148d9c Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Mon, 10 Aug 2026 20:32:10 +0100 Subject: [PATCH 13/46] ImageLoaderStore uses S3.getObject to release it's client usage. --- image-loader/app/lib/ImageLoaderStore.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/image-loader/app/lib/ImageLoaderStore.scala b/image-loader/app/lib/ImageLoaderStore.scala index db2e2f5792..7d2cfeb328 100644 --- a/image-loader/app/lib/ImageLoaderStore.scala +++ b/image-loader/app/lib/ImageLoaderStore.scala @@ -32,8 +32,7 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati } def getS3Object(key: String)(implicit logMarker: LogMarker): ResponseInputStream[GetObjectResponse] = handleNotFound(key) { - client.getObject( - GetObjectRequest.builder().bucket(config.maybeIngestBucket.get).key(key).build()) + getObject(config.maybeIngestBucket.get, key) } { logger.error(logMarker, s"Attempted to read $key from ingest bucket, but it does not exist.") } From 7f5ca31b74eecc8a98476e5d9b6ac10a12009283 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Mon, 10 Aug 2026 20:46:13 +0100 Subject: [PATCH 14/46] S3 provides copy. --- .../main/scala/com/gu/mediaservice/lib/aws/S3.scala | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 297e52ae6b..664e64c323 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -7,10 +7,10 @@ import org.joda.time.{DateTime, DateTimeZone} import software.amazon.awssdk.core.ResponseInputStream import software.amazon.awssdk.core.sync.RequestBody import software.amazon.awssdk.regions.Region -import software.amazon.awssdk.services.s3.{S3Client, S3Configuration} import software.amazon.awssdk.services.s3.model._ import software.amazon.awssdk.services.s3.presigner.S3Presigner import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest +import software.amazon.awssdk.services.s3.{S3Client, S3Configuration} import java.io.File import java.net.{URI, URL} @@ -251,6 +251,16 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with def deleteVersion(bucket: Bucket, key: String, objectVersion: String): Unit = client.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(key).versionId(objectVersion).build()) + def copy(key: String, sourceBucket: String, destinationBucket: String): CopyObjectResponse = { + client.copyObject( + CopyObjectRequest.builder() + .sourceBucket(sourceBucket) + .sourceKey(key) + .destinationBucket(destinationBucket) + .destinationKey(key) + .build() + ) + } } object S3Ops { From 11e912681eb3f1e852fec875ad1b7639fc830b5c Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Mon, 10 Aug 2026 20:46:55 +0100 Subject: [PATCH 15/46] ImageLoaderStore uses S3.copy. --- image-loader/app/lib/ImageLoaderStore.scala | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/image-loader/app/lib/ImageLoaderStore.scala b/image-loader/app/lib/ImageLoaderStore.scala index 7d2cfeb328..6f5221ac3d 100644 --- a/image-loader/app/lib/ImageLoaderStore.scala +++ b/image-loader/app/lib/ImageLoaderStore.scala @@ -62,15 +62,10 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati req.url().toExternalForm } - def moveObjectToFailedBucket(key: String)(implicit logMarker: LogMarker) = handleNotFound(key){ - client.copyObject( - CopyObjectRequest.builder() - .sourceBucket(config.maybeIngestBucket.get) // TODO Naked get - make optional - .sourceKey(key) - .destinationBucket(config.maybeFailBucket.get) // TODO Naked get - make optional - .destinationKey(key) - .build() - ) + def moveObjectToFailedBucket(key: String)(implicit logMarker: LogMarker): Unit = handleNotFound(key){ + val sourceBucket = config.maybeIngestBucket.get // TODO Naked get - make optional + val destinationBucket = config.maybeFailBucket.get // TODO Naked get - make optional + copy(key, sourceBucket, destinationBucket) deleteObjectFromIngestBucket(key) } { logger.warn(logMarker, s"Attempted to copy $key from ingest bucket to fail bucket, but it does not exist.") From eac8ddc3985ac97dbe083bab42690b13904cff38 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Mon, 10 Aug 2026 20:47:18 +0100 Subject: [PATCH 16/46] S3 client is private. S3 is the only user of client. Sets up for S3 to be able to pull the client from S3Bucket. --- common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 664e64c323..d79eed3cb7 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -72,7 +72,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with type Key = String type UserMetadata = Map[String, String] - lazy val client: S3Client = S3Ops.buildS3Client(config) + private lazy val client: S3Client = S3Ops.buildS3Client(config) lazy val presigner = S3Ops.buildPresignerClientV2(config) def signUrl( bucket: Bucket, From b143d190fb4f7393a1cf7dc98689adb4fd790160 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Fri, 28 Aug 2026 21:44:15 +0100 Subject: [PATCH 17/46] S3 provides presignPutObject so that ImageLoaderStore does not need a reference to the presigner. presigner is private. --- .../src/main/scala/com/gu/mediaservice/lib/aws/S3.scala | 8 ++++++-- image-loader/app/lib/ImageLoaderStore.scala | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index d79eed3cb7..58d85da171 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -9,7 +9,7 @@ import software.amazon.awssdk.core.sync.RequestBody import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.s3.model._ import software.amazon.awssdk.services.s3.presigner.S3Presigner -import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest +import software.amazon.awssdk.services.s3.presigner.model.{GetObjectPresignRequest, PresignedPutObjectRequest, PutObjectPresignRequest} import software.amazon.awssdk.services.s3.{S3Client, S3Configuration} import java.io.File @@ -73,7 +73,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with type UserMetadata = Map[String, String] private lazy val client: S3Client = S3Ops.buildS3Client(config) - lazy val presigner = S3Ops.buildPresignerClientV2(config) + private lazy val presigner = S3Ops.buildPresignerClientV2(config) def signUrl( bucket: Bucket, url: URI, @@ -127,6 +127,10 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with req.url() } + def presignPutObject(putObjectPresignRequest: PutObjectPresignRequest): PresignedPutObjectRequest = { + presigner.presignPutObject(putObjectPresignRequest) + } + def getObject(bucket: Bucket, url: URI): ResponseInputStream[GetObjectResponse]= { // get path and remove leading `/` val key: Key = url.getPath.drop(1) diff --git a/image-loader/app/lib/ImageLoaderStore.scala b/image-loader/app/lib/ImageLoaderStore.scala index 6f5221ac3d..065eadfdea 100644 --- a/image-loader/app/lib/ImageLoaderStore.scala +++ b/image-loader/app/lib/ImageLoaderStore.scala @@ -58,7 +58,7 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati .signatureDuration(duration) .build(); - val req = presigner.presignPutObject(putObjectPresignRequest) + val req = presignPutObject(putObjectPresignRequest) req.url().toExternalForm } From 3925b386ca4c3f43ad99b8c518be83cf7c21d4df Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Thu, 13 Aug 2026 08:52:58 +0100 Subject: [PATCH 18/46] Clean up; rename to s3; not a s3Client. --- media-api/app/MediaApiComponents.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/media-api/app/MediaApiComponents.scala b/media-api/app/MediaApiComponents.scala index ed62917800..efc6b77a79 100644 --- a/media-api/app/MediaApiComponents.scala +++ b/media-api/app/MediaApiComponents.scala @@ -18,7 +18,7 @@ class MediaApiComponents(context: Context) extends GridComponents(context, new M val messageSender = new ThrallMessageSender(config.thrallKinesisStreamConfig) val mediaApiMetrics = new MediaApiMetrics(config, actorSystem, applicationLifecycle) - val s3Client = new S3(config) + private val s3 = new S3(config) val usageQuota = new UsageQuota(config, actorSystem.scheduler) usageQuota.quotaStore.update() @@ -28,12 +28,12 @@ class MediaApiComponents(context: Context) extends GridComponents(context, new M val elasticSearch = new ElasticSearch(config, mediaApiMetrics, config.esConfig, () => usageQuota.usageStore.overQuotaAgencies, actorSystem.scheduler, new InstancesClient(config, wsClient)) // TODO needs to move somewhere more instance aware elasticSearch.ensureIndexExistsAndAliasAssigned() - val imageResponse = new ImageResponse(config, s3Client, usageQuota) + val imageResponse = new ImageResponse(config, s3, usageQuota) val softDeletedMetadataTable = new SoftDeletedMetadataTable(config) val embedder = new Embedder(new Bedrock(config), new SimpleSqsMessageConsumer(config.queueUrl, config)) - val mediaApi = new MediaApi(auth, messageSender, softDeletedMetadataTable, elasticSearch, imageResponse, config, controllerComponents, s3Client, mediaApiMetrics, wsClient, authorisation, embedder, usageEvents) + val mediaApi = new MediaApi(auth, messageSender, softDeletedMetadataTable, elasticSearch, imageResponse, config, controllerComponents, s3, mediaApiMetrics, wsClient, authorisation, embedder, usageEvents) val suggestionController = new SuggestionController(auth, elasticSearch, controllerComponents) val aggController = new AggregationController(auth, elasticSearch, controllerComponents) val usageController = new UsageController(auth, config, elasticSearch, usageQuota, controllerComponents) From a8a07a774da0e5db9e2558b475fb34683a078344 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Fri, 28 Aug 2026 22:01:44 +0100 Subject: [PATCH 19/46] For consistency all usages of S3 are injected. --- .../main/scala/com/gu/mediaservice/lib/BaseStore.scala | 4 +--- .../scala/com/gu/mediaservice/lib/auth/KeyStore.scala | 5 +++-- cropper/app/CropperComponents.scala | 5 ++++- cropper/app/lib/Crops.scala | 4 +--- cropper/test/lib/CropsTest.scala | 10 ++++++---- image-loader/app/ImageLoaderComponents.scala | 6 ++++-- image-loader/app/model/Projector.scala | 4 ++-- media-api/app/MediaApiComponents.scala | 2 +- media-api/app/lib/QuotaStore.scala | 6 ++++-- media-api/app/lib/UsageQuota.scala | 9 ++++++--- media-api/app/lib/UsageStore.scala | 6 ++++-- .../auth/provider/ApiKeyAuthenticationProvider.scala | 3 ++- .../lib/auth/ApiKeyAuthenticationProviderTest.scala | 4 +++- 13 files changed, 41 insertions(+), 27 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala index f72d2a1400..8913b02513 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala @@ -12,11 +12,9 @@ import scala.concurrent.duration._ import scala.util.control.NonFatal -abstract class BaseStore[TStoreKey, TStoreVal](bucket: String, config: CommonConfig)(implicit ec: ExecutionContext) +abstract class BaseStore[TStoreKey, TStoreVal](bucket: String, config: CommonConfig, s3: S3)(implicit ec: ExecutionContext) extends GridLogging { - val s3 = new S3(config) - protected val store: AtomicReference[Map[TStoreKey, TStoreVal]] = new AtomicReference(Map.empty) protected val lastUpdated: AtomicReference[DateTime] = new AtomicReference(DateTime.now()) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala index 49f48bcbec..a0c9d438b9 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala @@ -1,14 +1,15 @@ package com.gu.mediaservice.lib.auth import com.gu.mediaservice.lib.BaseStore +import com.gu.mediaservice.lib.aws.S3 import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.model.Instance import scala.concurrent.duration._ import scala.concurrent.{Await, ExecutionContext} -class KeyStore(bucket: String, config: CommonConfig)(implicit ec: ExecutionContext) - extends BaseStore[String, ApiAccessor](bucket, config)(ec) { +class KeyStore(bucket: String, config: CommonConfig, s3: S3)(implicit ec: ExecutionContext) + extends BaseStore[String, ApiAccessor](bucket, config, s3)(ec) { def lookupIdentity(key: String)(implicit instance: Instance): Option[ApiAccessor] = store.get().get(instance.id + "/" + key) diff --git a/cropper/app/CropperComponents.scala b/cropper/app/CropperComponents.scala index b05381cb06..85a150fca8 100644 --- a/cropper/app/CropperComponents.scala +++ b/cropper/app/CropperComponents.scala @@ -1,4 +1,5 @@ import com.gu.mediaservice.GridClient +import com.gu.mediaservice.lib.aws.S3 import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.management.Management import com.gu.mediaservice.lib.play.GridComponents @@ -13,7 +14,9 @@ class CropperComponents(context: Context) extends GridComponents(context, new Cr val store = new CropStore(config) val imageOperations = new ImageOperations(context.environment.rootPath.getAbsolutePath) - val crops = new Crops(config, store, imageOperations, config.imageBucket) + private val s3 = new S3(config) + + val crops = new Crops(config, store, imageOperations, config.imageBucket, s3) val notifications = new Notifications(config) private val gridClient = GridClient(config.services, config.services.cropperBaseUri)(wsClient) diff --git a/cropper/app/lib/Crops.scala b/cropper/app/lib/Crops.scala index 6c2be74d21..2eb027499f 100644 --- a/cropper/app/lib/Crops.scala +++ b/cropper/app/lib/Crops.scala @@ -17,7 +17,7 @@ case object InvalidCropRequest extends Exception("Crop request invalid for image case class MasterCrop(sizing: Future[Asset], file: File, dimensions: Dimensions, aspectRatio: Float) -class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOperations, imageBucket: String)(implicit ec: ExecutionContext) extends GridLogging { +class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOperations, imageBucket: String, s3: S3)(implicit ec: ExecutionContext) extends GridLogging { import Files._ private val cropQuality = 75d @@ -26,8 +26,6 @@ class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOpera // We don't overly care about output crop file sizes here, but prefer a fast output, so turn it right down. private val pngCropQuality = 1d - private val s3 = new S3(config) - def outputFilename(source: SourceImage, bounds: Bounds, outputWidth: Int, fileType: MimeType, isMaster: Boolean = false)(implicit instance: Instance): String = { val masterString: String = if (isMaster) "master/" else "" instance.id + "/" + s"${source.id}/${Crop.getCropId(bounds)}/$masterString$outputWidth${fileType.fileExtension}" diff --git a/cropper/test/lib/CropsTest.scala b/cropper/test/lib/CropsTest.scala index e348b9a4a6..912feca0d9 100644 --- a/cropper/test/lib/CropsTest.scala +++ b/cropper/test/lib/CropsTest.scala @@ -1,5 +1,6 @@ package lib +import com.gu.mediaservice.lib.aws.S3 import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.model._ import org.scalatest.funspec.AnyFunSpec @@ -51,24 +52,25 @@ class CropsTest extends AnyFunSpec with Matchers with MockitoSugar { private val bounds: Bounds = Bounds(10, 20, 30, 40) private val outputWidth = 1234 private val imageBucket = "crops-bucket" + private val s3 = new S3(config) it("should should construct a correct address for a master jpg") { - val outputFilename = new Crops(config, store, imageOperations, imageBucket) + val outputFilename = new Crops(config, store, imageOperations, imageBucket, s3) .outputFilename(source, bounds, outputWidth, Jpeg, isMaster = true) outputFilename shouldBe "an-instance/test/10_20_30_40/master/1234.jpg" } it("should should construct a correct address for a non-master jpg") { - val outputFilename = new Crops(config, store, imageOperations, imageBucket) + val outputFilename = new Crops(config, store, imageOperations, imageBucket, s3) .outputFilename(source, bounds, outputWidth, Jpeg) outputFilename shouldBe "an-instance/test/10_20_30_40/1234.jpg" } it("should should construct a correct address for a non-master tiff") { - val outputFilename = new Crops(config, store, imageOperations, imageBucket) + val outputFilename = new Crops(config, store, imageOperations, imageBucket, s3) .outputFilename(source, bounds, outputWidth, Tiff) outputFilename shouldBe "an-instance/test/10_20_30_40/1234.tiff" } it("should should construct a correct address for a non-master png") { - val outputFilename = new Crops(config, store, imageOperations, imageBucket) + val outputFilename = new Crops(config, store, imageOperations, imageBucket, s3) .outputFilename(source, bounds, outputWidth, Png) outputFilename shouldBe "an-instance/test/10_20_30_40/1234.png" } diff --git a/image-loader/app/ImageLoaderComponents.scala b/image-loader/app/ImageLoaderComponents.scala index 9d4a739b26..c6529ecbb1 100644 --- a/image-loader/app/ImageLoaderComponents.scala +++ b/image-loader/app/ImageLoaderComponents.scala @@ -1,5 +1,5 @@ import com.gu.mediaservice.GridClient -import com.gu.mediaservice.lib.aws.{Bedrock, Embedder, S3Vectors, SimpleSqsMessageConsumer} +import com.gu.mediaservice.lib.aws.{Bedrock, Embedder, S3, S3Vectors, SimpleSqsMessageConsumer} import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.GridLogging import com.gu.mediaservice.lib.play.GridComponents @@ -34,8 +34,10 @@ class ImageLoaderComponents(context: Context) extends GridComponents(context, ne new Embedder(new Bedrock(config), new SimpleSqsMessageConsumer(queueUrl, config)) } + private val s3 = new S3(config) + val uploader = new Uploader(store, config, imageOperations, notifications, maybeEmbedder, imageProcessor, gridClient, auth) - val projector = Projector(config, imageOperations, imageProcessor, auth, maybeEmbedder) + val projector = Projector(config, s3, imageOperations, imageProcessor, auth, maybeEmbedder) val quarantineUploader: Option[QuarantineUploader] = config.maybeQuarantineBucket.map(_ => new QuarantineUploader(new QuarantineStore(config), config) ) diff --git a/image-loader/app/model/Projector.scala b/image-loader/app/model/Projector.scala index cc469d886b..b50a04b5aa 100644 --- a/image-loader/app/model/Projector.scala +++ b/image-loader/app/model/Projector.scala @@ -29,8 +29,8 @@ object Projector { import Uploader.toImageUploadOpsCfg - def apply(config: ImageLoaderConfig, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, maybeEmbedder: Option[Embedder])(implicit ec: ExecutionContext): Projector - = new Projector(toImageUploadOpsCfg(config), new S3(config), imageOps, processor, auth, maybeEmbedder) + def apply(config: ImageLoaderConfig, s3: S3, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, maybeEmbedder: Option[Embedder])(implicit ec: ExecutionContext): Projector + = new Projector(toImageUploadOpsCfg(config), s3, imageOps, processor, auth, maybeEmbedder) } case class S3FileExtractedMetadata( diff --git a/media-api/app/MediaApiComponents.scala b/media-api/app/MediaApiComponents.scala index efc6b77a79..b73420d5ff 100644 --- a/media-api/app/MediaApiComponents.scala +++ b/media-api/app/MediaApiComponents.scala @@ -20,7 +20,7 @@ class MediaApiComponents(context: Context) extends GridComponents(context, new M private val s3 = new S3(config) - val usageQuota = new UsageQuota(config, actorSystem.scheduler) + val usageQuota = new UsageQuota(config, actorSystem.scheduler, s3) usageQuota.quotaStore.update() usageQuota.scheduleUpdates() applicationLifecycle.addStopHook(() => Future{usageQuota.stopUpdates()}) diff --git a/media-api/app/lib/QuotaStore.scala b/media-api/app/lib/QuotaStore.scala index c79486fbd1..8e8052096a 100644 --- a/media-api/app/lib/QuotaStore.scala +++ b/media-api/app/lib/QuotaStore.scala @@ -1,6 +1,7 @@ package lib import com.gu.mediaservice.lib.BaseStore +import com.gu.mediaservice.lib.aws.S3 import play.api.libs.json.Json import scala.concurrent.ExecutionContext @@ -8,8 +9,9 @@ import scala.concurrent.ExecutionContext class QuotaStore( quotaFile: String, bucket: String, - config: MediaApiConfig - )(implicit ec: ExecutionContext) extends BaseStore[String, SupplierUsageQuota](bucket, config)(ec) { + config: MediaApiConfig, + s3: S3 + )(implicit ec: ExecutionContext) extends BaseStore[String, SupplierUsageQuota](bucket, config, s3)(ec) { def getQuota: Map[String, SupplierUsageQuota] = store.get() diff --git a/media-api/app/lib/UsageQuota.scala b/media-api/app/lib/UsageQuota.scala index bcd4e93e21..75762e2d69 100644 --- a/media-api/app/lib/UsageQuota.scala +++ b/media-api/app/lib/UsageQuota.scala @@ -2,6 +2,7 @@ package lib import org.apache.pekko.actor.Scheduler import com.gu.mediaservice.lib.FeatureToggle +import com.gu.mediaservice.lib.aws.S3 import com.gu.mediaservice.model.UsageRights import scala.concurrent.Await @@ -12,17 +13,19 @@ import scala.util.Try case class ImageNotFound() extends Exception("Image not found") case class NoUsageQuota() extends Exception("No usage found for this image") -class UsageQuota(config: MediaApiConfig, scheduler: Scheduler) { +class UsageQuota(config: MediaApiConfig, scheduler: Scheduler, s3: S3) { val quotaStore = new QuotaStore( config.quotaStoreConfig.storeKey, config.quotaStoreConfig.storeBucket, - config + config, + s3 ) val usageStore = new UsageStore( config.usageMailBucket, config, - quotaStore + quotaStore, + s3 ) def scheduleUpdates(): Unit = { diff --git a/media-api/app/lib/UsageStore.scala b/media-api/app/lib/UsageStore.scala index 3490455caf..65b3ab3bde 100644 --- a/media-api/app/lib/UsageStore.scala +++ b/media-api/app/lib/UsageStore.scala @@ -1,6 +1,7 @@ package lib import com.gu.mediaservice.lib.BaseStore +import com.gu.mediaservice.lib.aws.S3 import com.gu.mediaservice.lib.logging.GridLogging import com.gu.mediaservice.model.{Agencies, Agency, UsageRights} import com.gu.mediaservice.model.usage.{DigitalUsage, PrintUsage, PublishedUsageStatus, RemovedUsageStatus, UnknownUsageStatus, Usage, UsageStatus, UsageType} @@ -62,8 +63,9 @@ object UsageStore extends GridLogging { class UsageStore( bucket: String, config: MediaApiConfig, - quotaStore: QuotaStore -)(implicit val ec: ExecutionContext) extends BaseStore[String, SupplierUsageStatus](bucket, config) with GridLogging { + quotaStore: QuotaStore, + s3: S3 +)(implicit val ec: ExecutionContext) extends BaseStore[String, SupplierUsageStatus](bucket, config, s3) with GridLogging { def getUsageStatusForUsageRights(usageRights: UsageRights): Future[SupplierUsageStatus] = { usageRights match { diff --git a/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala b/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala index e42c25c718..b74fc8b41f 100644 --- a/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala +++ b/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala @@ -2,6 +2,7 @@ package com.gu.mediaservice.lib.auth.provider import com.gu.mediaservice.lib.auth.Authentication.{MachinePrincipal, Principal} import com.gu.mediaservice.lib.auth.provider.ApiKeyAuthenticationProvider.{ApiKeyInstance, KindeIdKey} import com.gu.mediaservice.lib.auth.{ApiAccessor, KeyStore} +import com.gu.mediaservice.lib.aws.S3 import com.gu.mediaservice.lib.config.InstanceForRequest import com.gu.mediaservice.lib.events.UsageEvents import com.gu.mediaservice.model.Instance @@ -25,7 +26,7 @@ class ApiKeyAuthenticationProvider(configuration: Configuration, resources: Auth var keyStorePlaceholder: Option[KeyStore] = _ override def initialise(): Unit = { - val store = new KeyStore(configuration.get[String]("authKeyStoreBucket"), resources.commonConfig) + val store = new KeyStore(configuration.get[String]("authKeyStoreBucket"), resources.commonConfig, new S3(resources.commonConfig)) store.scheduleUpdates(resources.actorSystem.scheduler) keyStorePlaceholder = Some(store) } diff --git a/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala b/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala index 3ba9231943..d9f036be40 100644 --- a/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala +++ b/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala @@ -3,6 +3,7 @@ package com.gu.mediaservice.lib.auth import org.apache.pekko.actor.ActorSystem import com.gu.mediaservice.lib.auth.Authentication.MachinePrincipal import com.gu.mediaservice.lib.auth.provider.{ApiKeyAuthenticationProvider, Authenticated, AuthenticationProviderResources, Invalid, NotAuthenticated, NotAuthorised} +import com.gu.mediaservice.lib.aws.S3 import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources} import com.gu.mediaservice.lib.events.UsageEvents import com.gu.mediaservice.model.Instance @@ -36,6 +37,7 @@ class ApiKeyAuthenticationProviderTest extends AsyncFreeSpec with Matchers with private val providerConfig = Configuration.empty private val controllerComponents: DefaultControllerComponents = DefaultControllerComponents(null, null, null, null, null, global) private val resources = AuthenticationProviderResources(config, actorSystem, wsClient, controllerComponents, mock[Authorisation], mock[CookieSigner], mock[UsageEvents] ) + private val s3 = new S3(config) private val provider = new ApiKeyAuthenticationProvider(providerConfig, resources) { override def initialise(): Unit = { /* do nothing */ } @@ -43,7 +45,7 @@ class ApiKeyAuthenticationProviderTest extends AsyncFreeSpec with Matchers with Future.successful(()) } - override def keyStore: KeyStore = new KeyStore("not-used", resources.commonConfig) { + override def keyStore: KeyStore = new KeyStore("not-used", resources.commonConfig, s3) { override def lookupIdentity(key: String)(implicit instance: Instance): Option[ApiAccessor] = { key match { case "key-chuckle" => Some(ApiAccessor("brothers", Internal)) From f189f7d0f2bb627ae0603a0245b81b35d9d23086 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Fri, 28 Aug 2026 22:16:49 +0100 Subject: [PATCH 20/46] imageBucket moves up to CommonConfig to reduce duplication of bucket init. --- .../scala/com/gu/mediaservice/lib/config/CommonConfig.scala | 2 ++ common-lib/src/test/resources/application.conf | 1 + cropper/app/lib/CropperConfig.scala | 2 -- image-loader/app/lib/ImageLoaderConfig.scala | 2 -- media-api/app/lib/MediaApiConfig.scala | 1 - rest-lib/src/test/resources/application.conf | 1 + thrall/app/lib/ThrallConfig.scala | 2 -- 7 files changed, 4 insertions(+), 7 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala index 70167400d4..3592df85e2 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala @@ -60,6 +60,8 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientBui lazy val softDeletedMetadataTable: String = string("dynamo.table.softDelete.metadata") + val imageBucket: String = string("s3.image.bucket") + val maybeIngestSqsQueueUrl: Option[String] = stringOpt("sqs.ingest.queue.url") val maybeIngestBucket: Option[String] = stringOpt("s3.ingest.bucket") val maybeFailBucket: Option[String] = stringOpt("s3.fail.bucket") diff --git a/common-lib/src/test/resources/application.conf b/common-lib/src/test/resources/application.conf index 05d1db98e6..4ee28f88b1 100644 --- a/common-lib/src/test/resources/application.conf +++ b/common-lib/src/test/resources/application.conf @@ -3,6 +3,7 @@ grid.appName: "test" thrall.kinesis.stream.name: "not-used" thrall.kinesis.lowPriorityStream.name: "not-used" domain.root: "notused.example.com" +s3.image.bucket: "not-used" image.processors = [ "com.gu.mediaservice.lib.cleanup.GuardianMetadataCleaners", diff --git a/cropper/app/lib/CropperConfig.scala b/cropper/app/lib/CropperConfig.scala index 96d043d60a..ef762ed2a1 100644 --- a/cropper/app/lib/CropperConfig.scala +++ b/cropper/app/lib/CropperConfig.scala @@ -7,8 +7,6 @@ import java.io.File class CropperConfig(resources: GridConfigResources) extends CommonConfig(resources) { - val imageBucket: String = string("s3.image.bucket") - val imgPublishingBucket = string("publishing.image.bucket") val canDownloadCrop: Boolean = boolean("canDownloadCrop") diff --git a/image-loader/app/lib/ImageLoaderConfig.scala b/image-loader/app/lib/ImageLoaderConfig.scala index 37d43aa078..46f349a8c6 100644 --- a/image-loader/app/lib/ImageLoaderConfig.scala +++ b/image-loader/app/lib/ImageLoaderConfig.scala @@ -10,8 +10,6 @@ import play.api.inject.ApplicationLifecycle import scala.concurrent.duration.FiniteDuration class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(resources) with StrictLogging { - val imageBucket: String = string("s3.image.bucket") - val maybeImageReplicaBucket: Option[String] = stringOpt("s3.image.replicaBucket") val thumbnailBucket: String = string("s3.thumb.bucket") diff --git a/media-api/app/lib/MediaApiConfig.scala b/media-api/app/lib/MediaApiConfig.scala index 5b48ec4a37..82afb8a935 100644 --- a/media-api/app/lib/MediaApiConfig.scala +++ b/media-api/app/lib/MediaApiConfig.scala @@ -28,7 +28,6 @@ class MediaApiConfig(resources: GridConfigResources) extends CommonConfigWithEla //Lazy allows this to be empty and not break things unless used somewhere lazy val imgPublishingBucket = string("publishing.image.bucket") - val imageBucket: String = string("s3.image.bucket") val thumbBucket: String = string("s3.thumb.bucket") val cloudFrontDomainThumbBucket: Option[String] = stringOpt("cloudfront.domain.thumbbucket") diff --git a/rest-lib/src/test/resources/application.conf b/rest-lib/src/test/resources/application.conf index 930c6a00d5..02678a3f05 100644 --- a/rest-lib/src/test/resources/application.conf +++ b/rest-lib/src/test/resources/application.conf @@ -3,3 +3,4 @@ grid.appName: "test" thrall.kinesis.stream.name: "not-used" thrall.kinesis.lowPriorityStream.name: "not-used" domain.root: "notused.example.com" +s3.image.bucket: "not-used" diff --git a/thrall/app/lib/ThrallConfig.scala b/thrall/app/lib/ThrallConfig.scala index 477cfc5d3e..7b706ceb65 100644 --- a/thrall/app/lib/ThrallConfig.scala +++ b/thrall/app/lib/ThrallConfig.scala @@ -56,8 +56,6 @@ object KinesisReceiverConfig { } class ThrallConfig(resources: GridConfigResources) extends CommonConfigWithElastic(resources) { - val imageBucket: String = string("s3.image.bucket") - val thumbnailBucket: String = string("s3.thumb.bucket") val maybeReaperBucket: Option[String] = stringOpt("s3.reaper.bucket") From 354aeb7696784d6341bceadcd5ad89bb980d0466 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Fri, 28 Aug 2026 22:34:28 +0100 Subject: [PATCH 21/46] thumbnailBucket moves up to CommonConfig to reduce duplication of bucket init. --- .../scala/com/gu/mediaservice/lib/config/CommonConfig.scala | 1 + common-lib/src/test/resources/application.conf | 1 + image-loader/app/lib/ImageLoaderConfig.scala | 2 -- media-api/app/lib/ImageResponse.scala | 2 +- media-api/app/lib/MediaApiConfig.scala | 2 -- rest-lib/src/test/resources/application.conf | 1 + thrall/app/lib/ThrallConfig.scala | 2 -- 7 files changed, 4 insertions(+), 7 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala index 3592df85e2..b988b85fdc 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala @@ -61,6 +61,7 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientBui lazy val softDeletedMetadataTable: String = string("dynamo.table.softDelete.metadata") val imageBucket: String = string("s3.image.bucket") + val thumbnailBucket: String = string("s3.thumb.bucket") val maybeIngestSqsQueueUrl: Option[String] = stringOpt("sqs.ingest.queue.url") val maybeIngestBucket: Option[String] = stringOpt("s3.ingest.bucket") diff --git a/common-lib/src/test/resources/application.conf b/common-lib/src/test/resources/application.conf index 4ee28f88b1..b510ffad0f 100644 --- a/common-lib/src/test/resources/application.conf +++ b/common-lib/src/test/resources/application.conf @@ -4,6 +4,7 @@ thrall.kinesis.stream.name: "not-used" thrall.kinesis.lowPriorityStream.name: "not-used" domain.root: "notused.example.com" s3.image.bucket: "not-used" +s3.thumb.bucket: "not-used" image.processors = [ "com.gu.mediaservice.lib.cleanup.GuardianMetadataCleaners", diff --git a/image-loader/app/lib/ImageLoaderConfig.scala b/image-loader/app/lib/ImageLoaderConfig.scala index 46f349a8c6..77077dde61 100644 --- a/image-loader/app/lib/ImageLoaderConfig.scala +++ b/image-loader/app/lib/ImageLoaderConfig.scala @@ -12,8 +12,6 @@ import scala.concurrent.duration.FiniteDuration class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(resources) with StrictLogging { val maybeImageReplicaBucket: Option[String] = stringOpt("s3.image.replicaBucket") - val thumbnailBucket: String = string("s3.thumb.bucket") - val lowerEnvironmentSamplingPercentageAsDecimal = intOpt("s3.sampling.percentage").getOrElse(1) / 100.0 val maybeLowerEnvironmentQueueBucketToSampleInto = stringOpt("s3.sampling.targetBucket") diff --git a/media-api/app/lib/ImageResponse.scala b/media-api/app/lib/ImageResponse.scala index a8cd9df3bd..73824d0169 100644 --- a/media-api/app/lib/ImageResponse.scala +++ b/media-api/app/lib/ImageResponse.scala @@ -83,7 +83,7 @@ class ImageResponse(config: MediaApiConfig, s3Client: S3, usageQuota: UsageQuota val pngUrl: Option[String] = pngFileUri .map(s3Client.signUrl(config.imageBucket, _, image, imageType = OptimisedPng)) - def s3SignedThumbUrl = s3Client.signUrl(config.thumbBucket, fileUri, image, imageType = Thumbnail) + def s3SignedThumbUrl = s3Client.signUrl(config.thumbnailBucket, fileUri, image, imageType = Thumbnail) val thumbUrl = config.cloudFrontDomainThumbBucket .map(domain => s"https://$domain${fileUri.getPath}") diff --git a/media-api/app/lib/MediaApiConfig.scala b/media-api/app/lib/MediaApiConfig.scala index 82afb8a935..e9557b7e0c 100644 --- a/media-api/app/lib/MediaApiConfig.scala +++ b/media-api/app/lib/MediaApiConfig.scala @@ -28,8 +28,6 @@ class MediaApiConfig(resources: GridConfigResources) extends CommonConfigWithEla //Lazy allows this to be empty and not break things unless used somewhere lazy val imgPublishingBucket = string("publishing.image.bucket") - val thumbBucket: String = string("s3.thumb.bucket") - val cloudFrontDomainThumbBucket: Option[String] = stringOpt("cloudfront.domain.thumbbucket") val cloudFrontPrivateKeyBucket: Option[String] = stringOpt("cloudfront.private-key.bucket") val cloudFrontPrivateKeyBucketKey: Option[String] = stringOpt("cloudfront.private-key.key") diff --git a/rest-lib/src/test/resources/application.conf b/rest-lib/src/test/resources/application.conf index 02678a3f05..d6e40ffbb8 100644 --- a/rest-lib/src/test/resources/application.conf +++ b/rest-lib/src/test/resources/application.conf @@ -4,3 +4,4 @@ thrall.kinesis.stream.name: "not-used" thrall.kinesis.lowPriorityStream.name: "not-used" domain.root: "notused.example.com" s3.image.bucket: "not-used" +s3.thumb.bucket: "not-used" diff --git a/thrall/app/lib/ThrallConfig.scala b/thrall/app/lib/ThrallConfig.scala index 7b706ceb65..88ad70e0e0 100644 --- a/thrall/app/lib/ThrallConfig.scala +++ b/thrall/app/lib/ThrallConfig.scala @@ -56,8 +56,6 @@ object KinesisReceiverConfig { } class ThrallConfig(resources: GridConfigResources) extends CommonConfigWithElastic(resources) { - val thumbnailBucket: String = string("s3.thumb.bucket") - val maybeReaperBucket: Option[String] = stringOpt("s3.reaper.bucket") val maybeReaperCountPerRun: Option[Int] = intOpt("reaper.countPerRun") From 777b28a58907135fee8dc4cfc16f164f2d1e8f3a Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Fri, 28 Aug 2026 22:38:36 +0100 Subject: [PATCH 22/46] crops bucket moves up to CommonConfig to reduce duplication of bucket init. --- .../scala/com/gu/mediaservice/lib/config/CommonConfig.scala | 3 +++ cropper/app/lib/CropperConfig.scala | 2 -- media-api/app/lib/MediaApiConfig.scala | 3 --- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala index b988b85fdc..8f1e89a93c 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala @@ -63,6 +63,9 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientBui val imageBucket: String = string("s3.image.bucket") val thumbnailBucket: String = string("s3.thumb.bucket") + // Lazy allows this to be empty and not break things unless used somewhere + lazy val imgPublishingBucket: String = string("publishing.image.bucket") + val maybeIngestSqsQueueUrl: Option[String] = stringOpt("sqs.ingest.queue.url") val maybeIngestBucket: Option[String] = stringOpt("s3.ingest.bucket") val maybeFailBucket: Option[String] = stringOpt("s3.fail.bucket") diff --git a/cropper/app/lib/CropperConfig.scala b/cropper/app/lib/CropperConfig.scala index ef762ed2a1..176fdeed94 100644 --- a/cropper/app/lib/CropperConfig.scala +++ b/cropper/app/lib/CropperConfig.scala @@ -7,8 +7,6 @@ import java.io.File class CropperConfig(resources: GridConfigResources) extends CommonConfig(resources) { - val imgPublishingBucket = string("publishing.image.bucket") - val canDownloadCrop: Boolean = boolean("canDownloadCrop") val imgPublishingHost = string("publishing.image.host") diff --git a/media-api/app/lib/MediaApiConfig.scala b/media-api/app/lib/MediaApiConfig.scala index e9557b7e0c..3952c0f2b6 100644 --- a/media-api/app/lib/MediaApiConfig.scala +++ b/media-api/app/lib/MediaApiConfig.scala @@ -25,9 +25,6 @@ class MediaApiConfig(resources: GridConfigResources) extends CommonConfigWithEla val quotaStoreKey: String = string("quota.store.key") val quotaStoreConfig: StoreConfig = StoreConfig(configBucket, quotaStoreKey) - //Lazy allows this to be empty and not break things unless used somewhere - lazy val imgPublishingBucket = string("publishing.image.bucket") - val cloudFrontDomainThumbBucket: Option[String] = stringOpt("cloudfront.domain.thumbbucket") val cloudFrontPrivateKeyBucket: Option[String] = stringOpt("cloudfront.private-key.bucket") val cloudFrontPrivateKeyBucketKey: Option[String] = stringOpt("cloudfront.private-key.key") From e6f65810470841307b686077e15536048ebd5834 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Fri, 28 Aug 2026 22:41:21 +0100 Subject: [PATCH 23/46] Clean up; remove lazy on imgPublishingBucket. Cannot see a reason why you'd not set this; cropping is not an optional feature. --- .../scala/com/gu/mediaservice/lib/config/CommonConfig.scala | 4 +--- common-lib/src/test/resources/application.conf | 1 + media-api/test/lib/elasticsearch/Fixtures.scala | 1 + rest-lib/src/test/resources/application.conf | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala index 8f1e89a93c..620ec94e8d 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala @@ -62,9 +62,7 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientBui val imageBucket: String = string("s3.image.bucket") val thumbnailBucket: String = string("s3.thumb.bucket") - - // Lazy allows this to be empty and not break things unless used somewhere - lazy val imgPublishingBucket: String = string("publishing.image.bucket") + val imgPublishingBucket: String = string("publishing.image.bucket") val maybeIngestSqsQueueUrl: Option[String] = stringOpt("sqs.ingest.queue.url") val maybeIngestBucket: Option[String] = stringOpt("s3.ingest.bucket") diff --git a/common-lib/src/test/resources/application.conf b/common-lib/src/test/resources/application.conf index b510ffad0f..92e1d78ac7 100644 --- a/common-lib/src/test/resources/application.conf +++ b/common-lib/src/test/resources/application.conf @@ -5,6 +5,7 @@ thrall.kinesis.lowPriorityStream.name: "not-used" domain.root: "notused.example.com" s3.image.bucket: "not-used" s3.thumb.bucket: "not-used" +publishing.image.bucket: "not-used" image.processors = [ "com.gu.mediaservice.lib.cleanup.GuardianMetadataCleaners", diff --git a/media-api/test/lib/elasticsearch/Fixtures.scala b/media-api/test/lib/elasticsearch/Fixtures.scala index f2b5a0e495..70375788be 100644 --- a/media-api/test/lib/elasticsearch/Fixtures.scala +++ b/media-api/test/lib/elasticsearch/Fixtures.scala @@ -37,6 +37,7 @@ trait Fixtures { "es6.url", "s3.image.bucket", "s3.thumb.bucket", + "publishing.image.bucket", "grid.stage", "grid.appName", "instance.service.my", diff --git a/rest-lib/src/test/resources/application.conf b/rest-lib/src/test/resources/application.conf index d6e40ffbb8..1bff942991 100644 --- a/rest-lib/src/test/resources/application.conf +++ b/rest-lib/src/test/resources/application.conf @@ -5,3 +5,4 @@ thrall.kinesis.lowPriorityStream.name: "not-used" domain.root: "notused.example.com" s3.image.bucket: "not-used" s3.thumb.bucket: "not-used" +publishing.image.bucket: "not-used" From 3eb3ed57999d1590d97841ffc5826c2800754c87 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Fri, 28 Aug 2026 22:59:33 +0100 Subject: [PATCH 24/46] Clean up; unused raw s3Client. --- thrall/app/ThrallComponents.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/thrall/app/ThrallComponents.scala b/thrall/app/ThrallComponents.scala index 9d5f8e4d0c..0f908c99a4 100644 --- a/thrall/app/ThrallComponents.scala +++ b/thrall/app/ThrallComponents.scala @@ -91,7 +91,6 @@ class ThrallComponents(context: Context) extends GridComponents(context, new Thr val streamRunning: Future[Done] = thrallStreamProcessor.run() - val s3 = S3Ops.buildS3Client(config) val s3Vectors = new S3Vectors(config) Source.repeat(()).throttle(1, per = 5.minute).map(_ => { From 3ca6b8cbbe62e8e87db5198ced46f0205e7eec9f Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Sat, 29 Aug 2026 10:17:22 +0100 Subject: [PATCH 25/46] buildS3Client bypasses withAWSCredentials and takes explicit endpoint and use path style URLs parameters rather than isDev and useLocalAuth. --- .../com/gu/mediaservice/lib/aws/S3.scala | 20 ++++++++++++------- .../auth/PandaAuthenticationProvider.scala | 2 +- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 58d85da171..450c5f56aa 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -267,19 +267,25 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with } } -object S3Ops { +object S3Ops extends GridLogging { // TODO make this localstack friendly // TODO: Make this region aware - i.e. RegionUtils.getRegion(region).getServiceEndpoint(AmazonS3.ENDPOINT_PREFIX) val s3Endpoint = "s3.amazonaws.com" - def buildS3Client(config: CommonConfig, localstackAware: Boolean = true, maybeRegionOverride: Option[Region] = None): S3Client = { - val builder = config.awsLocalEndpoint match { - case Some(_) if config.isDev => - S3Client.builder().forcePathStyle(true) - case _ => S3Client.builder() + def buildS3Client(config: CommonConfig, endpointOverride: Option[String] = None, usesPathStyleURLs: Boolean = false, maybeRegionOverride: Option[Region] = None): S3Client = { + val builder = S3Client.builder() + .credentialsProvider(config.awsCredentials) + .region(maybeRegionOverride.getOrElse(config.awsRegion)) + .forcePathStyle(usesPathStyleURLs) + + val withEndpoint = endpointOverride match { + case Some(endpoint) => + logger.info(s"creating S3 client with endpoint override: $endpoint") + builder.endpointOverride(new URI(endpoint)) + case _ => builder } - config.withAWSCredentials(builder, localstackAware, maybeRegionOverride).build() + withEndpoint.build() } def buildPresignerClientV2(config: CommonConfig, localstackAware: Boolean = true, maybeRegionOverride: Option[Region] = None): S3Presigner = { diff --git a/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PandaAuthenticationProvider.scala b/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PandaAuthenticationProvider.scala index d98cefe4ef..ab5b36f80d 100644 --- a/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PandaAuthenticationProvider.scala +++ b/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PandaAuthenticationProvider.scala @@ -161,7 +161,7 @@ class PandaAuthenticationProvider( system = providerConfiguration.getOptional[String]("panda.system").getOrElse("media-service"), bucketName = providerConfiguration.getOptional[String]("panda.bucketName").getOrElse("pan-domain-auth-settings"), settingsFileKey = providerConfiguration.getOptional[String]("panda.settingsFileKey").getOrElse(s"$domain.settings"), - s3Client = S3Ops.buildS3Client(resources.commonConfig, localstackAware=resources.commonConfig.useLocalAuth) + s3Client = S3Ops.buildS3Client(resources.commonConfig, usesPathStyleURLs = resources.commonConfig.useLocalAuth) ) } From 24dadc2673209d10d8bfaae07ee20a9f88e6ff97 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Sat, 29 Aug 2026 10:27:49 +0100 Subject: [PATCH 26/46] buildS3Presigner bypasses withAWSCredentials and takes explicit endpoint and use path style URLs parameters rather than localstackAware. --- .../scala/com/gu/mediaservice/lib/aws/S3.scala | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 450c5f56aa..0be7a0ebb5 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -288,18 +288,22 @@ object S3Ops extends GridLogging { withEndpoint.build() } - def buildPresignerClientV2(config: CommonConfig, localstackAware: Boolean = true, maybeRegionOverride: Option[Region] = None): S3Presigner = { + def buildPresignerClientV2(config: CommonConfig, endpointOverride: Option[String] = None, usesPathStyleURLs: Boolean = false, maybeRegionOverride: Option[Region] = None): S3Presigner = { val builder = S3Presigner.builder() .credentialsProvider(config.awsCredentials) - .region(config.awsRegion) - - config.awsLocalEndpointUri match { - case Some(endpoint) if config.isDev => builder.endpointOverride(endpoint) - .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()).build() - case _ => builder.build() + .region(maybeRegionOverride.getOrElse(config.awsRegion)) + .serviceConfiguration(S3Configuration.builder() + .pathStyleAccessEnabled(usesPathStyleURLs) + .build()) + val withEndpoint = endpointOverride match { + case Some(endpoint) => + logger.info(s"creating S3 presigner with endpoint override: $endpoint") + builder.endpointOverride(new URI(endpoint)) + case _ => builder } + withEndpoint.build() } } From f6285498547f86bc71abb86a1b7ad91c17a5a74f Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Sun, 9 Aug 2026 13:01:29 +0100 Subject: [PATCH 27/46] Add our fat S3Bucket object. --- .../main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala new file mode 100644 index 0000000000..3f049d94d1 --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala @@ -0,0 +1,6 @@ +package com.gu.mediaservice.lib.aws + +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.presigner.S3Presigner + +case class S3Bucket(bucket: String, endPoint: String, usesPathStyleURLs: Boolean, client: S3Client, presigner: S3Presigner) From d597e6ed7d6e78efb2d190e39b0174c5f765d284 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Sat, 29 Aug 2026 10:54:14 +0100 Subject: [PATCH 28/46] Claude generated migration of Bucket to S3Bucket. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use client and presigners attached to the S3Bucket. Left as String deliberately: ImageLoaderConfig.maybeImageReplicaBucket — it drives a raw replicaS3: S3Client, not the S3/S3Bucket abstraction. Same for panda/permissions provider buckets. --- .../com/gu/mediaservice/lib/BaseStore.scala | 4 +- .../lib/ImageIngestOperations.scala | 24 ++-- .../lib/ImageQuarantineOperations.scala | 4 +- .../gu/mediaservice/lib/ImageStorage.scala | 6 +- .../gu/mediaservice/lib/S3ImageStorage.scala | 18 +-- .../gu/mediaservice/lib/auth/KeyStore.scala | 4 +- .../com/gu/mediaservice/lib/aws/S3.scala | 107 +++++++++--------- .../gu/mediaservice/lib/aws/S3Bucket.scala | 23 ++++ .../lib/config/CommonConfig.scala | 16 +-- cropper/app/lib/CropStore.scala | 2 +- cropper/app/lib/Crops.scala | 4 +- cropper/test/lib/CropsTest.scala | 4 +- .../controllers/ImageLoaderController.scala | 2 +- image-loader/app/lib/ImageLoaderStore.scala | 5 +- image-loader/app/model/Projector.scala | 12 +- image-loader/app/model/Uploader.scala | 6 +- .../test/scala/lib/ResourceHelpers.scala | 6 + .../test/scala/model/ImageUploadTest.scala | 2 +- .../test/scala/model/ProjectorTest.scala | 2 +- media-api/app/lib/MediaApiConfig.scala | 7 +- media-api/app/lib/QuotaStore.scala | 4 +- media-api/app/lib/UsageStore.scala | 4 +- .../ApiKeyAuthenticationProvider.scala | 4 +- .../ApiKeyAuthenticationProviderTest.scala | 4 +- thrall/app/lib/ThrallConfig.scala | 4 +- 25 files changed, 153 insertions(+), 125 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala index 8913b02513..cd7df655a2 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala @@ -1,6 +1,6 @@ package com.gu.mediaservice.lib -import com.gu.mediaservice.lib.aws.S3 +import com.gu.mediaservice.lib.aws.{S3, S3Bucket} import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.GridLogging import org.apache.pekko.actor.{Cancellable, Scheduler} @@ -12,7 +12,7 @@ import scala.concurrent.duration._ import scala.util.control.NonFatal -abstract class BaseStore[TStoreKey, TStoreVal](bucket: String, config: CommonConfig, s3: S3)(implicit ec: ExecutionContext) +abstract class BaseStore[TStoreKey, TStoreVal](bucket: S3Bucket, config: CommonConfig, s3: S3)(implicit ec: ExecutionContext) extends GridLogging { protected val store: AtomicReference[Map[TStoreKey, TStoreVal]] = new AtomicReference(Map.empty) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala index 1f65d3f441..4055e89787 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala @@ -1,7 +1,7 @@ package com.gu.mediaservice.lib -import com.gu.mediaservice.lib.aws.S3Object +import com.gu.mediaservice.lib.aws.{S3Bucket, S3Object} import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.LogMarker import com.gu.mediaservice.model.{Instance, MimeType} @@ -19,7 +19,7 @@ object ImageIngestOperations { private def snippetForId(id: String) = id.take(6).mkString("/") + "/" + id } -class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config: CommonConfig, isVersionedS3: Boolean = false) +class ImageIngestOperations(imageBucket: S3Bucket, thumbnailBucket: S3Bucket, config: CommonConfig, isVersionedS3: Boolean = false) extends S3ImageStorage(config) with StrictLogging { import ImageIngestOperations.{fileKeyFromId, optimisedPngKeyFromId} @@ -34,7 +34,7 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config private def storeOriginalImage(storableImage: StorableOriginalImage) (implicit logMarker: LogMarker): Future[S3Object] = { val instanceSpecificKey = instanceAwareOriginalImageKey(storableImage) - logger.info(s"Storing original image to instance specific key:$imageBucket / $instanceSpecificKey") + logger.info(s"Storing original image to instance specific key:${imageBucket.bucket} / $instanceSpecificKey") storeImage(imageBucket, instanceSpecificKey, storableImage.file, Some(storableImage.mimeType), storableImage.meta, overwrite = false) } @@ -42,7 +42,7 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config private def storeThumbnailImage(storableImage: StorableThumbImage) (implicit logMarker: LogMarker): Future[S3Object] = { val instanceSpecificKey = instanceAwareThumbnailImageKey(storableImage) - logger.info(s"Storing thumbnail to instance specific key: $thumbnailBucket / $instanceSpecificKey") + logger.info(s"Storing thumbnail to instance specific key: ${thumbnailBucket.bucket} / $instanceSpecificKey") storeImage(thumbnailBucket, instanceSpecificKey, storableImage.file, Some(storableImage.mimeType), overwrite = true) } @@ -50,12 +50,12 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config private def storeOptimisedImage(storableImage: StorableOptimisedImage) (implicit logMarker: LogMarker): Future[S3Object] = { val instanceSpecificKey = optimisedPngKeyFromId(storableImage.id)(storableImage.instance) - logger.info(s"Storing optimised image to instance specific key: $thumbnailBucket / $instanceSpecificKey") + logger.info(s"Storing optimised image to instance specific key: ${thumbnailBucket.bucket} / $instanceSpecificKey") storeImage(imageBucket, instanceSpecificKey, storableImage.file, Some(storableImage.mimeType), overwrite = true) } - private def bulkDelete(bucket: String, keys: List[String]): Future[Map[String, Boolean]] = keys match { + private def bulkDelete(bucket: S3Bucket, keys: List[String]): Future[Map[String, Boolean]] = keys match { case Nil => Future.successful(Map.empty) case _ => Future { deleteObjects(bucket, keys) @@ -90,8 +90,8 @@ sealed trait ImageWrapper { val instance: Instance } sealed trait StorableImage extends ImageWrapper { - def toProjectedS3Object(thumbBucket: String): S3Object = S3Object( - thumbBucket, + def toProjectedS3Object(thumbBucket: S3Bucket): S3Object = S3Object( + thumbBucket.bucket, ImageIngestOperations.fileKeyFromId(id)(instance), file, Some(mimeType), @@ -102,8 +102,8 @@ sealed trait StorableImage extends ImageWrapper { case class StorableThumbImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage case class StorableOriginalImage(id: String, file: File, mimeType: MimeType, lastModified: DateTime, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage { - override def toProjectedS3Object(thumbBucket: String): S3Object = S3Object( - thumbBucket, + override def toProjectedS3Object(thumbBucket: S3Bucket): S3Object = S3Object( + thumbBucket.bucket, ImageIngestOperations.fileKeyFromId(id)(instance), file, Some(mimeType), @@ -112,8 +112,8 @@ case class StorableOriginalImage(id: String, file: File, mimeType: MimeType, las ) } case class StorableOptimisedImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage { - override def toProjectedS3Object(thumbBucket: String): S3Object = S3Object( - thumbBucket, + override def toProjectedS3Object(thumbBucket: S3Bucket): S3Object = S3Object( + thumbBucket.bucket, ImageIngestOperations.optimisedPngKeyFromId(id)(instance), file, Some(mimeType), diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageQuarantineOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageQuarantineOperations.scala index 0cc3a146a0..eaf6b82979 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageQuarantineOperations.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageQuarantineOperations.scala @@ -2,13 +2,13 @@ package com.gu.mediaservice.lib import java.io.File import com.gu.mediaservice.lib.config.CommonConfig -import com.gu.mediaservice.lib.aws.S3Object +import com.gu.mediaservice.lib.aws.{S3Bucket, S3Object} import com.gu.mediaservice.lib.logging.LogMarker import com.gu.mediaservice.model.{Instance, MimeType} import scala.concurrent.Future -class ImageQuarantineOperations(quarantineBucket: String, config: CommonConfig, isVersionedS3: Boolean = false) +class ImageQuarantineOperations(quarantineBucket: S3Bucket, config: CommonConfig, isVersionedS3: Boolean = false) extends S3ImageStorage(config) { def storeQuarantineImage(id: String, file: File, mimeType: Option[MimeType], meta: Map[String, String] = Map.empty) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageStorage.scala index 1e57a3513a..1369b5e1eb 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageStorage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageStorage.scala @@ -6,7 +6,7 @@ import java.io.File import scala.concurrent.{ExecutionContext, Future} import scala.concurrent.duration._ import scala.language.postfixOps -import com.gu.mediaservice.lib.aws.S3Object +import com.gu.mediaservice.lib.aws.{S3Bucket, S3Object} import com.gu.mediaservice.lib.logging.LogMarker import com.gu.mediaservice.model.MimeType @@ -37,9 +37,9 @@ trait ImageStorage { /** Store a copy of the given file and return the URI of that copy. * The file can safely be deleted afterwards. */ - def storeImage(bucket: String, id: String, file: File, mimeType: Option[MimeType], + def storeImage(bucket: S3Bucket, id: String, file: File, mimeType: Option[MimeType], meta: Map[String, String] = Map.empty, overwrite: Boolean) (implicit logMarker: LogMarker): Future[S3Object] - def deleteImage(bucket: String, id: String)(implicit logMarker: LogMarker): Future[Unit] + def deleteImage(bucket: S3Bucket, id: String)(implicit logMarker: LogMarker): Future[Unit] } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala index bacd17b615..b7c3e925e2 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala @@ -1,6 +1,6 @@ package com.gu.mediaservice.lib -import com.gu.mediaservice.lib.aws.S3 +import com.gu.mediaservice.lib.aws.{S3, S3Bucket} import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} import com.gu.mediaservice.model.MimeType @@ -14,10 +14,10 @@ import scala.jdk.CollectionConverters._ class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage with GridLogging { private val cacheSetting = Some(cacheForever) - def storeImage(bucket: String, id: String, file: File, mimeType: Option[MimeType], + def storeImage(bucket: S3Bucket, id: String, file: File, mimeType: Option[MimeType], meta: Map[String, String] = Map.empty, overwrite: Boolean) (implicit logMarker: LogMarker) = { - logger.info(logMarker, s"bucket: $bucket, id: $id, meta: $meta") + logger.info(logMarker, s"bucket: ${bucket.bucket}, id: $id, meta: $meta") val eventualObject = if (overwrite) { store(bucket, id, file, mimeType, meta, cacheSetting) } else { @@ -27,20 +27,20 @@ class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage eventualObject } - def deleteImage(bucket: String, key: String)(implicit logMarker: LogMarker) = Future { + def deleteImage(bucket: S3Bucket, key: String)(implicit logMarker: LogMarker) = Future { deleteObject(bucket, key) - logger.info(logMarker, s"Deleted image $key from bucket $bucket") + logger.info(logMarker, s"Deleted image $key from bucket ${bucket.bucket}") } - def deleteVersionedImage(bucket: String, id: String)(implicit logMarker: LogMarker) = Future { + def deleteVersionedImage(bucket: S3Bucket, id: String)(implicit logMarker: LogMarker) = Future { val objectVersion = getMetadata(bucket, id).objectVersion.getOrElse( - throw new IllegalStateException(s"No version id found for $id in bucket $bucket") + throw new IllegalStateException(s"No version id found for $id in bucket ${bucket.bucket}") ) deleteVersion(bucket, id, objectVersion) - logger.info(logMarker, s"Deleted image $id from bucket $bucket (version: $objectVersion)") + logger.info(logMarker, s"Deleted image $id from bucket ${bucket.bucket} (version: $objectVersion)") } - def deleteFolder(bucket: String, id: String)(implicit logMarker: LogMarker): Future[Unit] = list(bucket, id).map { files => + def deleteFolder(bucket: S3Bucket, id: String)(implicit logMarker: LogMarker): Future[Unit] = list(bucket, id).map { files => logger.info(s"Found ${files.size} files to delete in folder $id") files.foreach(file => deleteObject(bucket, file.uri.getPath.stripPrefix("/"))) logger.info(logMarker, s"Deleting images in folder $id from bucket $bucket") diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala index a0c9d438b9..f95e075e21 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala @@ -1,14 +1,14 @@ package com.gu.mediaservice.lib.auth import com.gu.mediaservice.lib.BaseStore -import com.gu.mediaservice.lib.aws.S3 +import com.gu.mediaservice.lib.aws.{S3, S3Bucket} import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.model.Instance import scala.concurrent.duration._ import scala.concurrent.{Await, ExecutionContext} -class KeyStore(bucket: String, config: CommonConfig, s3: S3)(implicit ec: ExecutionContext) +class KeyStore(bucket: S3Bucket, config: CommonConfig, s3: S3)(implicit ec: ExecutionContext) extends BaseStore[String, ApiAccessor](bucket, config, s3)(ec) { def lookupIdentity(key: String)(implicit instance: Instance): Option[ApiAccessor] = store.get().get(instance.id + "/" + key) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 0be7a0ebb5..6a62ee6e8e 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -68,14 +68,11 @@ object S3Metadata { case class S3ObjectMetadata(contentType: Option[MimeType], cacheControl: Option[String], lastModified: Option[DateTime]) class S3(config: CommonConfig) extends GridLogging with ContentDisposition with RoundedExpiration { - type Bucket = String type Key = String type UserMetadata = Map[String, String] - private lazy val client: S3Client = S3Ops.buildS3Client(config) - private lazy val presigner = S3Ops.buildPresignerClientV2(config) def signUrl( - bucket: Bucket, + bucket: S3Bucket, url: URI, image: Image, expiration: DateTime = cachableExpiration(), @@ -91,7 +88,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with val remainingSeconds = Math.max(1, (targetExpirationMillis - nowMillis) / 1000) val getObjectRequest = GetObjectRequest.builder() - .bucket(bucket) + .bucket(bucket.bucket) .key(key) .responseContentDisposition(contentDisposition) .build() @@ -101,11 +98,11 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with .signatureDuration(Duration.ofSeconds(remainingSeconds)) .build() - val req = presigner.presignGetObject(getObjectPresignRequest) + val req = bucket.presigner.presignGetObject(getObjectPresignRequest) req.url().toExternalForm } - def signUrlTony(bucket: Bucket, url: URI, expiration: DateTime = cachableExpiration()): URL = { + def signUrlTony(bucket: S3Bucket, url: URI, expiration: DateTime = cachableExpiration()): URL = { // get path and remove leading `/` val key: Key = url.getPath.drop(1) @@ -114,7 +111,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with val remainingSeconds = Math.max(1, (targetExpirationMillis - nowMillis) / 1000) val getObjectRequest = GetObjectRequest.builder() - .bucket(bucket) + .bucket(bucket.bucket) .key(key) .build() @@ -123,108 +120,108 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with .signatureDuration(Duration.ofSeconds(remainingSeconds)) .build() - val req = presigner.presignGetObject(getObjectPresignRequest) + val req = bucket.presigner.presignGetObject(getObjectPresignRequest) req.url() } - def presignPutObject(putObjectPresignRequest: PutObjectPresignRequest): PresignedPutObjectRequest = { - presigner.presignPutObject(putObjectPresignRequest) + def presignPutObject(bucket: S3Bucket, putObjectPresignRequest: PutObjectPresignRequest): PresignedPutObjectRequest = { + bucket.presigner.presignPutObject(putObjectPresignRequest) } - def getObject(bucket: Bucket, url: URI): ResponseInputStream[GetObjectResponse]= { + def getObject(bucket: S3Bucket, url: URI): ResponseInputStream[GetObjectResponse]= { // get path and remove leading `/` val key: Key = url.getPath.drop(1) - client.getObject(GetObjectRequest.builder().key(key).bucket(bucket).build()) + bucket.client.getObject(GetObjectRequest.builder().key(key).bucket(bucket.bucket).build()) } - def getObject(bucket: Bucket, key: String): ResponseInputStream[GetObjectResponse] = { - client.getObject(GetObjectRequest.builder().key(key).bucket(bucket).build()) + def getObject(bucket: S3Bucket, key: String): ResponseInputStream[GetObjectResponse] = { + bucket.client.getObject(GetObjectRequest.builder().key(key).bucket(bucket.bucket).build()) } - def getObjectAsString(bucket: Bucket, key: String): Option[String] = { + def getObjectAsString(bucket: S3Bucket, key: String): Option[String] = { try { - val stream = client.getObject(GetObjectRequest.builder().key(key).bucket(bucket).build()); + val stream = bucket.client.getObject(GetObjectRequest.builder().key(key).bucket(bucket.bucket).build()); Some(new String(stream.readAllBytes(), StandardCharsets.UTF_8)) } catch { case e: NoSuchKeyException => - logger.warn(s"Cannot find key: $key in bucket: $bucket") + logger.warn(s"Cannot find key: $key in bucket: ${bucket.bucket}") None } } - def putString(bucket: String, key: String, fileContents: String) = { - client.putObject(PutObjectRequest.builder().bucket(bucket).key(key).build(), RequestBody.fromString(fileContents)) + def putString(bucket: S3Bucket, key: String, fileContents: String) = { + bucket.client.putObject(PutObjectRequest.builder().bucket(bucket.bucket).key(key).build(), RequestBody.fromString(fileContents)) } - def store(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) + def store(bucket: S3Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) (implicit ex: ExecutionContext, logMarker: LogMarker): Future[S3Object] = Future { val fileMarkers = Map( - "bucket" -> bucket, + "bucket" -> bucket.bucket, ) val markers = logMarker ++ fileMarkers - val reqBuilder = PutObjectRequest.builder().key(id).bucket(bucket) + val reqBuilder = PutObjectRequest.builder().key(id).bucket(bucket.bucket) cacheControl.foreach(c => reqBuilder.cacheControl(c)) mimeType.foreach(m => reqBuilder.contentType(m.name)) reqBuilder.metadata(meta.asJava) val req = reqBuilder.build() Stopwatch(s"S3 client.putObject ($req)"){ - client.putObject(req, RequestBody.fromFile(file)) + bucket.client.putObject(req, RequestBody.fromFile(file)) // once we've completed the PUT read back to ensure that we are returning reality - val metadata = client.headObject( - HeadObjectRequest.builder().key(id).bucket(bucket).build() + val metadata = bucket.client.headObject( + HeadObjectRequest.builder().key(id).bucket(bucket.bucket).build() ) - S3Object(bucket, id, metadata.contentLength(), S3Metadata(metadata)) + S3Object(bucket.bucket, id, metadata.contentLength(), S3Metadata(metadata)) }(markers) } - def storeIfNotPresent(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) + def storeIfNotPresent(bucket: S3Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) (implicit ex: ExecutionContext, logMarker: LogMarker): Future[S3Object] = { Future { - Some(client.headObject( - HeadObjectRequest.builder().key(id).bucket(bucket).build() + Some(bucket.client.headObject( + HeadObjectRequest.builder().key(id).bucket(bucket.bucket).build() )) }.recover { // translate this exception into the object not existing case _: NoSuchKeyException => None }.flatMap { case Some(metadata) => - logger.info(logMarker, s"Skipping storing of S3 file $id as key is already present in bucket $bucket") - Future.successful(S3Object(bucket, id, metadata.contentLength(), S3Metadata(metadata))) + logger.info(logMarker, s"Skipping storing of S3 file $id as key is already present in bucket ${bucket.bucket}") + Future.successful(S3Object(bucket.bucket, id, metadata.contentLength(), S3Metadata(metadata))) case None => store(bucket, id, file, mimeType, meta, cacheControl) } } - def list(bucket: Bucket, prefixDir: String) + def list(bucket: S3Bucket, prefixDir: String) (implicit ex: ExecutionContext): Future[List[S3Object]] = Future { - val req = ListObjectsV2Request.builder().bucket(bucket).prefix(s"$prefixDir/").build() - val listing = client.listObjectsV2(req) + val req = ListObjectsV2Request.builder().bucket(bucket.bucket).prefix(s"$prefixDir/").build() + val listing = bucket.client.listObjectsV2(req) val s3Objects = listing.contents().asScala.toList s3Objects.map(s3Object => { - S3Object(bucket, s3Object.key(), size = s3Object.size(), metadata = getMetadata(bucket, s3Object.key())) + S3Object(bucket.bucket, s3Object.key(), size = s3Object.size(), metadata = getMetadata(bucket, s3Object.key())) }) } - def getMetadata(bucket: Bucket, key: Key): S3Metadata = { - val meta = client.headObject(HeadObjectRequest.builder().key(key).bucket(bucket).build()) + def getMetadata(bucket: S3Bucket, key: Key): S3Metadata = { + val meta = bucket.client.headObject(HeadObjectRequest.builder().key(key).bucket(bucket.bucket).build()) S3Metadata(meta) } - def syncFindKey(bucket: Bucket, prefixName: String): Option[Key] = { - val req = ListObjectsV2Request.builder().bucket(bucket).prefix(s"$prefixName-").build() - val objects = client.listObjectsV2(req).contents().asScala.toList + def syncFindKey(bucket: S3Bucket, prefixName: String): Option[Key] = { + val req = ListObjectsV2Request.builder().bucket(bucket.bucket).prefix(s"$prefixName-").build() + val objects = bucket.client.listObjectsV2(req).contents().asScala.toList objects.headOption.map(_.key()) } - def doesObjectExist(bucket: Bucket, key: String) = { + def doesObjectExist(bucket: S3Bucket, key: String) = { try { - client.headObject( - HeadObjectRequest.builder().key(key).bucket(bucket).build() + bucket.client.headObject( + HeadObjectRequest.builder().key(key).bucket(bucket.bucket).build() ) true } catch { @@ -232,17 +229,17 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with } } - def deleteObject(bucket: Bucket, key: String): Unit = - client.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(key).build()) + def deleteObject(bucket: S3Bucket, key: String): Unit = + bucket.client.deleteObject(DeleteObjectRequest.builder().bucket(bucket.bucket).key(key).build()) - def deleteObjects(bucket: Bucket, keys: List[String]): Map[String, Boolean] = { + def deleteObjects(bucket: S3Bucket, keys: List[String]): Map[String, Boolean] = { val objects: util.List[ObjectIdentifier] = keys.map { key => ObjectIdentifier.builder() .key(key) .build() }.asJava - val response = client.deleteObjects( - DeleteObjectsRequest.builder().bucket(bucket) + val response = bucket.client.deleteObjects( + DeleteObjectsRequest.builder().bucket(bucket.bucket) .delete(Delete.builder().objects(objects).build()) .build() ) @@ -252,15 +249,15 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with }.toMap } - def deleteVersion(bucket: Bucket, key: String, objectVersion: String): Unit = - client.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(key).versionId(objectVersion).build()) + def deleteVersion(bucket: S3Bucket, key: String, objectVersion: String): Unit = + bucket.client.deleteObject(DeleteObjectRequest.builder().bucket(bucket.bucket).key(key).versionId(objectVersion).build()) - def copy(key: String, sourceBucket: String, destinationBucket: String): CopyObjectResponse = { - client.copyObject( + def copy(key: String, sourceBucket: S3Bucket, destinationBucket: S3Bucket): CopyObjectResponse = { + sourceBucket.client.copyObject( CopyObjectRequest.builder() - .sourceBucket(sourceBucket) + .sourceBucket(sourceBucket.bucket) .sourceKey(key) - .destinationBucket(destinationBucket) + .destinationBucket(destinationBucket.bucket) .destinationKey(key) .build() ) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala index 3f049d94d1..16b6e69d74 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala @@ -1,6 +1,29 @@ package com.gu.mediaservice.lib.aws +import com.gu.mediaservice.lib.config.CommonConfig import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3.presigner.S3Presigner case class S3Bucket(bucket: String, endPoint: String, usesPathStyleURLs: Boolean, client: S3Client, presigner: S3Presigner) + +object S3Bucket { + + /** + * Build a bucket that talks to the endpoint implied by the current environment - i.e. the localstack + * endpoint (with path style URLs) when running in DEV, otherwise the real AWS S3 endpoint. + */ + def apply(bucket: String, config: CommonConfig): S3Bucket = { + val endpointOverride = config.awsLocalEndpoint + val usesPathStyleURLs = endpointOverride.isDefined + apply(bucket, config, endpointOverride, usesPathStyleURLs) + } + + def apply(bucket: String, config: CommonConfig, endpointOverride: Option[String], usesPathStyleURLs: Boolean): S3Bucket = + S3Bucket( + bucket = bucket, + endPoint = endpointOverride.getOrElse(S3Ops.s3Endpoint), + usesPathStyleURLs = usesPathStyleURLs, + client = S3Ops.buildS3Client(config, endpointOverride, usesPathStyleURLs), + presigner = S3Ops.buildPresignerClientV2(config, endpointOverride, usesPathStyleURLs) + ) +} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala index 620ec94e8d..bf26b0b039 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala @@ -1,6 +1,6 @@ package com.gu.mediaservice.lib.config -import com.gu.mediaservice.lib.aws.{AwsClientBuilderUtils, KinesisSenderConfig} +import com.gu.mediaservice.lib.aws.{AwsClientBuilderUtils, KinesisSenderConfig, S3Bucket} import com.gu.mediaservice.model.UsageRightsSpec import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging @@ -60,17 +60,17 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientBui lazy val softDeletedMetadataTable: String = string("dynamo.table.softDelete.metadata") - val imageBucket: String = string("s3.image.bucket") - val thumbnailBucket: String = string("s3.thumb.bucket") - val imgPublishingBucket: String = string("publishing.image.bucket") + val imageBucket: S3Bucket = S3Bucket(string("s3.image.bucket"), this) + val thumbnailBucket: S3Bucket = S3Bucket(string("s3.thumb.bucket"), this) + val imgPublishingBucket: S3Bucket = S3Bucket(string("publishing.image.bucket"), this) val maybeIngestSqsQueueUrl: Option[String] = stringOpt("sqs.ingest.queue.url") - val maybeIngestBucket: Option[String] = stringOpt("s3.ingest.bucket") - val maybeFailBucket: Option[String] = stringOpt("s3.fail.bucket") + val maybeIngestBucket: Option[S3Bucket] = stringOpt("s3.ingest.bucket").map(S3Bucket(_, this)) + val maybeFailBucket: Option[S3Bucket] = stringOpt("s3.fail.bucket").map(S3Bucket(_, this)) - val maybeQuarantineBucket: Option[String] = stringOpt("s3.quarantine.bucket") + val maybeQuarantineBucket: Option[S3Bucket] = stringOpt("s3.quarantine.bucket").map(S3Bucket(_, this)) - val maybeBucketForUIUploads: Option[String] = maybeQuarantineBucket orElse maybeIngestBucket + val maybeBucketForUIUploads: Option[S3Bucket] = maybeQuarantineBucket orElse maybeIngestBucket val maybeUploadLimitInBytes: Option[Int] = intOpt("upload.limit.mb").map(_ * 1024 * 1024) diff --git a/cropper/app/lib/CropStore.scala b/cropper/app/lib/CropStore.scala index 749764337e..dcb6ada98f 100644 --- a/cropper/app/lib/CropStore.scala +++ b/cropper/app/lib/CropStore.scala @@ -78,7 +78,7 @@ class CropStore(config: CropperConfig) extends S3ImageStorage(config) with CropS def translateImgHost(uri: URI): URI = new URI("https", config.imgPublishingHost, uri.getPath, uri.getFragment) - private def folderForImagesCrops(id: Bucket, instance: Instance) = { + private def folderForImagesCrops(id: String, instance: Instance) = { instance.id + "/" + id } diff --git a/cropper/app/lib/Crops.scala b/cropper/app/lib/Crops.scala index 2eb027499f..215987d00d 100644 --- a/cropper/app/lib/Crops.scala +++ b/cropper/app/lib/Crops.scala @@ -3,7 +3,7 @@ package lib import java.io.File import com.gu.mediaservice.lib.metadata.FileMetadataHelper import com.gu.mediaservice.lib.Files -import com.gu.mediaservice.lib.aws.S3 +import com.gu.mediaservice.lib.aws.{S3, S3Bucket} import com.gu.mediaservice.lib.imaging.{ExportResult, ImageOperations} import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch} import com.gu.mediaservice.model._ @@ -17,7 +17,7 @@ case object InvalidCropRequest extends Exception("Crop request invalid for image case class MasterCrop(sizing: Future[Asset], file: File, dimensions: Dimensions, aspectRatio: Float) -class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOperations, imageBucket: String, s3: S3)(implicit ec: ExecutionContext) extends GridLogging { +class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOperations, imageBucket: S3Bucket, s3: S3)(implicit ec: ExecutionContext) extends GridLogging { import Files._ private val cropQuality = 75d diff --git a/cropper/test/lib/CropsTest.scala b/cropper/test/lib/CropsTest.scala index 912feca0d9..b834611a7a 100644 --- a/cropper/test/lib/CropsTest.scala +++ b/cropper/test/lib/CropsTest.scala @@ -1,6 +1,6 @@ package lib -import com.gu.mediaservice.lib.aws.S3 +import com.gu.mediaservice.lib.aws.{S3, S3Bucket} import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.model._ import org.scalatest.funspec.AnyFunSpec @@ -51,7 +51,7 @@ class CropsTest extends AnyFunSpec with Matchers with MockitoSugar { private val source: SourceImage = SourceImage("test", mock[Asset], valid = true, mock[ImageMetadata], mock[FileMetadata]) private val bounds: Bounds = Bounds(10, 20, 30, 40) private val outputWidth = 1234 - private val imageBucket = "crops-bucket" + private val imageBucket = S3Bucket("crops-bucket", "s3.amazonaws.com", usesPathStyleURLs = false, client = null, presigner = null) private val s3 = new S3(config) it("should should construct a correct address for a master jpg") { diff --git a/image-loader/app/controllers/ImageLoaderController.scala b/image-loader/app/controllers/ImageLoaderController.scala index 2c85b621cb..7c8548a88d 100644 --- a/image-loader/app/controllers/ImageLoaderController.scala +++ b/image-loader/app/controllers/ImageLoaderController.scala @@ -460,7 +460,7 @@ class ImageLoaderController(auth: Authentication, logger.info(context, "image found") Ok(Json.toJson(img)).as(ArgoMediaType) case None => - val s3Path = "s3://" + config.imageBucket + "/" + ImageIngestOperations.fileKeyFromId(imageId) + val s3Path = "s3://" + config.imageBucket.bucket + "/" + ImageIngestOperations.fileKeyFromId(imageId) logger.info(context, "image not found") respondError(NotFound, "image-not-found", s"Could not find image: $imageId in s3 at $s3Path") } recover { diff --git a/image-loader/app/lib/ImageLoaderStore.scala b/image-loader/app/lib/ImageLoaderStore.scala index 065eadfdea..584ac87a76 100644 --- a/image-loader/app/lib/ImageLoaderStore.scala +++ b/image-loader/app/lib/ImageLoaderStore.scala @@ -48,8 +48,9 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati } def generatePreSignedUploadUrl(filename: String, duration: Duration, uploadedBy: String, mediaId: String)(implicit instance: Instance): String = { + val bucket = config.maybeBucketForUIUploads.get val putObjectRequest = PutObjectRequest.builder() - .bucket(config.maybeBucketForUIUploads.get).key(s"${instance.id}/$uploadedBy/$filename").metadata(Map( + .bucket(bucket.bucket).key(s"${instance.id}/$uploadedBy/$filename").metadata(Map( "media-id" -> mediaId).asJava) .build() val putObjectPresignRequest = @@ -58,7 +59,7 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati .signatureDuration(duration) .build(); - val req = presignPutObject(putObjectPresignRequest) + val req = presignPutObject(bucket, putObjectPresignRequest) req.url().toExternalForm } diff --git a/image-loader/app/model/Projector.scala b/image-loader/app/model/Projector.scala index b50a04b5aa..fdaab7b496 100644 --- a/image-loader/app/model/Projector.scala +++ b/image-loader/app/model/Projector.scala @@ -2,7 +2,7 @@ package model import com.gu.mediaservice.lib.ImageIngestOperations.{fileKeyFromId, optimisedPngKeyFromId} import com.gu.mediaservice.lib.auth.Authentication -import com.gu.mediaservice.lib.aws.{Embedder, S3} +import com.gu.mediaservice.lib.aws.{Embedder, S3, S3Bucket} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.config.InstanceForRequest import com.gu.mediaservice.lib.imaging.ImageOperations @@ -93,9 +93,9 @@ class Projector(config: ImageUploadOpsCfg, val s3Key = fileKeyFromId(imageId) if (!s3.doesObjectExist(config.originalFileBucket, s3Key)) - throw new NoSuchImageExistsInS3(config.originalFileBucket, s3Key) + throw new NoSuchImageExistsInS3(config.originalFileBucket.bucket, s3Key) - val s3Source = Stopwatch(s"object exists, getting s3 object at s3://${config.originalFileBucket}/$s3Key to perform Image projection"){ + val s3Source = Stopwatch(s"object exists, getting s3 object at s3://${config.originalFileBucket.bucket}/$s3Key to perform Image projection"){ s3.getObject(config.originalFileBucket, s3Key) }(logMarker) @@ -203,13 +203,13 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg, } private def fetchFile( - bucket: String, key: String, outFile: File + bucket: S3Bucket, key: String, outFile: File )(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[(File, MimeType)]] = { - logger.info(logMarker, s"Trying fetch existing image from S3 bucket - $bucket at key $key") + logger.info(logMarker, s"Trying fetch existing image from S3 bucket - ${bucket.bucket} at key $key") val doesFileExist = Future { s3.doesObjectExist(bucket, key) } recover { case _ => false } doesFileExist.flatMap { case false => - logger.warn(logMarker, s"image did not exist in bucket $bucket at key $key") + logger.warn(logMarker, s"image did not exist in bucket ${bucket.bucket} at key $key") Future.successful(None) // falls back to creating from original file case true => val obj = s3.getObject(bucket, key) diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala index e50089c98d..65f58181ec 100644 --- a/image-loader/app/model/Uploader.scala +++ b/image-loader/app/model/Uploader.scala @@ -7,7 +7,7 @@ import com.gu.mediaservice.lib.ImageIngestOperations.fileKeyFromId import com.gu.mediaservice.lib._ import com.gu.mediaservice.lib.argo.ArgoHelpers import com.gu.mediaservice.lib.auth.Authentication -import com.gu.mediaservice.lib.aws.{Embedder, EmbedderMessage, S3Object, UpdateMessage} +import com.gu.mediaservice.lib.aws.{Embedder, EmbedderMessage, S3Bucket, S3Object, UpdateMessage} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.formatting._ import com.gu.mediaservice.lib.imaging.ImageOperations @@ -67,8 +67,8 @@ case class ImageUploadOpsCfg( thumbWidth: Int, thumbQuality: Double, transcodedMimeTypes: List[MimeType], - originalFileBucket: String, - thumbBucket: String + originalFileBucket: S3Bucket, + thumbBucket: S3Bucket ) case class ImageUploadOpsDependencies( diff --git a/image-loader/test/scala/lib/ResourceHelpers.scala b/image-loader/test/scala/lib/ResourceHelpers.scala index 2abae0ce4d..2d4c762da0 100644 --- a/image-loader/test/scala/lib/ResourceHelpers.scala +++ b/image-loader/test/scala/lib/ResourceHelpers.scala @@ -1,5 +1,7 @@ package test.lib +import com.gu.mediaservice.lib.aws.S3Bucket + import java.io.File object ResourceHelpers { @@ -8,4 +10,8 @@ object ResourceHelpers { new File(getClass.getResource(s"/$resourcePath").toURI) } + /** A bucket with no working client/presigner, for tests that never touch S3. */ + def dummyBucket(name: String): S3Bucket = + S3Bucket(name, "s3.amazonaws.com", usesPathStyleURLs = false, client = null, presigner = null) + } diff --git a/image-loader/test/scala/model/ImageUploadTest.scala b/image-loader/test/scala/model/ImageUploadTest.scala index 9711519a0c..f29397ad6d 100644 --- a/image-loader/test/scala/model/ImageUploadTest.scala +++ b/image-loader/test/scala/model/ImageUploadTest.scala @@ -32,7 +32,7 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar { private implicit val logMarker: MockLogMarker = new MockLogMarker() // For mime type info, see https://github.com/guardian/grid/pull/2568 val tempDir = new File("/tmp") - val mockConfig: ImageUploadOpsCfg = ImageUploadOpsCfg(tempDir, 256, 85d, List(Tiff), "img-bucket", "thumb-bucket") + val mockConfig: ImageUploadOpsCfg = ImageUploadOpsCfg(tempDir, 256, 85d, List(Tiff), ResourceHelpers.dummyBucket("img-bucket"), ResourceHelpers.dummyBucket("thumb-bucket")) /** * @todo: I flailed about until I found a path that worked, but diff --git a/image-loader/test/scala/model/ProjectorTest.scala b/image-loader/test/scala/model/ProjectorTest.scala index 3c9e27e87e..b7194928c9 100644 --- a/image-loader/test/scala/model/ProjectorTest.scala +++ b/image-loader/test/scala/model/ProjectorTest.scala @@ -41,7 +41,7 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc private val imageOperations = new ImageOperations(ctxPath) - private val config = ImageUploadOpsCfg(new File("/tmp"), 256, 85d, Nil, "img-bucket", "thumb-bucket") + private val config = ImageUploadOpsCfg(new File("/tmp"), 256, 85d, Nil, dummyBucket("img-bucket"), dummyBucket("thumb-bucket")) private val maybeEmbedder = None diff --git a/media-api/app/lib/MediaApiConfig.scala b/media-api/app/lib/MediaApiConfig.scala index 3952c0f2b6..13ea92abaa 100644 --- a/media-api/app/lib/MediaApiConfig.scala +++ b/media-api/app/lib/MediaApiConfig.scala @@ -1,5 +1,6 @@ package lib +import com.gu.mediaservice.lib.aws.S3Bucket import com.gu.mediaservice.lib.config.{CommonConfigWithElastic, GridConfigResources} import com.gu.mediaservice.lib.elasticsearch.filters import com.sksamuel.elastic4s.ElasticApi.{matchPhraseQuery, should} @@ -14,13 +15,13 @@ import scala.collection.immutable import scala.util.Try case class StoreConfig( - storeBucket: String, + storeBucket: S3Bucket, storeKey: String ) class MediaApiConfig(resources: GridConfigResources) extends CommonConfigWithElastic(resources) { - val configBucket: String = string("s3.config.bucket") - val usageMailBucket: String = string("s3.usagemail.bucket") + val configBucket: S3Bucket = S3Bucket(string("s3.config.bucket"), this) + val usageMailBucket: S3Bucket = S3Bucket(string("s3.usagemail.bucket"), this) val quotaStoreKey: String = string("quota.store.key") val quotaStoreConfig: StoreConfig = StoreConfig(configBucket, quotaStoreKey) diff --git a/media-api/app/lib/QuotaStore.scala b/media-api/app/lib/QuotaStore.scala index 8e8052096a..9b380e6f32 100644 --- a/media-api/app/lib/QuotaStore.scala +++ b/media-api/app/lib/QuotaStore.scala @@ -1,14 +1,14 @@ package lib import com.gu.mediaservice.lib.BaseStore -import com.gu.mediaservice.lib.aws.S3 +import com.gu.mediaservice.lib.aws.{S3, S3Bucket} import play.api.libs.json.Json import scala.concurrent.ExecutionContext class QuotaStore( quotaFile: String, - bucket: String, + bucket: S3Bucket, config: MediaApiConfig, s3: S3 )(implicit ec: ExecutionContext) extends BaseStore[String, SupplierUsageQuota](bucket, config, s3)(ec) { diff --git a/media-api/app/lib/UsageStore.scala b/media-api/app/lib/UsageStore.scala index 65b3ab3bde..8e229a3e45 100644 --- a/media-api/app/lib/UsageStore.scala +++ b/media-api/app/lib/UsageStore.scala @@ -1,7 +1,7 @@ package lib import com.gu.mediaservice.lib.BaseStore -import com.gu.mediaservice.lib.aws.S3 +import com.gu.mediaservice.lib.aws.{S3, S3Bucket} import com.gu.mediaservice.lib.logging.GridLogging import com.gu.mediaservice.model.{Agencies, Agency, UsageRights} import com.gu.mediaservice.model.usage.{DigitalUsage, PrintUsage, PublishedUsageStatus, RemovedUsageStatus, UnknownUsageStatus, Usage, UsageStatus, UsageType} @@ -61,7 +61,7 @@ object UsageStore extends GridLogging { } class UsageStore( - bucket: String, + bucket: S3Bucket, config: MediaApiConfig, quotaStore: QuotaStore, s3: S3 diff --git a/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala b/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala index b74fc8b41f..d315662b74 100644 --- a/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala +++ b/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala @@ -2,7 +2,7 @@ package com.gu.mediaservice.lib.auth.provider import com.gu.mediaservice.lib.auth.Authentication.{MachinePrincipal, Principal} import com.gu.mediaservice.lib.auth.provider.ApiKeyAuthenticationProvider.{ApiKeyInstance, KindeIdKey} import com.gu.mediaservice.lib.auth.{ApiAccessor, KeyStore} -import com.gu.mediaservice.lib.aws.S3 +import com.gu.mediaservice.lib.aws.{S3, S3Bucket} import com.gu.mediaservice.lib.config.InstanceForRequest import com.gu.mediaservice.lib.events.UsageEvents import com.gu.mediaservice.model.Instance @@ -26,7 +26,7 @@ class ApiKeyAuthenticationProvider(configuration: Configuration, resources: Auth var keyStorePlaceholder: Option[KeyStore] = _ override def initialise(): Unit = { - val store = new KeyStore(configuration.get[String]("authKeyStoreBucket"), resources.commonConfig, new S3(resources.commonConfig)) + val store = new KeyStore(S3Bucket(configuration.get[String]("authKeyStoreBucket"), resources.commonConfig), resources.commonConfig, new S3(resources.commonConfig)) store.scheduleUpdates(resources.actorSystem.scheduler) keyStorePlaceholder = Some(store) } diff --git a/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala b/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala index d9f036be40..72d906d1f4 100644 --- a/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala +++ b/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala @@ -3,7 +3,7 @@ package com.gu.mediaservice.lib.auth import org.apache.pekko.actor.ActorSystem import com.gu.mediaservice.lib.auth.Authentication.MachinePrincipal import com.gu.mediaservice.lib.auth.provider.{ApiKeyAuthenticationProvider, Authenticated, AuthenticationProviderResources, Invalid, NotAuthenticated, NotAuthorised} -import com.gu.mediaservice.lib.aws.S3 +import com.gu.mediaservice.lib.aws.{S3, S3Bucket} import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources} import com.gu.mediaservice.lib.events.UsageEvents import com.gu.mediaservice.model.Instance @@ -45,7 +45,7 @@ class ApiKeyAuthenticationProviderTest extends AsyncFreeSpec with Matchers with Future.successful(()) } - override def keyStore: KeyStore = new KeyStore("not-used", resources.commonConfig, s3) { + override def keyStore: KeyStore = new KeyStore(S3Bucket("not-used", "s3.amazonaws.com", usesPathStyleURLs = false, client = null, presigner = null), resources.commonConfig, s3) { override def lookupIdentity(key: String)(implicit instance: Instance): Option[ApiAccessor] = { key match { case "key-chuckle" => Some(ApiAccessor("brothers", Internal)) diff --git a/thrall/app/lib/ThrallConfig.scala b/thrall/app/lib/ThrallConfig.scala index 88ad70e0e0..8518a44205 100644 --- a/thrall/app/lib/ThrallConfig.scala +++ b/thrall/app/lib/ThrallConfig.scala @@ -1,6 +1,6 @@ package lib -import com.gu.mediaservice.lib.aws.AwsClientBuilderUtils +import com.gu.mediaservice.lib.aws.{AwsClientBuilderUtils, S3Bucket} import com.gu.mediaservice.lib.cleanup.ReapableEligibiltyResources import com.gu.mediaservice.lib.config.{CommonConfigWithElastic, GridConfigResources, ReapableEligibilityLoader} import com.gu.mediaservice.lib.elasticsearch.ReapableEligibility @@ -56,7 +56,7 @@ object KinesisReceiverConfig { } class ThrallConfig(resources: GridConfigResources) extends CommonConfigWithElastic(resources) { - val maybeReaperBucket: Option[String] = stringOpt("s3.reaper.bucket") + val maybeReaperBucket: Option[S3Bucket] = stringOpt("s3.reaper.bucket").map(S3Bucket(_, this)) val maybeReaperCountPerRun: Option[Int] = intOpt("reaper.countPerRun") val metadataTopicArn: String = string("indexed.image.sns.topic.arn") From 19c738cf74494d052aca654f9374dfc94ac504f9 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Sat, 29 Aug 2026 11:25:31 +0100 Subject: [PATCH 29/46] bucket.name --- .../lib/ImageIngestOperations.scala | 12 ++--- .../gu/mediaservice/lib/S3ImageStorage.scala | 8 ++-- .../com/gu/mediaservice/lib/aws/S3.scala | 48 +++++++++---------- .../gu/mediaservice/lib/aws/S3Bucket.scala | 10 ++-- .../controllers/ImageLoaderController.scala | 2 +- image-loader/app/lib/ImageLoaderStore.scala | 2 +- image-loader/app/model/Projector.scala | 8 ++-- 7 files changed, 45 insertions(+), 45 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala index 4055e89787..8050996063 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala @@ -34,7 +34,7 @@ class ImageIngestOperations(imageBucket: S3Bucket, thumbnailBucket: S3Bucket, co private def storeOriginalImage(storableImage: StorableOriginalImage) (implicit logMarker: LogMarker): Future[S3Object] = { val instanceSpecificKey = instanceAwareOriginalImageKey(storableImage) - logger.info(s"Storing original image to instance specific key:${imageBucket.bucket} / $instanceSpecificKey") + logger.info(s"Storing original image to instance specific key:${imageBucket.name} / $instanceSpecificKey") storeImage(imageBucket, instanceSpecificKey, storableImage.file, Some(storableImage.mimeType), storableImage.meta, overwrite = false) } @@ -42,7 +42,7 @@ class ImageIngestOperations(imageBucket: S3Bucket, thumbnailBucket: S3Bucket, co private def storeThumbnailImage(storableImage: StorableThumbImage) (implicit logMarker: LogMarker): Future[S3Object] = { val instanceSpecificKey = instanceAwareThumbnailImageKey(storableImage) - logger.info(s"Storing thumbnail to instance specific key: ${thumbnailBucket.bucket} / $instanceSpecificKey") + logger.info(s"Storing thumbnail to instance specific key: ${thumbnailBucket.name} / $instanceSpecificKey") storeImage(thumbnailBucket, instanceSpecificKey, storableImage.file, Some(storableImage.mimeType), overwrite = true) } @@ -50,7 +50,7 @@ class ImageIngestOperations(imageBucket: S3Bucket, thumbnailBucket: S3Bucket, co private def storeOptimisedImage(storableImage: StorableOptimisedImage) (implicit logMarker: LogMarker): Future[S3Object] = { val instanceSpecificKey = optimisedPngKeyFromId(storableImage.id)(storableImage.instance) - logger.info(s"Storing optimised image to instance specific key: ${thumbnailBucket.bucket} / $instanceSpecificKey") + logger.info(s"Storing optimised image to instance specific key: ${thumbnailBucket.name} / $instanceSpecificKey") storeImage(imageBucket, instanceSpecificKey, storableImage.file, Some(storableImage.mimeType), overwrite = true) } @@ -91,7 +91,7 @@ sealed trait ImageWrapper { } sealed trait StorableImage extends ImageWrapper { def toProjectedS3Object(thumbBucket: S3Bucket): S3Object = S3Object( - thumbBucket.bucket, + thumbBucket.name, ImageIngestOperations.fileKeyFromId(id)(instance), file, Some(mimeType), @@ -103,7 +103,7 @@ sealed trait StorableImage extends ImageWrapper { case class StorableThumbImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage case class StorableOriginalImage(id: String, file: File, mimeType: MimeType, lastModified: DateTime, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage { override def toProjectedS3Object(thumbBucket: S3Bucket): S3Object = S3Object( - thumbBucket.bucket, + thumbBucket.name, ImageIngestOperations.fileKeyFromId(id)(instance), file, Some(mimeType), @@ -113,7 +113,7 @@ case class StorableOriginalImage(id: String, file: File, mimeType: MimeType, las } case class StorableOptimisedImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage { override def toProjectedS3Object(thumbBucket: S3Bucket): S3Object = S3Object( - thumbBucket.bucket, + thumbBucket.name, ImageIngestOperations.optimisedPngKeyFromId(id)(instance), file, Some(mimeType), diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala index b7c3e925e2..f419f7f430 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala @@ -17,7 +17,7 @@ class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage def storeImage(bucket: S3Bucket, id: String, file: File, mimeType: Option[MimeType], meta: Map[String, String] = Map.empty, overwrite: Boolean) (implicit logMarker: LogMarker) = { - logger.info(logMarker, s"bucket: ${bucket.bucket}, id: $id, meta: $meta") + logger.info(logMarker, s"bucket: ${bucket.name}, id: $id, meta: $meta") val eventualObject = if (overwrite) { store(bucket, id, file, mimeType, meta, cacheSetting) } else { @@ -29,15 +29,15 @@ class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage def deleteImage(bucket: S3Bucket, key: String)(implicit logMarker: LogMarker) = Future { deleteObject(bucket, key) - logger.info(logMarker, s"Deleted image $key from bucket ${bucket.bucket}") + logger.info(logMarker, s"Deleted image $key from bucket ${bucket.name}") } def deleteVersionedImage(bucket: S3Bucket, id: String)(implicit logMarker: LogMarker) = Future { val objectVersion = getMetadata(bucket, id).objectVersion.getOrElse( - throw new IllegalStateException(s"No version id found for $id in bucket ${bucket.bucket}") + throw new IllegalStateException(s"No version id found for $id in bucket ${bucket.name}") ) deleteVersion(bucket, id, objectVersion) - logger.info(logMarker, s"Deleted image $id from bucket ${bucket.bucket} (version: $objectVersion)") + logger.info(logMarker, s"Deleted image $id from bucket ${bucket.name} (version: $objectVersion)") } def deleteFolder(bucket: S3Bucket, id: String)(implicit logMarker: LogMarker): Future[Unit] = list(bucket, id).map { files => diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 6a62ee6e8e..1ef7e13a22 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -88,7 +88,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with val remainingSeconds = Math.max(1, (targetExpirationMillis - nowMillis) / 1000) val getObjectRequest = GetObjectRequest.builder() - .bucket(bucket.bucket) + .bucket(bucket.name) .key(key) .responseContentDisposition(contentDisposition) .build() @@ -111,7 +111,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with val remainingSeconds = Math.max(1, (targetExpirationMillis - nowMillis) / 1000) val getObjectRequest = GetObjectRequest.builder() - .bucket(bucket.bucket) + .bucket(bucket.name) .key(key) .build() @@ -131,26 +131,26 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with def getObject(bucket: S3Bucket, url: URI): ResponseInputStream[GetObjectResponse]= { // get path and remove leading `/` val key: Key = url.getPath.drop(1) - bucket.client.getObject(GetObjectRequest.builder().key(key).bucket(bucket.bucket).build()) + bucket.client.getObject(GetObjectRequest.builder().key(key).bucket(bucket.name).build()) } def getObject(bucket: S3Bucket, key: String): ResponseInputStream[GetObjectResponse] = { - bucket.client.getObject(GetObjectRequest.builder().key(key).bucket(bucket.bucket).build()) + bucket.client.getObject(GetObjectRequest.builder().key(key).bucket(bucket.name).build()) } def getObjectAsString(bucket: S3Bucket, key: String): Option[String] = { try { - val stream = bucket.client.getObject(GetObjectRequest.builder().key(key).bucket(bucket.bucket).build()); + val stream = bucket.client.getObject(GetObjectRequest.builder().key(key).bucket(bucket.name).build()); Some(new String(stream.readAllBytes(), StandardCharsets.UTF_8)) } catch { case e: NoSuchKeyException => - logger.warn(s"Cannot find key: $key in bucket: ${bucket.bucket}") + logger.warn(s"Cannot find key: $key in bucket: ${bucket.name}") None } } def putString(bucket: S3Bucket, key: String, fileContents: String) = { - bucket.client.putObject(PutObjectRequest.builder().bucket(bucket.bucket).key(key).build(), RequestBody.fromString(fileContents)) + bucket.client.putObject(PutObjectRequest.builder().bucket(bucket.name).key(key).build(), RequestBody.fromString(fileContents)) } def store(bucket: S3Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) @@ -158,11 +158,11 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with Future { val fileMarkers = Map( - "bucket" -> bucket.bucket, + "bucket" -> bucket.name, ) val markers = logMarker ++ fileMarkers - val reqBuilder = PutObjectRequest.builder().key(id).bucket(bucket.bucket) + val reqBuilder = PutObjectRequest.builder().key(id).bucket(bucket.name) cacheControl.foreach(c => reqBuilder.cacheControl(c)) mimeType.foreach(m => reqBuilder.contentType(m.name)) reqBuilder.metadata(meta.asJava) @@ -172,10 +172,10 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with bucket.client.putObject(req, RequestBody.fromFile(file)) // once we've completed the PUT read back to ensure that we are returning reality val metadata = bucket.client.headObject( - HeadObjectRequest.builder().key(id).bucket(bucket.bucket).build() + HeadObjectRequest.builder().key(id).bucket(bucket.name).build() ) - S3Object(bucket.bucket, id, metadata.contentLength(), S3Metadata(metadata)) + S3Object(bucket.name, id, metadata.contentLength(), S3Metadata(metadata)) }(markers) } @@ -183,15 +183,15 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with (implicit ex: ExecutionContext, logMarker: LogMarker): Future[S3Object] = { Future { Some(bucket.client.headObject( - HeadObjectRequest.builder().key(id).bucket(bucket.bucket).build() + HeadObjectRequest.builder().key(id).bucket(bucket.name).build() )) }.recover { // translate this exception into the object not existing case _: NoSuchKeyException => None }.flatMap { case Some(metadata) => - logger.info(logMarker, s"Skipping storing of S3 file $id as key is already present in bucket ${bucket.bucket}") - Future.successful(S3Object(bucket.bucket, id, metadata.contentLength(), S3Metadata(metadata))) + logger.info(logMarker, s"Skipping storing of S3 file $id as key is already present in bucket ${bucket.name}") + Future.successful(S3Object(bucket.name, id, metadata.contentLength(), S3Metadata(metadata))) case None => store(bucket, id, file, mimeType, meta, cacheControl) } @@ -200,28 +200,28 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with def list(bucket: S3Bucket, prefixDir: String) (implicit ex: ExecutionContext): Future[List[S3Object]] = Future { - val req = ListObjectsV2Request.builder().bucket(bucket.bucket).prefix(s"$prefixDir/").build() + val req = ListObjectsV2Request.builder().bucket(bucket.name).prefix(s"$prefixDir/").build() val listing = bucket.client.listObjectsV2(req) val s3Objects = listing.contents().asScala.toList s3Objects.map(s3Object => { - S3Object(bucket.bucket, s3Object.key(), size = s3Object.size(), metadata = getMetadata(bucket, s3Object.key())) + S3Object(bucket.name, s3Object.key(), size = s3Object.size(), metadata = getMetadata(bucket, s3Object.key())) }) } def getMetadata(bucket: S3Bucket, key: Key): S3Metadata = { - val meta = bucket.client.headObject(HeadObjectRequest.builder().key(key).bucket(bucket.bucket).build()) + val meta = bucket.client.headObject(HeadObjectRequest.builder().key(key).bucket(bucket.name).build()) S3Metadata(meta) } def syncFindKey(bucket: S3Bucket, prefixName: String): Option[Key] = { - val req = ListObjectsV2Request.builder().bucket(bucket.bucket).prefix(s"$prefixName-").build() + val req = ListObjectsV2Request.builder().bucket(bucket.name).prefix(s"$prefixName-").build() val objects = bucket.client.listObjectsV2(req).contents().asScala.toList objects.headOption.map(_.key()) } def doesObjectExist(bucket: S3Bucket, key: String) = { try { bucket.client.headObject( - HeadObjectRequest.builder().key(key).bucket(bucket.bucket).build() + HeadObjectRequest.builder().key(key).bucket(bucket.name).build() ) true } catch { @@ -230,7 +230,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with } def deleteObject(bucket: S3Bucket, key: String): Unit = - bucket.client.deleteObject(DeleteObjectRequest.builder().bucket(bucket.bucket).key(key).build()) + bucket.client.deleteObject(DeleteObjectRequest.builder().bucket(bucket.name).key(key).build()) def deleteObjects(bucket: S3Bucket, keys: List[String]): Map[String, Boolean] = { val objects: util.List[ObjectIdentifier] = keys.map { key => @@ -239,7 +239,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with .build() }.asJava val response = bucket.client.deleteObjects( - DeleteObjectsRequest.builder().bucket(bucket.bucket) + DeleteObjectsRequest.builder().bucket(bucket.name) .delete(Delete.builder().objects(objects).build()) .build() ) @@ -250,14 +250,14 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with } def deleteVersion(bucket: S3Bucket, key: String, objectVersion: String): Unit = - bucket.client.deleteObject(DeleteObjectRequest.builder().bucket(bucket.bucket).key(key).versionId(objectVersion).build()) + bucket.client.deleteObject(DeleteObjectRequest.builder().bucket(bucket.name).key(key).versionId(objectVersion).build()) def copy(key: String, sourceBucket: S3Bucket, destinationBucket: S3Bucket): CopyObjectResponse = { sourceBucket.client.copyObject( CopyObjectRequest.builder() - .sourceBucket(sourceBucket.bucket) + .sourceBucket(sourceBucket.name) .sourceKey(key) - .destinationBucket(destinationBucket.bucket) + .destinationBucket(destinationBucket.name) .destinationKey(key) .build() ) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala index 16b6e69d74..822b38182c 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala @@ -4,7 +4,7 @@ import com.gu.mediaservice.lib.config.CommonConfig import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3.presigner.S3Presigner -case class S3Bucket(bucket: String, endPoint: String, usesPathStyleURLs: Boolean, client: S3Client, presigner: S3Presigner) +case class S3Bucket(name: String, endPoint: String, usesPathStyleURLs: Boolean, client: S3Client, presigner: S3Presigner) object S3Bucket { @@ -12,15 +12,15 @@ object S3Bucket { * Build a bucket that talks to the endpoint implied by the current environment - i.e. the localstack * endpoint (with path style URLs) when running in DEV, otherwise the real AWS S3 endpoint. */ - def apply(bucket: String, config: CommonConfig): S3Bucket = { + def apply(name: String, config: CommonConfig): S3Bucket = { val endpointOverride = config.awsLocalEndpoint val usesPathStyleURLs = endpointOverride.isDefined - apply(bucket, config, endpointOverride, usesPathStyleURLs) + apply(name, config, endpointOverride, usesPathStyleURLs) } - def apply(bucket: String, config: CommonConfig, endpointOverride: Option[String], usesPathStyleURLs: Boolean): S3Bucket = + def apply(name: String, config: CommonConfig, endpointOverride: Option[String], usesPathStyleURLs: Boolean): S3Bucket = S3Bucket( - bucket = bucket, + name = name, endPoint = endpointOverride.getOrElse(S3Ops.s3Endpoint), usesPathStyleURLs = usesPathStyleURLs, client = S3Ops.buildS3Client(config, endpointOverride, usesPathStyleURLs), diff --git a/image-loader/app/controllers/ImageLoaderController.scala b/image-loader/app/controllers/ImageLoaderController.scala index 7c8548a88d..5a4a96b88b 100644 --- a/image-loader/app/controllers/ImageLoaderController.scala +++ b/image-loader/app/controllers/ImageLoaderController.scala @@ -460,7 +460,7 @@ class ImageLoaderController(auth: Authentication, logger.info(context, "image found") Ok(Json.toJson(img)).as(ArgoMediaType) case None => - val s3Path = "s3://" + config.imageBucket.bucket + "/" + ImageIngestOperations.fileKeyFromId(imageId) + val s3Path = "s3://" + config.imageBucket.name + "/" + ImageIngestOperations.fileKeyFromId(imageId) logger.info(context, "image not found") respondError(NotFound, "image-not-found", s"Could not find image: $imageId in s3 at $s3Path") } recover { diff --git a/image-loader/app/lib/ImageLoaderStore.scala b/image-loader/app/lib/ImageLoaderStore.scala index 584ac87a76..39fd800e9c 100644 --- a/image-loader/app/lib/ImageLoaderStore.scala +++ b/image-loader/app/lib/ImageLoaderStore.scala @@ -50,7 +50,7 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati val bucket = config.maybeBucketForUIUploads.get val putObjectRequest = PutObjectRequest.builder() - .bucket(bucket.bucket).key(s"${instance.id}/$uploadedBy/$filename").metadata(Map( + .bucket(bucket.name).key(s"${instance.id}/$uploadedBy/$filename").metadata(Map( "media-id" -> mediaId).asJava) .build() val putObjectPresignRequest = diff --git a/image-loader/app/model/Projector.scala b/image-loader/app/model/Projector.scala index fdaab7b496..217abb0edd 100644 --- a/image-loader/app/model/Projector.scala +++ b/image-loader/app/model/Projector.scala @@ -93,9 +93,9 @@ class Projector(config: ImageUploadOpsCfg, val s3Key = fileKeyFromId(imageId) if (!s3.doesObjectExist(config.originalFileBucket, s3Key)) - throw new NoSuchImageExistsInS3(config.originalFileBucket.bucket, s3Key) + throw new NoSuchImageExistsInS3(config.originalFileBucket.name, s3Key) - val s3Source = Stopwatch(s"object exists, getting s3 object at s3://${config.originalFileBucket.bucket}/$s3Key to perform Image projection"){ + val s3Source = Stopwatch(s"object exists, getting s3 object at s3://${config.originalFileBucket.name}/$s3Key to perform Image projection"){ s3.getObject(config.originalFileBucket, s3Key) }(logMarker) @@ -205,11 +205,11 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg, private def fetchFile( bucket: S3Bucket, key: String, outFile: File )(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[(File, MimeType)]] = { - logger.info(logMarker, s"Trying fetch existing image from S3 bucket - ${bucket.bucket} at key $key") + logger.info(logMarker, s"Trying fetch existing image from S3 bucket - ${bucket.name} at key $key") val doesFileExist = Future { s3.doesObjectExist(bucket, key) } recover { case _ => false } doesFileExist.flatMap { case false => - logger.warn(logMarker, s"image did not exist in bucket ${bucket.bucket} at key $key") + logger.warn(logMarker, s"image did not exist in bucket ${bucket.name} at key $key") Future.successful(None) // falls back to creating from original file case true => val obj = s3.getObject(bucket, key) From fbfe33e58704e0ea543cdc560e05d1b2f50577f3 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Sat, 29 Aug 2026 12:21:26 +0100 Subject: [PATCH 30/46] maybeImageReplicaBucket migrates to S3Bucket. ImageLoaderController loses raw S3Client. --- .../com/gu/mediaservice/lib/aws/S3Bucket.scala | 9 +++++---- .../controllers/ImageLoaderController.scala | 18 ++++++------------ image-loader/app/lib/ImageLoaderConfig.scala | 7 ++++++- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala index 822b38182c..87b1d55fec 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala @@ -1,6 +1,7 @@ package com.gu.mediaservice.lib.aws import com.gu.mediaservice.lib.config.CommonConfig +import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3.presigner.S3Presigner @@ -15,15 +16,15 @@ object S3Bucket { def apply(name: String, config: CommonConfig): S3Bucket = { val endpointOverride = config.awsLocalEndpoint val usesPathStyleURLs = endpointOverride.isDefined - apply(name, config, endpointOverride, usesPathStyleURLs) + apply(name, config, endpointOverride, usesPathStyleURLs, None) } - def apply(name: String, config: CommonConfig, endpointOverride: Option[String], usesPathStyleURLs: Boolean): S3Bucket = + def apply(name: String, config: CommonConfig, endpointOverride: Option[String], usesPathStyleURLs: Boolean, maybeRegionOverride: Option[Region]): S3Bucket = S3Bucket( name = name, endPoint = endpointOverride.getOrElse(S3Ops.s3Endpoint), usesPathStyleURLs = usesPathStyleURLs, - client = S3Ops.buildS3Client(config, endpointOverride, usesPathStyleURLs), - presigner = S3Ops.buildPresignerClientV2(config, endpointOverride, usesPathStyleURLs) + client = S3Ops.buildS3Client(config, endpointOverride, usesPathStyleURLs, maybeRegionOverride), + presigner = S3Ops.buildPresignerClientV2(config, endpointOverride, usesPathStyleURLs, maybeRegionOverride) ) } diff --git a/image-loader/app/controllers/ImageLoaderController.scala b/image-loader/app/controllers/ImageLoaderController.scala index 5a4a96b88b..be3eda2905 100644 --- a/image-loader/app/controllers/ImageLoaderController.scala +++ b/image-loader/app/controllers/ImageLoaderController.scala @@ -1,9 +1,5 @@ package controllers -import org.apache.pekko.Done -import org.apache.pekko.stream.Materializer -import org.apache.pekko.stream.scaladsl.Source -import software.amazon.awssdk.services.sqs.model.{Message => SQSMessage} import com.drew.imaging.ImageProcessingException import com.gu.mediaservice.GridClient import com.gu.mediaservice.lib.ImageIngestOperations.fileKeyFromId @@ -12,7 +8,7 @@ import com.gu.mediaservice.lib.argo.model.Link import com.gu.mediaservice.lib.auth.Authentication.{MachinePrincipal, OnBehalfOfPrincipal, UserPrincipal} import com.gu.mediaservice.lib.auth._ import com.gu.mediaservice.lib.auth.provider.ApiKeyAuthenticationProvider -import com.gu.mediaservice.lib.aws.{S3Ops, SimpleSqsMessageConsumer, SqsHelpers} +import com.gu.mediaservice.lib.aws.{S3Bucket, SimpleSqsMessageConsumer, SqsHelpers} import com.gu.mediaservice.lib.config.InstanceForRequest import com.gu.mediaservice.lib.events.UsageEvents import com.gu.mediaservice.lib.formatting.printDateTime @@ -38,9 +34,7 @@ import play.api.inject.ApplicationLifecycle import play.api.libs.json.Json import play.api.libs.ws.WSClient import play.api.mvc._ -import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.cloudwatch.model.Dimension -import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3.model.{GetObjectRequest, HeadObjectRequest, NoSuchKeyException} import software.amazon.awssdk.services.sqs.model.{Message => SQSMessage} @@ -637,10 +631,10 @@ class ImageLoaderController(auth: Authentication, } } - lazy val replicaS3: S3Client = S3Ops.buildS3Client(config, maybeRegionOverride = Some(Region.US_WEST_1)) - def doesObjectExist(bucket: String, key: String) = { + // TODO is this a duplicate with S3? + def doesObjectExist(bucket: S3Bucket, key: String) = { try { - replicaS3.headObject(HeadObjectRequest.builder().bucket(bucket).key(key).build()) + bucket.client.headObject(HeadObjectRequest.builder().bucket(bucket.name).key(key).build()) true } catch { case _: NoSuchKeyException => false @@ -674,8 +668,8 @@ class ImageLoaderController(auth: Authentication, logger.info(logMarker, s"Restoring image $imageId from replica bucket $replicaBucket (key: $s3Key)") - val replicaObject = replicaS3.getObject( - GetObjectRequest.builder().bucket(replicaBucket).key(s3Key).build() + val replicaObject = replicaBucket.client.getObject( + GetObjectRequest.builder().bucket(replicaBucket.name).key(s3Key).build() ) val lastModified = replicaObject.response().lastModified() val metaMap = replicaObject.response().metadata().asScala.toMap diff --git a/image-loader/app/lib/ImageLoaderConfig.scala b/image-loader/app/lib/ImageLoaderConfig.scala index 77077dde61..c6436af9ad 100644 --- a/image-loader/app/lib/ImageLoaderConfig.scala +++ b/image-loader/app/lib/ImageLoaderConfig.scala @@ -1,16 +1,21 @@ package lib +import com.gu.mediaservice.lib.aws.{S3Bucket, S3Ops} + import java.io.File import com.gu.mediaservice.lib.cleanup.{ComposedImageProcessor, ImageProcessor, ImageProcessorResources} import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources, ImageProcessorLoader} import com.gu.mediaservice.model._ import com.typesafe.scalalogging.StrictLogging import play.api.inject.ApplicationLifecycle +import software.amazon.awssdk.regions.Region import scala.concurrent.duration.FiniteDuration class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(resources) with StrictLogging { - val maybeImageReplicaBucket: Option[String] = stringOpt("s3.image.replicaBucket") + val maybeImageReplicaBucket: Option[S3Bucket] = stringOpt("s3.image.replicaBucket").map{ replicaBucketName => + S3Bucket.apply(replicaBucketName, this, None, usesPathStyleURLs = false, maybeRegionOverride = Some(Region.US_WEST_1)) + } val lowerEnvironmentSamplingPercentageAsDecimal = intOpt("s3.sampling.percentage").getOrElse(1) / 100.0 val maybeLowerEnvironmentQueueBucketToSampleInto = stringOpt("s3.sampling.targetBucket") From 80599ca6e613756e83fe600f6d411d717d823d4c Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Sat, 29 Aug 2026 12:40:01 +0100 Subject: [PATCH 31/46] S3Object takes a full S3Bucket. --- .../gu/mediaservice/lib/ImageIngestOperations.scala | 6 +++--- .../main/scala/com/gu/mediaservice/lib/aws/S3.scala | 12 ++++++------ image-loader/test/scala/model/ImageUploadTest.scala | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala index 8050996063..3bff1d70d4 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala @@ -91,7 +91,7 @@ sealed trait ImageWrapper { } sealed trait StorableImage extends ImageWrapper { def toProjectedS3Object(thumbBucket: S3Bucket): S3Object = S3Object( - thumbBucket.name, + thumbBucket, ImageIngestOperations.fileKeyFromId(id)(instance), file, Some(mimeType), @@ -103,7 +103,7 @@ sealed trait StorableImage extends ImageWrapper { case class StorableThumbImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage case class StorableOriginalImage(id: String, file: File, mimeType: MimeType, lastModified: DateTime, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage { override def toProjectedS3Object(thumbBucket: S3Bucket): S3Object = S3Object( - thumbBucket.name, + thumbBucket, ImageIngestOperations.fileKeyFromId(id)(instance), file, Some(mimeType), @@ -113,7 +113,7 @@ case class StorableOriginalImage(id: String, file: File, mimeType: MimeType, las } case class StorableOptimisedImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage { override def toProjectedS3Object(thumbBucket: S3Bucket): S3Object = S3Object( - thumbBucket.name, + thumbBucket, ImageIngestOperations.optimisedPngKeyFromId(id)(instance), file, Some(mimeType), diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 1ef7e13a22..369a04879e 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -23,15 +23,15 @@ import scala.jdk.CollectionConverters._ case class S3Object(uri: URI, size: Long, metadata: S3Metadata) object S3Object { - def objectUrl(bucket: String, key: String): URI = { + def objectUrl(bucket: S3Bucket, key: String): URI = { val bucketUrl = s"$bucket.${S3Ops.s3Endpoint}" new URI("http", bucketUrl, s"/$key", null) } - def apply(bucket: String, key: String, size: Long, metadata: S3Metadata): S3Object = + def apply(bucket: S3Bucket, key: String, size: Long, metadata: S3Metadata): S3Object = apply(objectUrl(bucket, key), size, metadata) - def apply(bucket: String, key: String, file: File, mimeType: Option[MimeType], lastModified: Option[DateTime], + def apply(bucket: S3Bucket, key: String, file: File, mimeType: Option[MimeType], lastModified: Option[DateTime], meta: Map[String, String] = Map.empty, cacheControl: Option[String] = None): S3Object = { S3Object( bucket, @@ -175,7 +175,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with HeadObjectRequest.builder().key(id).bucket(bucket.name).build() ) - S3Object(bucket.name, id, metadata.contentLength(), S3Metadata(metadata)) + S3Object(bucket, id, metadata.contentLength(), S3Metadata(metadata)) }(markers) } @@ -191,7 +191,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with }.flatMap { case Some(metadata) => logger.info(logMarker, s"Skipping storing of S3 file $id as key is already present in bucket ${bucket.name}") - Future.successful(S3Object(bucket.name, id, metadata.contentLength(), S3Metadata(metadata))) + Future.successful(S3Object(bucket, id, metadata.contentLength(), S3Metadata(metadata))) case None => store(bucket, id, file, mimeType, meta, cacheControl) } @@ -204,7 +204,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with val listing = bucket.client.listObjectsV2(req) val s3Objects = listing.contents().asScala.toList s3Objects.map(s3Object => { - S3Object(bucket.name, s3Object.key(), size = s3Object.size(), metadata = getMetadata(bucket, s3Object.key())) + S3Object(bucket, s3Object.key(), size = s3Object.size(), metadata = getMetadata(bucket, s3Object.key())) }) } diff --git a/image-loader/test/scala/model/ImageUploadTest.scala b/image-loader/test/scala/model/ImageUploadTest.scala index f29397ad6d..874c4b70d7 100644 --- a/image-loader/test/scala/model/ImageUploadTest.scala +++ b/image-loader/test/scala/model/ImageUploadTest.scala @@ -1,7 +1,7 @@ package model import com.drew.imaging.ImageProcessingException -import com.gu.mediaservice.lib.aws.{S3Metadata, S3Object, S3ObjectMetadata} +import com.gu.mediaservice.lib.aws.{S3Bucket, S3Metadata, S3Object, S3ObjectMetadata} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.LogMarker @@ -53,7 +53,7 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar { def mockStore = (a: StorableImage) => Future.successful( - S3Object("madeupname", "madeupkey", a.file, Some(a.mimeType), None, a.meta, None) + mockS3Object ) def storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object] = mockStore From ede5823eb3200c3c0854e9891e199b58a8fc2e8a Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Thu, 13 Aug 2026 12:43:29 +0100 Subject: [PATCH 32/46] S3Object.objectUrl uses bucket end point. --- common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 369a04879e..814244ad85 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -24,7 +24,7 @@ case class S3Object(uri: URI, size: Long, metadata: S3Metadata) object S3Object { def objectUrl(bucket: S3Bucket, key: String): URI = { - val bucketUrl = s"$bucket.${S3Ops.s3Endpoint}" + val bucketUrl = s"${bucket.name}.${bucket.endPoint}" new URI("http", bucketUrl, s"/$key", null) } From 532a0a8ba9a93684e1754f7acdd1b855f5770db4 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Thu, 13 Aug 2026 12:52:05 +0100 Subject: [PATCH 33/46] S3Object.objectUrl moves to S3Bucket. --- .../scala/com/gu/mediaservice/lib/aws/S3.scala | 6 +----- .../com/gu/mediaservice/lib/aws/S3Bucket.scala | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 814244ad85..26e637d6d6 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -23,13 +23,9 @@ import scala.jdk.CollectionConverters._ case class S3Object(uri: URI, size: Long, metadata: S3Metadata) object S3Object { - def objectUrl(bucket: S3Bucket, key: String): URI = { - val bucketUrl = s"${bucket.name}.${bucket.endPoint}" - new URI("http", bucketUrl, s"/$key", null) - } def apply(bucket: S3Bucket, key: String, size: Long, metadata: S3Metadata): S3Object = - apply(objectUrl(bucket, key), size, metadata) + apply(bucket.objectUrl(key), size, metadata) def apply(bucket: S3Bucket, key: String, file: File, mimeType: Option[MimeType], lastModified: Option[DateTime], meta: Map[String, String] = Map.empty, cacheControl: Option[String] = None): S3Object = { diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala index 87b1d55fec..dccbf5f920 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala @@ -4,15 +4,23 @@ import com.gu.mediaservice.lib.config.CommonConfig import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3.presigner.S3Presigner +import java.net.URI -case class S3Bucket(name: String, endPoint: String, usesPathStyleURLs: Boolean, client: S3Client, presigner: S3Presigner) +case class S3Bucket(name: String, endPoint: String, usesPathStyleURLs: Boolean, client: S3Client, presigner: S3Presigner) { + + def objectUrl(key: String): URI = { + val bucketUrl = s"$name.$endPoint" + new URI("http", bucketUrl, s"/$key", null) + } + +} object S3Bucket { /** - * Build a bucket that talks to the endpoint implied by the current environment - i.e. the localstack - * endpoint (with path style URLs) when running in DEV, otherwise the real AWS S3 endpoint. - */ + * Build a bucket that talks to the endpoint implied by the current environment - i.e. the localstack + * endpoint (with path style URLs) when running in DEV, otherwise the real AWS S3 endpoint. + */ def apply(name: String, config: CommonConfig): S3Bucket = { val endpointOverride = config.awsLocalEndpoint val usesPathStyleURLs = endpointOverride.isDefined From 7f19fa6826784cb29f2a9dc9cc2392dad99ccf03 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Thu, 13 Aug 2026 12:54:00 +0100 Subject: [PATCH 34/46] S3Bucket.objectUrl is path style URLs aware. --- .../scala/com/gu/mediaservice/lib/aws/S3Bucket.scala | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala index dccbf5f920..44418ccecd 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala @@ -9,8 +9,16 @@ import java.net.URI case class S3Bucket(name: String, endPoint: String, usesPathStyleURLs: Boolean, client: S3Client, presigner: S3Presigner) { def objectUrl(key: String): URI = { - val bucketUrl = s"$name.$endPoint" - new URI("http", bucketUrl, s"/$key", null) + val bucketBaseURL = bucketURL() + new URI("http", bucketBaseURL.getHost, bucketBaseURL.getPath + key, null) + } + + private def bucketURL(): URI = { + if (usesPathStyleURLs) { + new URI("https", endPoint, s"/$name/", null) + } else { + new URI("https", s"$name.$endPoint", "/", null) + } } } From ede402997b1c27715f28689bc1c0ad1dd8629824 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Thu, 13 Aug 2026 13:10:22 +0100 Subject: [PATCH 35/46] Extract S3 URL to key onto S3Bucket. --- .../main/scala/com/gu/mediaservice/lib/aws/S3.scala | 10 ++++------ .../scala/com/gu/mediaservice/lib/aws/S3Bucket.scala | 5 +++++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 26e637d6d6..5c6783d652 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -74,8 +74,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with expiration: DateTime = cachableExpiration(), imageType: ImageFileType = Source ): String = { - // Fix key extraction (use stripPrefix to avoid corrupting relative paths) - val key: Key = url.getPath.stripPrefix("/") + val key: Key = bucket.keyFromURL(url) val contentDisposition = getContentDisposition(image, imageType, config.shortenDownloadFilename) @@ -99,8 +98,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with } def signUrlTony(bucket: S3Bucket, url: URI, expiration: DateTime = cachableExpiration()): URL = { - // get path and remove leading `/` - val key: Key = url.getPath.drop(1) + val key: Key = bucket.keyFromURL(url) val nowMillis = System.currentTimeMillis() val targetExpirationMillis = expiration.getMillis @@ -125,8 +123,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with } def getObject(bucket: S3Bucket, url: URI): ResponseInputStream[GetObjectResponse]= { - // get path and remove leading `/` - val key: Key = url.getPath.drop(1) + val key: Key = bucket.keyFromURL(url) bucket.client.getObject(GetObjectRequest.builder().key(key).bucket(bucket.name).build()) } @@ -258,6 +255,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with .build() ) } + } object S3Ops extends GridLogging { diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala index 44418ccecd..34ae814ba0 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala @@ -13,6 +13,11 @@ case class S3Bucket(name: String, endPoint: String, usesPathStyleURLs: Boolean, new URI("http", bucketBaseURL.getHost, bucketBaseURL.getPath + key, null) } + def keyFromURL(url: URI): String = { + // get path and remove leading `/` + url.getPath.drop(1) + } + private def bucketURL(): URI = { if (usesPathStyleURLs) { new URI("https", endPoint, s"/$name/", null) From d68fb88c317dbc7bc4a60bbb2295a7c3a3ed217c Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Thu, 13 Aug 2026 13:11:23 +0100 Subject: [PATCH 36/46] keyFromURL works for path based buckets. --- .../main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala index 34ae814ba0..0dc559754c 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala @@ -14,8 +14,12 @@ case class S3Bucket(name: String, endPoint: String, usesPathStyleURLs: Boolean, } def keyFromURL(url: URI): String = { - // get path and remove leading `/` - url.getPath.drop(1) + if (usesPathStyleURLs) { + url.getPath.drop(bucket.length + 2) + } else { + // get path and remove leading `/` + url.getPath.drop(1) + } } private def bucketURL(): URI = { From dffc804080422d37f702b23ed0c02321054a7f3a Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Sat, 29 Aug 2026 12:58:27 +0100 Subject: [PATCH 37/46] keyFromURL works for path based buckets. --- .../src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala index 0dc559754c..cd90095056 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala @@ -15,7 +15,7 @@ case class S3Bucket(name: String, endPoint: String, usesPathStyleURLs: Boolean, def keyFromURL(url: URI): String = { if (usesPathStyleURLs) { - url.getPath.drop(bucket.length + 2) + url.getPath.drop(name.length + 2) } else { // get path and remove leading `/` url.getPath.drop(1) From 34bf60288302abb76664210a6d81eb7badb75385 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Thu, 13 Aug 2026 13:16:09 +0100 Subject: [PATCH 38/46] getObject called by key only; callers should dereference the URIs first. --- .../src/main/scala/com/gu/mediaservice/lib/aws/S3.scala | 5 ----- media-api/app/controllers/MediaApi.scala | 8 +++++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 5c6783d652..f0a64bd935 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -122,11 +122,6 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with bucket.presigner.presignPutObject(putObjectPresignRequest) } - def getObject(bucket: S3Bucket, url: URI): ResponseInputStream[GetObjectResponse]= { - val key: Key = bucket.keyFromURL(url) - bucket.client.getObject(GetObjectRequest.builder().key(key).bucket(bucket.name).build()) - } - def getObject(bucket: S3Bucket, key: String): ResponseInputStream[GetObjectResponse] = { bucket.client.getObject(GetObjectRequest.builder().key(key).bucket(bucket.name).build()) } diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index ec46f0b043..93731aaeb0 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -44,7 +44,7 @@ class MediaApi( imageResponse: ImageResponse, config: MediaApiConfig, override val controllerComponents: ControllerComponents, - s3Client: S3, + s3Client: S3, // TODO rename to s3 mediaApiMetrics: MediaApiMetrics, ws: WSClient, authorisation: Authorisation, @@ -324,7 +324,8 @@ class MediaApi( val maybeResult = for { export <- source.exports.find(_.id.contains(exportId)) asset <- export.assets.find(_.dimensions.exists(_.width == width)) - s3Res = Try(s3Client.getObject(config.imgPublishingBucket, asset.file)) + key = config.imgPublishingBucket.keyFromURL(asset.file) + s3Res = Try(s3Client.getObject(config.imgPublishingBucket, key)) _ = s3Res.failed.foreach { ex => logger.error("Failed to fetch S3 object", ex) } @@ -461,7 +462,8 @@ class MediaApi( val apiKey = request.user.accessor logger.info(logMarker, s"Download original image: $id from user: ${Authentication.getIdentity(request.user)}") mediaApiMetrics.incrementImageDownload(apiKey, mediaApiMetrics.OriginalDownloadType) - val s3Object = s3Client.getObject(config.imageBucket, image.source.file) + val key = config.imageBucket.keyFromURL(image.source.file) + val s3Object = s3Client.getObject(config.imageBucket, key) val file = StreamConverters.fromInputStream(() => s3Object) val entity = HttpEntity.Streamed(file, image.source.size, image.source.mimeType.map(_.name)) From b96f8beb5371462b1187a62cda2e305c494993ea Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Thu, 13 Aug 2026 18:11:02 +0100 Subject: [PATCH 39/46] Called to S3 sign URLs need to resolve the URL to a key. --- .../src/main/scala/com/gu/mediaservice/lib/aws/S3.scala | 8 ++------ cropper/app/lib/CropStore.scala | 3 ++- cropper/app/lib/Crops.scala | 3 ++- media-api/app/controllers/MediaApi.scala | 3 ++- media-api/app/lib/ImageResponse.scala | 8 +++++--- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index f0a64bd935..a1975e21a8 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -69,13 +69,11 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with def signUrl( bucket: S3Bucket, - url: URI, + key: Key, image: Image, expiration: DateTime = cachableExpiration(), imageType: ImageFileType = Source ): String = { - val key: Key = bucket.keyFromURL(url) - val contentDisposition = getContentDisposition(image, imageType, config.shortenDownloadFilename) val nowMillis = System.currentTimeMillis() @@ -97,9 +95,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with req.url().toExternalForm } - def signUrlTony(bucket: S3Bucket, url: URI, expiration: DateTime = cachableExpiration()): URL = { - val key: Key = bucket.keyFromURL(url) - + def signUrlTony(bucket: S3Bucket, key: Key, expiration: DateTime = cachableExpiration()): URL = { val nowMillis = System.currentTimeMillis() val targetExpirationMillis = expiration.getMillis val remainingSeconds = Math.max(1, (targetExpirationMillis - nowMillis) / 1000) diff --git a/cropper/app/lib/CropStore.scala b/cropper/app/lib/CropStore.scala index dcb6ada98f..a7a99c0260 100644 --- a/cropper/app/lib/CropStore.scala +++ b/cropper/app/lib/CropStore.scala @@ -83,7 +83,8 @@ class CropStore(config: CropperConfig) extends S3ImageStorage(config) with CropS } private def signedCropAssetUrl(uri: URI): URI = { - signUrlTony(config.imgPublishingBucket, uri).toURI + val key = config.imgPublishingBucket.keyFromURL(uri) + signUrlTony(config.imgPublishingBucket, key).toURI } } diff --git a/cropper/app/lib/Crops.scala b/cropper/app/lib/Crops.scala index 215987d00d..32b848b02c 100644 --- a/cropper/app/lib/Crops.scala +++ b/cropper/app/lib/Crops.scala @@ -114,7 +114,8 @@ class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOpera val hasAlpha = apiImage.fileMetadata.colourModelInformation.get("hasAlpha").flatMap(a => Try(a.toBoolean).toOption).getOrElse(true) val cropType = Crops.cropType(mimeType, colourType, hasAlpha) - val secureUrl = s3.signUrlTony(imageBucket, secureFile) + val key = imageBucket.keyFromURL(secureFile) + val secureUrl = s3.signUrlTony(imageBucket, key) Stopwatch.async(s"making crop assets for ${apiImage.id} ${Crop.getCropId(source.bounds)}") { for { diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index 93731aaeb0..1bf79260d4 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -526,8 +526,9 @@ class MediaApi( logger.info(logMarker, s"Download optimised image: $id from user: ${Authentication.getIdentity(request.user)}") mediaApiMetrics.incrementImageDownload(apiKey, mediaApiMetrics.OptimisedDownloadType) + val key = config.imageBucket.keyFromURL(image.optimisedPng.getOrElse(image.source).file) val sourceImageUri = - new URI(s3Client.signUrl(config.imageBucket, image.optimisedPng.getOrElse(image.source).file, image, imageType = image.optimisedPng match { + new URI(s3Client.signUrl(config.imageBucket, key, image, imageType = image.optimisedPng match { case Some(_) => OptimisedPng case _ => Source })) diff --git a/media-api/app/lib/ImageResponse.scala b/media-api/app/lib/ImageResponse.scala index 73824d0169..7934546a7b 100644 --- a/media-api/app/lib/ImageResponse.scala +++ b/media-api/app/lib/ImageResponse.scala @@ -78,12 +78,14 @@ class ImageResponse(config: MediaApiConfig, s3Client: S3, usageQuota: UsageQuota val pngFileUri = image.optimisedPng.map(_.file) val fileUri = image.source.file + val imageKey = config.imageBucket.keyFromURL(fileUri) - val imageUrl = s3Client.signUrl(config.imageBucket, fileUri, image, imageType = Source) + val imageUrl = s3Client.signUrl(config.imageBucket, imageKey, image, imageType = Source) val pngUrl: Option[String] = pngFileUri - .map(s3Client.signUrl(config.imageBucket, _, image, imageType = OptimisedPng)) + .map(file => s3Client.signUrl(config.imageBucket, config.imageBucket.keyFromURL(file), image, imageType = OptimisedPng)) - def s3SignedThumbUrl = s3Client.signUrl(config.thumbnailBucket, fileUri, image, imageType = Thumbnail) + val thumbKey = config.thumbnailBucket.keyFromURL(fileUri) + def s3SignedThumbUrl = s3Client.signUrl(config.thumbnailBucket, thumbKey, image, imageType = Thumbnail) val thumbUrl = config.cloudFrontDomainThumbBucket .map(domain => s"https://$domain${fileUri.getPath}") From e4d4eaef42907c267e29ad117e5790b82164acf3 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Sat, 25 Jan 2025 11:50:27 +0000 Subject: [PATCH 40/46] Extract bucket base URL function. Upload bucket connectSource is supplied by the bucket object; knows about path style bucket URLs. Kahuna UI can connect to ingest bucket endpoint. --- .../main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala | 2 +- kahuna/app/lib/KahunaConfig.scala | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala index cd90095056..83493cfd0b 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala @@ -22,7 +22,7 @@ case class S3Bucket(name: String, endPoint: String, usesPathStyleURLs: Boolean, } } - private def bucketURL(): URI = { + def bucketURL(): URI = { if (usesPathStyleURLs) { new URI("https", endPoint, s"/$name/", null) } else { diff --git a/kahuna/app/lib/KahunaConfig.scala b/kahuna/app/lib/KahunaConfig.scala index d4b65b73ec..72ae076655 100644 --- a/kahuna/app/lib/KahunaConfig.scala +++ b/kahuna/app/lib/KahunaConfig.scala @@ -2,6 +2,7 @@ package lib import com.gu.mediaservice.lib.auth.Permissions.Pinboard import com.gu.mediaservice.lib.auth.SimplePermission +import com.gu.mediaservice.lib.aws.S3Bucket import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources} import com.gu.mediaservice.model.Instance import play.api.libs.json._ @@ -51,9 +52,8 @@ class KahunaConfig(resources: GridConfigResources) extends CommonConfig(resource val aiSearchResultLimit: Int = intOpt("ai.search.resultLimit").getOrElse(200) val frameAncestors: Set[String] = getStringSet("security.frameAncestors") - val connectSources: Set[String] = getStringSet("security.connectSources") ++ maybeBucketForUIUploads.map { bucket => - if (isDev) "https://localstack.media.local.dev-gutools.co.uk" - else s"https://$bucket.s3.$awsRegion.amazonaws.com" + val connectSources: Set[String] = getStringSet("security.connectSources") ++ maybeBucketForUIUploads.map { bucketForUIUploads => + bucketForUIUploads.bucketURL().toURL.toExternalForm } ++ telemetryUri val fontSources: Set[String] = getStringSet("security.fontSources") val imageSources: Set[String] = getStringSet("security.imageSources") From d9d4080096f2d8918aa66604a367d6aceea424b5 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Sat, 29 Aug 2026 13:13:40 +0100 Subject: [PATCH 41/46] Clean up; misleading parameter name. --- media-api/app/controllers/MediaApi.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index 1bf79260d4..e9e8ebd612 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -44,7 +44,7 @@ class MediaApi( imageResponse: ImageResponse, config: MediaApiConfig, override val controllerComponents: ControllerComponents, - s3Client: S3, // TODO rename to s3 + s3: S3, mediaApiMetrics: MediaApiMetrics, ws: WSClient, authorisation: Authorisation, @@ -325,7 +325,7 @@ class MediaApi( export <- source.exports.find(_.id.contains(exportId)) asset <- export.assets.find(_.dimensions.exists(_.width == width)) key = config.imgPublishingBucket.keyFromURL(asset.file) - s3Res = Try(s3Client.getObject(config.imgPublishingBucket, key)) + s3Res = Try(s3.getObject(config.imgPublishingBucket, key)) _ = s3Res.failed.foreach { ex => logger.error("Failed to fetch S3 object", ex) } @@ -463,7 +463,7 @@ class MediaApi( logger.info(logMarker, s"Download original image: $id from user: ${Authentication.getIdentity(request.user)}") mediaApiMetrics.incrementImageDownload(apiKey, mediaApiMetrics.OriginalDownloadType) val key = config.imageBucket.keyFromURL(image.source.file) - val s3Object = s3Client.getObject(config.imageBucket, key) + val s3Object = s3.getObject(config.imageBucket, key) val file = StreamConverters.fromInputStream(() => s3Object) val entity = HttpEntity.Streamed(file, image.source.size, image.source.mimeType.map(_.name)) @@ -528,7 +528,7 @@ class MediaApi( val key = config.imageBucket.keyFromURL(image.optimisedPng.getOrElse(image.source).file) val sourceImageUri = - new URI(s3Client.signUrl(config.imageBucket, key, image, imageType = image.optimisedPng match { + new URI(s3.signUrl(config.imageBucket, key, image, imageType = image.optimisedPng match { case Some(_) => OptimisedPng case _ => Source })) From ff51e4208c442edf1112d0d591b0494eee9f1da0 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Sat, 29 Aug 2026 13:28:07 +0100 Subject: [PATCH 42/46] keyFromURL usages. --- .../src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala | 2 +- .../src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala | 2 +- thrall/app/controllers/ReaperController.scala | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala index f419f7f430..6af6915496 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala @@ -42,7 +42,7 @@ class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage def deleteFolder(bucket: S3Bucket, id: String)(implicit logMarker: LogMarker): Future[Unit] = list(bucket, id).map { files => logger.info(s"Found ${files.size} files to delete in folder $id") - files.foreach(file => deleteObject(bucket, file.uri.getPath.stripPrefix("/"))) + files.foreach(file => deleteObject(bucket, bucket.keyFromURL(file.uri))) logger.info(logMarker, s"Deleting images in folder $id from bucket $bucket") } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala index f95e075e21..a99f7007c6 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala @@ -21,7 +21,7 @@ class KeyStore(bucket: S3Bucket, config: CommonConfig, s3: S3)(implicit ec: Exec private def fetchAll: Map[String, ApiAccessor] = { val objects = Await.result(s3.list(bucket, ""), 10.seconds) - val keys = objects.map(_.uri.getPath.stripPrefix("/")) + val keys = objects.map( s3Object => bucket.keyFromURL(s3Object.uri)) keys.flatMap(k => getS3Object(k).map(k -> ApiAccessor(_))).toMap } } diff --git a/thrall/app/controllers/ReaperController.scala b/thrall/app/controllers/ReaperController.scala index 2ed3246039..8e32cb5e78 100644 --- a/thrall/app/controllers/ReaperController.scala +++ b/thrall/app/controllers/ReaperController.scala @@ -215,7 +215,7 @@ class ReaperController( .filter(_.metadata.objectMetadata.lastModified.exists(_ isAfter now.minusHours(48))) .sortBy(_.metadata.objectMetadata.lastModified.map(_.getMillis)) .reverse - .map(_.uri.getPath.stripPrefix("/")) + .map(s3Object => reaperBucket.keyFromURL(s3Object.uri)) Ok(views.html.reaper(isPaused, INTERVAL.toString(), countOfImagesToReap, recentRecordKeys)) } From 34eac50cfbb82953a1f1e06e4f75f99f7de40ea2 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Sun, 16 Aug 2026 17:58:01 +0100 Subject: [PATCH 43/46] [gcp-buckets] make safe the S3 metadata to MimeType mapping. Fixes Unsupported mime type text/plain; charset=UTF-8 regression in KeyStore loading API keys. SDK v2 behaviour around auto-detection is different. --- .../src/main/scala/com/gu/mediaservice/lib/aws/S3.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index a1975e21a8..ff0563fc7a 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -19,6 +19,7 @@ import java.time.Duration import java.util import scala.concurrent.{ExecutionContext, Future} import scala.jdk.CollectionConverters._ +import scala.util.Try case class S3Object(uri: URI, size: Long, metadata: S3Metadata) @@ -49,10 +50,14 @@ case class S3Metadata(userMetadata: Map[String, String], objectMetadata: S3Objec object S3Metadata { def apply(meta: HeadObjectResponse): S3Metadata = { + val maybeMineType = Try { + Option(meta.contentType()).filterNot(_.toLowerCase == "application/octet-stream").map(MimeType.apply) + }.toOption.flatten + S3Metadata( meta.metadata().asScala.toMap, S3ObjectMetadata( - contentType = Option(meta.contentType()).filterNot(_.toLowerCase == "application/octet-stream").map(MimeType.apply), + contentType = maybeMineType, cacheControl = Option(meta.cacheControl()), lastModified = Option(meta.lastModified()).map(l => new DateTime(l.toEpochMilli).withZone(DateTimeZone.UTC)) ), From 319db749e05a7d74e465da096fcae8caf64b9add Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Sat, 29 Aug 2026 21:14:02 +0100 Subject: [PATCH 44/46] bucket uri is a URI. --- .../scala/com/gu/mediaservice/lib/aws/S3Bucket.scala | 10 +++++----- cropper/test/lib/CropsTest.scala | 4 +++- image-loader/test/scala/lib/ResourceHelpers.scala | 3 ++- .../lib/auth/ApiKeyAuthenticationProviderTest.scala | 3 ++- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala index 83493cfd0b..e3d82b53d5 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala @@ -6,11 +6,11 @@ import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3.presigner.S3Presigner import java.net.URI -case class S3Bucket(name: String, endPoint: String, usesPathStyleURLs: Boolean, client: S3Client, presigner: S3Presigner) { +case class S3Bucket(name: String, endPoint: URI, usesPathStyleURLs: Boolean, client: S3Client, presigner: S3Presigner) { def objectUrl(key: String): URI = { val bucketBaseURL = bucketURL() - new URI("http", bucketBaseURL.getHost, bucketBaseURL.getPath + key, null) + new URI(bucketBaseURL.getScheme, bucketBaseURL.getHost, bucketBaseURL.getPath + key, null) } def keyFromURL(url: URI): String = { @@ -24,9 +24,9 @@ case class S3Bucket(name: String, endPoint: String, usesPathStyleURLs: Boolean, def bucketURL(): URI = { if (usesPathStyleURLs) { - new URI("https", endPoint, s"/$name/", null) + new URI(endPoint.getScheme, endPoint.getHost, s"/$name/", null) } else { - new URI("https", s"$name.$endPoint", "/", null) + new URI(endPoint.getScheme, s"$name.${endPoint.getHost}", "/", null) } } @@ -47,7 +47,7 @@ object S3Bucket { def apply(name: String, config: CommonConfig, endpointOverride: Option[String], usesPathStyleURLs: Boolean, maybeRegionOverride: Option[Region]): S3Bucket = S3Bucket( name = name, - endPoint = endpointOverride.getOrElse(S3Ops.s3Endpoint), + endPoint = new URI(endpointOverride.getOrElse(S3Ops.s3Endpoint)), usesPathStyleURLs = usesPathStyleURLs, client = S3Ops.buildS3Client(config, endpointOverride, usesPathStyleURLs, maybeRegionOverride), presigner = S3Ops.buildPresignerClientV2(config, endpointOverride, usesPathStyleURLs, maybeRegionOverride) diff --git a/cropper/test/lib/CropsTest.scala b/cropper/test/lib/CropsTest.scala index b834611a7a..b852f9c948 100644 --- a/cropper/test/lib/CropsTest.scala +++ b/cropper/test/lib/CropsTest.scala @@ -7,6 +7,8 @@ import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers import org.scalatestplus.mockito.MockitoSugar +import java.net.URI + class CropsTest extends AnyFunSpec with Matchers with MockitoSugar { import scala.concurrent.ExecutionContext.Implicits.global @@ -51,7 +53,7 @@ class CropsTest extends AnyFunSpec with Matchers with MockitoSugar { private val source: SourceImage = SourceImage("test", mock[Asset], valid = true, mock[ImageMetadata], mock[FileMetadata]) private val bounds: Bounds = Bounds(10, 20, 30, 40) private val outputWidth = 1234 - private val imageBucket = S3Bucket("crops-bucket", "s3.amazonaws.com", usesPathStyleURLs = false, client = null, presigner = null) + private val imageBucket = S3Bucket("crops-bucket", new URI("https://s3.amazonaws.com"), usesPathStyleURLs = false, client = null, presigner = null) private val s3 = new S3(config) it("should should construct a correct address for a master jpg") { diff --git a/image-loader/test/scala/lib/ResourceHelpers.scala b/image-loader/test/scala/lib/ResourceHelpers.scala index 2d4c762da0..716235d362 100644 --- a/image-loader/test/scala/lib/ResourceHelpers.scala +++ b/image-loader/test/scala/lib/ResourceHelpers.scala @@ -3,6 +3,7 @@ package test.lib import com.gu.mediaservice.lib.aws.S3Bucket import java.io.File +import java.net.URI object ResourceHelpers { @@ -12,6 +13,6 @@ object ResourceHelpers { /** A bucket with no working client/presigner, for tests that never touch S3. */ def dummyBucket(name: String): S3Bucket = - S3Bucket(name, "s3.amazonaws.com", usesPathStyleURLs = false, client = null, presigner = null) + S3Bucket(name, new URI("s3.amazonaws.com"), usesPathStyleURLs = false, client = null, presigner = null) } diff --git a/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala b/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala index 72d906d1f4..64ef909433 100644 --- a/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala +++ b/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala @@ -18,6 +18,7 @@ import play.api.mvc.DefaultControllerComponents import play.api.test.{FakeRequest, WsTestClient} import play.api.{Configuration, Environment} +import java.net.URI import scala.concurrent.ExecutionContext.global import scala.concurrent.Future @@ -45,7 +46,7 @@ class ApiKeyAuthenticationProviderTest extends AsyncFreeSpec with Matchers with Future.successful(()) } - override def keyStore: KeyStore = new KeyStore(S3Bucket("not-used", "s3.amazonaws.com", usesPathStyleURLs = false, client = null, presigner = null), resources.commonConfig, s3) { + override def keyStore: KeyStore = new KeyStore(S3Bucket("not-used", new URI("https://s3.amazonaws.com"), usesPathStyleURLs = false, client = null, presigner = null), resources.commonConfig, s3) { override def lookupIdentity(key: String)(implicit instance: Instance): Option[ApiAccessor] = { key match { case "key-chuckle" => Some(ApiAccessor("brothers", Internal)) From 782c88ae1538b6e3db937941afaa40e7f83b3ab4 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Mon, 31 Aug 2026 20:15:04 +0100 Subject: [PATCH 45/46] Log number of KeyStore objects found. --- .../src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala index a99f7007c6..9d87ca7384 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala @@ -21,6 +21,7 @@ class KeyStore(bucket: S3Bucket, config: CommonConfig, s3: S3)(implicit ec: Exec private def fetchAll: Map[String, ApiAccessor] = { val objects = Await.result(s3.list(bucket, ""), 10.seconds) + logger.info(s"fetchAll found ${objects.size} objects") val keys = objects.map( s3Object => bucket.keyFromURL(s3Object.uri)) keys.flatMap(k => getS3Object(k).map(k -> ApiAccessor(_))).toMap } From 80fb43b96085262ffaf1ba465b2fb1b66648e5a5 Mon Sep 17 00:00:00 2001 From: Tony McCrae Date: Mon, 31 Aug 2026 20:21:52 +0100 Subject: [PATCH 46/46] S3 listPaginating prefix with optional prefix. Fixes KeyStore loads no API keys. --- .../gu/mediaservice/lib/auth/KeyStore.scala | 2 +- .../com/gu/mediaservice/lib/aws/S3.scala | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala index 9d87ca7384..fdf7c30d96 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala @@ -20,7 +20,7 @@ class KeyStore(bucket: S3Bucket, config: CommonConfig, s3: S3)(implicit ec: Exec } private def fetchAll: Map[String, ApiAccessor] = { - val objects = Await.result(s3.list(bucket, ""), 10.seconds) + val objects = Await.result(s3.listPaginating(bucket, None), 10.seconds) logger.info(s"fetchAll found ${objects.size} objects") val keys = objects.map( s3Object => bucket.keyFromURL(s3Object.uri)) keys.flatMap(k => getS3Object(k).map(k -> ApiAccessor(_))).toMap diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index ff0563fc7a..a0ec134d53 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -17,6 +17,7 @@ import java.net.{URI, URL} import java.nio.charset.StandardCharsets import java.time.Duration import java.util +import scala.annotation.tailrec import scala.concurrent.{ExecutionContext, Future} import scala.jdk.CollectionConverters._ import scala.util.Try @@ -197,6 +198,28 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with }) } + def listPaginating(bucket: S3Bucket, prefixDir: Option[String]) + (implicit ex: ExecutionContext): Future[List[S3Object]] = + Future { + @tailrec + def pageThrough(continuationToken: Option[String], accumulated: List[S3Object]): List[S3Object] = { + val reqBuilder = prefixDir.map { prefix => + ListObjectsV2Request.builder().bucket(bucket.name).prefix(s"$prefix/") + }.getOrElse{ + ListObjectsV2Request.builder().bucket(bucket.name) + } + continuationToken.foreach(reqBuilder.continuationToken) + val listing = bucket.client.listObjectsV2(reqBuilder.build()) + val s3Objects = listing.contents().asScala.toList.map(s3Object => + S3Object(bucket, s3Object.key(), size = s3Object.size(), metadata = getMetadata(bucket, s3Object.key())) + ) + val all = accumulated ++ s3Objects + if (listing.isTruncated) pageThrough(Some(listing.nextContinuationToken()), all) else all + } + + pageThrough(None, Nil) + } + def getMetadata(bucket: S3Bucket, key: Key): S3Metadata = { val meta = bucket.client.headObject(HeadObjectRequest.builder().key(key).bucket(bucket.name).build()) S3Metadata(meta)