persistence.onlyTheseCollections
Used by the reaper…
diff --git a/image-loader/app/ImageLoaderComponents.scala b/image-loader/app/ImageLoaderComponents.scala
index c5d088a8309..9d4a739b263 100644
--- a/image-loader/app/ImageLoaderComponents.scala
+++ b/image-loader/app/ImageLoaderComponents.scala
@@ -1,5 +1,5 @@
import com.gu.mediaservice.GridClient
-import com.gu.mediaservice.lib.aws.{Bedrock, S3Vectors, SimpleSqsMessageConsumer, Embedder}
+import com.gu.mediaservice.lib.aws.{Bedrock, Embedder, S3Vectors, SimpleSqsMessageConsumer}
import com.gu.mediaservice.lib.imaging.ImageOperations
import com.gu.mediaservice.lib.logging.GridLogging
import com.gu.mediaservice.lib.play.GridComponents
@@ -43,7 +43,7 @@ class ImageLoaderComponents(context: Context) extends GridComponents(context, ne
val metrics = new ImageLoaderMetrics(config, actorSystem, applicationLifecycle)
val controller = new ImageLoaderController(
- auth, downloader, store, maybeIngestQueue, uploadStatusTable, notifications, config, uploader, quarantineUploader, projector, controllerComponents, gridClient, authorisation, metrics, applicationLifecycle)
+ auth, downloader, store, maybeIngestQueue, uploadStatusTable, config, uploader, quarantineUploader, projector, controllerComponents, gridClient, authorisation, metrics, usageEvents, wsClient, applicationLifecycle)
val uploadStatusController = new UploadStatusController(auth, uploadStatusTable, config, controllerComponents, authorisation)
val imageLoaderManagement = new ImageLoaderManagement(controllerComponents, buildInfo, controller.maybeIngestQueueAndProcessor)
diff --git a/image-loader/app/controllers/ImageLoaderController.scala b/image-loader/app/controllers/ImageLoaderController.scala
index cde5129fe82..2c85b621cba 100644
--- a/image-loader/app/controllers/ImageLoaderController.scala
+++ b/image-loader/app/controllers/ImageLoaderController.scala
@@ -9,31 +9,40 @@ import com.gu.mediaservice.GridClient
import com.gu.mediaservice.lib.ImageIngestOperations.fileKeyFromId
import com.gu.mediaservice.lib.argo.ArgoHelpers
import com.gu.mediaservice.lib.argo.model.Link
-import com.gu.mediaservice.lib.auth.Authentication.OnBehalfOfPrincipal
+import com.gu.mediaservice.lib.auth.Authentication.{MachinePrincipal, OnBehalfOfPrincipal, UserPrincipal}
import com.gu.mediaservice.lib.auth._
+import com.gu.mediaservice.lib.auth.provider.ApiKeyAuthenticationProvider
import com.gu.mediaservice.lib.aws.{S3Ops, SimpleSqsMessageConsumer, SqsHelpers}
+import com.gu.mediaservice.lib.config.InstanceForRequest
+import com.gu.mediaservice.lib.events.UsageEvents
import com.gu.mediaservice.lib.formatting.printDateTime
+import com.gu.mediaservice.lib.instances.Instances
import com.gu.mediaservice.lib.logging.{FALLBACK, LogMarker, MarkerMap}
import com.gu.mediaservice.lib.play.RequestLoggingFilter
import com.gu.mediaservice.lib.{DateTimeUtils, ImageIngestOperations, ImageStorageProps}
-import com.gu.mediaservice.model.{UnsupportedMimeTypeException, UploadInfo}
-import org.scanamo.{ConditionNotMet, ScanamoError}
+import com.gu.mediaservice.model.{Instance, UnsupportedMimeTypeException, UploadInfo}
import lib.FailureResponse.Response
+import lib._
import lib.imaging.{MimeTypeDetection, NoSuchImageExistsInS3, UserImageLoaderException}
import lib.storage.{ImageLoaderStore, S3FileDoesNotExistException}
-import lib._
import model.upload.UploadRequest
import model.{Projector, QuarantineUploader, S3FileExtractedMetadata, S3IngestObject, StatusType, UploadStatus, UploadStatusRecord, UploadStatusUri, Uploader}
+import org.apache.pekko.Done
+import org.apache.pekko.stream.Materializer
+import org.apache.pekko.stream.scaladsl.Source
import org.joda.time.{DateTime, DateTimeZone}
+import org.scanamo.{ConditionNotMet, ScanamoError}
import play.api.data.Form
import play.api.data.Forms._
import play.api.inject.ApplicationLifecycle
import play.api.libs.json.Json
+import play.api.libs.ws.WSClient
import play.api.mvc._
import software.amazon.awssdk.regions.Region
import software.amazon.awssdk.services.cloudwatch.model.Dimension
import software.amazon.awssdk.services.s3.S3Client
import software.amazon.awssdk.services.s3.model.{GetObjectRequest, HeadObjectRequest, NoSuchKeyException}
+import software.amazon.awssdk.services.sqs.model.{Message => SQSMessage}
import java.io.{File, FileOutputStream}
import java.net.URI
@@ -49,8 +58,7 @@ class ImageLoaderController(auth: Authentication,
store: ImageLoaderStore,
maybeIngestQueue: Option[SimpleSqsMessageConsumer],
uploadStatusTable: UploadStatusTable,
- notifications: Notifications,
- config: ImageLoaderConfig,
+ val config: ImageLoaderConfig,
uploader: Uploader,
quarantineUploader: Option[QuarantineUploader],
projector: Projector,
@@ -58,9 +66,11 @@ class ImageLoaderController(auth: Authentication,
gridClient: GridClient,
authorisation: Authorisation,
metrics: ImageLoaderMetrics,
+ usageEvents: UsageEvents,
+ val wsClient: WSClient,
applicationLifecycle: ApplicationLifecycle)
(implicit val ec: ExecutionContext, materializer: Materializer)
- extends BaseController with ArgoHelpers with SqsHelpers {
+ extends BaseController with ArgoHelpers with SqsHelpers with InstanceForRequest with Instances {
private val AuthenticatedAndAuthorised = auth andThen authorisation.CommonActionFilters.authorisedForUpload
@@ -116,26 +126,27 @@ class ImageLoaderController(auth: Authentication,
(ingestQueue, processor)
}
- private lazy val indexResponse: Result = {
+ private def indexResponse(instance: Instance): Result = {
val indexData = Map("description" -> "This is the Loader Service")
val indexLinks = List(
- Link("prepare", s"${config.rootUri}/prepare"),
- Link("uploadStatus", s"${config.rootUri}/uploadStatus/{id}"),
- Link("uploadStatuses", s"${config.rootUri}/uploadStatuses"),
- Link("load", s"${config.rootUri}/images{?uploadedBy,identifiers,uploadTime,filename}"),
- Link("import", s"${config.rootUri}/imports{?uri,uploadedBy,identifiers,uploadTime,filename}")
+ Link("prepare", s"${config.rootUri(instance)}/prepare"),
+ Link("uploadStatus", s"${config.rootUri(instance)}/uploadStatus/{id}"),
+ Link("uploadStatuses", s"${config.rootUri(instance)}/uploadStatuses"),
+ Link("load", s"${config.rootUri(instance)}/images{?uploadedBy,identifiers,uploadTime,filename}"),
+ Link("import", s"${config.rootUri(instance)}/imports{?uri,uploadedBy,identifiers,uploadTime,filename}")
)
respond(indexData, indexLinks)
}
- def index: Action[AnyContent] = AuthenticatedAndAuthorised { indexResponse }
-
- private def quarantineOrStoreImage(uploadRequest: UploadRequest)(implicit logMarker: LogMarker) = {
- quarantineUploader.map(_.quarantineFile(uploadRequest)).getOrElse(for { uploadStatusUri <- uploader.storeFile(uploadRequest)} yield{uploadStatusUri.toJsObject})
+ def index: Action[AnyContent] = AuthenticatedAndAuthorised { request =>
+ indexResponse(instanceOf(request))
}
- private def handleMessageFromIngestBucket(sqsMessage:SQSMessage)(basicLogMarker: LogMarker): Future[Unit] = Future[Future[Unit]]{
+ private def quarantineOrStoreImage(uploadRequest: UploadRequest)(implicit logMarker: LogMarker, instance: Instance) = {
+ quarantineUploader.map(_.quarantineFile(uploadRequest)(instance)).getOrElse(for { uploadStatusUri <- uploader.storeFile(uploadRequest)} yield{uploadStatusUri.toJsObject})
+ }
+ private def handleMessageFromIngestBucket(sqsMessage: SQSMessage)(basicLogMarker: LogMarker): Future[Unit] = {
logger.info(basicLogMarker, sqsMessage.toString)
extractS3KeyFromSqsMessage(sqsMessage) match {
@@ -144,9 +155,22 @@ class ImageLoaderController(auth: Authentication,
logger.error(basicLogMarker, s"Failed to parse s3 data from SQS message", exception)
Future.unit
case Success(key) =>
- val s3IngestObject = S3IngestObject(key, store)(basicLogMarker)
+ val pathComponents = key.split("/")
+ val instanceId = pathComponents.head
+ val path = pathComponents.drop(1).mkString("/")
+ logger.info(s"Instance and key: $instanceId / $path")
+
+ val eventualIsValidInstanceId = getInstances().map { instances =>
+ instances.map(_.id).contains(instanceId)
+ }
- val isUiUpload = s3IngestObject.maybeMediaIdFromUiUpload.isDefined
+ eventualIsValidInstanceId.flatMap { isValidInstanceSpecificKey =>
+ if (isValidInstanceSpecificKey) {
+ try {
+ implicit val instance: Instance = Instance(id = instanceId)
+
+ val s3IngestObject = S3IngestObject(key, store)(basicLogMarker)
+ val isUiUpload = s3IngestObject.maybeMediaIdFromUiUpload.isDefined
implicit val logMarker: LogMarker = basicLogMarker ++ Map(
"uploadedBy" -> s3IngestObject.uploadedBy,
@@ -160,51 +184,62 @@ class ImageLoaderController(auth: Authentication,
Dimension.builder().name("IsUiUpload").value(isUiUpload.toString).build(),
)
- val approximateReceiveCount = getApproximateReceiveCount(sqsMessage)
-
- if(config.maybeUploadLimitInBytes.exists(_ < s3IngestObject.contentLength)){
- val errorMessage = s"File size exceeds the maximum allowed size (${config.maybeUploadLimitInBytes.get / 1_000_000}MB). Moving to fail bucket."
- logger.warn(logMarker, errorMessage)
- store.moveObjectToFailedBucket(s3IngestObject.key)
- s3IngestObject.maybeMediaIdFromUiUpload foreach { imageId =>
- uploadStatusTable.updateStatus( // fire & forget, since there's nothing else we can do
- imageId, UploadStatus(StatusType.Failed, Some(errorMessage))
- )
- }
- metrics.failedIngestsFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
- Future.unit
- }
- else if (approximateReceiveCount > 2) {
- metrics.abandonedMessagesFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
- val errorMessage = s"File processing has been attempted $approximateReceiveCount times. Moving to fail bucket."
- logger.warn(logMarker, errorMessage)
- store.moveObjectToFailedBucket(s3IngestObject.key)
- s3IngestObject.maybeMediaIdFromUiUpload foreach { imageId =>
- uploadStatusTable.updateStatus( // fire & forget, since there's nothing else we can do
- imageId, UploadStatus(StatusType.Failed, Some(errorMessage))
- )
- }
- Future.unit
- } else {
- attemptToProcessIngestedFile(s3IngestObject, isUiUpload)(logMarker) map { digestedFile =>
- metrics.successfulIngestsFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
- logger.info(logMarker, s"Successfully processed image ${digestedFile.file.getName}")
- store.deleteObjectFromIngestBucket(s3IngestObject.key)
- } recover {
- case _: UnsupportedMimeTypeException =>
- metrics.failedIngestsFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
- logger.info(logMarker, s"Unsupported mime type. Moving straight to fail bucket.")
- store.moveObjectToFailedBucket(s3IngestObject.key)
- case t: Throwable =>
- metrics.failedIngestsFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
- logger.error(logMarker, s"Failed to process file. Moving to fail bucket.", t)
- store.moveObjectToFailedBucket(s3IngestObject.key)
+ val approximateReceiveCount = getApproximateReceiveCount(sqsMessage)
+
+ if (config.maybeUploadLimitInBytes.exists(_ < s3IngestObject.contentLength)) {
+ val errorMessage = s"File size exceeds the maximum allowed size (${config.maybeUploadLimitInBytes.get / 1_000_000}MB). Moving to fail bucket."
+ logger.warn(logMarker, errorMessage)
+ store.moveObjectToFailedBucket(s3IngestObject.key)
+ s3IngestObject.maybeMediaIdFromUiUpload foreach { imageId =>
+ uploadStatusTable.updateStatus( // fire & forget, since there's nothing else we can do
+ imageId, UploadStatus(StatusType.Failed, Some(errorMessage))
+ )
+ }
+ metrics.failedIngestsFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
+ Future.unit
+ }
+ else if (approximateReceiveCount > 2) {
+ metrics.abandonedMessagesFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
+ val errorMessage = s"File processing has been attempted $approximateReceiveCount times. Moving to fail bucket."
+ logger.warn(logMarker, errorMessage)
+ store.moveObjectToFailedBucket(s3IngestObject.key)
+ s3IngestObject.maybeMediaIdFromUiUpload foreach { imageId =>
+ uploadStatusTable.updateStatus( // fire & forget, since there's nothing else we can do
+ imageId, UploadStatus(StatusType.Failed, Some(errorMessage))
+ )
+ }
+ Future.unit
+ } else {
+ attemptToProcessIngestedFile(s3IngestObject, isUiUpload)(logMarker)(instance) map { digestedFile =>
+ metrics.successfulIngestsFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
+ usageEvents.successfulIngestFromQueue(instance = instance, image = digestedFile.digest, filesize = s3IngestObject.contentLength )
+ logger.info(logMarker, s"Successfully processed image ${digestedFile.file.getName}")
+ store.deleteObjectFromIngestBucket(s3IngestObject.key)
+ } recover {
+ case _: UnsupportedMimeTypeException =>
+ metrics.failedIngestsFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
+ logger.info(logMarker, s"Unsupported mime type. Moving straight to fail bucket.")
+ store.moveObjectToFailedBucket(s3IngestObject.key)
+ case t: Throwable =>
+ metrics.failedIngestsFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
+ logger.error(logMarker, s"Failed to process file. Moving to fail bucket.", t)
+ store.moveObjectToFailedBucket(s3IngestObject.key)
+ }
+ }
+ }
+ catch {
+ case t: Throwable =>
+ logger.error("Uncaught throw:", t)
+ Future.unit
+ }
+ } else {
+ Future.unit
}
}
}
- }.flatten
+ }
- private def attemptToProcessIngestedFile(s3IngestObject:S3IngestObject, isUiUpload: Boolean)(initialLogMarker:LogMarker): Future[DigestedFile] = {
+ private def attemptToProcessIngestedFile(s3IngestObject:S3IngestObject, isUiUpload: Boolean)(initialLogMarker:LogMarker)(implicit instance: Instance): Future[DigestedFile] = {
logger.info(initialLogMarker, "Attempting to process file")
val tempFile = createTempFile("s3IngestBucketFile")(initialLogMarker)
@@ -218,12 +253,14 @@ class ImageLoaderController(auth: Authentication,
"mediaId" -> digestedFile.digest
)
+ val filename = s3IngestObject.filename
val futureUploadStatusUri = uploadDigestedFileToStore(
digestedFileFuture = Future(digestedFile),
uploadedBy = s3IngestObject.uploadedBy,
identifiers = s3IngestObject.identifiers,
uploadTime = Some(s3IngestObject.uploadTime.toString) , // upload time as iso string - uploader uses DateTimeUtils.fromValueOrNow
- filename = Some(s3IngestObject.filename)
+ filename = Some(filename),
+ isFeedUpload = s3IngestObject.isFeedUpload,
)
// under all circumstances, remove the temp files
@@ -239,6 +276,8 @@ class ImageLoaderController(auth: Authentication,
}
def getPreSignedUploadUrlsAndTrack: Action[AnyContent] = AuthenticatedAndAuthorised.async { request =>
+ implicit val instance: Instance = instanceOf(request)
+
val expiration = DateTimeUtils.now().plusHours(1)
val signatureDuration = Duration.ofHours(1)
@@ -249,6 +288,7 @@ class ImageLoaderController(auth: Authentication,
Future.sequence(
mediaIdToFilenameMap.map{case (mediaId, filename) =>
+ logger.info(s"Preparing file upload for instance $instance: $mediaId / $filename")
val preSignedUrl = store.generatePreSignedUploadUrl(filename, signatureDuration, uploadedBy, mediaId)
@@ -261,9 +301,19 @@ class ImageLoaderController(auth: Authentication,
StatusType.Prepared,
errorMessage = None,
expires = expiration.toEpochSecond, // TTL in case upload is never completed by client
- )).map(_ =>
+ instance = instance.id
+ )).map { _ =>
+ val user = request.user match {
+ case u: UserPrincipal => u.attributes.get(ApiKeyAuthenticationProvider.KindeIdKey)
+ case _ => None
+ }
+ val apiKey = request.user match {
+ case m: MachinePrincipal => Some(m.accessor.identity)
+ case _ => None
+ }
+ usageEvents.prepareUpload(instance = instance, image = mediaId, user = user, apiKey = apiKey)
mediaId -> preSignedUrl
- )
+ }
}
)
.map(_.toMap)
@@ -271,7 +321,7 @@ class ImageLoaderController(auth: Authentication,
.map(Ok(_))
}
- def loadImage(uploadedBy: Option[String], identifiers: Option[String], uploadTime: Option[String], filename: Option[String]): Action[DigestedFile] = {
+ def loadImage(uploadedBy: Option[String], identifiers: Option[String], uploadTime: Option[String], filename: Option[String]): Action[DigestedFile] = {
val uploadTimeToRecord = DateTimeUtils.fromValueOrNow(uploadTime)
val initialContext = MarkerMap(
@@ -288,7 +338,8 @@ class ImageLoaderController(auth: Authentication,
logger.info(initialContext, "body parsed")
val bodyParser = DigestBodyParser.create(tempFile)
- AuthenticatedAndAuthorised.async(bodyParser) { req =>
+ AuthenticatedAndAuthorised.async(bodyParser) { req: Authentication.Request[DigestedFile] =>
+ implicit val instance: Instance = instanceOf(req)
val uploadedByToRecord = uploadedBy.getOrElse(Authentication.getIdentity(req.user))
implicit val context: LogMarker =
@@ -299,23 +350,38 @@ class ImageLoaderController(auth: Authentication,
val uploadStatus = if(config.maybeQuarantineBucket.isDefined) StatusType.Pending else StatusType.Completed
val uploadExpiry = Instant.now.getEpochSecond + config.uploadStatusExpiry.toSeconds
- val record = UploadStatusRecord(req.body.digest, filename, uploadedByToRecord, printDateTime(uploadTimeToRecord), identifiers, uploadStatus, None, uploadExpiry)
+ val record = UploadStatusRecord(req.body.digest, filename, uploadedByToRecord, printDateTime(uploadTimeToRecord), identifiers, uploadStatus, None, uploadExpiry, instance.id)
+ logger.info(s"Loading image for instance ${instance.id}: record ${record.id} / $filename")
+
val result = for {
uploadRequest <- uploader.loadFile(
req.body,
uploadedByToRecord,
identifiers.map(Json.parse(_).as[Map[String, String]]).getOrElse(Map.empty),
uploadTimeToRecord,
- filename.flatMap(_.trim.nonEmptyOpt)
+ filename.flatMap(_.trim.nonEmptyOpt),
+ instance,
+ isFeedUpload = false,
)
_ <- uploadStatusTable.setStatus(record)
- result <- quarantineOrStoreImage(uploadRequest)
+
+ result <- quarantineOrStoreImage(uploadRequest)(context, instance)
+
} yield result
result.onComplete( _ => Try { deleteTempFile(tempFile) } )
result map { r =>
val result = Accepted(r).as(ArgoMediaType)
logger.info(context, "loadImage request end")
+ val user = req.user match {
+ case u: UserPrincipal => u.attributes.get(ApiKeyAuthenticationProvider.KindeIdKey)
+ case _ => None
+ }
+ val apiKey = req.user match {
+ case m: MachinePrincipal => Some(m.accessor.identity)
+ case _ => None
+ }
+ usageEvents.uploadImage(instance = instance, image = req.body.digest, filesize = req.body.file.length(), apiKey = apiKey, user = user)
result
} recover {
case NonFatal(e) =>
@@ -347,6 +413,7 @@ class ImageLoaderController(auth: Authentication,
val bodyParser = DigestBodyParser.create(tempFile)
AuthenticatedAndAuthorised.async(bodyParser) { req =>
+ implicit val instance: Instance = instanceOf(req)
val allIdentifiers = identifiers.map(Json.parse(_).as[Map[String, String]]).getOrElse(Map.empty) ++ Map(
ImageStorageProps.derivativeOfMediaIdsIdentifierKey -> derivativeOfMediaIds
@@ -372,12 +439,14 @@ class ImageLoaderController(auth: Authentication,
// Fetch
def projectImageBy(imageId: String): Action[AnyContent] = {
+
val initialContext = MarkerMap(
"imageId" -> imageId,
"requestType" -> "image-projection"
)
val tempFile = createTempFile(s"projection-$imageId")(initialContext)
auth.async { req =>
+ implicit val instance: Instance = instanceOf(req)
implicit val context: LogMarker = initialContext ++ Map(
"requestId" -> RequestLoggingFilter.getRequestId(req)
)
@@ -408,9 +477,11 @@ class ImageLoaderController(auth: Authentication,
uploadedBy: Option[String],
identifiers: Option[String],
uploadTime: Option[String],
- filename: Option[String]
+ filename: Option[String],
): Action[AnyContent] = {
AuthenticatedAndAuthorised.async { request =>
+ implicit val instance: Instance = instanceOf(request)
+
implicit val context: MarkerMap = MarkerMap(
"requestType" -> "import-image",
"key-tier" -> request.user.accessor.tier.toString,
@@ -418,7 +489,7 @@ class ImageLoaderController(auth: Authentication,
"requestId" -> RequestLoggingFilter.getRequestId(request)
)
- logger.info(context, "importImage request start")
+ logger.info(context, "importImage request start for $uri into instance $instance")
val tempFile = createTempFile("download")
val digestedFileFuture = for {
@@ -431,7 +502,8 @@ class ImageLoaderController(auth: Authentication,
uploadedBy.getOrElse(Authentication.getIdentity(request.user)),
identifiers.map(Json.parse(_).as[Map[String, String]]).getOrElse(Map.empty),
uploadTime,
- filename
+ filename,
+ isFeedUpload = false,
)
// under all circumstances, remove the temp files
@@ -458,8 +530,9 @@ class ImageLoaderController(auth: Authentication,
uploadedBy: String,
identifiers: Map[String, String],
uploadTime: Option[String],
- filename: Option[String]
- )(implicit logMarker:LogMarker): Future[UploadStatusUri] = {
+ filename: Option[String],
+ isFeedUpload: Boolean
+ )(implicit logMarker:LogMarker, instance: Instance): Future[UploadStatusUri] = {
for {
digestedFile <- digestedFileFuture
@@ -473,6 +546,8 @@ class ImageLoaderController(auth: Authentication,
).getOrElse(identifiers),
uploadTime = DateTimeUtils.fromValueOrNow(maybeStatus.map(_.uploadTime).orElse(uploadTime)),
filename = maybeStatus.flatMap(_.fileName).orElse(filename).flatMap(_.trim.nonEmptyOpt),
+ instance,
+ isFeedUpload = isFeedUpload
)
result <- uploader.storeFile(uploadRequest)
} yield {
@@ -484,7 +559,7 @@ class ImageLoaderController(auth: Authentication,
private def resolveUploadAndUpdateStatus (
uploadResultFuture: Future[UploadStatusUri],
digestedFileFuture: Future[DigestedFile],
- )(implicit logMarker:LogMarker):Future[Either[Response,UploadStatusUri]] = {
+ )(implicit logMarker:LogMarker, instance: Instance):Future[Either[Response,UploadStatusUri]] = {
// combine the import result and digest file together into a single future
uploadResultFuture.transformWith { // note that we use transformWith instead of zip here as we are still interested in value of digestedFile even if the import fails
maybeImportResult =>
@@ -523,7 +598,7 @@ class ImageLoaderController(auth: Authentication,
private def updateUploadStatusTable(
uploadAttempt: Future[UploadStatusUri],
digestedFile: DigestedFile
- )(implicit logMarker: LogMarker): Future[Unit] = {
+ )(implicit logMarker: LogMarker, instance: Instance): Future[Unit] = {
def reportFailure(error: Throwable): Unit = {
val errorMessage = s"an error occurred while updating image upload status, error:$error"
@@ -574,6 +649,7 @@ class ImageLoaderController(auth: Authentication,
private case class RestoreFromReplicaForm(imageId: String)
def restoreFromReplica: Action[AnyContent] = AuthenticatedAndAuthorised.async { implicit request =>
+ implicit val instance: Instance = instanceOf(request)
val imageId = Form(
mapping(
@@ -624,7 +700,8 @@ class ImageLoaderController(auth: Authentication,
metadata.uploadTime,
metadata.uploadedBy,
metadata.identifiers,
- UploadInfo(metadata.uploadFileName)
+ UploadInfo(metadata.uploadFileName, metadata.isFeedUpload),
+ instance,
),
gridClient,
auth.getOnBehalfOfPrincipal(request.user)
@@ -636,7 +713,7 @@ class ImageLoaderController(auth: Authentication,
future.map { _ =>
logger.info(logMarker, s"Restored image $imageId from replica bucket $replicaBucket (key: $s3Key)")
- Redirect(s"${config.kahunaUri}/images/$imageId")
+ Redirect(s"${config.kahunaUri(instance)}/images/$imageId")
}
case _ =>
Future.successful(NotFound("Image not found in replica bucket"))
diff --git a/image-loader/app/controllers/UploadStatusController.scala b/image-loader/app/controllers/UploadStatusController.scala
index 4bde0c8b3be..c449f8d2ffb 100644
--- a/image-loader/app/controllers/UploadStatusController.scala
+++ b/image-loader/app/controllers/UploadStatusController.scala
@@ -3,8 +3,10 @@ package controllers
import com.gu.mediaservice.lib.argo.ArgoHelpers
import com.gu.mediaservice.lib.auth._
+import com.gu.mediaservice.lib.config.InstanceForRequest
import lib._
import model.{StatusType, UploadStatus}
+import com.gu.mediaservice.model.Instance
import org.scanamo.{ConditionNotMet, ScanamoError}
import play.api.mvc._
@@ -18,13 +20,14 @@ class UploadStatusController(auth: Authentication,
authorisation: Authorisation
)
(implicit val ec: ExecutionContext)
- extends BaseController with ArgoHelpers {
+ extends BaseController with ArgoHelpers with InstanceForRequest {
- def getUploadStatus(imageId: String) = auth.async {
+ def getUploadStatus(imageId: String) = auth.async { request =>
+ implicit val instance: Instance = instanceOf(request)
store.getStatus(imageId)
.map {
case Some(Right(record)) => respond(UploadStatus(record.status, record.errorMessage),
- uri = Some(URI.create(s"${config.apiUri}/images/${imageId}")))
+ uri = Some(URI.create(s"${config.apiUri(instance)}/images/$imageId")))
case Some(Left(error)) => respondError(BadRequest, "cannot-get", s"Cannot get upload status ${error}")
case None => respondNotFound(s"No upload status found for image id: ${imageId}")
}
@@ -32,6 +35,7 @@ class UploadStatusController(auth: Authentication,
}
def updateUploadStatus(imageId: String) = (auth andThen authorisation.CommonActionFilters.authorisedForUpload).async(parse.json[UploadStatus]) { request =>
+ implicit val instance: Instance = instanceOf(request)
request.body match {
case UploadStatus(StatusType.Failed, None) =>
Future.successful(respondError(
@@ -55,6 +59,7 @@ class UploadStatusController(auth: Authentication,
def getUploadsBy(user:String): Action[AnyContent] = getUploads(Some(user))
private def getUploads(maybeUser: Option[String]): Action[AnyContent] = auth.async { req =>
+ implicit val instance: Instance = instanceOf(req)
store.queryByUser(maybeUser.getOrElse(req.user.accessor.identity))
.map(list => respond(list))
}
diff --git a/image-loader/app/lib/ImageLoaderConfig.scala b/image-loader/app/lib/ImageLoaderConfig.scala
index 9417585a471..37d43aa0781 100644
--- a/image-loader/app/lib/ImageLoaderConfig.scala
+++ b/image-loader/app/lib/ImageLoaderConfig.scala
@@ -24,9 +24,9 @@ class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(res
val thumbWidth: Int = 256
val thumbQuality: Double = 85d // out of 100
- val rootUri: String = services.loaderBaseUri
- val apiUri: String = services.apiBaseUri
- val kahunaUri: String = services.kahunaBaseUri
+ val rootUri: Instance => String = services.loaderBaseUri
+ val apiUri: Instance => String = services.apiBaseUri
+ val kahunaUri: Instance => String = services.kahunaBaseUri
val transcodedMimeTypes: List[MimeType] = getStringSet("transcoded.mime.types").toList.map(MimeType(_))
val supportedMimeTypes: List[MimeType] = List(Jpeg, Png) ::: transcodedMimeTypes
@@ -36,6 +36,7 @@ class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(res
val shouldEmbed: Boolean = boolean("s3.vectors.shouldEmbed")
val maybeImageEmbedderQueueUrl: Option[String] = stringOpt("sqs.image.embedder.queue.url")
+
/**
* Load in the chain of image processors from config. This can be a list of
* companion objects, class names, both with and without config.
diff --git a/image-loader/app/lib/ImageLoaderStore.scala b/image-loader/app/lib/ImageLoaderStore.scala
index 3758c583ff9..455a8337164 100644
--- a/image-loader/app/lib/ImageLoaderStore.scala
+++ b/image-loader/app/lib/ImageLoaderStore.scala
@@ -1,29 +1,32 @@
package lib.storage
-import software.amazon.awssdk.services.s3.model.{CopyObjectRequest, DeleteObjectRequest, GetObjectRequest, GetObjectResponse, PutObjectRequest, S3Exception}
-
-import java.time.Duration
-import scala.jdk.CollectionConverters.MapHasAsJava
-import software.amazon.awssdk.core.ResponseInputStream
-import software.amazon.awssdk.services.s3.presigner.S3Presigner
-import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest
import lib.ImageLoaderConfig
import com.gu.mediaservice.lib
-import com.gu.mediaservice.lib.logging.LogMarker
+import com.gu.mediaservice.lib.aws
+import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker}
+import com.gu.mediaservice.model.Instance
+import software.amazon.awssdk.core.ResponseInputStream
+import software.amazon.awssdk.services.s3.model._
+import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest
import java.io.File
+import java.time.Duration
+import scala.concurrent.Future
+import scala.jdk.CollectionConverters.MapHasAsJava
class S3FileDoesNotExistException extends Exception()
-class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperations(config.imageBucket, config.thumbnailBucket, config) {
+class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperations(config.imageBucket, config.thumbnailBucket, config) with GridLogging {
private def handleNotFound[T](key: String)(doWork: => T)(loggingIfNotFound: => Unit): T = {
try {
doWork
} catch {
- case e: S3Exception if e.statusCode() == 404 || e.statusCode() == 403 =>
+ case e: S3Exception if e.statusCode() == 404 || e.statusCode() == 403 => {
+ logger.warn(s"AmazonS3Exception ${e.statusCode()} for key '$key'")
loggingIfNotFound
throw new S3FileDoesNotExistException
+ }
case other: Throwable => throw other
}
}
@@ -35,19 +38,19 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati
logger.error(logMarker, s"Attempted to read $key from ingest bucket, but it does not exist.")
}
- def queueS3Object(uploader: String, filename: String, s3Meta: Map[String, String], file: File)(implicit logMarker: LogMarker) = {
+ def queueS3Object(uploader: String, filename: String, s3Meta: Map[String, String], file: File)(implicit logMarker: LogMarker, instance: Instance): Future[aws.S3Object] = {
store(
config.maybeIngestBucket.get,
- s"$uploader/$filename",
+ s"${instance.id}/$uploader/$filename",
file,
mimeType = None, // we don't care as this is just the queue bucket
meta = s3Meta,
)
}
- def generatePreSignedUploadUrl(filename: String, duration: Duration, uploadedBy: String, mediaId: String): String = {
+ def generatePreSignedUploadUrl(filename: String, duration: Duration, uploadedBy: String, mediaId: String)(implicit instance: Instance): String = {
val putObjectRequest = PutObjectRequest.builder()
- .bucket(config.maybeBucketForUIUploads.get).key(s"$uploadedBy/$filename").metadata(Map(
+ .bucket(config.maybeBucketForUIUploads.get).key(s"${instance.id}/$uploadedBy/$filename").metadata(Map(
"media-id" -> mediaId).asJava)
.build()
val putObjectPresignRequest =
@@ -63,9 +66,9 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati
def moveObjectToFailedBucket(key: String)(implicit logMarker: LogMarker) = handleNotFound(key){
client.copyObject(
CopyObjectRequest.builder()
- .sourceBucket(config.maybeIngestBucket.get)
+ .sourceBucket(config.maybeIngestBucket.get) // TODO Naked get - make optional
.sourceKey(key)
- .destinationBucket(config.maybeFailBucket.get)
+ .destinationBucket(config.maybeFailBucket.get) // TODO Naked get - make optional
.destinationKey(key)
.build()
)
diff --git a/image-loader/app/lib/UploadStatusTable.scala b/image-loader/app/lib/UploadStatusTable.scala
index d193b0a3e83..52a44d3401c 100644
--- a/image-loader/app/lib/UploadStatusTable.scala
+++ b/image-loader/app/lib/UploadStatusTable.scala
@@ -1,51 +1,56 @@
package lib
+import com.gu.mediaservice.lib.logging.GridLogging
+import com.gu.mediaservice.model.Instance
+import model.StatusType.{Prepared, Queued}
+import model.{StatusType, UploadStatus, UploadStatusRecord}
import org.scanamo._
-import org.scanamo.syntax._
import org.scanamo.generic.auto._
-import model.StatusType.{Prepared, Queued}
-import model.{UploadStatus, UploadStatusRecord}
-import software.amazon.awssdk.services.dynamodb.{DynamoDbAsyncClient, DynamoDbAsyncClientBuilder}
+import org.scanamo.syntax._
+import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.Future
-class UploadStatusTable(config: ImageLoaderConfig) {
+class UploadStatusTable(config: ImageLoaderConfig) extends GridLogging {
+
+ implicit val statusTypeFormat: Typeclass[StatusType] =
+ DynamoFormat.coercedXmap[StatusType, String, IllegalArgumentException](StatusType.apply, _.name)
val client = config.withAWSCredentials(DynamoDbAsyncClient.builder()).build()
val scanamo = ScanamoAsync(client)
private val uploadStatusTable = Table[UploadStatusRecord](config.uploadStatusTable)
- def getStatus(imageId: String) = {
- scanamo.exec(uploadStatusTable.get("id" === imageId))
+ def getStatus(imageId: String)(implicit instance: Instance) = {
+ scanamo.exec(uploadStatusTable.get("instance" === instance.id and "id" === imageId))
}
def setStatus(uploadStatus: UploadStatusRecord) = {
scanamo.exec(uploadStatusTable.put(uploadStatus))
}
- def updateStatus(imageId: String, updateRequest: UploadStatus) = {
- val updateExpression = updateRequest.errorMessage match {
- case Some(error) => set("status", updateRequest.status) and set("errorMessage", error)
- case None => set("status", updateRequest.status)
+ def updateStatus(imageId: String, updateStatus: UploadStatus)(implicit instance: Instance) = {
+ val updateExpression = updateStatus.errorMessage match {
+ case Some(error) => set("status", updateStatus.status) and set("errorMessage", error)
+ case None => set("status", updateStatus.status)
}
val uploadStatusTableWithCondition =
- if(updateRequest.status == Queued) // can only transition to Queued status from Prepared status
- uploadStatusTable.when(attributeExists("id") and ("status" === Prepared.toString))
+ if(updateStatus.status == Queued) // can only transition to Queued status from Prepared status
+ uploadStatusTable.when(attributeExists("id") and attributeExists("instance") and "status" === Prepared.name)
else
- uploadStatusTable.when(attributeExists("id"))
+ uploadStatusTable.when(attributeExists("id") and attributeExists("instance"))
scanamo.exec(
uploadStatusTableWithCondition
.update(
- key = "id" === imageId,
+ "id" === imageId and "instance" === instance.id,
update = updateExpression
)
)
}
- def queryByUser(user: String): Future[List[UploadStatusRecord]] = {
- scanamo.exec(uploadStatusTable.scan()).map {
+ def queryByUser(user: String)(implicit instance: Instance): Future[List[UploadStatusRecord]] = {
+ scanamo.exec(uploadStatusTable.query("instance" === instance.id)).map {
case Nil => List.empty[UploadStatusRecord]
case recordsAndErrors => {
recordsAndErrors
diff --git a/image-loader/app/model/Projector.scala b/image-loader/app/model/Projector.scala
index 41dcf25fa00..cc469d886bd 100644
--- a/image-loader/app/model/Projector.scala
+++ b/image-loader/app/model/Projector.scala
@@ -1,30 +1,29 @@
package model
-import software.amazon.awssdk.core.ResponseInputStream
-import software.amazon.awssdk.services.s3.model.GetObjectResponse
-
-import java.io.{File, FileOutputStream}
-import com.gu.mediaservice.{GridClient, ImageDataMerger}
-import com.gu.mediaservice.lib.auth.Authentication
import com.gu.mediaservice.lib.ImageIngestOperations.{fileKeyFromId, optimisedPngKeyFromId}
-import com.gu.mediaservice.lib.{ImageIngestOperations, ImageStorageProps, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage}
-import com.gu.mediaservice.lib.aws.{Embedder, S3, S3Object}
+import com.gu.mediaservice.lib.auth.Authentication
+import com.gu.mediaservice.lib.aws.{Embedder, S3}
import com.gu.mediaservice.lib.cleanup.ImageProcessor
+import com.gu.mediaservice.lib.config.InstanceForRequest
import com.gu.mediaservice.lib.imaging.ImageOperations
import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch}
import com.gu.mediaservice.lib.net.URI
-import com.gu.mediaservice.model.{Image, MimeType, UploadInfo}
+import com.gu.mediaservice.lib._
+import com.gu.mediaservice.model.{Image, Instance, MimeType, UploadInfo}
+import com.gu.mediaservice.{GridClient, ImageDataMerger}
import lib.imaging.{MimeTypeDetection, NoSuchImageExistsInS3}
import lib.{DigestedFile, ImageLoaderConfig}
import model.upload.UploadRequest
import org.apache.commons.io.IOUtils
import org.joda.time.{DateTime, DateTimeZone}
-import play.api.libs.ws.WSRequest
+import _root_.play.api.libs.ws.WSRequest
+import software.amazon.awssdk.core.ResponseInputStream
+import software.amazon.awssdk.services.s3.model.GetObjectResponse
-import java.time.Instant
-import scala.jdk.CollectionConverters._
+import java.io.{File, FileOutputStream}
import scala.concurrent.duration.Duration
import scala.concurrent.{Await, ExecutionContext, Future}
+import scala.jdk.CollectionConverters._
object Projector {
@@ -38,11 +37,12 @@ case class S3FileExtractedMetadata(
uploadedBy: String,
uploadTime: DateTime,
uploadFileName: Option[String],
- identifiers: Map[String, String]
+ identifiers: Map[String, String],
+ isFeedUpload: Option[Boolean],
)
object S3FileExtractedMetadata {
-
+
def apply(lastModified: DateTime, userMetadata: Map[String, String]): S3FileExtractedMetadata = {
val fileUserMetadata = userMetadata.map { case (key, value) =>
// Fix up the contents of the metadata.
@@ -63,6 +63,7 @@ object S3FileExtractedMetadata {
}.map{ case (key, value) =>
key.stripPrefix(ImageStorageProps.identifierMetadataKeyPrefix) -> value
}
+ val isFeedUpload = fileUserMetadata.get(ImageStorageProps.isFeedUploadMetadataKey).map(_.toBoolean)
val uploadFileName = fileUserMetadata.get(ImageStorageProps.filenameMetadataKey)
@@ -71,6 +72,7 @@ object S3FileExtractedMetadata {
uploadTime = uploadTime,
uploadFileName = uploadFileName,
identifiers = identifiers,
+ isFeedUpload = isFeedUpload,
)
}
}
@@ -80,12 +82,12 @@ class Projector(config: ImageUploadOpsCfg,
imageOps: ImageOperations,
processor: ImageProcessor,
auth: Authentication,
- maybeEmbedder: Option[Embedder]) extends GridLogging {
+ maybeEmbedder: Option[Embedder]) extends GridLogging with InstanceForRequest {
private val imageUploadProjectionOps = new ImageUploadProjectionOps(config, imageOps, processor, s3, maybeEmbedder)
def projectS3ImageById(imageId: String, tempFile: File, gridClient: GridClient, onBehalfOfFn: WSRequest => WSRequest)
- (implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[Image]] = {
+ (implicit ec: ExecutionContext, logMarker: LogMarker, instance: Instance): Future[Option[Image]] = {
Future {
import ImageIngestOperations.fileKeyFromId
val s3Key = fileKeyFromId(imageId)
@@ -125,11 +127,11 @@ class Projector(config: ImageUploadOpsCfg,
extractedS3Meta: S3FileExtractedMetadata,
gridClient: GridClient,
onBehalfOfFn: WSRequest => WSRequest)
- (implicit ec: ExecutionContext, logMarker: LogMarker): Future[Image] = {
+ (implicit ec: ExecutionContext, logMarker: LogMarker, instance: Instance): Future[Image] = {
val DigestedFile(tempFile_, id_) = srcFileDigest
val identifiers_ = extractedS3Meta.identifiers
- val uploadInfo_ = UploadInfo(filename = extractedS3Meta.uploadFileName)
+ val uploadInfo_ = UploadInfo(filename = extractedS3Meta.uploadFileName, isFeedUpload = extractedS3Meta.isFeedUpload)
MimeTypeDetection.guessMimeType(tempFile_) match {
case util.Left(unsupported) => Future.failed(unsupported)
@@ -141,7 +143,8 @@ class Projector(config: ImageUploadOpsCfg,
uploadTime = extractedS3Meta.uploadTime,
uploadedBy = extractedS3Meta.uploadedBy,
identifiers = identifiers_,
- uploadInfo = uploadInfo_
+ uploadInfo = uploadInfo_,
+ instance = instance, // TODO careful with this one!
)
imageUploadProjectionOps.projectImageFromUploadRequest(uploadRequest) flatMap (
@@ -158,12 +161,12 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg,
maybeEmbedder: Option[Embedder],
) extends GridLogging {
- import Uploader.{fromUploadRequestShared, toMetaMap}
+ import Uploader.fromUploadRequestShared
def projectImageFromUploadRequest(uploadRequest: UploadRequest)
(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Image] = {
- val dependenciesWithProjectionsOnly = ImageUploadOpsDependencies(
+ val dependenciesWithProjectionsOnly: ImageUploadOpsDependencies = ImageUploadOpsDependencies(
config,
imageOps,
projectOriginalFileAsS3Model,
@@ -186,17 +189,15 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg,
Future.successful(storableOptimisedImage.toProjectedS3Object(config.originalFileBucket))
private def fetchThumbFile(
- imageId: String, outFile: File
- )(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[(File, MimeType)]] = {
- val key = fileKeyFromId(imageId)
-
+ imageId: String, outFile: File, instance: Instance)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[(File, MimeType)]] = {
+ val key = fileKeyFromId(imageId)(instance)
fetchFile(config.thumbBucket, key, outFile)
}
private def fetchOptimisedFile(
- imageId: String, outFile: File
+ imageId: String, outFile: File, instance: Instance
)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[(File, MimeType)]] = {
- val key = optimisedPngKeyFromId(imageId)
+ val key = optimisedPngKeyFromId(imageId)(instance)
fetchFile(config.originalFileBucket, key, outFile)
}
diff --git a/image-loader/app/model/QuarantineUploader.scala b/image-loader/app/model/QuarantineUploader.scala
index af75e44a73c..b18079981ee 100644
--- a/image-loader/app/model/QuarantineUploader.scala
+++ b/image-loader/app/model/QuarantineUploader.scala
@@ -18,6 +18,8 @@ import play.api.Logger
import play.api.libs.json.{JsObject, Json}
import com.gu.mediaservice.lib.formatting._
import model.upload.UploadRequest
+import play.api.mvc.{AnyContent, Request}
+
import java.net.URLEncoder
import java.nio.charset.StandardCharsets
import scala.concurrent.{ExecutionContext, Future}
@@ -28,6 +30,7 @@ class QuarantineUploader(val store: QuarantineStore,
private def storeQuarantineFile(uploadRequest: UploadRequest)
(implicit logMarker: LogMarker) = {
+ implicit val instance: Instance = uploadRequest.instance
val meta = Uploader.toMetaMap(uploadRequest)
store.storeQuarantineImage(
uploadRequest.imageId,
@@ -37,7 +40,7 @@ class QuarantineUploader(val store: QuarantineStore,
)
}
- def quarantineFile(uploadRequest: UploadRequest)(
+ def quarantineFile(uploadRequest: UploadRequest)(instance: Instance) (
implicit ec: ExecutionContext,
logMarker: LogMarker): Future[JsObject] = {
@@ -45,7 +48,7 @@ class QuarantineUploader(val store: QuarantineStore,
for {
_ <- storeQuarantineFile(uploadRequest)
- uri = s"${config.rootUri}/uploadStatus/${uploadRequest.imageId}"
+ uri = s"${config.rootUri(instance)}/uploadStatus/${uploadRequest.imageId}"
} yield {
Json.obj("uri" -> uri)
}
diff --git a/image-loader/app/model/S3IngestObject.scala b/image-loader/app/model/S3IngestObject.scala
index c6179d83635..65127032207 100644
--- a/image-loader/app/model/S3IngestObject.scala
+++ b/image-loader/app/model/S3IngestObject.scala
@@ -15,7 +15,8 @@ case class S3IngestObject (
uploadTime: java.util.Date,
contentLength: Long,
getInputStream: () => java.io.InputStream,
- identifiers: Map[String, String] = Map.empty
+ identifiers: Map[String, String] = Map.empty,
+ isFeedUpload: Boolean,
)
object S3IngestObject {
@@ -25,19 +26,34 @@ object S3IngestObject {
val s3Object = store.getS3Object(key)
val metadata = s3Object.response().metadata()
+
+ val mediaIdFromUiUpload = metadata.asScala.get("media-id")
+ val isFeedUpload = mediaIdFromUiUpload.isEmpty // TODO Not concise
+
S3IngestObject(
key,
- uploadedBy = keyParts.head,
+ uploadedBy = uploadedFromPath(keyParts),
filename = keyParts.last,
- maybeMediaIdFromUiUpload = metadata.asScala.toMap.get("media-id"), // set by the client in upload in manager.js
+ maybeMediaIdFromUiUpload = mediaIdFromUiUpload, // set by the client in upload in manager.js
uploadTime = new Date(s3Object.response().lastModified().toEpochMilli),
contentLength = s3Object.response().contentLength(),
getInputStream = () => s3Object,
identifiers = metadata.asScala.collect{
case (key, value) if key.startsWith(ImageStorageProps.identifierMetadataKeyPrefix) =>
key.stripPrefix(ImageStorageProps.identifierMetadataKeyPrefix) -> value
- }.toMap
+ }.toMap,
+ isFeedUpload = isFeedUpload
)
}
+
+ def uploadedFromPath(keyParts: Array[String]): String = {
+ val indexOfFeedProviderName = if (keyParts.contains("feeds")) {
+ keyParts.indexOf("feeds") + 1
+ } else {
+ 0
+ }
+
+ keyParts(Seq(indexOfFeedProviderName, 0).max)
+ }
}
diff --git a/image-loader/app/model/UploadStatus.scala b/image-loader/app/model/UploadStatus.scala
index 044acacf713..af28678e23f 100644
--- a/image-loader/app/model/UploadStatus.scala
+++ b/image-loader/app/model/UploadStatus.scala
@@ -10,7 +10,8 @@ case class UploadStatusRecord(
identifiers: Option[String],
status: StatusType,
errorMessage: Option[String],
- expires: Long
+ expires: Long,
+ instance: String
)
object UploadStatusRecord {
diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala
index 6e6c7acc26b..6b3f722d715 100644
--- a/image-loader/app/model/Uploader.scala
+++ b/image-loader/app/model/Uploader.scala
@@ -1,14 +1,12 @@
package model
-import com.gu.mediaservice.{GridClient, ImageDataMerger}
+import _root_.play.api.libs.json.Json
+import _root_.play.api.libs.ws.WSRequest
import com.gu.mediaservice.lib.Files.createTempFile
import com.gu.mediaservice.lib.ImageIngestOperations.fileKeyFromId
-
-import java.io.File
-import java.nio.file.{Files, Path}
+import com.gu.mediaservice.lib._
import com.gu.mediaservice.lib.argo.ArgoHelpers
import com.gu.mediaservice.lib.auth.Authentication
-import com.gu.mediaservice.lib.{BrowserViewableImage, ImageStorageProps, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage}
import com.gu.mediaservice.lib.aws.{Embedder, EmbedderMessage, S3Object, UpdateMessage}
import com.gu.mediaservice.lib.cleanup.ImageProcessor
import com.gu.mediaservice.lib.formatting._
@@ -19,19 +17,18 @@ import com.gu.mediaservice.lib.metadata.{FileMetadataHelper, ImageMetadataConver
import com.gu.mediaservice.lib.net.URI
import com.gu.mediaservice.model._
import com.gu.mediaservice.syntax.MessageSubjects
-import lib.{DigestedFile, ImageLoaderConfig, Notifications}
+import com.gu.mediaservice.{GridClient, ImageDataMerger}
import lib.imaging.{FileMetadataReader, MimeTypeDetection}
import lib.storage.ImageLoaderStore
+import lib.{DigestedFile, ImageLoaderConfig, Notifications}
import model.Uploader.{fromUploadRequestShared, toImageUploadOpsCfg}
import model.upload.{OptimiseOps, OptimiseWithPngQuant, UploadRequest}
import org.joda.time.DateTime
-import play.api.libs.json.Json
-import play.api.libs.ws.WSRequest
import software.amazon.awssdk.services.s3.model.CopyObjectRequest
-import scala.collection.compat._
+import java.io.File
+import java.nio.file.Files
import scala.concurrent.{ExecutionContext, Future}
-import scala.jdk.CollectionConverters.MapHasAsScala
case class ImageUpload(uploadRequest: UploadRequest, image: Image)
@@ -80,8 +77,8 @@ case class ImageUploadOpsDependencies(
storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object],
storeOrProjectThumbFile: StorableThumbImage => Future[S3Object],
storeOrProjectOptimisedImage: StorableOptimisedImage => Future[S3Object],
- tryFetchThumbFile: (String, File) => Future[Option[(File, MimeType)]] = (_, _) => Future.successful(None),
- tryFetchOptimisedFile: (String, File) => Future[Option[(File, MimeType)]] = (_, _) => Future.successful(None),
+ tryFetchThumbFile: (String, File, Instance) => Future[Option[(File, MimeType)]] = (_, _, _) => Future.successful(None),
+ tryFetchOptimisedFile: (String, File, Instance) => Future[Option[(File, MimeType)]] = (_, _, _) => Future.successful(None),
)
@@ -141,6 +138,7 @@ object Uploader extends GridLogging {
case Some(a) => a
case None => throw new Exception("File of unknown and undetectable mime type")
}
+ logger.info("Original Mime type: " + originalMimeType)
val tempDirForRequest: File = Files.createTempDirectory(deps.config.tempDir.toPath, "upload").toFile
@@ -153,7 +151,8 @@ object Uploader extends GridLogging {
uploadRequest.tempFile,
originalMimeType,
uploadRequest.uploadTime,
- toMetaMap(uploadRequest)
+ toMetaMap(uploadRequest),
+ uploadRequest.instance
)
val sourceStoreFuture = storeOrProjectOriginalFile(storableOriginalImage)
val eventualBrowserViewableImage = createBrowserViewableFileFuture(uploadRequest, tempDirForRequest, deps)
@@ -165,10 +164,10 @@ object Uploader extends GridLogging {
optimisedFileMetadata <- FileMetadataReader.fromIPTCHeadersWithColorInfo(browserViewableImage)
sourceDimensions <- sourceDimensionsFuture
sourceOrientationMetadata <- sourceOrientationMetadataFuture
- thumbViewableImage <- createThumbFuture(optimisedFileMetadata, colourModelFuture, browserViewableImage, deps, tempDirForRequest, orientationMetadata = sourceOrientationMetadata)
+ thumbViewableImage <- createThumbFuture(optimisedFileMetadata, colourModelFuture, browserViewableImage, deps, tempDirForRequest, uploadRequest.instance, orientationMetadata = sourceOrientationMetadata)
s3Thumb <- storeOrProjectThumbFile(thumbViewableImage)
maybeStorableOptimisedImage <- getStorableOptimisedImage(
- tempDirForRequest, optimiseOps, browserViewableImage, optimisedFileMetadata, deps.tryFetchOptimisedFile)
+ tempDirForRequest, optimiseOps, browserViewableImage, optimisedFileMetadata, deps.tryFetchOptimisedFile, uploadRequest.instance)
s3PngOption <- maybeStorableOptimisedImage match {
case Some(storableOptimisedImage) => storeOrProjectOptimisedFile(storableOptimisedImage).map(a=>Some(a))
case None => Future.successful(None)
@@ -212,12 +211,13 @@ object Uploader extends GridLogging {
optimiseOps: OptimiseOps,
browserViewableImage: BrowserViewableImage,
optimisedFileMetadata: FileMetadata,
- tryFetchOptimisedFile: (String, File) => Future[Option[(File, MimeType)]]
+ tryFetchOptimisedFile: (String, File, Instance) => Future[Option[(File, MimeType)]],
+ instance: Instance
)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[StorableOptimisedImage]] = {
if (optimiseOps.shouldOptimise(Some(browserViewableImage.mimeType), optimisedFileMetadata)) {
for {
tempFile <- createTempFile("optimisedpng-", optimisedMimeType.fileExtension, tempDir)
- maybeDownloadedOptimisedFile <- tryFetchOptimisedFile(browserViewableImage.id, tempFile)
+ maybeDownloadedOptimisedFile <- tryFetchOptimisedFile(browserViewableImage.id, tempFile, instance)
(optimisedFile, optimisedMimeType) <- {
maybeDownloadedOptimisedFile match {
case Some(optData) => Future.successful(optData)
@@ -239,10 +239,11 @@ object Uploader extends GridLogging {
def toMetaMap(uploadRequest: UploadRequest): Map[String, String] = {
val baseMeta = Map(
ImageStorageProps.uploadedByMetadataKey -> uploadRequest.uploadedBy,
- ImageStorageProps.uploadTimeMetadataKey -> printDateTime(uploadRequest.uploadTime)
+ ImageStorageProps.uploadTimeMetadataKey -> printDateTime(uploadRequest.uploadTime),
) ++
uploadRequest.identifiersMeta ++
- uploadRequest.uploadInfo.filename.map(ImageStorageProps.filenameMetadataKey -> _)
+ uploadRequest.uploadInfo.filename.map(ImageStorageProps.filenameMetadataKey -> _) ++
+ uploadRequest.uploadInfo.isFeedUpload.map(ImageStorageProps.isFeedUploadMetadataKey -> _.toString)
baseMeta.view.mapValues(URI.encode).toMap
}
@@ -259,7 +260,8 @@ object Uploader extends GridLogging {
browserViewableImage: BrowserViewableImage,
deps: ImageUploadOpsDependencies,
tempDir: File,
- orientationMetadata: Option[OrientationMetadata],
+ instance: Instance,
+ orientationMetadata: Option[OrientationMetadata]
)(implicit ec: ExecutionContext, logMarker: LogMarker) = {
import deps._
@@ -281,7 +283,7 @@ object Uploader extends GridLogging {
for {
tempFile <- createTempFile(s"thumb-", thumbMimeType.fileExtension, tempDir)
- maybeThumbFile <- deps.tryFetchThumbFile(browserViewableImage.id, tempFile)
+ maybeThumbFile <- deps.tryFetchThumbFile(browserViewableImage.id, tempFile, instance)
(thumb, thumbMimeType) <- {
maybeThumbFile match {
case Some(thumbData) => Future.successful(thumbData)
@@ -307,14 +309,16 @@ object Uploader extends GridLogging {
uploadRequest.imageId,
file = file,
mimeType = mimeType,
- isTransformedFromSource = true
+ isTransformedFromSource = true,
+ instance = uploadRequest.instance
)
case Some(mimeType) =>
Future.successful(
BrowserViewableImage(
uploadRequest.imageId,
file = uploadRequest.tempFile,
- mimeType = mimeType)
+ mimeType = mimeType,
+ instance = uploadRequest.instance)
)
case None => Future.failed(new Exception("This file is not an image with an identifiable mime type"))
}
@@ -350,7 +354,7 @@ class Uploader(
isReplacement: Boolean
)(
mediaIdToAddUsageTo: String
- ) = {
+ )(implicit instance: Instance) = {
gridClient.postUsage(
usageType = "child",
data = Json.obj(
@@ -367,7 +371,7 @@ class Uploader(
}
private def fromUploadRequest(uploadRequest: UploadRequest)
- (implicit logMarker: LogMarker): Future[ImageUpload] = {
+ (implicit logMarker: LogMarker, instance: Instance): Future[ImageUpload] = {
val sideEffectDependencies = ImageUploadOpsDependencies(toImageUploadOpsCfg(config), imageOps,
storeSource, storeThumbnail, storeOptimisedImage)
Stopwatch.async("finalImage") {
@@ -406,7 +410,9 @@ class Uploader(
uploadedBy: String,
identifiers: Map[String, String],
uploadTime: DateTime,
- filename: Option[String])
+ filename: Option[String],
+ instance: Instance,
+ isFeedUpload: Boolean)
(implicit ec:ExecutionContext,
logMarker: LogMarker): Future[UploadRequest] = Future {
val DigestedFile(tempFile, id) = digestedFile
@@ -429,20 +435,21 @@ class Uploader(
uploadTime = uploadTime,
uploadedBy = uploadedBy,
identifiers = identifiersMap,
- uploadInfo = UploadInfo(filename)
+ uploadInfo = UploadInfo(filename, Some(isFeedUpload)),
+ instance = instance
)
}
}
def storeFile(uploadRequest: UploadRequest)
(implicit ec:ExecutionContext,
- logMarker: LogMarker): Future[UploadStatusUri] = {
+ logMarker: LogMarker, instance: Instance): Future[UploadStatusUri] = {
logger.info(logMarker, "Storing file")
for {
imageUpload <- fromUploadRequest(uploadRequest)
- updateMessage = UpdateMessage(subject = Image, image = Some(imageUpload.image))
+ updateMessage = UpdateMessage(subject = Image, image = Some(imageUpload.image), instance = uploadRequest.instance)
_ <- Future { notifications.publish(updateMessage) }
// Send the optimised PNG to the embedder if there is one (e.g. for TIFFs),
// otherwise send the original image.
@@ -464,6 +471,7 @@ class Uploader(
mimeTypeForEmbedder,
s3BucketForEmbedder,
s3KeyForEmbedder,
+ instance.id
))
// TODO: centralise where all these URLs are constructed
} yield {
@@ -487,7 +495,7 @@ class Uploader(
}
}
- UploadStatusUri(s"${config.rootUri}/uploadStatus/${uploadRequest.imageId}")
+ UploadStatusUri(s"${config.rootUri(instance)}/uploadStatus/${uploadRequest.imageId}")
}
}
@@ -496,13 +504,14 @@ class Uploader(
gridClient: GridClient,
onBehalfOfFn: WSRequest => WSRequest)
(implicit ec: ExecutionContext,
- logMarker: LogMarker): Future[Unit] = for {
+ logMarker: LogMarker,
+ instance: Instance): Future[Unit] = for {
imageUpload <- fromUploadRequest(uploadRequest)
imageWithoutUserEdits = imageUpload.image
imageWithUserEditsApplied <- ImageDataMerger.aggregate(imageWithoutUserEdits, gridClient, onBehalfOfFn)
_ <- Future {
notifications.publish(
- UpdateMessage(subject = Image, image = Some(imageWithUserEditsApplied))
+ UpdateMessage(subject = Image, image = Some(imageWithUserEditsApplied), instance = uploadRequest.instance)
)
}
} yield ()
diff --git a/image-loader/app/model/upload/UploadRequest.scala b/image-loader/app/model/upload/UploadRequest.scala
index cf5fb3848c6..57b83e72c0c 100644
--- a/image-loader/app/model/upload/UploadRequest.scala
+++ b/image-loader/app/model/upload/UploadRequest.scala
@@ -4,7 +4,7 @@ import com.gu.mediaservice.lib.ImageStorageProps
import java.io.File
import java.util.UUID
-import com.gu.mediaservice.model.{MimeType, UploadInfo}
+import com.gu.mediaservice.model.{Instance, MimeType, UploadInfo}
import net.logstash.logback.marker.{LogstashMarker, Markers}
import org.joda.time.format.ISODateTimeFormat
import org.joda.time.{DateTime, DateTimeZone}
@@ -19,6 +19,7 @@ case class UploadRequest(
uploadedBy: String,
identifiers: Map[String, String],
uploadInfo: UploadInfo,
+ instance: Instance
) {
val identifiersMeta: Map[String, String] = identifiers.map { case (k, v) =>
diff --git a/image-loader/test/scala/model/ImageUploadTest.scala b/image-loader/test/scala/model/ImageUploadTest.scala
index 2430a911a2f..9711519a0cf 100644
--- a/image-loader/test/scala/model/ImageUploadTest.scala
+++ b/image-loader/test/scala/model/ImageUploadTest.scala
@@ -1,15 +1,12 @@
package model
-import java.io.File
-import java.net.URI
-import java.util.UUID
import com.drew.imaging.ImageProcessingException
-import com.gu.mediaservice.lib.{StorableImage, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage}
-import com.gu.mediaservice.lib.aws.{EmbedderMessage, S3Metadata, S3Object, S3ObjectMetadata, S3Ops}
+import com.gu.mediaservice.lib.aws.{S3Metadata, S3Object, S3ObjectMetadata}
import com.gu.mediaservice.lib.cleanup.ImageProcessor
import com.gu.mediaservice.lib.imaging.ImageOperations
import com.gu.mediaservice.lib.logging.LogMarker
-import com.gu.mediaservice.model.{FileMetadata, Jpeg, MimeType, Png, Tiff, UploadInfo}
+import com.gu.mediaservice.lib.{StorableImage, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage}
+import com.gu.mediaservice.model._
import lib.imaging.MimeTypeDetection
import model.upload.{OptimiseWithPngQuant, UploadRequest}
import org.joda.time.DateTime
@@ -17,10 +14,11 @@ import org.scalatest.Assertion
import org.scalatest.funsuite.AsyncFunSuite
import org.scalatest.matchers.should.Matchers
import org.scalatestplus.mockito.MockitoSugar
-import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse
import test.lib.ResourceHelpers
-import java.nio.file.Path
+import java.io.File
+import java.net.URI
+import java.util.UUID
import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Failure, Success}
@@ -71,7 +69,7 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar {
)
val tempFile = ResourceHelpers.fileAt(fileName)
- val ul = UploadInfo(None)
+ val ul = UploadInfo(None, None)
val uploadRequest = UploadRequest(
imageId = randomId,
@@ -80,7 +78,8 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar {
uploadTime = DateTime.now(),
uploadedBy = "uploadedBy",
identifiers = Map(),
- uploadInfo = ul
+ uploadInfo = ul,
+ Instance("an-instance"),
)
val futureImage = Uploader.uploadAndStoreImage(
diff --git a/image-loader/test/scala/model/ProjectorTest.scala b/image-loader/test/scala/model/ProjectorTest.scala
index 4de3767ef3f..3c9e27e87ea 100644
--- a/image-loader/test/scala/model/ProjectorTest.scala
+++ b/image-loader/test/scala/model/ProjectorTest.scala
@@ -9,7 +9,7 @@ import com.gu.mediaservice.lib.aws.S3
import com.gu.mediaservice.lib.cleanup.ImageProcessor
import com.gu.mediaservice.lib.imaging.ImageOperations
import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap}
-import com.gu.mediaservice.model._
+import com.gu.mediaservice.model.{Instance, _}
import com.gu.mediaservice.model.leases.LeasesByMedia
import lib.DigestedFile
import org.joda.time.{DateTime, DateTimeZone}
@@ -23,6 +23,7 @@ import org.scalatestplus.mockito.MockitoSugar
import play.api.libs.json.{JsArray, JsString}
import software.amazon.awssdk.services.s3.S3Client
import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse
+import play.api.mvc.RequestHeader
import test.lib.ResourceHelpers
import java.nio.file.Path
@@ -48,6 +49,8 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc
private val auth = mock[Authentication]
private val projector = new Projector(config, s3, imageOperations, ImageProcessor.identity, auth, maybeEmbedder)
+ private implicit val instance: Instance = Instance("an-instance")
+
// FIXME temporary ignored as test is not executable in CI/CD machine
// because graphic lib files like srgb.icc, cmyk.icc are in root directory instead of resources
// this test is passing when running on local machine
@@ -135,7 +138,7 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc
softDeletedMetadata = None,
lastModified = Some(new DateTime("2020-01-24T17:36:08.456Z").withZone(DateTimeZone.UTC)),
identifiers = Map(),
- uploadInfo = UploadInfo(Some("getty.jpg")),
+ uploadInfo = UploadInfo(Some("getty.jpg"), Some(true)),
source = Asset(new URI("http://img-bucket.s3.amazonaws.com/i/d/1/2/3/" + id),
Some(12666),
Some(Jpeg),
@@ -200,9 +203,11 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc
uploadTime = uploadTime,
uploadFileName = uploadFileName,
identifiers = Map.empty,
+ isFeedUpload = Some(true),
)
implicit val logMarker: LogMarker = MarkerMap()
+ implicit val instance: Instance = Instance("an-instance")
val gridClient = mock[GridClient]
when(gridClient.getUsages(id, identity)).thenReturn(Future.successful(Nil))
diff --git a/image-loader/test/scala/model/S3IngestObjectTest.scala b/image-loader/test/scala/model/S3IngestObjectTest.scala
new file mode 100644
index 00000000000..ff0512ef379
--- /dev/null
+++ b/image-loader/test/scala/model/S3IngestObjectTest.scala
@@ -0,0 +1,14 @@
+package scala.model
+
+import model.S3IngestObject
+import org.scalatest.flatspec.AnyFlatSpec
+import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper
+
+class S3IngestObjectTest extends AnyFlatSpec {
+
+ it should "infer uploader from folder after feeds" in {
+ val keyParts = "fingerpost/feeds/20250226/PA/205350.SOCCER-Kilmarnock--20514432_PICTURES_PRI13.jpg".split("/")
+ S3IngestObject.uploadedFromPath(keyParts) shouldBe "20250226"
+ }
+
+}
diff --git a/kahuna/app/InstanceSpecificSecurityHeaderFilter.scala b/kahuna/app/InstanceSpecificSecurityHeaderFilter.scala
new file mode 100644
index 00000000000..3a28c8b1ec9
--- /dev/null
+++ b/kahuna/app/InstanceSpecificSecurityHeaderFilter.scala
@@ -0,0 +1,17 @@
+import com.gu.mediaservice.lib.config.InstanceForRequest
+import lib.KahunaConfig
+import play.api.Configuration
+import play.api.mvc.{EssentialAction, EssentialFilter, RequestHeader}
+import play.filters.headers._
+
+class InstanceSpecificSecurityHeaderFilter(config: KahunaConfig, playConfig: Configuration)
+ extends EssentialFilter with InstanceForRequest {
+
+ override def apply(next: EssentialAction): EssentialAction = (req: RequestHeader) => {
+ val instance = instanceOf(req)
+ val kahunaSecurityConfig = KahunaSecurityConfig.apply(config, playConfig, instance)
+ val instanceSpecificSecurityHeadersFilter = SecurityHeadersFilter(kahunaSecurityConfig: SecurityHeadersConfig).apply(next)
+ instanceSpecificSecurityHeadersFilter.apply(req)
+ }
+
+}
diff --git a/kahuna/app/KahunaComponents.scala b/kahuna/app/KahunaComponents.scala
index 9c179e485c5..619db511599 100644
--- a/kahuna/app/KahunaComponents.scala
+++ b/kahuna/app/KahunaComponents.scala
@@ -1,17 +1,27 @@
import com.gu.mediaservice.lib.net.URI
-import com.gu.mediaservice.lib.play.GridComponents
+import com.gu.mediaservice.lib.play.{ConnectionBrokenFilter, GridComponents, RequestLoggingFilter, RequestMetricFilter}
+import com.gu.mediaservice.model.Instance
import controllers.{AssetsComponents, KahunaController}
import lib.KahunaConfig
import play.api.ApplicationLoader.Context
import play.api.Configuration
+import play.api.mvc.EssentialFilter
import play.filters.headers.SecurityHeadersConfig
import router.Routes
class KahunaComponents(context: Context) extends GridComponents(context, new KahunaConfig(_)) with AssetsComponents {
- final override lazy val securityHeadersConfig: SecurityHeadersConfig = KahunaSecurityConfig(config, context.initialConfiguration)
-
final override val buildInfo = utils.buildinfo.BuildInfo
+ override def httpFilters: Seq[EssentialFilter] = Seq(
+ instanceSpecificCorsFilter,
+ // csrfFilter, TODO Ineffective as gateway is not setting correct hostname headers!
+ new InstanceSpecificSecurityHeaderFilter(config, context.initialConfiguration),
+ gzipFilter,
+ new RequestLoggingFilter(materializer),
+ new ConnectionBrokenFilter(materializer),
+ new RequestMetricFilter(config, materializer, actorSystem, applicationLifecycle)
+ )
+
val controller = new KahunaController(auth, config, controllerComponents, authorisation)
final override val router = new Routes(httpErrorHandler, controller, assets, management)
@@ -19,31 +29,30 @@ class KahunaComponents(context: Context) extends GridComponents(context, new Kah
}
object KahunaSecurityConfig {
- def apply(config: KahunaConfig, playConfig: Configuration): SecurityHeadersConfig = {
+ def apply(config: KahunaConfig, playConfig: Configuration, instance: Instance): SecurityHeadersConfig = {
val base = SecurityHeadersConfig.fromConfiguration(playConfig)
val services = List(
- config.services.apiBaseUri,
- config.services.loaderBaseUri,
- config.services.cropperBaseUri,
- config.services.metadataBaseUri,
- config.services.imgopsBaseUri,
- config.services.usageBaseUri,
- config.services.collectionsBaseUri,
- config.services.leasesBaseUri,
- config.services.authBaseUri,
+ config.services.apiBaseUri(instance),
+ config.services.loaderBaseUri(instance),
+ config.services.cropperBaseUri(instance),
+ config.services.metadataBaseUri(instance),
+ config.services.imgopsBaseUri(instance),
+ config.services.usageBaseUri(instance),
+ config.services.collectionsBaseUri(instance),
+ config.services.leasesBaseUri(instance),
+ config.services.authBaseUri(instance),
config.services.guardianWitnessBaseUri
)
- val frameSources = s"frame-src ${config.services.authBaseUri} ${config.services.kahunaBaseUri} https://accounts.google.com https://www.youtube.com ${config.scriptsToLoad.map(_.host).mkString(" ")}"
+ val frameSources = s"frame-src https://accounts.google.com https://www.youtube.com ${config.scriptsToLoad.map(_.host).mkString(" ")}"
val frameAncestors = s"frame-ancestors ${config.frameAncestors.mkString(" ")}"
- val connectSources = s"connect-src 'self' ${(services :+ config.imageOrigin).mkString(" ")} ${config.connectSources.mkString(" ")}"
+ val connectSources = s"connect-src 'self' ${config.connectSources.mkString(" ")}"
- val imageSources = s"img-src ${List(
+ val imageSources = s"img-src ${List(
"data:",
"blob:",
- URI.ensureSecure(config.services.imgopsBaseUri).toString,
- URI.ensureSecure(config.fullOrigin).toString,
+ URI.ensureSecure(config.services.imgopsBaseUri(instance)).toString,
URI.ensureSecure(config.thumbOrigin).toString,
URI.ensureSecure(config.cropOrigin).toString,
URI.ensureSecure("app.getsentry.com").toString,
diff --git a/kahuna/app/controllers/KahunaController.scala b/kahuna/app/controllers/KahunaController.scala
index 816350e42e6..78b21840d2d 100644
--- a/kahuna/app/controllers/KahunaController.scala
+++ b/kahuna/app/controllers/KahunaController.scala
@@ -3,13 +3,14 @@ package controllers
import com.gu.mediaservice.lib.argo.ArgoHelpers
import com.gu.mediaservice.lib.auth.Authentication.Principal
import com.gu.mediaservice.lib.auth.{Authentication, Authorisation, BaseControllerWithLoginRedirects}
-import lib.{ExampleSwitch, FeatureSwitches, KahunaConfig, UseCqlChips}
+import lib.{ExampleSwitch, FeatureSwitches, KahunaConfig, KahunaClientServiceUrls, UseCqlChips}
import play.api.mvc.ControllerComponents
import play.api.libs.json._
import scala.concurrent.ExecutionContext
import com.gu.mediaservice.lib.config.FieldAlias._
-import com.gu.mediaservice.lib.config.Services
+import com.gu.mediaservice.lib.config.{InstanceForRequest, Services}
+import com.gu.mediaservice.model.Instance
import play.api.mvc.Security.AuthenticatedRequest
import play.twirl.api.Html
@@ -20,13 +21,14 @@ class KahunaController(
authorisation: Authorisation
)(
implicit val ec: ExecutionContext
-) extends BaseControllerWithLoginRedirects with ArgoHelpers {
+) extends BaseControllerWithLoginRedirects with ArgoHelpers with InstanceForRequest {
override def auth: Authentication = authentication
override def services: Services = config.services
def index(ignored: String) = withOptionalLoginRedirect { request =>
+ implicit val instance: Instance = instanceOf(request)
val maybeUser: Option[Authentication.Principal] = request match {
case authedRequest: AuthenticatedRequest[_, _] => authedRequest.user match {
@@ -55,19 +57,27 @@ class KahunaController(
val metadataTemplates: String = Json.toJson(config.metadataTemplates).toString()
val announcements: String = Json.toJson(config.announcements).toString()
val interimFilterOptions: String = Json.toJson(config.interimFilterOptions).toString()
- val returnUri = config.rootUri + okPath
+ val returnUri = config.rootUri(instance) + okPath
val costFilterLabel = config.costFilterLabel.getOrElse("Free to use only")
val costFilterChargeable = config.costFilterChargeable.getOrElse(false)
val maybeOrgOwnedValue =
if(config.shouldDisplayOrgOwnedCountAndFilterCheckbox)
- Html(s""""${config.staffPhotographerOrganisation}-owned"""")
+ Html(""""owned"""")
else
Html("undefined")
val imageTypes = Json.toJson(config.imageTypes).toString()
val agencyPicksIngredients = Json.toJson(config.agencyPicksIngredients).toString()
+ val rootUri = config.rootUri(instance)
+
+ val kahunaClientServiceUrls = KahunaClientServiceUrls(
+ rootUri = rootUri,
+ mediaApiUri = config.mediaApiUri(instance),
+ authUri = config.authUri(instance)
+ )
+
Ok(views.html.main(
- s"${config.authUri}/login?redirectUri=$returnUri",
+ s"${config.authUri(instance)}/login?redirectUri=$returnUri",
fieldAliases,
scriptsToLoad,
domainMetadataSpecs,
@@ -81,12 +91,13 @@ class KahunaController(
config,
featureSwitchesJson,
imageTypes,
- agencyPicksIngredients
+ agencyPicksIngredients,
+ kahunaClientServiceUrls
))
}
def quotas = authentication { req =>
- Ok(views.html.quotas(config.mediaApiUri))
+ Ok(views.html.quotas(config.mediaApiUri(instanceOf(req))))
}
def notifications = authentication { req =>
diff --git a/kahuna/app/lib/KahunaClientServiceUrls.scala b/kahuna/app/lib/KahunaClientServiceUrls.scala
new file mode 100644
index 00000000000..d2a2ec0c223
--- /dev/null
+++ b/kahuna/app/lib/KahunaClientServiceUrls.scala
@@ -0,0 +1,3 @@
+package lib
+
+case class KahunaClientServiceUrls(rootUri: String, mediaApiUri: String, authUri: String)
diff --git a/kahuna/app/lib/KahunaConfig.scala b/kahuna/app/lib/KahunaConfig.scala
index 1daa8623d4e..d4b65b73ecf 100644
--- a/kahuna/app/lib/KahunaConfig.scala
+++ b/kahuna/app/lib/KahunaConfig.scala
@@ -3,6 +3,7 @@ package lib
import com.gu.mediaservice.lib.auth.Permissions.Pinboard
import com.gu.mediaservice.lib.auth.SimplePermission
import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources}
+import com.gu.mediaservice.model.Instance
import play.api.libs.json._
case class ScriptToLoad(
@@ -14,16 +15,14 @@ case class ScriptToLoad(
)
class KahunaConfig(resources: GridConfigResources) extends CommonConfig(resources) {
- val rootUri: String = services.kahunaBaseUri
- val mediaApiUri: String = services.apiBaseUri
- val authUri: String = services.authBaseUri
+ val rootUri: Instance => String = services.kahunaBaseUri
+ def mediaApiUri: Instance => String = services.apiBaseUri
+ val authUri: Instance => String = services.authBaseUri
val sentryDsn: Option[String] = stringOpt("sentry.dsn").filterNot(_.isEmpty)
val thumbOrigin: String = string("origin.thumb")
- val fullOrigin: String = string("origin.full")
val cropOrigin: String = string("origin.crops")
- val imageOrigin: String = string("origin.images")
val costFilterLabel: Option[String] = stringOpt("costFilter.label")
val costFilterChargeable: Option[Boolean] = booleanOpt("costFilter.chargeable")
diff --git a/kahuna/app/views/main.scala.html b/kahuna/app/views/main.scala.html
index b09de2ed432..6591004c7e3 100644
--- a/kahuna/app/views/main.scala.html
+++ b/kahuna/app/views/main.scala.html
@@ -1,6 +1,7 @@
@import lib.ScriptToLoad
@import lib.KahunaConfig
+@import lib.KahunaClientServiceUrls
@(
reauthUri: String,
fieldAliases: String,
@@ -16,7 +17,8 @@
kahunaConfig: KahunaConfig,
featureSwitches: String,
imageTypes: String,
- agencyPicksIngredients: String
+ agencyPicksIngredients: String,
+ kahunaClientServiceUrls: KahunaClientServiceUrls
)
@@ -29,15 +31,15 @@
-
-
+
+
-
+
-
+
@kahunaConfig.sentryDsn.map { dsn => }
@@ -53,7 +55,7 @@
|