diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 58a8b9dca12..2ae66882420 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,7 +77,7 @@ jobs: npm test - name: Image Embedder bundling run: ./scripts/embedder-deploy/build.sh - - uses: guardian/actions-riff-raff@v4.2.4 + - uses: guardian/actions-riff-raff@v4.3.3 if: "! github.event.pull_request.head.repo.fork" with: githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/sbt-dependency-graph.yaml b/.github/workflows/sbt-dependency-graph.yaml index f9aad713eca..68729d8a7be 100644 --- a/.github/workflows/sbt-dependency-graph.yaml +++ b/.github/workflows/sbt-dependency-graph.yaml @@ -19,7 +19,7 @@ jobs: java-version: 17 - name: Install sbt id: sbt - uses: sbt/setup-sbt@3e125ece5c3e5248e18da9ed8d2cce3d335ec8dd # v1.1.14 + uses: sbt/setup-sbt@4ed7b7ec4bfa2074fe48554c09e341267397247c # v1.2.0 - name: Submit dependencies id: submit uses: scalacenter/sbt-dependency-submission@f43202114d7522a4b233e052f82c2eea8d658134 # v3.2.1 diff --git a/auth/app/auth/AuthComponents.scala b/auth/app/auth/AuthComponents.scala index 1077bb81186..0882a1086ac 100644 --- a/auth/app/auth/AuthComponents.scala +++ b/auth/app/auth/AuthComponents.scala @@ -1,6 +1,6 @@ package auth -import com.gu.mediaservice.lib.management.{InnerServiceStatusCheckController, Management} +import com.gu.mediaservice.lib.management.Management import com.gu.mediaservice.lib.play.GridComponents import play.api.ApplicationLoader.Context import play.api.{Configuration, Environment} @@ -14,10 +14,9 @@ class AuthComponents(context: Context) extends GridComponents(context, new AuthC val controller = new AuthController(auth, providers, config, controllerComponents, authorisation) val permissionsAwareManagement = new Management(controllerComponents, buildInfo) - val InnerServiceStatusCheckController = new InnerServiceStatusCheckController(auth, controllerComponents, config.services, wsClient) - override val router = new Routes(httpErrorHandler, controller, permissionsAwareManagement, InnerServiceStatusCheckController) + override val router = new Routes(httpErrorHandler, controller, permissionsAwareManagement) } object AuthHttpConfig { diff --git a/auth/app/auth/AuthConfig.scala b/auth/app/auth/AuthConfig.scala index c16150cf8ad..be228e5e44e 100644 --- a/auth/app/auth/AuthConfig.scala +++ b/auth/app/auth/AuthConfig.scala @@ -1,8 +1,10 @@ package auth import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources} +import com.gu.mediaservice.model.Instance class AuthConfig(resources: GridConfigResources) extends CommonConfig(resources) { - val rootUri: String = services.authBaseUri - val mediaApiUri: String = services.apiBaseUri + val rootUri: Instance => String = services.authBaseUri + val rootInstanceUri: Instance => String = services.authBaseInstanceUri + val mediaApiUri: Instance => String = services.apiBaseUri } diff --git a/auth/app/auth/AuthController.scala b/auth/app/auth/AuthController.scala index 73c180991ac..86bfd76f0e9 100644 --- a/auth/app/auth/AuthController.scala +++ b/auth/app/auth/AuthController.scala @@ -6,12 +6,14 @@ import com.gu.mediaservice.lib.auth.Authentication.{InnerServicePrincipal, Machi import com.gu.mediaservice.lib.auth.Permissions.{DeleteImage, ShowPaid, UploadImages} import com.gu.mediaservice.lib.auth.provider.AuthenticationProviders import com.gu.mediaservice.lib.auth.{Authentication, Authorisation, Internal} +import com.gu.mediaservice.lib.config.InstanceForRequest import com.gu.mediaservice.lib.guardian.auth.PandaAuthenticationProvider +import com.gu.mediaservice.model.Instance import play.api.libs.json.Json -import play.api.mvc.{BaseController, ControllerComponents, Result} +import play.api.mvc.{AnyContent, BaseController, ControllerComponents, Request, Result} import java.net.URI -import java.util.Date +import java.time.Instant import scala.concurrent.{ExecutionContext, Future} import scala.util.Try @@ -19,30 +21,36 @@ class AuthController(auth: Authentication, providers: AuthenticationProviders, v override val controllerComponents: ControllerComponents, authorisation: Authorisation)(implicit ec: ExecutionContext) extends BaseController - with ArgoHelpers { + with ArgoHelpers with InstanceForRequest { - val indexResponse = { + def indexResponse()(implicit instance: Instance) = { val indexData = Map("description" -> "This is the Auth API") val indexLinks = List( - Link("root", config.mediaApiUri), - Link("login", config.services.loginUriTemplate), - Link("ui:logout", s"${config.rootUri}/logout"), - Link("session", s"${config.rootUri}/session") + Link("root", config.mediaApiUri(instance)), + Link("login", config.services.loginUriTemplate(instance)), + Link("ui:logout", s"${config.rootUri(instance)}/logout"), + Link("session", s"${config.rootInstanceUri(instance)}/session") ) respond(indexData, indexLinks) } def cookieMonster = auth { request => providers.userProvider match { - case panda: PandaAuthenticationProvider =>{ - val cookieBatter = panda.readAuthenticatedUser(request).map(user => panda.generateCookie(user.copy(expires = new Date().getTime))) + case panda: PandaAuthenticationProvider => + val cookieBatter = panda.readAuthenticatedUser(request) + // Note the cookie monster does not expire the cookie itself, but instead expires the panda token stored + // by the cookie. The cookie will remain in the browser storage, but if decoded will declare that it expired + // at the epoch in 1970. + .map(user => panda.generateCookie(user.copy(expires = Instant.ofEpochMilli(0L)))) cookieBatter.fold(respond("Me want cookie."))(cookie => respond("Cookies are a sometimes food.").withCookies(cookie)) - } case _ => respond("Me want cookie.") } } - def index = auth { indexResponse } + def index = auth { request => + implicit val instance: Instance = instanceOf(request) + indexResponse() + } def session = auth { request => val showPaid = authorisation.hasPermissionTo(ShowPaid)(request.user) diff --git a/auth/conf/routes b/auth/conf/routes index 9a58260935c..4423cc15e99 100644 --- a/auth/conf/routes +++ b/auth/conf/routes @@ -16,7 +16,6 @@ GET /cookieMonster auth.AuthController.cookieMonster # Management GET /management/healthcheck com.gu.mediaservice.lib.management.Management.healthCheck GET /management/manifest com.gu.mediaservice.lib.management.Management.manifest -GET /management/whoAmI com.gu.mediaservice.lib.management.InnerServiceStatusCheckController.whoAmI(depth: Int) # Shoo robots away GET /robots.txt com.gu.mediaservice.lib.management.Management.disallowRobots diff --git a/build.sbt b/build.sbt index b27ebb4a5bb..610fc7ceb16 100644 --- a/build.sbt +++ b/build.sbt @@ -1,11 +1,10 @@ -import play.sbt.PlayImport.PlayKeys._ +import com.typesafe.sbt.packager.docker.* +import play.sbt.PlayImport.PlayKeys.* import sbt.Package.FixedTimestamp -import scala.sys.process._ +import scala.collection.JavaConverters.* +import scala.sys.process.* import scala.util.control.NonFatal -import scala.collection.JavaConverters._ - -import com.typesafe.sbt.packager.debian.JDebPackaging // We need to keep the timestamps to allow caching headers to work as expected on assets. // The below should work, but some problem in one of the plugins (possible the play plugin? or sbt-web?) causes @@ -53,6 +52,7 @@ val commonSettings = Seq( "org.scalatestplus" %% "mockito-3-4" % "3.1.4.0" % Test, "org.mockito" % "mockito-core" % "2.18.0" % Test, "org.scalamock" %% "scalamock" % "5.1.0" % Test, + "org.testcontainers" % "localstack" % "1.21.4" % Test ), dependencyOverrides ++= jacksonOverrides, @@ -76,9 +76,10 @@ Global / concurrentRestrictions := Seq( Tags.limitAll(12) ) -val awsSdkVersion = "1.12.470" -val awsSdkV2Version = "2.42.25" -val elastic4sVersion = "8.18.2" +val awsSdkVersion = "1.12.797" +val awsSdkV2Version = "2.44.13" +val elastic4sVersion = "8.19.1" +val awsKclVersion = "3.4.3" val okHttpVersion = "3.12.1" val bbcBuildProcess: Boolean = System.getenv().asScala.get("BUILD_ORG").contains("bbc") @@ -90,22 +91,25 @@ val maybeBBCLib: Option[sbt.ProjectReference] = if(bbcBuildProcess) Some(bbcProj lazy val commonLib = project("common-lib").settings( libraryDependencies ++= Seq( + "com.google.guava" % "guava" % "33.5.0-jre", "com.gu" %% "editorial-permissions-client" % "4.0.0", - "com.gu" %% "pan-domain-auth-play_3-0" % "9.0.0", + "com.gu" %% "pan-domain-auth-play_3-0" % "19.0.0", "com.amazonaws" % "aws-java-sdk-iam" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-s3" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-ec2" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-sqs" % awsSdkVersion, + "software.amazon.awssdk" % "sqs" % awsSdkV2Version, "com.amazonaws" % "aws-java-sdk-sns" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-sts" % awsSdkVersion, - "com.amazonaws" % "aws-java-sdk-dynamodb" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-kinesis" % awsSdkVersion, + "software.amazon.awssdk" % "s3" % awsSdkV2Version, "nl.gn0s1s" %% "elastic4s-core" % elastic4sVersion, "nl.gn0s1s" %% "elastic4s-client-esjava" % elastic4sVersion, "nl.gn0s1s" %% "elastic4s-domain" % elastic4sVersion, "com.gu" %% "thrift-serializer" % "5.0.2", "org.scalaz" %% "scalaz-core" % "7.3.8", "org.im4java" % "im4java" % "1.4.0", + "app.photofox.vips-ffm" % "vips-ffm-core" % "1.9.6", "com.gu" % "kinesis-logback-appender" % "1.4.4", "net.logstash.logback" % "logstash-logback-encoder" % "5.0", logback, // play-logback; needed when running the scripts @@ -124,8 +128,10 @@ lazy val commonLib = project("common-lib").settings( // declare explicit dependency on desired version of aws sdk v2 bedrock runtime "software.amazon.awssdk" % "bedrockruntime" % awsSdkV2Version, "software.amazon.awssdk" % "s3vectors" % awsSdkV2Version, + "com.adobe.xmp" % "xmpcore" % "6.1.11", ws, - "org.testcontainers" % "elasticsearch" % "1.21.4" % Test + "com.google.genai" % "google-genai" % "1.53.0" exclude("com.fasterxml.jackson.module", "jackson-module-kotlin"), + "org.testcontainers" % "testcontainers-elasticsearch" % "2.0.2" % Test, ), dependencyOverrides += "ch.qos.logback" % "logback-classic" % "1.2.13" % Test ) @@ -142,9 +148,9 @@ lazy val auth = playProject("auth", 9011) lazy val collections = playProject("collections", 9010) -lazy val cropper = playProject("cropper", 9006) +lazy val cropper = playImageLoaderProject("cropper", 9006) -lazy val imageLoader = playProject("image-loader", 9003).settings { +lazy val imageLoader = playImageLoaderProject("image-loader", 9003).settings { libraryDependencies ++= Seq( "org.apache.tika" % "tika-core" % "3.2.3", "com.drewnoakes" % "metadata-extractor" % "2.19.0" @@ -176,39 +182,33 @@ lazy val thrall = playProject("thrall", 9002) pipelineStages := Seq(digest, gzip), libraryDependencies ++= Seq( "org.codehaus.groovy" % "groovy-json" % "3.0.7", - // TODO upgrading kcl to v3? check if you can remove avro override below - "software.amazon.kinesis" % "amazon-kinesis-client" % "2.6.1", + "software.amazon.kinesis" % "amazon-kinesis-client" % awsKclVersion, // explicit dependencies on kinesis and dynamodb to upgrade the versions used by kcl "software.amazon.awssdk" % "kinesis" % awsSdkV2Version, "software.amazon.awssdk" % "dynamodb" % awsSdkV2Version, - "com.gu" %% "kcl-pekko-stream" % "0.1.0", - "org.testcontainers" % "elasticsearch" % "1.19.2" % Test, - "com.google.protobuf" % "protobuf-java" % "3.19.6" + "com.gu" %% "kcl-pekko-stream" % "0.1.2", + "org.testcontainers" % "testcontainers-elasticsearch" % "2.0.2" % Test, + "com.google.protobuf" % "protobuf-java" % "3.19.6", + "software.amazon.awssdk" % "sqs" % awsSdkV2Version, + "org.apache.pekko" %% "pekko-connectors-sqs" % "1.0.2" ), - // amazon-kinesis-client 2.6.0 brings in a critically vulnerable version of apache avro, - // but we cannot upgrade amazon-kinesis-client further without performing the v2->v3 upgrade https://docs.aws.amazon.com/streams/latest/dev/kcl-migration-from-2-3.html dependencyOverrides ++= Seq( - "org.apache.avro" % "avro" % "1.11.4", - "org.apache.pekko" %% "pekko-stream" % "1.0.3" + "org.apache.pekko" %% "pekko-stream" % "1.0.3", + "org.apache.pekko" %% "pekko-http" % "1.0.1", + "org.apache.pekko" %% "pekko-http-core" % "1.0.1", + "org.apache.pekko" %% "pekko-parsing" % "1.0.1" ) ) lazy val usage = playProject("usage", 9009).settings( libraryDependencies ++= Seq( - "com.gu" %% "content-api-client-default" % "32.0.0", - "com.gu" %% "content-api-client-aws" % "0.7.6", "io.reactivex" %% "rxscala" % "0.27.0", - // amazon-kinesis-client brings in a critical vulnerability warning through apache avro, resolved in versions 1.11.4 and 1.12.0. - // updating amazon-kinesis-client? check if the override below can be removed - "software.amazon.kinesis" % "amazon-kinesis-client" % "3.0.2", + "software.amazon.kinesis" % "amazon-kinesis-client" % awsKclVersion, // explicit dependencies on kinesis and dynamodb to upgrade the versions used by kcl "software.amazon.awssdk" % "kinesis" % awsSdkV2Version, "software.amazon.awssdk" % "dynamodb" % awsSdkV2Version, "com.google.protobuf" % "protobuf-java" % "3.19.6" ), - dependencyOverrides ++= Seq( - "org.apache.avro" % "avro" % "1.11.4", - ) ) lazy val scripts = project("scripts") @@ -251,39 +251,53 @@ val buildInfo = Seq( ) def playProject(projectName: String, port: Int, path: Option[String] = None): Project = { - val commonProject = project(projectName, path) - .enablePlugins(PlayScala, JDebPackaging, SystemdPlugin, BuildInfoPlugin) + project(projectName, path) + .enablePlugins(PlayScala, BuildInfoPlugin, DockerPlugin) .dependsOn(restLib) .settings(commonSettings ++ buildInfo ++ Seq( + dockerBaseImage := "eclipse-temurin:25", + dockerExposedPorts := Seq(port), playDefaultPort := port, - debianPackageDependencies := Seq("java11-runtime-headless"), - Linux / maintainer := "Guardian Developers ", - Linux / packageSummary := description.value, - packageDescription := description.value, bashScriptEnvConfigLocation := Some("/etc/environment"), - Debian / makeEtcDefault := None, - Debian / packageBin := { - val originalFileName = (Debian / packageBin).value - val (base, ext) = originalFileName.baseAndExt - val newBase = base.replace(s"_${version.value}_all","") - val newFileName = file(originalFileName.getParent) / s"$newBase.$ext" - IO.move(originalFileName, newFileName) - println(s"Renamed $originalFileName to $newFileName") - newFileName - }, Universal / mappings ++= Seq( file("common-lib/src/main/resources/application.conf") -> "conf/application.conf", file("common-lib/src/main/resources/logback.xml") -> "conf/logback.xml" ), Universal / javaOptions ++= Seq( "-Dpidfile.path=/dev/null", - s"-Dconfig.file=/usr/share/$projectName/conf/application.conf", - s"-Dlogger.file=/usr/share/$projectName/conf/logback.xml", - "-J-Xlog:gc*", - s"-J-Xlog:gc:/var/log/$projectName/gc.log" - ) - )) - //Add the BBC library dependency if defined - maybeBBCLib.fold(commonProject){commonProject.dependsOn(_)} + s"-Dconfig.file=/opt/docker/conf/application.conf", + s"-Dlogger.file=/opt/docker/conf/logback.xml", + "-XX:+PrintCommandLineFlags", "-XX:MaxRAMPercentage=40" + )) + ) +} + +def playImageLoaderProject(projectName: String, port: Int, path: Option[String] = None): Project = { + project(projectName, path) + .enablePlugins(PlayScala, BuildInfoPlugin, DockerPlugin) + .dependsOn(restLib) + .settings(commonSettings ++ buildInfo ++ Seq( + dockerBaseImage := "eu.gcr.io/grid-301122/jdk-vips:25-8.18.3", + dockerExposedPorts := Seq(port), + dockerCommands ++= Seq( + Cmd("ENV", "LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so") + ), + playDefaultPort := port, + + bashScriptEnvConfigLocation := Some("/etc/environment"), + Universal / mappings ++= Seq( + file("common-lib/src/main/resources/application.conf") -> "conf/application.conf", + file("common-lib/src/main/resources/logback.xml") -> "conf/logback.xml", + file("image-loader/cmyk.icc") -> "cmyk.icc", + file("image-loader/facebook-TINYsRGB_c2.icc") -> "facebook-TINYsRGB_c2.icc", + file("image-loader/grayscale.icc") -> "grayscale.icc", + file("image-loader/srgb.icc") -> "srgb.icc" + ), + Universal / javaOptions ++= Seq( + "-Dpidfile.path=/dev/null", + s"-Dconfig.file=/opt/docker/conf/application.conf", + s"-Dlogger.file=/opt/docker/conf/logback.xml", + "-XX:+PrintCommandLineFlags", "-XX:MaxRAMPercentage=20" + ))) } diff --git a/cdk/package-lock.json b/cdk/package-lock.json index 093ab660eec..f798c6935f8 100644 --- a/cdk/package-lock.json +++ b/cdk/package-lock.json @@ -7439,9 +7439,9 @@ "license": "MIT" }, "node_modules/fast-xml-builder": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.4.tgz", - "integrity": "sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.2.0.tgz", + "integrity": "sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==", "dev": true, "funding": [ { @@ -7451,7 +7451,8 @@ ], "license": "MIT", "dependencies": { - "path-expression-matcher": "^1.1.3" + "path-expression-matcher": "^1.5.0", + "xml-naming": "^0.1.0" } }, "node_modules/fast-xml-parser": { @@ -10138,9 +10139,9 @@ } }, "node_modules/path-expression-matcher": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.4.0.tgz", - "integrity": "sha512-s4DQMxIdhj3jLFWd9LxHOplj4p9yQ4ffMGowFf3cpEgrrJjEhN0V5nxw4Ye1EViAGDoL4/1AeO6qHpqYPOzE4Q==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.5.0.tgz", + "integrity": "sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==", "dev": true, "funding": [ { @@ -12577,6 +12578,22 @@ } } }, + "node_modules/xml-naming": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.1.0.tgz", + "integrity": "sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/cloudbuild-kahuna-only.yaml b/cloudbuild-kahuna-only.yaml new file mode 100644 index 00000000000..8907d469693 --- /dev/null +++ b/cloudbuild-kahuna-only.yaml @@ -0,0 +1,25 @@ +options: + machineType: 'N1_HIGHCPU_8' +steps: + - name: 'node:24-alpine' + entrypoint: 'npm' + dir: 'kahuna' + args: [ 'install' ] + - name: 'node:24-alpine' + entrypoint: 'npm' + dir: 'kahuna' + args: [ 'run', 'test' ] + - name: 'node:24-alpine' + entrypoint: 'npm' + dir: 'kahuna' + args: [ 'run', 'dist' ] + + - name: 'gcr.io/$PROJECT_ID/scala-sbt:1.11.7-jdk-25' + args: ['kahuna/docker:publishLocal'] + env: + - 'DOCKER_API_VERSION=1.41' + + - name: 'gcr.io/cloud-builders/docker' + args: ['tag', 'kahuna:0.1', 'eu.gcr.io/$PROJECT_ID/kahuna:$BRANCH_NAME'] + - name: 'gcr.io/cloud-builders/docker' + args: ['push', 'eu.gcr.io/$PROJECT_ID/kahuna:$BRANCH_NAME'] diff --git a/cloudbuild.yaml b/cloudbuild.yaml new file mode 100644 index 00000000000..afff3526883 --- /dev/null +++ b/cloudbuild.yaml @@ -0,0 +1,69 @@ +options: + machineType: 'N1_HIGHCPU_8' +steps: + - name: 'node:24-alpine' + entrypoint: 'npm' + dir: 'kahuna' + args: [ 'install' ] + - name: 'node:24-alpine' + entrypoint: 'npm' + dir: 'kahuna' + args: [ 'run', 'test' ] + - name: 'node:24-alpine' + entrypoint: 'npm' + dir: 'kahuna' + args: [ 'run', 'dist' ] + + - name: 'gcr.io/$PROJECT_ID/scala-sbt:1.11.7-jdk-25' + args: ['docker:publishLocal'] + env: + - 'DOCKER_API_VERSION=1.41' + - name: 'gcr.io/cloud-builders/docker' + args: ['tag', 'auth:0.1', 'eu.gcr.io/$PROJECT_ID/auth:$BRANCH_NAME'] + - name: 'gcr.io/cloud-builders/docker' + args: ['push', 'eu.gcr.io/$PROJECT_ID/auth:$BRANCH_NAME'] + + - name: 'gcr.io/cloud-builders/docker' + args: ['tag', 'cropper:0.1', 'eu.gcr.io/$PROJECT_ID/cropper:$BRANCH_NAME'] + - name: 'gcr.io/cloud-builders/docker' + args: ['push', 'eu.gcr.io/$PROJECT_ID/cropper:$BRANCH_NAME'] + + - name: 'gcr.io/cloud-builders/docker' + args: ['tag', 'collections:0.1', 'eu.gcr.io/$PROJECT_ID/collections:$BRANCH_NAME'] + - name: 'gcr.io/cloud-builders/docker' + args: ['push', 'eu.gcr.io/$PROJECT_ID/collections:$BRANCH_NAME'] + + - name: 'gcr.io/cloud-builders/docker' + args: ['tag', 'image-loader:0.1', 'eu.gcr.io/$PROJECT_ID/image-loader:$BRANCH_NAME'] + - name: 'gcr.io/cloud-builders/docker' + args: ['push', 'eu.gcr.io/$PROJECT_ID/image-loader:$BRANCH_NAME'] + + - name: 'gcr.io/cloud-builders/docker' + args: ['tag', 'kahuna:0.1', 'eu.gcr.io/$PROJECT_ID/kahuna:$BRANCH_NAME'] + - name: 'gcr.io/cloud-builders/docker' + args: ['push', 'eu.gcr.io/$PROJECT_ID/kahuna:$BRANCH_NAME'] + + - name: 'gcr.io/cloud-builders/docker' + args: ['tag', 'leases:0.1', 'eu.gcr.io/$PROJECT_ID/leases:$BRANCH_NAME'] + - name: 'gcr.io/cloud-builders/docker' + args: ['push', 'eu.gcr.io/$PROJECT_ID/leases:$BRANCH_NAME'] + + - name: 'gcr.io/cloud-builders/docker' + args: ['tag', 'media-api:0.1', 'eu.gcr.io/$PROJECT_ID/media-api:$BRANCH_NAME'] + - name: 'gcr.io/cloud-builders/docker' + args: ['push', 'eu.gcr.io/$PROJECT_ID/media-api:$BRANCH_NAME'] + + - name: 'gcr.io/cloud-builders/docker' + args: ['tag', 'metadata-editor:0.1', 'eu.gcr.io/$PROJECT_ID/metadata-editor:$BRANCH_NAME'] + - name: 'gcr.io/cloud-builders/docker' + args: ['push', 'eu.gcr.io/$PROJECT_ID/metadata-editor:$BRANCH_NAME'] + + - name: 'gcr.io/cloud-builders/docker' + args: ['tag', 'thrall:0.1', 'eu.gcr.io/$PROJECT_ID/thrall:$BRANCH_NAME'] + - name: 'gcr.io/cloud-builders/docker' + args: ['push', 'eu.gcr.io/$PROJECT_ID/thrall:$BRANCH_NAME'] + + - name: 'gcr.io/cloud-builders/docker' + args: ['tag', 'usage:0.1', 'eu.gcr.io/$PROJECT_ID/usage:$BRANCH_NAME'] + - name: 'gcr.io/cloud-builders/docker' + args: ['push', 'eu.gcr.io/$PROJECT_ID/usage:$BRANCH_NAME'] diff --git a/collections/app/CollectionsComponents.scala b/collections/app/CollectionsComponents.scala index c679179bf6d..66755c0423a 100644 --- a/collections/app/CollectionsComponents.scala +++ b/collections/app/CollectionsComponents.scala @@ -1,23 +1,22 @@ -import com.gu.mediaservice.lib.management.InnerServiceStatusCheckController import com.gu.mediaservice.lib.play.GridComponents import controllers.{CollectionsController, ImageCollectionsController} import lib.{CollectionsConfig, CollectionsMetrics, Notifications} import play.api.ApplicationLoader.Context import router.Routes +import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient import store.{CollectionsStore, ImageCollectionsStore} class CollectionsComponents(context: Context) extends GridComponents(context, new CollectionsConfig(_)) { final override val buildInfo = utils.buildinfo.BuildInfo - val collectionsStore = new CollectionsStore(config) - val imageCollectionsStore = new ImageCollectionsStore(config) + private val collectionsStore = new CollectionsStore(config.collectionsTable, config.withAWSCredentialsV2(DynamoDbAsyncClient.builder()).build()) + val imageCollectionsStore = new ImageCollectionsStore(config.imageCollectionsTable, config.withAWSCredentialsV2(DynamoDbAsyncClient.builder()).build()) val metrics = new CollectionsMetrics(config, actorSystem, applicationLifecycle) val notifications = new Notifications(config) val collections = new CollectionsController(auth, config, collectionsStore, controllerComponents) - val imageCollections = new ImageCollectionsController(auth, config, notifications, imageCollectionsStore, controllerComponents) - val InnerServiceStatusCheckController = new InnerServiceStatusCheckController(auth, controllerComponents, config.services, wsClient) + val imageCollections = new ImageCollectionsController(auth, notifications, imageCollectionsStore, controllerComponents) - override val router = new Routes(httpErrorHandler, collections, imageCollections, management, InnerServiceStatusCheckController) + override val router = new Routes(httpErrorHandler, collections, imageCollections, management) } diff --git a/collections/app/controllers/CollectionsController.scala b/collections/app/controllers/CollectionsController.scala index 1a3a4c8986e..92142ec9f07 100644 --- a/collections/app/controllers/CollectionsController.scala +++ b/collections/app/controllers/CollectionsController.scala @@ -6,16 +6,18 @@ import com.gu.mediaservice.lib.argo.model.{EmbeddedEntity, Link} import com.gu.mediaservice.lib.auth.Authentication import com.gu.mediaservice.lib.auth.Authentication.getIdentity import com.gu.mediaservice.lib.collections.CollectionsManager -import com.gu.mediaservice.model.{ActionData, Collection} +import com.gu.mediaservice.lib.config.InstanceForRequest +import com.gu.mediaservice.model.{ActionData, Collection, Instance} import lib.CollectionsConfig import model.Node import org.joda.time.DateTime import play.api.libs.functional.syntax._ import play.api.libs.json._ -import play.api.mvc.{BaseController, ControllerComponents} +import play.api.mvc.{BaseController, ControllerComponents, Request} import store.{CollectionsStore, CollectionsStoreError} import com.gu.mediaservice.lib.net.{URI => UriOps} import software.amazon.awssdk.services.dynamodb.model.AttributeValue +import com.gu.mediaservice.lib.net.{URI => UriOps} import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future @@ -31,31 +33,31 @@ object AppIndex { } class CollectionsController(authenticated: Authentication, config: CollectionsConfig, store: CollectionsStore, - val controllerComponents: ControllerComponents) extends BaseController with ArgoHelpers { + val controllerComponents: ControllerComponents) extends BaseController with ArgoHelpers with InstanceForRequest { import CollectionsManager.{getCssColour, isValidPathBit, pathToUri, uriToPath} // Stupid name clash between Argo and Play import com.gu.mediaservice.lib.argo.model.{Action => ArgoAction} - def uri(u: String) = URI.create(u) - val collectionUri = uri(s"${config.rootUri}/collections") - def collectionUri(p: List[String] = Nil) = { + private def uri(u: String) = URI.create(u) + private def collectionUri()(implicit instance: Instance) = uri(s"${config.rootUri(instance)}/collections") + private def collectionUri(p: List[String] = Nil)(implicit instance: Instance) = { val path = if(p.nonEmpty) s"/${pathToUri(p)}" else "" - uri(s"${config.rootUri}/collections$path") + uri(s"${config.rootUri(instance)}/collections$path") } - val appIndex = AppIndex("media-collections", "The one stop shop for collections") - val indexLinks = List(Link("collections", collectionUri.toString)) + private val appIndex = AppIndex("media-collections", "The one stop shop for collections") + private def indexLinks()(implicit instance: Instance) = List(Link("collections", collectionUri().toString)) - def getNodeAction(n: Node[Collection]): Option[Link] = Some(Link("collection", collectionUri(n.fullPath).toString)) - def addChildAction(pathId: List[String] = Nil): Option[ArgoAction] = Some(ArgoAction("add-child", collectionUri(pathId), "POST")) - def addChildAction(n: Node[Collection]): Option[ArgoAction] = addChildAction(n.fullPath) - def removeNodeAction(n: Node[Collection]): Option[ArgoAction] = if (n.children.nonEmpty) None else Some( + private def getNodeAction(n: Node[Collection])(implicit instance: Instance): Option[Link] = Some(Link("collection", collectionUri(n.fullPath).toString)) + private def addChildAction(pathId: List[String] = Nil)(implicit instance: Instance): Option[ArgoAction] = Some(ArgoAction("add-child", collectionUri(pathId), "POST")) + private def addChildAction(n: Node[Collection])(implicit instance: Instance): Option[ArgoAction] = addChildAction(n.fullPath) + private def removeNodeAction(n: Node[Collection])(implicit instance: Instance): Option[ArgoAction] = if (n.children.nonEmpty) None else Some( ArgoAction("remove", collectionUri(n.fullPath), "DELETE") ) def index = authenticated { req => - respond(appIndex, links = indexLinks) + respond(appIndex, links = indexLinks()(instanceOf(req))) } def collectionNotFound(path: String) = @@ -70,15 +72,16 @@ class CollectionsController(authenticated: Authentication, config: CollectionsCo def storeError(message: String) = respondError(InternalServerError, "collection-store-error", message) - def getActions(n: Node[Collection]): List[ArgoAction] = { + def getActions(n: Node[Collection])(implicit instance: Instance): List[ArgoAction] = { List(addChildAction(n), removeNodeAction(n)).flatten } - def getLinks(n: Node[Collection]): List[Link] = { + private def getLinks(n: Node[Collection])(implicit instance: Instance): List[Link] = { List(getNodeAction(n)).flatten } def correctedCollections = authenticated.async { req => + implicit val instance: Instance = instanceOf(req) store.getAll flatMap { collections => val tree = Node.fromList[Collection]( collections, @@ -100,14 +103,15 @@ class CollectionsController(authenticated: Authentication, config: CollectionsCo } } - def allCollections = store.getAll.map { collections => + def allCollections()(implicit instance: Instance)= store.getAll.map { collections => Node.fromList[Collection]( collections, (collection) => collection.path, (collection) => collection.description) } - def getCollection(collectionPathId: String) = authenticated.async { + def getCollection(collectionPathId: String) = authenticated.async { request => + implicit val instance: Instance = instanceOf(request) store.get(uriToPath(collectionPathId)).map { case Some(collection) => val node = Node(collection.path.last, Nil, collection.path, collection.path, Some(collection)) @@ -120,7 +124,18 @@ class CollectionsController(authenticated: Authentication, config: CollectionsCo } def getCollections = authenticated.async { req => - allCollections.map { tree => + implicit val instance: Instance = instanceOf(req) + implicit def asArgo: Writes[Node[Collection]] = ( + (__ \ "basename").write[String] ~ + (__ \ "children").lazyWrite[CollectionsEntity](Writes[CollectionsEntity] + // This is so we don't have to rewrite the Write[Seq[T]] + (seq => Json.toJson(seq))).contramap(collectionsEntity(_: List[Node[Collection]])) ~ + (__ \ "fullPath").write[List[String]] ~ + (__ \ "data").writeNullable[Collection] ~ + (__ \ "cssColour").writeNullable[String] + )(node => (node.basename, node.children, node.fullPath, node.data, getCssColour(node.fullPath))) + + allCollections().map { tree => respond( Json.toJson(tree)(asArgo), actions = List(addChildAction()).flatten @@ -134,6 +149,7 @@ class CollectionsController(authenticated: Authentication, config: CollectionsCo def addChildToRoot = addChildTo(None) def addChildToCollection(collectionPathId: String) = addChildTo(Some(collectionPathId)) def addChildTo(collectionPathId: Option[String]) = authenticated.async(parse.json) { req => + implicit val instance: Instance = instanceOf(req) (req.body \ "data").asOpt[String] map { child => if (isValidPathBit(child)) { val path = collectionPathId.map(uriToPath).getOrElse(Nil) :+ child @@ -153,8 +169,8 @@ class CollectionsController(authenticated: Authentication, config: CollectionsCo } type MaybeTree = Option[Node[Collection]] - def hasChildren(path: List[String]): Future[Boolean] = - allCollections.map { tree => + private def hasChildren(path: List[String])(implicit instance: Instance) = + allCollections().map { tree => // Traverse the tree using the path val maybeTree = path @@ -168,6 +184,7 @@ class CollectionsController(authenticated: Authentication, config: CollectionsCo } def removeCollection(collectionPath: String) = authenticated.async { req => + implicit val instance: Instance = instanceOf(req) val path = CollectionsManager.uriToPath(UriOps.encodePlus(collectionPath)) hasChildren(path).flatMap { noRemove => @@ -195,18 +212,8 @@ class CollectionsController(authenticated: Authentication, config: CollectionsCo )(node => (node.basename, node.children, node.fullPath, node.data)) type CollectionsEntity = Seq[EmbeddedEntity[Node[Collection]]] - implicit def asArgo: Writes[Node[Collection]] = ( - (__ \ "basename").write[String] ~ - (__ \ "children").lazyWrite[CollectionsEntity](Writes[CollectionsEntity] - // This is so we don't have to rewrite the Write[Seq[T]] - (seq => Json.toJson(seq))).contramap(collectionsEntity) ~ - (__ \ "fullPath").write[List[String]] ~ - (__ \ "data").writeNullable[Collection] ~ - (__ \ "cssColour").writeNullable[String] - )(node => (node.basename, node.children, node.fullPath, node.data, getCssColour(node.fullPath))) - - def collectionsEntity(nodes: List[Node[Collection]]): CollectionsEntity = { + private def collectionsEntity(nodes: List[Node[Collection]])(implicit instance: Instance): CollectionsEntity = { nodes.map(n => EmbeddedEntity(collectionUri(n.fullPath), Some(n), links = getLinks(n), actions = getActions(n))) } diff --git a/collections/app/controllers/ImageCollectionsController.scala b/collections/app/controllers/ImageCollectionsController.scala index d1e65c464fe..ae26f7746bf 100644 --- a/collections/app/controllers/ImageCollectionsController.scala +++ b/collections/app/controllers/ImageCollectionsController.scala @@ -5,12 +5,12 @@ import com.gu.mediaservice.lib.auth.Authentication import com.gu.mediaservice.lib.auth.Authentication.getIdentity import com.gu.mediaservice.lib.aws.{NoItemFound, UpdateMessage} import com.gu.mediaservice.lib.collections.CollectionsManager +import com.gu.mediaservice.lib.config.InstanceForRequest import com.gu.mediaservice.lib.net.{URI => UriOps} -import com.gu.mediaservice.model.{ActionData, Collection} +import com.gu.mediaservice.model.{ActionData, Collection, Instance} import com.gu.mediaservice.syntax.MessageSubjects -import lib.{CollectionsConfig, Notifications} +import lib.Notifications import org.joda.time.DateTime -import play.api.libs.json.Json import play.api.mvc.{BaseController, ControllerComponents} import store.ImageCollectionsStore @@ -18,14 +18,15 @@ import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future -class ImageCollectionsController(authenticated: Authentication, config: CollectionsConfig, notifications: Notifications, +class ImageCollectionsController(authenticated: Authentication, notifications: Notifications, imageCollectionsStore: ImageCollectionsStore, override val controllerComponents: ControllerComponents) - extends BaseController with MessageSubjects with ArgoHelpers { + extends BaseController with MessageSubjects with ArgoHelpers with InstanceForRequest { import CollectionsManager.onlyLatest def getCollections(id: String) = authenticated.async { req => + implicit val instance: Instance = instanceOf(req) imageCollectionsStore.get(id).map { collections => respond(onlyLatest(collections)) } recover { @@ -34,6 +35,7 @@ class ImageCollectionsController(authenticated: Authentication, config: Collecti } def addCollection(id: String) = authenticated.async(parse.json) { req => + implicit val instance: Instance = instanceOf(req) (req.body \ "data").asOpt[List[String]].map { path => val collection = Collection.build(path, ActionData(getIdentity(req.user), DateTime.now())) imageCollectionsStore.add(id, collection) @@ -44,6 +46,7 @@ class ImageCollectionsController(authenticated: Authentication, config: Collecti def removeCollection(id: String, collectionString: String) = authenticated.async { req => + implicit val instance: Instance = instanceOf(req) val path = CollectionsManager.uriToPath(UriOps.encodePlus(collectionString)) // We do a get to be able to find the index of the current collection, then remove it. // Given that we're using Dynamo Lists this seemed like a decent way to do it. @@ -63,9 +66,9 @@ class ImageCollectionsController(authenticated: Authentication, config: Collecti } } - def publish(id: String)(collections: List[Collection]): List[Collection] = { + def publish(id: String)(collections: List[Collection])(implicit instance: Instance): List[Collection] = { val onlyLatestCollections = onlyLatest(collections) - val updateMessage = UpdateMessage(subject = SetImageCollections, id = Some(id), collections = Some(onlyLatestCollections)) + val updateMessage = UpdateMessage(subject = SetImageCollections, id = Some(id), collections = Some(onlyLatestCollections), instance = instance) notifications.publish(updateMessage) onlyLatestCollections } diff --git a/collections/app/lib/CollectionsConfig.scala b/collections/app/lib/CollectionsConfig.scala index 9570409bc2d..20dcb8ff52e 100644 --- a/collections/app/lib/CollectionsConfig.scala +++ b/collections/app/lib/CollectionsConfig.scala @@ -1,11 +1,12 @@ package lib import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources} +import com.gu.mediaservice.model.Instance class CollectionsConfig(resources: GridConfigResources) extends CommonConfig(resources) { val collectionsTable = string("dynamo.table.collections") val imageCollectionsTable = string("dynamo.table.imageCollections") - val rootUri = services.collectionsBaseUri + val rootUri: Instance => String = services.collectionsBaseUri } diff --git a/collections/app/store/CollectionsStore.scala b/collections/app/store/CollectionsStore.scala index fe7416c27c3..52ef88ed4d7 100644 --- a/collections/app/store/CollectionsStore.scala +++ b/collections/app/store/CollectionsStore.scala @@ -1,24 +1,20 @@ package store +import cats.implicits._ import com.gu.mediaservice.lib.collections.CollectionsManager -import com.gu.mediaservice.model.{ActionData, Collection} -import lib.CollectionsConfig +import com.gu.mediaservice.model.{ActionData, Collection, Instance} import org.joda.time.DateTime import org.scanamo.generic.auto.genericDerivedFormat -import org.scanamo.{DynamoFormat, ScanamoAsync, Table} import org.scanamo.syntax._ +import org.scanamo.{DynamoFormat, ScanamoAsync, Table} import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future -import cats.implicits._ -import org.scanamo.generic.semiauto.FieldName -case class Record(id: String, collection: Collection) +case class Record(id: String, collection: Collection, instance: String) -class CollectionsStore(config: CollectionsConfig) extends DynamoHelpers { - override val tableName: FieldName = config.collectionsTable - lazy val client: DynamoDbAsyncClient = config.withAWSCredentialsV2(DynamoDbAsyncClient.builder()).build() +class CollectionsStore(val tableName: String, client: DynamoDbAsyncClient) extends DynamoHelpers { import org.scanamo.generic.semiauto._ implicit val dateTimeFormat: Typeclass[DateTime] = DynamoFormat.coercedXmap[DateTime, String, IllegalArgumentException](DateTime.parse, _.toString) @@ -28,24 +24,24 @@ class CollectionsStore(config: CollectionsConfig) extends DynamoHelpers { private lazy val collectionsTable = Table[Record](tableName) - def getAll: Future[List[Collection]] = { - ScanamoAsync(client).exec(collectionsTable.scan()).map(_.sequence).flatMap(res => + def getAll(implicit instance: Instance): Future[List[Collection]] = { + ScanamoAsync(client).exec(collectionsTable.query("instance" === instance.id)).map(_.sequence).flatMap(res => handleResponse(res)(records => records.map(_.collection)) ) } - def add(collection: Collection): Future[Collection] = { + def add(collection: Collection)(implicit instance: Instance): Future[Collection] = { ScanamoAsync(client).exec( collectionsTable.update( - "id" === collection.pathId, + "id" === collection.pathId and "instance" === instance.id, set("collection", collection) ) ).flatMap(res => handleResponse(res)(record => record.collection)) } - def get(collectionPath: List[String]): Future[Option[Collection]] = { + def get(collectionPath: List[String])(implicit instance: Instance): Future[Option[Collection]] = { val path = CollectionsManager.pathToPathId(collectionPath) - ScanamoAsync(client).exec(collectionsTable.get("id" === path)).flatMap(maybeEither => + ScanamoAsync(client).exec(collectionsTable.get("id" === path and "instance" === instance.id)).flatMap(maybeEither => maybeEither.fold[Future[Option[Collection]]]( Future.successful(None) )(res => @@ -54,9 +50,9 @@ class CollectionsStore(config: CollectionsConfig) extends DynamoHelpers { ) } - def remove(collectionPath: List[String]): Future[Unit] = { + def remove(collectionPath: List[String])(implicit instance: Instance): Future[Unit] = { val path = CollectionsManager.pathToPathId(collectionPath) - ScanamoAsync(client).exec(collectionsTable.delete("id" === path)) + ScanamoAsync(client).exec(collectionsTable.delete("id" === path and "instance" === instance.id)) } } diff --git a/collections/app/store/ImageCollectionsStore.scala b/collections/app/store/ImageCollectionsStore.scala index e0dcb611c12..a9970d03a9f 100644 --- a/collections/app/store/ImageCollectionsStore.scala +++ b/collections/app/store/ImageCollectionsStore.scala @@ -1,6 +1,6 @@ package store -import com.gu.mediaservice.model.{ActionData, Collection} +import com.gu.mediaservice.model.{ActionData, Collection, Instance} import lib.CollectionsConfig import org.joda.time.DateTime import org.scanamo.generic.auto.genericDerivedFormat @@ -14,12 +14,9 @@ import org.scanamo.syntax._ import scala.concurrent.ExecutionContext.Implicits.global -case class ImageRecord(id: String, collections: List[Collection]) +case class ImageRecord(id: String, instance: String, collections: List[Collection]) -class ImageCollectionsStore(config: CollectionsConfig) extends DynamoHelpers { - - override val tableName = config.imageCollectionsTable - lazy val client: DynamoDbAsyncClient = config.withAWSCredentialsV2(DynamoDbAsyncClient.builder()).build() +class ImageCollectionsStore(val tableName: String, val client: DynamoDbAsyncClient) extends DynamoHelpers { import org.scanamo.generic.semiauto._ implicit val dateTimeFormat: Typeclass[DateTime] = @@ -30,8 +27,8 @@ class ImageCollectionsStore(config: CollectionsConfig) extends DynamoHelpers { private lazy val imageCollectionsTable = Table[ImageRecord](tableName) - def get(id: String): Future[List[Collection]] = { - ScanamoAsync(client).exec(imageCollectionsTable.get("id" === id)).flatMap(maybeEither => + def get(id: String)(implicit instance: Instance): Future[List[Collection]] = { + ScanamoAsync(client).exec(imageCollectionsTable.get("id" === id and "instance" === instance.id)).flatMap(maybeEither => maybeEither.fold[Future[List[Collection]]]( Future.failed(NoItemFound) )(res => @@ -40,14 +37,14 @@ class ImageCollectionsStore(config: CollectionsConfig) extends DynamoHelpers { ) } - def add(id: String, collection: Collection): Future[List[Collection]] = { - ScanamoAsync(client).exec(imageCollectionsTable.update("id" === id, append("collections", collection))).flatMap(res => { + def add(id: String, collection: Collection)(implicit instance: Instance): Future[List[Collection]] = { + ScanamoAsync(client).exec(imageCollectionsTable.update("id" === id and "instance" === instance.id, append("collections", collection))).flatMap(res => { handleResponse(res)(res => res.collections) }) } - def update(id: String, collections: List[Collection]): Future[List[Collection]] = { - ScanamoAsync(client).exec(imageCollectionsTable.update("id" === id, set("collections", collections))).flatMap(res => { + def update(id: String, collections: List[Collection])(implicit instance: Instance): Future[List[Collection]] = { + ScanamoAsync(client).exec(imageCollectionsTable.update("id" === id and "instance" === instance.id, set("collections", collections))).flatMap(res => { handleResponse(res)(res => res.collections) }) } diff --git a/collections/conf/routes b/collections/conf/routes index 21acc8e1243..8fd9d4424dc 100644 --- a/collections/conf/routes +++ b/collections/conf/routes @@ -16,7 +16,6 @@ POST /corrected-collections controllers.CollectionsC # Management GET /management/healthcheck com.gu.mediaservice.lib.management.Management.healthCheck GET /management/manifest com.gu.mediaservice.lib.management.Management.manifest -GET /management/whoAmI com.gu.mediaservice.lib.management.InnerServiceStatusCheckController.whoAmI(depth: Int) # Shoo robots away GET /robots.txt com.gu.mediaservice.lib.management.Management.disallowRobots diff --git a/collections/test/store/CollectionsStoreTest.scala b/collections/test/store/CollectionsStoreTest.scala new file mode 100644 index 00000000000..78c2f5a9e71 --- /dev/null +++ b/collections/test/store/CollectionsStoreTest.scala @@ -0,0 +1,117 @@ +package store + +import com.gu.mediaservice.model.{ActionData, Collection, Instance} +import org.joda.time.DateTime +import org.scalatest.BeforeAndAfterAll +import org.scalatest.concurrent.ScalaFutures +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers +import org.scalatest.time.{Millis, Seconds, Span} +import org.testcontainers.containers.localstack.LocalStackContainer +import org.testcontainers.containers.localstack.LocalStackContainer.Service.DYNAMODB +import org.testcontainers.utility.DockerImageName +import software.amazon.awssdk.auth.credentials.{AwsBasicCredentials, StaticCredentialsProvider} +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient +import software.amazon.awssdk.services.dynamodb.model._ + +import java.util.UUID +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future +import scala.jdk.CollectionConverters._ + + +class CollectionsStoreTest extends AnyFunSpec with Matchers with ScalaFutures with BeforeAndAfterAll { + + implicit val defaultPatience: PatienceConfig = PatienceConfig(timeout = Span(2, Seconds), interval = Span(100, Millis)) + + private val dynamoContainer = new LocalStackContainer(DockerImageName.parse("localstack/localstack:1.4.0")).withServices(DYNAMODB) + dynamoContainer.start() + + private val dynamoClient = DynamoDbAsyncClient.builder(). + endpointOverride(dynamoContainer.getEndpointOverride(DYNAMODB)). + region(Region.of(dynamoContainer.getRegion)). + credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(dynamoContainer.getAccessKey, dynamoContainer.getSecretKey))).build() + + private implicit val instance: Instance = Instance("an-instance") + + private val collectionsTable = "test-collections-table-" + UUID.randomUUID().toString + private val collectionsTableForAllTest = "test-collections-table-" + UUID.randomUUID().toString + private val store = new CollectionsStore(collectionsTable, dynamoClient) + private val storeForAllTest = new CollectionsStore(collectionsTableForAllTest, dynamoClient) + + override def beforeAll(): Unit = { + def createTableRequestFor(tableName: String): CreateTableRequest = { + val attributeDefinitions = List( + AttributeDefinition.builder.attributeName("id").attributeType(ScalarAttributeType.S).build(), + AttributeDefinition.builder.attributeName("instance").attributeType(ScalarAttributeType.S).build() + ) + val keySchema = List( + KeySchemaElement.builder.attributeName("instance").keyType(KeyType.HASH).build(), + KeySchemaElement.builder.attributeName("id").keyType(KeyType.RANGE).build() + ) + val provisionedThroughput = ProvisionedThroughput.builder.readCapacityUnits(1L).writeCapacityUnits(1L).build() + val request = CreateTableRequest.builder + .tableName(tableName) + .attributeDefinitions(attributeDefinitions.asJava) + .keySchema(keySchema.asJava) + .provisionedThroughput(provisionedThroughput) + .build() + request + } + + dynamoClient.createTable(createTableRequestFor(collectionsTable)).get() + dynamoClient.createTable(createTableRequestFor(collectionsTableForAllTest)).get() + } + + override def afterAll(): Unit = { + super.afterAll() + dynamoContainer.stop() + } + + describe("CollectionsStore") { + val collection = Collection(List("a", "b"), ActionData("author", DateTime.now()), "description") + + it("should be able to add a collection") { + val eventualResult = store.add(collection) + whenReady(eventualResult) { c => + c.pathId should be("a/b") + c.description should be("description") + } + } + + it("should be able to get a collection") { + val eventualResult = store.get(List("a", "b")) + whenReady(eventualResult) { c => + c.get.pathId should be("a/b") + c.get.description should be("description") + } + } + + it("should be able to get all collections") { + val collection1 = Collection(List("e", "f"), ActionData("author1", DateTime.now()), "description1") + val collection2 = Collection(List("g", "h"), ActionData("author2", DateTime.now()), "description2") + + val eventualAdded = Future.sequence(Seq(storeForAllTest.add(collection1), storeForAllTest.add(collection2))) + + val eventualResult = eventualAdded.flatMap(_ => storeForAllTest.getAll) + + whenReady(eventualResult) { collections => + collections.size should be(2) + collections.map(_.pathId) should contain allOf("e/f", "g/h") + } + } + + it("should be able to remove a collection") { + val eventualResult = store.remove(List("a", "b")).flatMap(_ => store.get(List("a", "b"))) + whenReady(eventualResult) { c => + c should be(None) + } + + val eventualReadback = store.get(List("a", "b")) + whenReady(eventualReadback) { r => + r should be(None) + } + } + } +} diff --git a/collections/test/store/ImageCollectionsStoreTest.scala b/collections/test/store/ImageCollectionsStoreTest.scala new file mode 100644 index 00000000000..77e366f42ba --- /dev/null +++ b/collections/test/store/ImageCollectionsStoreTest.scala @@ -0,0 +1,107 @@ +package store + +import com.gu.mediaservice.model.{ActionData, Collection, Instance} +import org.joda.time.DateTime +import org.scalatest.BeforeAndAfterAll +import org.scalatest.concurrent.ScalaFutures +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers +import org.scalatest.time.{Millis, Seconds, Span} +import org.testcontainers.containers.localstack.LocalStackContainer +import org.testcontainers.containers.localstack.LocalStackContainer.Service.DYNAMODB +import org.testcontainers.utility.DockerImageName +import software.amazon.awssdk.auth.credentials.{AwsBasicCredentials, StaticCredentialsProvider} +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient +import software.amazon.awssdk.services.dynamodb.model._ + +import java.util.UUID +import scala.jdk.CollectionConverters._ + +class ImageCollectionsStoreTest extends AnyFunSpec with Matchers with ScalaFutures with BeforeAndAfterAll { + + implicit val defaultPatience: PatienceConfig = PatienceConfig(timeout = Span(5, Seconds), interval = Span(500, Millis)) + + private val dynamoContainer = new LocalStackContainer(DockerImageName.parse("localstack/localstack:1.4.0")).withServices(DYNAMODB) + dynamoContainer.start() + + private val dynamoClient = DynamoDbAsyncClient.builder(). + endpointOverride(dynamoContainer.getEndpointOverride(DYNAMODB)). + region(Region.of(dynamoContainer.getRegion)). + credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(dynamoContainer.getAccessKey, dynamoContainer.getSecretKey))).build() + + private implicit val instance: Instance = Instance("an-instance") + + private val imageCollectionsTable = "test-image-collections-table-" + UUID.randomUUID().toString + private val store = new ImageCollectionsStore(imageCollectionsTable, dynamoClient) + + override def beforeAll(): Unit = { + val attributeDefinitions = List( + AttributeDefinition.builder.attributeName("id").attributeType(ScalarAttributeType.S).build(), + AttributeDefinition.builder.attributeName("instance").attributeType(ScalarAttributeType.S).build() + ) + val keySchema = List( + KeySchemaElement.builder.attributeName("instance").keyType(KeyType.HASH).build(), + KeySchemaElement.builder.attributeName("id").keyType(KeyType.RANGE).build() + ) + val provisionedThroughput = ProvisionedThroughput.builder.readCapacityUnits(1L).writeCapacityUnits(1L).build() + val request = CreateTableRequest.builder + .tableName(imageCollectionsTable) + .attributeDefinitions(attributeDefinitions.asJava) + .keySchema(keySchema.asJava) + .provisionedThroughput(provisionedThroughput) + .build() + dynamoClient.createTable(request).get() + } + + override def afterAll(): Unit = { + super.afterAll() + dynamoContainer.stop() + } + + describe("ImageCollectionsStore") { + val imageId = "test-image-id" + val collection1 = Collection(List("a", "b"), ActionData("author", DateTime.now()), "description 1") + val collection2 = Collection(List("c", "d"), ActionData("author", DateTime.now()), "description 2") + + it("should be able to add image to a collection") { + val eventualAddedResponse = store.add(imageId, collection1) + whenReady(eventualAddedResponse) { response => + response.size should be(1) + response.head.pathId should be("a/b") + } + + // Read back + val eventuallyReloaded = store.get(imageId) + whenReady(eventuallyReloaded) { collections => + collections.size should be(1) + collections.head.pathId should be("a/b") + } + + // Add another collection + whenReady(store.add(imageId, collection2)) { collections => + collections.size should be(2) + collections.map(_.pathId) should contain allOf("a/b", "c/d") + } + + whenReady(store.get(imageId)) { collections => + collections.size should be(2) + collections.head.pathId should be("a/b") + collections.last.pathId should be("c/d") + } + + // Update to replace all for this image + val newCollection = Collection(List("e", "f"), ActionData("new-author", DateTime.now()), "new description") + val eventuallyUpdated = store.update(imageId, List(newCollection)) + whenReady(eventuallyUpdated) { collections => + collections.size should be(1) + collections.head.pathId should be("e/f") + } + + whenReady(store.get(imageId)) { collections => + collections.size should be(1) + collections.head.pathId should be("e/f") + } + } + } +} diff --git a/common-lib/src/main/resources/logback.xml b/common-lib/src/main/resources/logback.xml index 733a850c110..47e666377ff 100644 --- a/common-lib/src/main/resources/logback.xml +++ b/common-lib/src/main/resources/logback.xml @@ -2,30 +2,6 @@ - - - - - - - - - - - ${LOGS_LOCATION}/application.log - - - ${LOGS_LOCATION}/application.log.%d{yyyy-MM-dd}.%i.gz - 100MB - 7 - 500MB - - - - %date - [%level] - from %logger in %thread markers=%marker %n%message%n%xException%n - - - @@ -39,12 +15,7 @@ - - - - - - + diff --git a/common-lib/src/main/scala/com/gu/mediaservice/GridClient.scala b/common-lib/src/main/scala/com/gu/mediaservice/GridClient.scala index c9058ea2850..2085112a384 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/GridClient.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/GridClient.scala @@ -3,14 +3,14 @@ package com.gu.mediaservice import java.net.URL import com.gu.mediaservice.GridClient.{Error, Found, NotFound, Response} import com.gu.mediaservice.lib.config.Services -import com.gu.mediaservice.model.{Collection, Crop, Edits, Image, ImageMetadata, ImageStatusRecord, SyndicationRights} +import com.gu.mediaservice.model.{Collection, Crop, Edits, Image, ImageMetadata, ImageStatusRecord, Instance, SourceImage, SyndicationRights} import com.gu.mediaservice.model.leases.LeasesByMedia import com.gu.mediaservice.model.usage.Usage import com.typesafe.scalalogging.LazyLogging import play.api.http.HeaderNames -import play.api.libs.json.{JsArray, JsObject, JsValue, Json, Reads} +import play.api.libs.json.{JsArray, JsObject, JsString, JsValue, Json, Reads} -import scala.concurrent.duration.{Duration, DurationInt} +import scala.concurrent.duration.{Duration, DurationInt, SECONDS} import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success, Try} import play.api.libs.ws.{WSClient, WSRequest, WSResponse} @@ -104,12 +104,13 @@ class GridClient(services: Services)(implicit wsClient: WSClient) extends LazyLo * process before returning data. * See also https://www.playframework.com/documentation/2.6.x/ScalaWS#Configuring-Timeouts */ - def makeGetRequestAsync(url: URL, authFn: WSRequest => WSRequest, requestTimeout: Option[Duration] = None) + def makeGetRequestAsync(url: URL, authFn: WSRequest => WSRequest, requestTimeout: Option[Duration] = None, + queryStringParameters: Option[Seq[(String, String)]] = None) (implicit ec: ExecutionContext): Future[Response] = { - val request: WSRequest = wsClient.url(url.toString) + val request: WSRequest = wsClient.url(url.toString).withQueryStringParameters(queryStringParameters.getOrElse(Seq.empty): _*) val requestWithTimeout = requestTimeout.fold(request)(request.withRequestTimeout) val authorisedRequest = authFn(requestWithTimeout) - authorisedRequest.get().map { response => validateResponse(response, url)} + authorisedRequest.get().map { response => validateResponse(response, url) } } private def validateResponse( @@ -133,8 +134,8 @@ class GridClient(services: Services)(implicit wsClient: WSClient) extends LazyLo } } - def getProjectionDiff(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext): Future[Option[JsValue]] = { - val url = new URL(s"${services.apiBaseUri}/images/$mediaId/projection/diff") + def getProjectionDiff(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext, instance: Instance): Future[Option[JsValue]] = { + val url = new URL(s"${services.apiBaseUri(instance)}/images/$mediaId/projection/diff") makeGetRequestAsync(url, authFn, requestTimeout = Some(120.seconds)).map { case Found(json, _) => Some(json) case NotFound(_, _) => None @@ -143,14 +144,15 @@ class GridClient(services: Services)(implicit wsClient: WSClient) extends LazyLo } def getImageLoaderProjection(mediaId: String, authFn: WSRequest => WSRequest) - (implicit ec: ExecutionContext): Future[Option[Image]] = { - getImageLoaderProjection(mediaId, services.projectionBaseUri, authFn) + (implicit ec: ExecutionContext, instance: Instance): Future[Option[Image]] = { + getImageLoaderProjection(mediaId, services.projectionBaseUri(instance), authFn) } def getImageLoaderProjection(mediaId: String, imageLoaderEndpoint: String, authFn: WSRequest => WSRequest) (implicit ec: ExecutionContext): Future[Option[Image]] = { - logger.info("attempt to get image projection from image-loader") - val url = new URL(s"$imageLoaderEndpoint/images/project/$mediaId") + val projectUrl = s"$imageLoaderEndpoint/images/project/$mediaId" + logger.info(s"attempt to get image projection from image-loader: $projectUrl") + val url = new URL(projectUrl) makeGetRequestAsync(url, authFn, requestTimeout = Some(300.seconds)) map { case Found(json, _) => Some(json.as[Image]) case NotFound(_, _) => None @@ -158,9 +160,9 @@ class GridClient(services: Services)(implicit wsClient: WSClient) extends LazyLo } } - def getLeases(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext): Future[LeasesByMedia] = { + def getLeases(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext, instance: Instance): Future[LeasesByMedia] = { logger.info("attempt to get leases") - val url = new URL(s"${services.leasesBaseUri}/leases/media/$mediaId") + val url = new URL(s"${services.leasesBaseUri(instance)}/leases/media/$mediaId") makeGetRequestAsync(url, authFn) map { case Found(json, _) => (json \ "data").as[LeasesByMedia] case NotFound(_, _) => LeasesByMedia.empty @@ -168,9 +170,9 @@ class GridClient(services: Services)(implicit wsClient: WSClient) extends LazyLo } } - def getCollections(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext): Future[List[Collection]] = { + def getCollections(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext, instance: Instance): Future[List[Collection]] = { logger.info("attempt to get collections") - val url = new URL(s"${services.collectionsBaseUri}/images/$mediaId") + val url = new URL(s"${services.collectionsBaseUri(instance)}/images/$mediaId") makeGetRequestAsync(url, authFn) map { case Found(json, _) => (json \ "data").as[List[Collection]] case NotFound(_, _) => Nil @@ -178,9 +180,24 @@ class GridClient(services: Services)(implicit wsClient: WSClient) extends LazyLo } } - def getEdits(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext): Future[Option[Edits]] = { + def createCollection(name: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext, instance: Instance): Future[Option[Collection]] = { + val url = new URL(s"${services.collectionsBaseUri(instance)}/collections") + val request = wsClient.url(url.toString).withRequestTimeout(Duration(10, SECONDS)) + val authorisedRequest = authFn(request) + val data = Json.obj("data" -> JsString(name)) + authorisedRequest.post(data).map { response => + logger.info("Got new collection response: " + response.body) + validateResponse(response, url) match { + case Found(json, _) => (json \ "data" \ "data").toOption.map(_.as[Collection]) + case NotFound(_, _) => None + case e@Error(_, _, _) => e.logErrorAndThrowException() + } + } + } + + def getEdits(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext, instance: Instance): Future[Option[Edits]] = { logger.info("attempt to get edits") - val url = new URL(s"${services.metadataBaseUri}/edits/$mediaId") + val url = new URL(s"${services.metadataBaseUri(instance)}/edits/$mediaId") makeGetRequestAsync(url, authFn) map { case Found(json, _) => Some((json \ "data").as[Edits]) case NotFound(_, _) => None @@ -188,9 +205,9 @@ class GridClient(services: Services)(implicit wsClient: WSClient) extends LazyLo } } - def getSoftDeletedMetadata(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext): Future[Option[ImageStatusRecord]] = { + def getSoftDeletedMetadata(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext, instance: Instance): Future[Option[ImageStatusRecord]] = { logger.info("attempt to get soft deleted metadata") - val url = new URL(s"${services.apiBaseUri}/images/$mediaId/softDeletedMetadata") + val url = new URL(s"${services.apiBaseUri(instance)}/images/$mediaId/softDeletedMetadata") makeGetRequestAsync(url, authFn) map { case Found(json, _) => Some((json \ "data").as[ImageStatusRecord]) case NotFound(_, _) => None @@ -198,9 +215,9 @@ class GridClient(services: Services)(implicit wsClient: WSClient) extends LazyLo } } - def getUploadedBy(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext): Future[Option[String]] = { + def getUploadedBy(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext, instance: Instance): Future[Option[String]] = { logger.info("attempt to get uploadedBy") - val url = new URL(s"${services.apiBaseUri}/images/$mediaId/uploadedBy") + val url = new URL(s"${services.apiBaseUri(instance)}/images/$mediaId/uploadedBy") makeGetRequestAsync(url, authFn) map { case Found(json, _) => Some((json \ "data").as[String]) case NotFound(_, _) => None @@ -208,9 +225,9 @@ class GridClient(services: Services)(implicit wsClient: WSClient) extends LazyLo } } - def getCrops(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext): Future[List[Crop]] = { + def getCrops(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext, instance: Instance): Future[List[Crop]] = { logger.info("attempt to get crops") - val url = new URL(s"${services.cropperBaseUri}/crops/$mediaId") + val url = new URL(s"${services.cropperBaseUri(instance)}/crops/$mediaId") makeGetRequestAsync(url, authFn) map { case Found(json, _) => (json \ "data").as[List[Crop]] case NotFound(_, _) => Nil @@ -218,7 +235,7 @@ class GridClient(services: Services)(implicit wsClient: WSClient) extends LazyLo } } - def getUsages(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext): Future[List[Usage]] = { + def getUsages(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext, instance: Instance): Future[List[Usage]] = { logger.info("attempt to get usages") def unpackUsagesFromEntityResponse(resBody: JsValue): List[JsValue] = { @@ -226,7 +243,7 @@ class GridClient(services: Services)(implicit wsClient: WSClient) extends LazyLo .map(entity => (entity.as[JsObject] \ "data").as[JsValue]).toList } - val url = new URL(s"${services.usageBaseUri}/usages/media/$mediaId") + val url = new URL(s"${services.usageBaseUri(instance)}/usages/media/$mediaId") makeGetRequestAsync(url, authFn) map { case Found(json, _) => unpackUsagesFromEntityResponse(json).map(_.as[Usage]) case NotFound(_, _) => Nil @@ -234,9 +251,19 @@ class GridClient(services: Services)(implicit wsClient: WSClient) extends LazyLo } } - def getMetadata(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext): Future[ImageMetadata] = { + def getSourceImage(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext, instance: Instance): Future[SourceImage] = { + logger.info("attempt to get image") + val url = new URL(s"${services.apiBaseUri(instance)}/images/$mediaId") + makeGetRequestAsync(url, authFn, queryStringParameters = Some(Seq("include" -> "fileMetadata"))) map { + case Found(json, _) => json.as[SourceImage] + case nf@NotFound(_, _) => Error(nf.status, url, nf.underlying).logErrorAndThrowException() + case e@Error(_, _, _) => e.logErrorAndThrowException() + } + } + + def getMetadata(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext, instance: Instance): Future[ImageMetadata] = { logger.info("attempt to get metadata") - val url = new URL(s"${services.apiBaseUri}/images/$mediaId") + val url = new URL(s"${services.apiBaseUri(instance)}/images/$mediaId") makeGetRequestAsync(url, authFn) map { case Found(json, _) => (json \ "data" \ "metadata").as[ImageMetadata] case nf@NotFound(_, _) => Error(nf.status, url, nf.underlying).logErrorAndThrowException() @@ -244,8 +271,8 @@ class GridClient(services: Services)(implicit wsClient: WSClient) extends LazyLo } } - def getSyndicationRights(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext) = { - val url = new URL(s"${services.metadataBaseUri}/metadata/$mediaId/syndication") + def getSyndicationRights(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext, instance: Instance) = { + val url = new URL(s"${services.metadataBaseUri(instance)}/metadata/$mediaId/syndication") makeGetRequestAsync(url, authFn) map { case Found(json, _) => Some((json \ "data").as[SyndicationRights]) case _: NotFound => None @@ -253,8 +280,8 @@ class GridClient(services: Services)(implicit wsClient: WSClient) extends LazyLo } } - def postUsage(usageType: String, data: JsObject, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext) = { - val url = new URL(s"${services.usageBaseUri}/usages/$usageType") + def postUsage(usageType: String, data: JsObject, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext, instance: Instance) = { + val url = new URL(s"${services.usageBaseUri(instance)}/usages/$usageType") val request: WSRequest = wsClient.url(url.toString) val authorisedRequest = authFn(request) authorisedRequest.post(Json.obj("data" -> data)).map { response => validateResponse(response, url)} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/ImageDataMerger.scala b/common-lib/src/main/scala/com/gu/mediaservice/ImageDataMerger.scala index 463c94500ea..ca9f75fd8ca 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/ImageDataMerger.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/ImageDataMerger.scala @@ -4,11 +4,12 @@ import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} import com.gu.mediaservice.model._ import org.joda.time.DateTime import play.api.libs.ws.WSRequest +import play.api.mvc.RequestHeader import scala.concurrent.{ExecutionContext, Future} object ImageDataMerger extends GridLogging { - def aggregate(image: Image, gridClient: GridClient, authFunction: WSRequest => WSRequest)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Image] = { + def aggregate(image: Image, gridClient: GridClient, authFunction: WSRequest => WSRequest)(implicit ec: ExecutionContext, logMarker: LogMarker, instance: Instance): Future[Image] = { logger.info(logMarker, s"starting to aggregate image") val mediaId = image.id // NB original metadata should already be added, cleaned, and copied to metadata. diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala index 0aeb698cccb..5a0f2eb36d4 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala @@ -1,7 +1,7 @@ package com.gu.mediaservice.lib import org.apache.pekko.actor.{Cancellable, Scheduler} -import com.gu.mediaservice.lib.aws.S3 +import com.gu.mediaservice.lib.aws.{S3, S3Bucket} import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.GridLogging import org.joda.time.DateTime @@ -14,7 +14,7 @@ import scala.concurrent.duration._ import scala.util.control.NonFatal -abstract class BaseStore[TStoreKey, TStoreVal](bucket: String, config: CommonConfig)(implicit ec: ExecutionContext) +abstract class BaseStore[TStoreKey, TStoreVal](bucket: S3Bucket, config: CommonConfig)(implicit ec: ExecutionContext) extends GridLogging { val s3 = new S3(config) @@ -25,15 +25,14 @@ abstract class BaseStore[TStoreKey, TStoreVal](bucket: String, config: CommonCon protected def getS3Object(key: String): Option[String] = s3.getObjectAsString(bucket, key) protected def getLatestS3Stream: Option[InputStream] = { - val objects = s3.client - .listObjects(bucket).getObjectSummaries.asScala + val objects = s3.listObjects(bucket).getObjectSummaries.asScala .filterNot(_.getKey == "AMAZON_SES_SETUP_NOTIFICATION") if (objects.nonEmpty) { val obj = objects.maxBy(_.getLastModified) logger.info(s"Latest key ${obj.getKey} in bucket $bucket") - val stream = s3.client.getObject(bucket, obj.getKey).getObjectContent + val stream = s3.getObject(bucket, obj).getObjectContent Some(stream) } else { logger.error(s"Bucket $bucket is empty") diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageId.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageId.scala index 9ec8633e275..022eacf212f 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageId.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageId.scala @@ -1,12 +1,19 @@ package com.gu.mediaservice.lib import _root_.play.api.libs.json._ +import com.gu.mediaservice.lib.logging.GridLogging -trait ImageId { +trait ImageId extends GridLogging { - def withImageId[A](image: JsValue)(f: String => A): A = { - (image \ "id").validate[String].asOpt.map(f).getOrElse { - sys.error(s"No id field present in message body: $image") + def withImageIdAndInstance[A](image: JsValue)(f: (String, String) => A): A = { + (for { + id <- (image \ "id").validate[String].asOpt + instance <- (image \ "instance").validate[String].asOpt + } yield { + (id, instance) + }).map((a: (String, String)) => f(a._1, a._2)) + .getOrElse { + sys.error(s"No id and/or instance field present in message body: $image") } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala index 0fdd57bb676..ce79b1828e0 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala @@ -1,25 +1,29 @@ package com.gu.mediaservice.lib -import com.amazonaws.services.s3.model.{DeleteObjectsRequest, MultiObjectDeleteException} +import com.amazonaws.services.s3.model +import com.amazonaws.services.s3.model.MultiObjectDeleteException import java.io.File import com.gu.mediaservice.lib.config.CommonConfig -import com.gu.mediaservice.lib.aws.S3Object +import com.gu.mediaservice.lib.aws.{S3Bucket, S3Object} import com.gu.mediaservice.lib.logging.LogMarker -import com.gu.mediaservice.model.{MimeType, Png} +import com.gu.mediaservice.model.{Instance, MimeType, Png} +import com.typesafe.scalalogging.StrictLogging import org.joda.time.DateTime import scala.concurrent.Future import scala.jdk.CollectionConverters._ object ImageIngestOperations { - def fileKeyFromId(id: String): String = id.take(6).mkString("/") + "/" + id + def fileKeyFromId(id: String)(implicit instance: Instance): String = instance.id + "/" + snippetForId(id) - def optimisedPngKeyFromId(id: String): String = "optimised/" + fileKeyFromId(id: String) + def optimisedPngKeyFromId(id: String)(implicit instance: Instance): String = instance.id + "/" + "optimised/" + snippetForId(id: String) + + private def snippetForId(id: String) = id.take(6).mkString("/") + "/" + id } -class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config: CommonConfig, isVersionedS3: Boolean = false) - extends S3ImageStorage(config) { +class ImageIngestOperations(imageBucket: S3Bucket, thumbnailBucket: S3Bucket, embeddingSourceBucket: S3Bucket, config: CommonConfig, isVersionedS3: Boolean = false) + extends S3ImageStorage(config) with StrictLogging { import ImageIngestOperations.{fileKeyFromId, optimisedPngKeyFromId} @@ -28,53 +32,104 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config case s:StorableOriginalImage => storeOriginalImage(s) case s:StorableThumbImage => storeThumbnailImage(s) case s:StorableOptimisedImage => storeOptimisedImage(s) + case s:StorableEmbeddingSourceImage => storeEmbeddingSourceImage(s) } private def storeOriginalImage(storableImage: StorableOriginalImage) - (implicit logMarker: LogMarker): Future[S3Object] = - storeImage(imageBucket, fileKeyFromId(storableImage.id), storableImage.file, Some(storableImage.mimeType), + (implicit logMarker: LogMarker): Future[S3Object] = { + val instanceSpecificKey = instanceAwareOriginalImageKey(storableImage) + logger.info(s"Storing original image to instance specific key:$imageBucket / $instanceSpecificKey") + storeImage(imageBucket, instanceSpecificKey, storableImage.file, Some(storableImage.mimeType), storableImage.meta, overwrite = false) + } private def storeThumbnailImage(storableImage: StorableThumbImage) - (implicit logMarker: LogMarker): Future[S3Object] = - storeImage(thumbnailBucket, fileKeyFromId(storableImage.id), storableImage.file, Some(storableImage.mimeType), + (implicit logMarker: LogMarker): Future[S3Object] = { + val instanceSpecificKey = instanceAwareThumbnailImageKey(storableImage) + logger.info(s"Storing thumbnail to instance specific key: ${thumbnailBucket.bucket} / $instanceSpecificKey") + storeImage(thumbnailBucket, instanceSpecificKey, storableImage.file, Some(storableImage.mimeType), overwrite = true) + } private def storeOptimisedImage(storableImage: StorableOptimisedImage) - (implicit logMarker: LogMarker): Future[S3Object] = - storeImage(imageBucket, optimisedPngKeyFromId(storableImage.id), storableImage.file, Some(storableImage.mimeType), + (implicit logMarker: LogMarker): Future[S3Object] = { + val instanceSpecificKey = optimisedPngKeyFromId(storableImage.id)(storableImage.instance) + logger.info(s"Storing optimised image to instance specific key: $thumbnailBucket / $instanceSpecificKey") + storeImage(imageBucket, instanceSpecificKey, storableImage.file, Some(storableImage.mimeType), overwrite = true) + } - private def bulkDelete(bucket: String, keys: List[String]): Future[Map[String, Boolean]] = keys match { + private def storeEmbeddingSourceImage(storableImage: StorableEmbeddingSourceImage) + (implicit logMarker: LogMarker): Future[S3Object] = { + val instanceSpecificKey = fileKeyFromId(storableImage.id)(storableImage.instance) + logger.info(s"Storing embedding source to instance specific key: ${embeddingSourceBucket.bucket} / $instanceSpecificKey") + storeImage(embeddingSourceBucket, instanceSpecificKey, storableImage.file, Some(storableImage.mimeType), + overwrite = true) + } + + def getEmbeddingStoreImage(key: String): model.S3Object = { + getObject(embeddingSourceBucket, key) + } + + private def bulkDelete(bucket: S3Bucket, keys: List[String]): Future[Map[String, Boolean]] = keys match { case Nil => Future.successful(Map.empty) - case _ => Future { - try { - client.deleteObjects( - new DeleteObjectsRequest(bucket).withKeys(keys: _*) - ) - keys.map { key => - key -> true - }.toMap - } catch { - case partialFailure: MultiObjectDeleteException => - logger.warn(s"Partial failure when deleting images from $bucket: ${partialFailure.getMessage} ${partialFailure.getErrors}") - val errorKeys = partialFailure.getErrors.asScala.map(_.getKey).toSet + case _ => + val bulkDeleteImplemented = bucket.endpoint != "storage.googleapis.com" + if (bulkDeleteImplemented) { + Future { + try { + logger.info(s"Bulk deleting S3 objects from ${bucket.bucket}: " + keys.mkString(",")) + deleteObjects(bucket, keys) + keys.map { key => + key -> true + }.toMap + } catch { + case partialFailure: MultiObjectDeleteException => + logger.warn(s"Partial failure when deleting images from $bucket: ${partialFailure.getMessage} ${partialFailure.getErrors}") + val errorKeys = partialFailure.getErrors.asScala.map(_.getKey).toSet + keys.map { key => + key -> !errorKeys.contains(key) + }.toMap + } + } + + } else { + Future.sequence { keys.map { key => - key -> !errorKeys.contains(key) - }.toMap + Future { + logger.info(s"Deleting S3 objects from ${bucket.bucket}: " + key) + try { + deleteObject(bucket, key) + (key, true) + } catch { + case e: Exception => + logger.debug(s"Failure when deleting images from $bucket: $key, ${e.getMessage}") + (key, false) + } + } + } + }.map(_.toMap) } - } } - def deleteOriginal(id: String)(implicit logMarker: LogMarker): Future[Unit] = if(isVersionedS3) deleteVersionedImage(imageBucket, fileKeyFromId(id)) else deleteImage(imageBucket, fileKeyFromId(id)) - def deleteOriginals(ids: Set[String]) = bulkDelete(imageBucket, ids.map(fileKeyFromId).toList) - def deleteThumbnail(id: String)(implicit logMarker: LogMarker): Future[Unit] = deleteImage(thumbnailBucket, fileKeyFromId(id)) - def deleteThumbnails(ids: Set[String]) = bulkDelete(thumbnailBucket, ids.map(fileKeyFromId).toList) - def deletePNG(id: String)(implicit logMarker: LogMarker): Future[Unit] = deleteImage(imageBucket, optimisedPngKeyFromId(id)) - def deletePNGs(ids: Set[String]) = bulkDelete(imageBucket, ids.map(optimisedPngKeyFromId).toList) + def deleteOriginal(id: String)(implicit logMarker: LogMarker, instance: Instance): Future[Unit] = if(isVersionedS3) deleteVersionedImage(imageBucket, fileKeyFromId(id)) else deleteImage(imageBucket, fileKeyFromId(id)) + def deleteOriginals(ids: Set[String])(implicit instance: Instance) = bulkDelete(imageBucket, ids.map(id => fileKeyFromId(id)).toList) + def deleteThumbnail(id: String)(implicit logMarker: LogMarker, instance: Instance): Future[Unit] = deleteImage(thumbnailBucket, fileKeyFromId(id)) + def deleteThumbnails(ids: Set[String])(implicit instance: Instance) = bulkDelete(thumbnailBucket, ids.map(id => fileKeyFromId(id)).toList) + def deletePNG(id: String)(implicit logMarker: LogMarker, instance: Instance): Future[Unit] = deleteImage(imageBucket, optimisedPngKeyFromId(id)) + def deletePNGs(ids: Set[String])(implicit instance: Instance) = bulkDelete(imageBucket, ids.map(id => optimisedPngKeyFromId(id)).toList) + + def doesOriginalExist(id: String)(implicit instance: Instance): Boolean = + doesObjectExist(imageBucket, fileKeyFromId(id)) + + private def instanceAwareOriginalImageKey(storableImage: StorableOriginalImage) = { + fileKeyFromId(storableImage.id)(storableImage.instance) + } + + private def instanceAwareThumbnailImageKey(storableImage: StorableThumbImage) = { + fileKeyFromId(storableImage.id)(storableImage.instance) + } - def doesOriginalExist(id: String): Boolean = - client.doesObjectExist(imageBucket, fileKeyFromId(id)) } sealed trait ImageWrapper { @@ -82,11 +137,12 @@ sealed trait ImageWrapper { val file: File val mimeType: MimeType val meta: Map[String, String] + val instance: Instance } sealed trait StorableImage extends ImageWrapper { - def toProjectedS3Object(thumbBucket: String): S3Object = S3Object( + def toProjectedS3Object(thumbBucket: S3Bucket): S3Object = S3Object( thumbBucket, - ImageIngestOperations.fileKeyFromId(id), + ImageIngestOperations.fileKeyFromId(id)(instance), file, Some(mimeType), lastModified = None, @@ -94,28 +150,37 @@ sealed trait StorableImage extends ImageWrapper { ) } -case class StorableThumbImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty) extends StorableImage -case class StorableOriginalImage(id: String, file: File, mimeType: MimeType, lastModified: DateTime, meta: Map[String, String] = Map.empty) extends StorableImage { - override def toProjectedS3Object(thumbBucket: String): S3Object = S3Object( +case class StorableThumbImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage +case class StorableOriginalImage(id: String, file: File, mimeType: MimeType, lastModified: DateTime, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage { + override def toProjectedS3Object(thumbBucket: S3Bucket): S3Object = S3Object( thumbBucket, - ImageIngestOperations.fileKeyFromId(id), + ImageIngestOperations.fileKeyFromId(id)(instance), file, Some(mimeType), lastModified = Some(lastModified), meta ) } -case class StorableOptimisedImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty) extends StorableImage { - override def toProjectedS3Object(thumbBucket: String): S3Object = S3Object( +case class StorableOptimisedImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage { + override def toProjectedS3Object(thumbBucket: S3Bucket): S3Object = S3Object( thumbBucket, - ImageIngestOperations.optimisedPngKeyFromId(id), + ImageIngestOperations.optimisedPngKeyFromId(id)(instance), + file, + Some(mimeType), + lastModified = None, + meta = meta + ) +} +case class StorableEmbeddingSourceImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage { + override def toProjectedS3Object(embeddingSourcesBucket: S3Bucket): S3Object = S3Object( + embeddingSourcesBucket, + ImageIngestOperations.fileKeyFromId(id)(instance), file, Some(mimeType), lastModified = None, meta = meta ) } - /** * @param id @@ -126,8 +191,9 @@ case class StorableOptimisedImage(id: String, file: File, mimeType: MimeType, me * Can be used in order to skip e.g. the stripping of incorrect colour profiles, * as in this case we have already inferred the profile upstream. */ -case class BrowserViewableImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty, isTransformedFromSource: Boolean = false) extends ImageWrapper { - def asStorableOptimisedImage = StorableOptimisedImage(id, file, mimeType, meta) - def asStorableThumbImage = StorableThumbImage(id, file, mimeType, meta) +case class BrowserViewableImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty, isTransformedFromSource: Boolean = false, instance: Instance) extends ImageWrapper { + def asStorableOptimisedImage = StorableOptimisedImage(id, file, mimeType, meta, instance) + def asStorableThumbImage = StorableThumbImage(id, file, mimeType, meta, instance) + def asStorableEmbeddingSourceImage = StorableEmbeddingSourceImage(id, file, mimeType, meta, instance) } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageQuarantineOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageQuarantineOperations.scala index 9de53d94905..eaf6b82979c 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageQuarantineOperations.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageQuarantineOperations.scala @@ -1,19 +1,18 @@ package com.gu.mediaservice.lib import java.io.File - import com.gu.mediaservice.lib.config.CommonConfig -import com.gu.mediaservice.lib.aws.S3Object +import com.gu.mediaservice.lib.aws.{S3Bucket, S3Object} import com.gu.mediaservice.lib.logging.LogMarker -import com.gu.mediaservice.model.MimeType +import com.gu.mediaservice.model.{Instance, MimeType} import scala.concurrent.Future -class ImageQuarantineOperations(quarantineBucket: String, config: CommonConfig, isVersionedS3: Boolean = false) +class ImageQuarantineOperations(quarantineBucket: S3Bucket, config: CommonConfig, isVersionedS3: Boolean = false) extends S3ImageStorage(config) { def storeQuarantineImage(id: String, file: File, mimeType: Option[MimeType], meta: Map[String, String] = Map.empty) - (implicit logMarker: LogMarker): Future[S3Object] = + (implicit logMarker: LogMarker, instance: Instance): Future[S3Object] = storeImage(quarantineBucket, ImageIngestOperations.fileKeyFromId(id), file, mimeType, meta, overwrite = true) } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageStorage.scala index f3016fddde1..010a3c0744d 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageStorage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageStorage.scala @@ -2,11 +2,10 @@ package com.gu.mediaservice.lib import java.util.concurrent.Executors import java.io.File - import scala.concurrent.{ExecutionContext, Future} import scala.concurrent.duration._ import scala.language.postfixOps -import com.gu.mediaservice.lib.aws.S3Object +import com.gu.mediaservice.lib.aws.{S3Bucket, S3Object} import com.gu.mediaservice.lib.logging.LogMarker import com.gu.mediaservice.model.MimeType @@ -19,6 +18,7 @@ object ImageStorageProps { val identifierMetadataKeyPrefix: String = "identifier!" val derivativeOfMediaIdsIdentifierKey: String = "derivative-of-media-ids" val replacesMediaIdIdentifierKey: String = "replaces-media-id" + val isFeedUploadMetadataKey = "is-feed-upload" } trait ImageStorage { @@ -36,9 +36,9 @@ trait ImageStorage { /** Store a copy of the given file and return the URI of that copy. * The file can safely be deleted afterwards. */ - def storeImage(bucket: String, id: String, file: File, mimeType: Option[MimeType], + def storeImage(bucket: S3Bucket, id: String, file: File, mimeType: Option[MimeType], meta: Map[String, String] = Map.empty, overwrite: Boolean) (implicit logMarker: LogMarker): Future[S3Object] - def deleteImage(bucket: String, id: String)(implicit logMarker: LogMarker): Future[Unit] + def deleteImage(bucket: S3Bucket, id: String)(implicit logMarker: LogMarker): Future[Unit] } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala index 7e164f876bb..a4d603f1a68 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala @@ -1,6 +1,6 @@ package com.gu.mediaservice.lib -import com.gu.mediaservice.lib.aws.S3 +import com.gu.mediaservice.lib.aws.{S3, S3Bucket} import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} import com.gu.mediaservice.model.MimeType @@ -14,10 +14,10 @@ import scala.concurrent.Future class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage with GridLogging { private val cacheSetting = Some(cacheForever) - def storeImage(bucket: String, id: String, file: File, mimeType: Option[MimeType], + def storeImage(bucket: S3Bucket, id: String, file: File, mimeType: Option[MimeType], meta: Map[String, String] = Map.empty, overwrite: Boolean) (implicit logMarker: LogMarker) = { - logger.info(logMarker, s"bucket: $bucket, id: $id, meta: $meta") + logger.info(logMarker, s"storeImage to bucket: ${bucket.bucket}, id: $id, meta: $meta") val eventualObject = if (overwrite) { store(bucket, id, file, mimeType, meta, cacheSetting) } else { @@ -27,20 +27,21 @@ class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage eventualObject } - def deleteImage(bucket: String, id: String)(implicit logMarker: LogMarker) = Future { - client.deleteObject(bucket, id) + def deleteImage(bucket: S3Bucket, id: String)(implicit logMarker: LogMarker) = Future { + deleteObject(bucket, id) logger.info(logMarker, s"Deleted image $id from bucket $bucket") } - def deleteVersionedImage(bucket: String, id: String)(implicit logMarker: LogMarker) = Future { - val objectVersion = client.getObjectMetadata(bucket, id).getVersionId - client.deleteVersion(bucket, id, objectVersion) + def deleteVersionedImage(bucket: S3Bucket, id: String)(implicit logMarker: LogMarker) = Future { + val objectVersion = getObjectMetadata(bucket, id).getVersionId + deleteVersion(bucket, id, objectVersion) logger.info(logMarker, s"Deleted image $id from bucket $bucket (version: $objectVersion)") } - def deleteFolder(bucket: String, id: String)(implicit logMarker: LogMarker) = Future { - val files = client.listObjects(bucket, id).getObjectSummaries.asScala - files.foreach(file => client.deleteObject(bucket, file.getKey)) + def deleteFolder(bucket: S3Bucket, id: String)(implicit logMarker: LogMarker) = Future { + val files = listObjects(bucket, id).getObjectSummaries.asScala + logger.info(s"Found ${files.size} files to delete in folder $id") + files.foreach(file => deleteObject(bucket, file.getKey)) logger.info(logMarker, s"Deleting images in folder $id from bucket $bucket") } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/ApiAccessor.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/ApiAccessor.scala index 3e1924451d5..4d321e82ca4 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/ApiAccessor.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/ApiAccessor.scala @@ -2,6 +2,7 @@ package com.gu.mediaservice.lib.auth import com.gu.mediaservice.lib.argo.ArgoHelpers import com.gu.mediaservice.lib.config.Services +import com.gu.mediaservice.model.Instance import play.api.mvc.{RequestHeader, Result} sealed trait Tier @@ -26,9 +27,12 @@ object ApiAccessor extends ArgoHelpers { ApiAccessor(name, tier) } - def hasAccess(apiKey: ApiAccessor, request: RequestHeader, services: Services): Boolean = apiKey.tier match { + def hasAccess(apiKey: ApiAccessor, request: RequestHeader, services: Services)(implicit instance: Instance): Boolean = apiKey.tier match { case Internal => true case ReadOnly => request.method == "GET" - case Syndication => request.method == "GET" && request.host == services.apiHost && request.path.startsWith("/images") + case Syndication => { + val isMediaApiRequest = request.uri.startsWith(services.apiBaseUri(instance)) // TODO check this! + request.method == "GET" && isMediaApiRequest && request.path.startsWith("/images") + } } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala index d3c90bc739e..ebc43bf09c2 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala @@ -1,24 +1,22 @@ package com.gu.mediaservice.lib.auth import com.gu.mediaservice.lib.BaseStore +import com.gu.mediaservice.lib.aws.S3Bucket import com.gu.mediaservice.lib.config.CommonConfig +import com.gu.mediaservice.model.Instance -import scala.jdk.CollectionConverters._ import scala.concurrent.ExecutionContext -class KeyStore(bucket: String, config: CommonConfig)(implicit ec: ExecutionContext) +class KeyStore(bucket: S3Bucket, config: CommonConfig)(implicit ec: ExecutionContext) extends BaseStore[String, ApiAccessor](bucket, config)(ec) { - def lookupIdentity(key: String): Option[ApiAccessor] = store.get().get(key) - - def findKey(prefix: String): Option[String] = s3.syncFindKey(bucket, prefix) + def lookupIdentity(key: String)(implicit instance: Instance): Option[ApiAccessor] = store.get().get(instance.id + "/" + key) def update(): Unit = { store.set(fetchAll) } private def fetchAll: Map[String, ApiAccessor] = { - val keys = s3.client.listObjects(bucket).getObjectSummaries.asScala.map(_.getKey) - keys.flatMap(k => getS3Object(k).map(k -> ApiAccessor(_))).toMap + s3.listObjectKeys(bucket).flatMap(k => getS3Object(k).map(k -> ApiAccessor(_))).toMap } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/AwsClientV1BuilderUtils.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/AwsClientV1BuilderUtils.scala index cdd4ec04d92..6545cf09c94 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/AwsClientV1BuilderUtils.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/AwsClientV1BuilderUtils.scala @@ -1,7 +1,6 @@ package com.gu.mediaservice.lib.aws -import com.amazonaws.auth.profile.ProfileCredentialsProvider -import com.amazonaws.auth.{AWSCredentialsProvider, AWSCredentialsProviderChain, InstanceProfileCredentialsProvider} +import com.amazonaws.auth.{AWSCredentialsProvider, AWSCredentialsProviderChain, EnvironmentVariableCredentialsProvider} import com.amazonaws.client.builder.AwsClientBuilder import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration import com.gu.mediaservice.lib.logging.GridLogging @@ -13,8 +12,7 @@ trait AwsClientV1BuilderUtils extends GridLogging { def awsRegion: String = "eu-west-1" def awsCredentials: AWSCredentialsProvider = new AWSCredentialsProviderChain( - new ProfileCredentialsProvider("media-service"), - InstanceProfileCredentialsProvider.getInstance() + new EnvironmentVariableCredentialsProvider(), ) final def awsEndpointConfiguration: Option[EndpointConfiguration] = awsLocalEndpoint match { diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/AwsClientV2BuilderUtils.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/AwsClientV2BuilderUtils.scala index e73d8d7408a..2aa2f166af4 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/AwsClientV2BuilderUtils.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/AwsClientV2BuilderUtils.scala @@ -1,7 +1,7 @@ package com.gu.mediaservice.lib.aws import com.gu.mediaservice.lib.logging.GridLogging -import software.amazon.awssdk.auth.credentials.{AwsCredentialsProvider, DefaultCredentialsProvider} +import software.amazon.awssdk.auth.credentials.{AwsCredentialsProvider, EnvironmentVariableCredentialsProvider} import software.amazon.awssdk.awscore.client.builder.AwsClientBuilder import software.amazon.awssdk.regions.Region @@ -13,7 +13,7 @@ trait AwsClientV2BuilderUtils extends GridLogging { def awsRegionV2: Region = Region.EU_WEST_1 - def awsCredentialsV2: AwsCredentialsProvider = DefaultCredentialsProvider.builder().profileName("media-service").build() + def awsCredentialsV2: AwsCredentialsProvider = EnvironmentVariableCredentialsProvider.create() final def withAWSCredentialsV2[T, S <: AwsClientBuilder[S, T]](builder: AwsClientBuilder[S, T], localstackAware: Boolean = true, maybeRegionOverride: Option[Region] = None): S = { val credentialedBuilder = builder.credentialsProvider(awsCredentialsV2).region(maybeRegionOverride.getOrElse(awsRegionV2)) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala index bb1a8dfaabb..b0cde6bb671 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala @@ -1,16 +1,17 @@ package com.gu.mediaservice.lib.aws -import software.amazon.awssdk.services.bedrockruntime.model._ -import software.amazon.awssdk.services.bedrockruntime._ import com.gu.mediaservice.lib.config.CommonConfig -import play.api.libs.json.Json -import software.amazon.awssdk.core.SdkBytes - -import java.net.URI +import com.gu.mediaservice.lib.embeddings.EmbeddingImplementation import com.gu.mediaservice.lib.logging.LogMarker +import com.gu.mediaservice.model.{CohereV4Embedding, Embedding, ImageMetadata, Jpeg, MimeType} +import org.apache.commons.codec.binary.Base64 import play.api.libs.json.OFormat.oFormatFromReadsAndOWrites import play.api.libs.json._ +import software.amazon.awssdk.core.SdkBytes +import software.amazon.awssdk.services.bedrockruntime._ +import software.amazon.awssdk.services.bedrockruntime.model._ +import java.net.URI import scala.concurrent.{ExecutionContext, Future} object Bedrock { @@ -22,10 +23,19 @@ object Bedrock { ) private implicit val bedrockTextRequestFormat: OFormat[BedrockTextRequest] = Json.format[BedrockTextRequest] + + case class BedrockImageRequest( + input_type: String, + embedding_types: List[String], + images: List[String], + output_dimension: Int + ) + + private implicit val bedrockImageRequestFormat: OFormat[BedrockImageRequest] = Json.format[BedrockImageRequest] } class Bedrock(config: CommonConfig) - extends AwsClientV2BuilderUtils { + extends EmbeddingImplementation with AwsClientV2BuilderUtils { // TODO: figure out what the more usual pattern for turning off localstack behaviour is override def awsLocalEndpointUri: Option[URI] = None @@ -37,7 +47,7 @@ class Bedrock(config: CommonConfig) .build() } - private def createRequestBody(inputData: String): InvokeModelRequest = { + private def createSearchQueryRequestBody(inputData: String): InvokeModelRequest = { val body = Bedrock.BedrockTextRequest( input_type = "search_query", embedding_types = List("float"), @@ -58,6 +68,30 @@ class Bedrock(config: CommonConfig) request } + private def createImageSearchDocumentRequestBody(base64Image: String, imageMimeType: MimeType): InvokeModelRequest = { + val body = Bedrock.BedrockImageRequest( + input_type = "search_document", + embedding_types = List("float"), + images = List( + s"`data:${imageMimeType.name};base64,$base64Image`" + ), + output_dimension = 1536 + ) + + val jsonBody = Json.toJson(body).toString() + + val request: InvokeModelRequest = { + InvokeModelRequest + .builder() + .accept("*/*") + .body(SdkBytes.fromUtf8String(jsonBody)) + .contentType("application/json") + .modelId("global.cohere.embed-v4:0") + .build() + } + request + } + private def sendBedrockEmbeddingRequest(requestBody: InvokeModelRequest)( implicit logMarker: LogMarker ): InvokeModelResponse = { @@ -77,7 +111,7 @@ class Bedrock(config: CommonConfig) } def createTextEmbedding(inputData: String)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[List[Float]] = { - val requestBody = createRequestBody(inputData) + val requestBody = createSearchQueryRequestBody(inputData) val bedrockFuture = Future { sendBedrockEmbeddingRequest(requestBody) } bedrockFuture.map { response => val responseBody = response.body().asUtf8String() @@ -91,4 +125,31 @@ class Bedrock(config: CommonConfig) embedding } } + + def createImageEmbeddings(source: Array[Byte], maybe_Metadata: Option[ImageMetadata])(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Embedding] = { + val base64ImageData = Base64.encodeBase64String(source) + val requestBody = createImageSearchDocumentRequestBody( + base64ImageData, embeddingSourceImageFormat().format + ) + val bedrockFuture = Future { + sendBedrockEmbeddingRequest(requestBody) + } + bedrockFuture.map { response => + val responseBody = response.body().asUtf8String() + val json = Json.parse(responseBody) + // Extract the embedding array (first element since it's an array of arrays) + val embeddings = (json \ "embeddings" \ "float")(0).as[List[Float]] + logger.info( + logMarker, + s"Successfully created image embedding. Vector size: ${embeddings.size}" + ) + embeddings + }.map { embeddings => + Embedding( + cohereEmbedV4 = Some(CohereV4Embedding(embeddings.map(_.toDouble))) + ) + } + } + + override def embeddingSourceImageFormat(): EmbeddingSourceImageFormat = EmbeddingSourceImageFormat(longestAxis = 3000, format = Jpeg, letterBox = false) } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/ContentDisposition.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/ContentDisposition.scala index da2a73ad96a..796a1fae85d 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/ContentDisposition.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/ContentDisposition.scala @@ -19,14 +19,16 @@ trait ContentDisposition extends GridLogging { getContentDisposition(filename, fallbackLatin1Filename(image, extension)) } - def getContentDisposition(image: Image, crop: Crop, asset: Asset, shortenDownloadFilename: Boolean): String = { - val cropId: String = crop.id.map(id => s"($id)").getOrElse("") + def getContentDisposition(image: Image, crop: Crop, asset: Asset): String = { val extension: String = getExtension(image, asset) - val dimensions: String = asset.dimensions.map(dims => s"(${dims.width} x ${dims.height})").getOrElse("") - val filenameSuffix: String = s"(${image.id})$cropId$dimensions$extension" - val filename = getBaseFilename(image, filenameSuffix, shortenDownloadFilename) + val filename = image.uploadInfo.filename match { + case Some(filename) => filename + case _ => image.id + } - getContentDisposition(filename, fallbackLatin1Filename(image, extension)) + // Drop original file's extension and replace with the crops actual extension + val withCropsExtension = removeExtension(filename) + extension + getContentDisposition(withCropsExtension, fallbackLatin1Filename(image, extension)) } private def getExtension(image: Image, asset: Asset): String = asset.mimeType match { diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala index 63e763fcaa6..6b1c8a06304 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala @@ -1,22 +1,12 @@ package com.gu.mediaservice.lib.aws -import java.util -import com.amazonaws.AmazonServiceException -import com.amazonaws.services.dynamodbv2.document.spec.{DeleteItemSpec, GetItemSpec, PutItemSpec, QuerySpec, UpdateItemSpec} -import com.amazonaws.services.dynamodbv2.document.utils.ValueMap -import com.amazonaws.services.dynamodbv2.document.{DynamoDB => AwsDynamoDB, _} -import com.amazonaws.services.dynamodbv2.model.{AttributeValue, DeleteItemRequest, KeysAndAttributes, ReturnValue} -import com.amazonaws.services.dynamodbv2.{AmazonDynamoDBAsync, AmazonDynamoDBAsyncClientBuilder} -import com.gu.mediaservice.lib.aws.DynamoDB.{deleteExpr, setExpr} -import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.GridLogging import org.joda.time.DateTime import play.api.libs.json._ import software.amazon.awssdk.enhanced.dynamodb._ import software.amazon.awssdk.enhanced.dynamodb.document.EnhancedDocument -import software.amazon.awssdk.enhanced.dynamodb.model import software.amazon.awssdk.services.dynamodb.DynamoDbClient -import software.amazon.awssdk.services.dynamodb.model.{UpdateItemRequest, AttributeValue => AttributeValueV2, ReturnValue => ReturnValueV2} +import software.amazon.awssdk.services.dynamodb.model.{BatchGetItemRequest, QueryRequest, UpdateItemRequest, AttributeValue => AttributeValueV2, KeysAndAttributes => KeysAndAttributesV2, ReturnValue => ReturnValueV2} import scala.annotation.tailrec import scala.concurrent.{ExecutionContext, Future} @@ -26,13 +16,12 @@ object NoItemFound extends Throwable("item not found") /** * A lightweight wrapper around AWS dynamo SDK for undertaking various operations - * @param config Common grid config including AWS credentials + * @param client2 DynamoDbClient client * @param tableName the table name for this instance of the dynamoDB wrapper * @param lastModifiedKey if set to a string the wrapper will maintain a last modified with that name on any update * @tparam T The type of this table */ -class DynamoDB[T](config: CommonConfig, tableName: String, lastModifiedKey: Option[String] = None) extends GridLogging { - lazy val client2: DynamoDbClient = config.withAWSCredentialsV2(DynamoDbClient.builder()).build() +class DynamoDB[T](client2: DynamoDbClient, tableName: String, lastModifiedKey: Option[String] = None) extends GridLogging { lazy val dynamo2: DynamoDbEnhancedClient = DynamoDbEnhancedClient.builder().dynamoDbClient(client2).build() lazy val tableSchema = TableSchema.documentSchemaBuilder() .addIndexPartitionKey(TableMetadata.primaryIndexName(), IdKey, AttributeValueType.S) @@ -40,10 +29,6 @@ class DynamoDB[T](config: CommonConfig, tableName: String, lastModifiedKey: Opti .build() lazy val table2 = dynamo2.table(tableName, tableSchema) - lazy val client: AmazonDynamoDBAsync = config.withAWSCredentials(AmazonDynamoDBAsyncClientBuilder.standard()).build() - lazy val dynamo = new AwsDynamoDB(client) - lazy val table: Table = dynamo.getTable(tableName) - private val IdKey = "id" private def itemKey(key: String) = Key.builder().partitionValue(key).build() @@ -59,14 +44,6 @@ class DynamoDB[T](config: CommonConfig, tableName: String, lastModifiedKey: Opti case None => Future.failed(NoItemFound) } - private def get(id: String, attribute: String)(implicit ex: ExecutionContext): Future[Item] = Future { - table.getItem( - new GetItemSpec() - .withPrimaryKey(IdKey, id) - .withAttributesToGet(attribute) - ) - } flatMap itemOrNotFound - private def docOrNotFound(docOrNull: EnhancedDocument): Future[EnhancedDocument] = { Option(docOrNull) match { case Some(doc) => Future.successful(doc) @@ -74,26 +51,9 @@ class DynamoDB[T](config: CommonConfig, tableName: String, lastModifiedKey: Opti } } - private def itemOrNotFound(itemOrNull: Item): Future[Item] = { - Option(itemOrNull) match { - case Some(item) => Future.successful(item) - case None => Future.failed(NoItemFound) - } - } - - def removeKey(id: String, key: String) - (implicit ex: ExecutionContext): Future[JsObject] = - update( - id, - s"REMOVE $key" - ) - def removeKeyV2(id: String, key: String)(implicit ex: ExecutionContext) = Future{ updateV2(id, DynamoDB.removeExpr(key, lastModifiedKey)) } - def deleteItem(id: String)(implicit ex: ExecutionContext): Future[Unit] = Future { - table.deleteItem(new DeleteItemSpec().withPrimaryKey(IdKey, id)) - } def deleteItemV2(id: String)(implicit ex: ExecutionContext): Future[Unit] = Future { table2.deleteItem( @@ -131,73 +91,73 @@ class DynamoDB[T](config: CommonConfig, tableName: String, lastModifiedKey: Opti def setAddV2(id: String, key: String, value: List[String])(implicit ex: ExecutionContext): Future[JsObject] = Future { updateV2(id, DynamoDB.addExpr(key, lastModifiedKey), AttributeValueV2.fromSs(value.asJava)) } - def batchGet(ids: List[String], attributeKey: String) - (implicit ex: ExecutionContext, rjs: Reads[T]): Future[Map[String, T]] = { + + def batchGetV2(ids: List[String], attributeKey: String) + (implicit ex: ExecutionContext, rjs: Reads[T]): Future[Map[String, T]] = { val keyChunkList = ids - .map(k => Map(IdKey -> new AttributeValue(k)).asJava) + .map(k => Map(IdKey -> AttributeValueV2.fromS(k)).asJava) .grouped(100) Future.traverse(keyChunkList) { keyChunk => { - val keysAndAttributes: KeysAndAttributes = new KeysAndAttributes().withKeys(keyChunk.asJava) - - @tailrec - def nextPageOfBatch(request: java.util.Map[String, KeysAndAttributes], acc: List[(String, T)]) - (implicit ex: ExecutionContext, rjs: Reads[T]): List[(String, T)] = { - if (request.isEmpty) acc - else { - logger.info(s"Fetching records for $request") - val response = client.batchGetItem(request) - val responses = response.getResponses - logger.info(s"Got responses of $responses") - val results = responses.get(tableName).asScala.toList - .flatMap(att => { - val attributes: util.Map[String, AnyRef] = ItemUtils.toSimpleMapValue(att) - logger.info(s"Obtained attributes of $attributes from response $att") - val json = asJsObject(Item.fromMap(attributes)) - val maybeT = (json \ attributeKey).asOpt[T] - logger.info(s"Obtained a T of $maybeT from json $json") - maybeT.map( - attributes.get(IdKey).toString -> _ - ) - }) - logger.info(s"Got $results for request") - nextPageOfBatch(response.getUnprocessedKeys, acc ::: results) + val keysAndAttributes: KeysAndAttributesV2 = KeysAndAttributesV2.builder().keys(keyChunk.asJava).build() + + @tailrec + def nextPageOfBatch(request: java.util.Map[String, KeysAndAttributesV2], acc: List[(String, T)]) + (implicit ex: ExecutionContext, rjs: Reads[T]): List[(String, T)] = { + if (request.isEmpty) acc + else { + logger.info(s"Fetching records for $request") + val response = client2.batchGetItem(BatchGetItemRequest.builder().requestItems(request).build()) + val responses = response.responses() + logger.info(s"Got responses of $responses") + val results = responses.get(tableName).asScala.toList + .flatMap(att => { + logger.info(s"Obtained attributes of $att from response") + val json = asJsObject(EnhancedDocument.fromAttributeValueMap(att)) + val maybeT = (json \ attributeKey).asOpt[T] + logger.info(s"Obtained a T of $maybeT from json $json") + maybeT.map( + att.get(IdKey).s() -> _ + ) + }) + logger.info(s"Got $results for request") + nextPageOfBatch(response.unprocessedKeys(), acc ::: results) + } } - } - Future { - nextPageOfBatch(Map(tableName -> keysAndAttributes).asJava, Nil).toMap + Future { + nextPageOfBatch(Map(tableName -> keysAndAttributes).asJava, Nil).toMap + } + } } - }} .map(chunkIterator => chunkIterator.fold(Map.empty)((acc, result) => acc ++ result)) } - // We cannot update, so make sure you send over the WHOLE document def jsonAddV2(id: String, key: String, value: Map[String, JsValue]) (implicit ex: ExecutionContext): Future[JsObject] = Future { updateV2( id, - setExpr(key, lastModifiedKey), + DynamoDB.setExpr(key, lastModifiedKey), AttributeValueV2.fromM(value.view.mapValues(DynamoDB.jsonToAttributeValue).toMap.asJava) ) } def setDeleteV2(id: String, key: String, value: String) (implicit ex: ExecutionContext): Future[JsObject] = Future { - updateV2(id, deleteExpr(key, lastModifiedKey), AttributeValueV2.fromSs(List(value).asJava)) + updateV2(id, DynamoDB.deleteExpr(key, lastModifiedKey), AttributeValueV2.fromSs(List(value).asJava)) } - def scanForId(indexName: String, keyname: String, key: String)(implicit ex: ExecutionContext) = Future { - val index = table.getIndex(indexName) - - val spec = new QuerySpec() - .withKeyConditionExpression(s"$keyname = :key") - .withValueMap(new ValueMap() - .withString(":key", key)) + def scanForIdV2(indexName: String, keyname: String, key: String)(implicit ex: ExecutionContext): Future[List[String]] = Future { + val request = QueryRequest.builder() + .tableName(tableName) + .indexName(indexName) + .keyConditionExpression(s"$keyname = :key") + .expressionAttributeValues(Map(":key" -> AttributeValueV2.fromS(key)).asJava) + .build() - val items: List[Item] = index.query(spec).iterator.asScala.toList - items map (a => a.getString("id")) + client2.query(request).items().asScala.toList + .flatMap(item => Option(item.get("id")).map(_.s())) } private def updateRequestBuilder(id: String, expression: String) = { @@ -208,11 +168,11 @@ class DynamoDB[T](config: CommonConfig, tableName: String, lastModifiedKey: Opti .tableName(tableName) } - def updateV2(id: String, expression: String, attribute: AttributeValueV2): JsObject = { + private def updateV2(id: String, expression: String, attribute: AttributeValueV2): JsObject = { updateV2(id, expression, Map(":value" -> attribute)) } - def updateV2(id: String, expression: String): JsObject = { + private def updateV2(id: String, expression: String): JsObject = { updateV2(id, expression, Map.empty[String, AttributeValueV2]) } @@ -226,49 +186,21 @@ class DynamoDB[T](config: CommonConfig, tableName: String, lastModifiedKey: Opti Json.parse(jsonString).as[JsObject] } - def update(id: String, expression: String, valueMap: ValueMap) - (implicit ex: ExecutionContext): Future[JsObject] = - update(id, expression, Some(valueMap)) - - def update(id: String, expression: String, valueMap: Option[ValueMap] = None) - (implicit ex: ExecutionContext): Future[JsObject] = Future { - - val baseUpdateSpec = new UpdateItemSpec(). - withPrimaryKey(IdKey, id). - withUpdateExpression(expression). - withReturnValues(ReturnValue.ALL_NEW). - withValueMap(valueMap.orNull) - - val updateSpec = lastModifiedKey.map { key => - DynamoDB.addLastModifiedUpdate(baseUpdateSpec, key, DateTime.now) - }.getOrElse(baseUpdateSpec) - - table.updateItem(updateSpec) - } map asJsObject - - - // FIXME: surely there must be a better way to convert? - def asJsObject(item: Item): JsObject = - jsonWithNullAsEmptyString(Json.parse(item.toJSON)).as[JsObject] - IdKey - def asJsObject(doc: EnhancedDocument): JsObject = jsonWithNullAsEmptyString(Json.parse(doc.toJson)).as[JsObject] - IdKey - def asJsObject(outcome: UpdateItemOutcome): JsObject = - Option(outcome.getItem) map asJsObject getOrElse Json.obj() - // FIXME: Dynamo accepts `null`, but not `""`. This is a well documented issue // around the community. This guard keeps the introduction of `null` fairly // fenced in this Dynamo play area. `null` is continual and big annoyance with AWS libs. // see: https://forums.aws.amazon.com/message.jspa?messageID=389032 // see: http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DataModel.html - def mapJsValue(jsValue: JsValue)(f: JsValue => JsValue): JsValue = jsValue match { + private def mapJsValue(jsValue: JsValue)(f: JsValue => JsValue): JsValue = jsValue match { case JsObject(items) => JsObject(items.map{ case (k, v) => k -> mapJsValue(v)(f) }) case JsArray(items) => JsArray(items.map(f)) case value => f(value) } - def jsonWithNullAsEmptyString(jsValue: JsValue): JsValue = mapJsValue(jsValue) { + private def jsonWithNullAsEmptyString(jsValue: JsValue): JsValue = mapJsValue(jsValue) { case JsNull => JsString("") case value => value } @@ -276,28 +208,6 @@ class DynamoDB[T](config: CommonConfig, tableName: String, lastModifiedKey: Opti } object DynamoDB { - def jsonToValueMap(json: JsObject): ValueMap = { - val valueMap = new ValueMap() - json.value map { case (key, value) => - value match { - case v: JsString => valueMap.withString(key, v.value) - case v: JsBoolean => valueMap.withBoolean(key, v.value) - case v: JsNumber => valueMap.withNumber(key, v.value) - case v: JsObject => valueMap.withMap(key, jsonToValueMap(v)) - - // TODO: Lists of different Types? JsArray is not type safe (because json lists aren't) - // so this leaves us in a bit of a pickle when converting them. So for now we only support - // List[String] - case v: JsArray => valueMap.withList(key, v.value.map { - case i: JsString => i.value - case i: JsValue => i.toString - }.asJava) - case _ => valueMap - } - } - valueMap - } - def jsonToAttributeValue(json: JsValue): AttributeValueV2 = { json match { case JsString(v) => AttributeValueV2.fromS(v) @@ -314,34 +224,6 @@ object DynamoDB { def caseClassToMap[T](caseClass: T)(implicit tjs: Writes[T]): Map[String, JsValue] = Json.toJson[T](caseClass).as[JsObject].as[Map[String, JsValue]] - def addLastModifiedUpdate(update: UpdateItemSpec, lastModifiedKey: String, lastModifiedDate: DateTime): UpdateItemSpec = { - val expression = update.getUpdateExpression - val valueMap: ValueMap = { - val m = new ValueMap() - Option(update.getValueMap).foreach { vm => - m.putAll(vm) - } - m - } - - val newExpression = { - val keyUpdate: String = s"$lastModifiedKey = :$lastModifiedKey" - if (expression.contains("SET ")) { - // add to existing clause - expression.replace("SET ", s"SET ${keyUpdate}, ") - } else { - // add SET clause to existing expression - s"SET $keyUpdate ${expression}" - } - } - - valueMap.put(s":$lastModifiedKey", lastModifiedDate.toString) - - update - .withUpdateExpression(newExpression) - .withValueMap(valueMap) - } - def setExpr[T](key: String, lastModifiedKey: Option[String]) = { val baseExpression = s"SET $key = :value" lastModifiedKey.fold(baseExpression)(lastModifiedKey => s"$baseExpression, $lastModifiedKey = :$lastModifiedKey") diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala index f3a5050fed0..38c2eca1b7f 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala @@ -1,33 +1,40 @@ package com.gu.mediaservice.lib.aws import com.amazonaws.services.sqs.model.SendMessageResult +import com.gu.mediaservice.lib.embeddings.EmbeddingImplementation import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} -import com.gu.mediaservice.model.{Jpeg, MimeType, Png, Tiff} +import com.gu.mediaservice.model.{Embedding, ImageMetadata, Jpeg, MimeType} import play.api.libs.json.{Json, OFormat} -import software.amazon.awssdk.services.s3vectors.model.QueryVectorsResponse -import software.amazon.awssdk.services.s3vectors.model.{QueryOutputVector, QueryVectorsResponse, VectorData} -import java.nio.file.{Files, Path} import scala.concurrent.{ExecutionContext, Future} -import scala.jdk.CollectionConverters.CollectionHasAsScala -case class EmbedderMessage(imageId: String, fileType: String, s3Bucket: String, s3Key: String) +case class EmbedderMessage(imageId: String, fileType: String, s3Bucket: String, s3Key: String, instance: String) object EmbedderMessage { implicit val format: OFormat[EmbedderMessage] = Json.format[EmbedderMessage] } -class Embedder(bedrock: Bedrock, sqs: SimpleSqsMessageConsumer)(implicit ec: ExecutionContext) extends GridLogging { +case class EmbeddingSourceImageFormat(longestAxis: Int, format: MimeType = Jpeg, letterBox: Boolean) + +class Embedder(embedding: EmbeddingImplementation, sqs: SimpleSqsMessageConsumer)(implicit ec: ExecutionContext) extends GridLogging { def createQueryEmbedding(query: String)(implicit logMarker: LogMarker): Future[List[Float]] = { logger.info(logMarker, s"Creating text embedding for query: $query") for { - embedding <- bedrock.createTextEmbedding(query) + embedding <- embedding.createTextEmbedding(query) } yield embedding } + def createImageEmbedding(source: Array[Byte], maybeMetadata: Option[ImageMetadata])(implicit logMarker: LogMarker): Future[Embedding] = { + logger.info(logMarker, s"Creating image embedding") + embedding.createImageEmbeddings(source, maybeMetadata) + } + def queueImageToEmbed(message: EmbedderMessage)(implicit logMarker: LogMarker) = { val messageBody = Json.stringify(Json.toJson(message)) val result: SendMessageResult = sqs.sendMessage(messageBody) logger.info(logMarker, s"Queued image for embedding with message ID: ${result.getMessageId}") } + + def embeddingSourceImageFormat(): EmbeddingSourceImageFormat = embedding.embeddingSourceImageFormat() + } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/InstanceAwareDynamoDB.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/InstanceAwareDynamoDB.scala new file mode 100644 index 00000000000..fded5dba41a --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/InstanceAwareDynamoDB.scala @@ -0,0 +1,228 @@ +package com.gu.mediaservice.lib.aws + +import com.gu.mediaservice.lib.aws.DynamoDB.deleteExpr +import com.gu.mediaservice.lib.logging.GridLogging +import com.gu.mediaservice.model.Instance +import org.joda.time.DateTime +import play.api.libs.json._ +import software.amazon.awssdk.enhanced.dynamodb._ +import software.amazon.awssdk.enhanced.dynamodb.document.EnhancedDocument +import software.amazon.awssdk.services.dynamodb.DynamoDbClient +import software.amazon.awssdk.services.dynamodb.model.{BatchGetItemRequest, QueryRequest, UpdateItemRequest, AttributeValue => AttributeValueV2, KeysAndAttributes => KeysAndAttributesV2, ReturnValue => ReturnValueV2} + +import scala.annotation.tailrec +import scala.concurrent.{ExecutionContext, Future} +import scala.jdk.CollectionConverters._ + +/** + * A lightweight wrapper around AWS dynamo SDK for undertaking various operations + * @param client2 DynamoDbClient client + * @param tableName the table name for this instance of the dynamoDB wrapper + * @param lastModifiedKey if set to a string the wrapper will maintain a last modified with that name on any update + * @tparam T The type of this table + */ +class InstanceAwareDynamoDB[T](client2: DynamoDbClient, tableName: String, lastModifiedKey: Option[String] = None) extends GridLogging { + lazy val dynamo2: DynamoDbEnhancedClient = DynamoDbEnhancedClient.builder().dynamoDbClient(client2).build() + lazy val tableSchema = TableSchema.documentSchemaBuilder() + .addIndexPartitionKey(TableMetadata.primaryIndexName(), InstanceKey, AttributeValueType.S) + .addIndexSortKey(TableMetadata.primaryIndexName(), IdKey, AttributeValueType.S) + .attributeConverterProviders(AttributeConverterProvider.defaultProvider()) + .build() + lazy val table2 = dynamo2.table(tableName, tableSchema) + + private val IdKey = "id" + private val InstanceKey = "instance" + + private def itemKey(id: String)(implicit instance: Instance) = { + Key.builder().partitionValue(instance.id).sortValue(id).build() + } + + def getV2(id: String)(implicit ex: ExecutionContext, instance: Instance): Future[JsObject] = Future { + table2.getItem(itemKey(id)) + } flatMap docOrNotFound map asJsObject + + private def getV2(id: String, attribute: String)(implicit ex: ExecutionContext, instance: Instance): Future[EnhancedDocument] = Future { + Option(table2.getItem(itemKey(id))).flatMap(doc => Option.when(doc.isPresent(attribute))(doc)) + } flatMap { + case Some(doc) => Future.successful(doc) + case None => Future.failed(NoItemFound) + } + + private def docOrNotFound(docOrNull: EnhancedDocument): Future[EnhancedDocument] = { + Option(docOrNull) match { + case Some(doc) => Future.successful(doc) + case None => Future.failed(NoItemFound) + } + } + + def removeKeyV2(id: String, key: String)(implicit ex: ExecutionContext, instance: Instance) = Future{ + updateV2(id, DynamoDB.removeExpr(key, lastModifiedKey)) + } + + def deleteItemV2(id: String)(implicit ex: ExecutionContext, instance: Instance): Future[Unit] = Future { + table2.deleteItem( + itemKey(id) + ) + } + def booleanGetV2(id: String, key: String) + (implicit ex: ExecutionContext, instance: Instance): Future[Boolean] = { + getV2(id, key).map(_.getBoolean(key).booleanValue()) + } + + def booleanSetV2(id: String, key: String, value: Boolean) + (implicit ex: ExecutionContext, instance: Instance): Future[JsObject] = Future { + updateV2( + id, + DynamoDB.setExpr(key, lastModifiedKey), + AttributeValueV2.fromBool(value) + ) + } + + def booleanSetOrRemoveV2(id: String, key: String, value: Boolean) + (implicit ex: ExecutionContext, instance: Instance): Future[JsObject] = + if (value) booleanSetV2(id, key, value) + else removeKeyV2(id, key) + + def stringSetV2(id: String, key: String, value: String)(implicit ex: ExecutionContext, instance: Instance): Future[JsObject] = Future { + updateV2(id, DynamoDB.setExpr(key, lastModifiedKey), AttributeValueV2.fromS(value)) + } + + def setGetV2(id: String, key: String) + (implicit ex: ExecutionContext, instance: Instance): Future[Set[String]] = { + getV2(id, key).map(_.getStringSet(key).asScala.toSet) + } + + def setAddV2(id: String, key: String, value: List[String])(implicit ex: ExecutionContext, instance: Instance): Future[JsObject] = Future { + updateV2(id, DynamoDB.addExpr(key, lastModifiedKey), AttributeValueV2.fromSs(value.asJava)) + } + + def batchGetV2(ids: List[String], attributeKey: String) + (implicit ex: ExecutionContext, rjs: Reads[T], instance: Instance): Future[Map[String, T]] = { + val keyChunkList = ids + .map(k => Map(IdKey -> AttributeValueV2.fromS(k), + InstanceKey -> AttributeValueV2.fromS(instance.id) + ).asJava) + .grouped(100) + + Future.traverse(keyChunkList) { keyChunk => { + val keysAndAttributes: KeysAndAttributesV2 = KeysAndAttributesV2.builder().keys(keyChunk.asJava).build() + + @tailrec + def nextPageOfBatch(request: java.util.Map[String, KeysAndAttributesV2], acc: List[(String, T)]) + (implicit ex: ExecutionContext, rjs: Reads[T]): List[(String, T)] = { + if (request.isEmpty) acc + else { + logger.info(s"Fetching records for $request") + val response = client2.batchGetItem(BatchGetItemRequest.builder().requestItems(request).build()) + val responses = response.responses() + logger.info(s"Got responses of $responses") + val results = responses.get(tableName).asScala.toList + .flatMap(att => { + logger.info(s"Obtained attributes of $att from response") + val json = asJsObject(EnhancedDocument.fromAttributeValueMap(att)) + val maybeT = (json \ attributeKey).asOpt[T] + logger.info(s"Obtained a T of $maybeT from json $json") + maybeT.map( + att.get(IdKey).s() -> _ + ) + }) + logger.info(s"Got $results for request") + nextPageOfBatch(response.unprocessedKeys(), acc ::: results) + } + } + + Future { + nextPageOfBatch(Map(tableName -> keysAndAttributes).asJava, Nil).toMap + } + } + } + .map(chunkIterator => chunkIterator.fold(Map.empty)((acc, result) => acc ++ result)) + } + + // We cannot update, so make sure you send over the WHOLE document + def jsonAddV2(id: String, key: String, value: Map[String, JsValue]) + (implicit ex: ExecutionContext, instance: Instance): Future[JsObject] = Future { + updateV2( + id, + setExpr(key, lastModifiedKey), + AttributeValueV2.fromM(value.view.mapValues(DynamoDB.jsonToAttributeValue).toMap.asJava) + ) + } + + def setDeleteV2(id: String, key: String, value: String) + (implicit ex: ExecutionContext, instance: Instance): Future[JsObject] = Future { + updateV2(id, deleteExpr(key, lastModifiedKey), AttributeValueV2.fromSs(List(value).asJava)) + } + + def scanForIdV2(indexName: String, keyname: String, key: String)(implicit ex: ExecutionContext, instance: Instance): Future[List[String]] = Future { + val request = QueryRequest.builder() + .tableName(tableName) + .indexName(indexName) + .keyConditionExpression(s"$keyname = :key AND instance = :instance") + .expressionAttributeValues(Map( + ":key" -> AttributeValueV2.fromS(key), + ":instance" -> AttributeValueV2.fromS(instance.id) + ).asJava) + .build() + + client2.query(request).items().asScala.toList + .flatMap(item => Option(item.get("id")).map(_.s())) + } + + private def updateRequestBuilder(id: String, expression: String)(implicit instance: Instance) = { + UpdateItemRequest.builder() + .key(Map( + InstanceKey -> AttributeValueV2.fromS(instance.id), + IdKey -> AttributeValueV2.fromS(id)).asJava + ) + .updateExpression(expression) + .returnValues(ReturnValueV2.ALL_NEW) + .tableName(tableName) + } + + def updateV2(id: String, expression: String, attribute: AttributeValueV2)(implicit instance: Instance) = { + val baseValuesMap = Map(":value" -> attribute) + val valuesMap = lastModifiedKey.fold(baseValuesMap)(key => baseValuesMap ++ Map(s":${key}" -> AttributeValueV2.fromS(DateTime.now().toString))) + val updateRequest = updateRequestBuilder(id, expression) + .expressionAttributeValues(valuesMap.asJava) + .build() + val updateItemResponse = client2.updateItem(updateRequest) + val jsonString = EnhancedDocument.fromAttributeValueMap(updateItemResponse.attributes()).toJson + Json.parse(jsonString).as[JsObject] + } + + def updateV2(id: String, expression: String)(implicit instance: Instance) = { + val valuesMap = lastModifiedKey.fold(Map.empty[String, AttributeValueV2])(key => Map(s":${key}" -> AttributeValueV2.fromS(DateTime.now().toString))) + val updateRequest = updateRequestBuilder(id, expression) + .expressionAttributeValues(valuesMap.asJava) + .build() + val updateItemResponse = client2.updateItem(updateRequest) + val jsonString = EnhancedDocument.fromAttributeValueMap(updateItemResponse.attributes()).toJson + Json.parse(jsonString).as[JsObject] + } + + def asJsObject(doc: EnhancedDocument): JsObject = + jsonWithNullAsEmptyString(Json.parse(doc.toJson)).as[JsObject] - IdKey - InstanceKey + + // FIXME: Dynamo accepts `null`, but not `""`. This is a well documented issue + // around the community. This guard keeps the introduction of `null` fairly + // fenced in this Dynamo play area. `null` is continual and big annoyance with AWS libs. + // see: https://forums.aws.amazon.com/message.jspa?messageID=389032 + // see: http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DataModel.html + def mapJsValue(jsValue: JsValue)(f: JsValue => JsValue): JsValue = jsValue match { + case JsObject(items) => JsObject(items.map{ case (k, v) => k -> mapJsValue(v)(f) }) + case JsArray(items) => JsArray(items.map(f)) + case value => f(value) + } + + def jsonWithNullAsEmptyString(jsValue: JsValue): JsValue = mapJsValue(jsValue) { + case JsNull => JsString("") + case value => value + } + + def setExpr[T](key: String, lastModifiedKey: Option[String]) = { + val baseExpression = s"SET $key = :value" + lastModifiedKey.fold(baseExpression)(lastModifiedKey => s"$baseExpression, $lastModifiedKey = :$lastModifiedKey") + } + +} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Kinesis.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Kinesis.scala index 811bfe64007..e4ef2dbac79 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Kinesis.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Kinesis.scala @@ -2,7 +2,7 @@ package com.gu.mediaservice.lib.aws import java.nio.ByteBuffer import java.util.UUID -import com.amazonaws.services.kinesis.model.PutRecordRequest +import com.amazonaws.services.kinesis.model.{PutRecordRequest, PutRecordsRequest, PutRecordsRequestEntry} import com.amazonaws.services.kinesis.{AmazonKinesis, AmazonKinesisClientBuilder} import com.gu.mediaservice.lib.json.JsonByteArrayUtil import com.gu.mediaservice.model.usage.UsageNotice @@ -10,8 +10,11 @@ import net.logstash.logback.marker.{LogstashMarker, Markers} import play.api.libs.json.{JodaWrites, Json, Writes} import com.amazonaws.auth.AWSCredentialsProvider import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} +import com.gu.mediaservice.model.Instance import org.joda.time.DateTime +import scala.jdk.CollectionConverters.SeqHasAsJava + case class KinesisSenderConfig( override val awsRegion: String, override val awsCredentials: AWSCredentialsProvider, @@ -32,12 +35,13 @@ class Kinesis(config: KinesisSenderConfig) extends GridLogging{ val partitionKey = UUID.randomUUID().toString implicit val yourJodaDateWrites: Writes[DateTime] = JodaWrites.JodaDateTimeWrites + implicit val iw: Writes[Instance] = Json.writes[Instance] implicit val unw: Writes[UsageNotice] = Json.writes[UsageNotice] val payload = JsonByteArrayUtil.toByteArray(message) val markers: LogstashMarker = message.toLogMarker.and(Markers.append("compressed-size", payload.length)) - logger.info(markers, "Publishing message to kinesis") + logger.info(markers, s"Publishing message to kinesis: ${config.streamName}") val data = ByteBuffer.wrap(payload) val request = new PutRecordRequest() @@ -55,5 +59,38 @@ class Kinesis(config: KinesisSenderConfig) extends GridLogging{ throw e } } + + def publish[T <: LogMarker](messages: Seq[T])(implicit messageWrites: Writes[T]): Unit = { + val records: Seq[PutRecordsRequestEntry] = messages.map { message => + val partitionKey = UUID.randomUUID().toString + implicit val yourJodaDateWrites: Writes[DateTime] = JodaWrites.JodaDateTimeWrites + implicit val iw: Writes[Instance] = Json.writes[Instance] + implicit val unw: Writes[UsageNotice] = Json.writes[UsageNotice] + + val payload = JsonByteArrayUtil.toByteArray(message) + val data = ByteBuffer.wrap(payload) + + val entry = new PutRecordsRequestEntry + entry.setPartitionKey(partitionKey) + entry.setData(data) + entry + } + + logger.info(s"Publishing ${messages.size} messages to kinesis: ${config.streamName}") + + val request = new PutRecordsRequest() + .withStreamName(config.streamName).withRecords(records.asJava) + + try { + val result = kinesisClient.putRecords(request) + logger.info(s"Published kinesis message: $result") + } catch { + case e: Exception => + logger.error(s"kinesis putRecord failed", e) + // propagate error forward to the client + throw e + } + } + } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index dc58e38f8ab..c001102e0d0 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -1,31 +1,31 @@ package com.gu.mediaservice.lib.aws -import com.amazonaws.services.s3.model._ +import com.amazonaws.{AmazonServiceException, ClientConfiguration} +import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials} +import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration +import com.amazonaws.services.s3.model.{Region => _, _} import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder, model} import com.amazonaws.util.IOUtils -import com.amazonaws.{AmazonServiceException, ClientConfiguration} import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch} import com.gu.mediaservice.model._ -import org.joda.time.{DateTime, Duration} +import org.joda.time.DateTime +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.s3.S3Client import java.io.File -import java.net.URI -import scala.jdk.CollectionConverters._ +import java.net.{URI, URL} import scala.concurrent.{ExecutionContext, Future} +import scala.jdk.CollectionConverters._ case class S3Object(uri: URI, size: Long, metadata: S3Metadata) object S3Object { - def objectUrl(bucket: String, key: String): URI = { - val bucketUrl = s"$bucket.${S3Ops.s3Endpoint}" - new URI("http", bucketUrl, s"/$key", null) - } - def apply(bucket: String, key: String, size: Long, metadata: S3Metadata): S3Object = - apply(objectUrl(bucket, key), size, metadata) + def apply(bucket: S3Bucket, key: String, size: Long, metadata: S3Metadata): S3Object = + apply(bucket.objectUrl(key), size, metadata) - def apply(bucket: String, key: String, file: File, mimeType: Option[MimeType], lastModified: Option[DateTime], + def apply(bucket: S3Bucket, key: String, file: File, mimeType: Option[MimeType], lastModified: Option[DateTime], meta: Map[String, String] = Map.empty, cacheControl: Option[String] = None): S3Object = { S3Object( bucket, @@ -61,32 +61,66 @@ object S3Metadata { case class S3ObjectMetadata(contentType: Option[MimeType], cacheControl: Option[String], lastModified: Option[DateTime]) class S3(config: CommonConfig) extends GridLogging with ContentDisposition with RoundedExpiration { - type Bucket = String type Key = String type UserMetadata = Map[String, String] - lazy val client: AmazonS3 = S3Ops.buildS3Client(config) + val AmazonAwsS3Endpoint: String = S3.AmazonAwsS3Endpoint - def signUrl(bucket: Bucket, url: URI, image: Image, expiration: DateTime = cachableExpiration(), imageType: ImageFileType = Source): String = { - // get path and remove leading `/` - val key: Key = url.getPath.drop(1) + private val amazonS3: AmazonS3 = S3Ops.buildS3Client(config) + private val googleS3: Option[AmazonS3] = S3Ops.buildGoogleS3Client(config) + private val localS3: Option[AmazonS3] = S3Ops.buildLocalS3Client(config) + def signUrl(bucket: S3Bucket, key: String, image: Image, expiration: DateTime = cachableExpiration(), imageType: ImageFileType = Source): String = { val contentDisposition = getContentDisposition(image, imageType, config.shortenDownloadFilename) val headers = new ResponseHeaderOverrides().withContentDisposition(contentDisposition) - val request = new GeneratePresignedUrlRequest(bucket, key).withExpiration(expiration.toDate).withResponseHeaders(headers) - client.generatePresignedUrl(request).toExternalForm + val request = new GeneratePresignedUrlRequest(bucket.bucket, key).withExpiration(expiration.toDate).withResponseHeaders(headers) + bucket.client.generatePresignedUrl(request).toExternalForm + } + + def signUrlTony(bucket: S3Bucket, key: String, expiration: DateTime = cachableExpiration()): URL = { + val request = new GeneratePresignedUrlRequest(bucket.bucket, key).withExpiration(expiration.toDate) + bucket.client.generatePresignedUrl(request) + } + + def copyObject(sourceBucket: S3Bucket, destinationBucket: S3Bucket, key: String): CopyObjectResult = { + // TODO check that source and destination share the same client + sourceBucket.client.copyObject(sourceBucket.bucket, key, destinationBucket.bucket, key) + } + + def generatePresignedRequest(request: GeneratePresignedUrlRequest, bucket: S3Bucket): URL = { + bucket.client.generatePresignedUrl(request) + } + + def deleteObject(bucket: S3Bucket, key: String): Unit = { + bucket.client.deleteObject(bucket.bucket, key) } - def getObject(bucket: Bucket, url: URI): model.S3Object = { - // get path and remove leading `/` - val key: Key = url.getPath.drop(1) - client.getObject(new GetObjectRequest(bucket, key)) + def deleteObjects(bucket: S3Bucket, keys: Seq[String]): DeleteObjectsResult = { + bucket.client.deleteObjects( + new DeleteObjectsRequest(bucket.bucket).withKeys(keys: _*) + ) + } + + def deleteVersion(bucket: S3Bucket, id: String, objectVersion: String): Unit = { + bucket.client.deleteVersion(bucket.bucket, id, objectVersion) + } + + def doesObjectExist(bucket: S3Bucket, key: String) = { + bucket.client.doesObjectExist(bucket.bucket, key) + } + + def getObject(bucket: S3Bucket, key: String): model.S3Object = { + bucket.client.getObject(new GetObjectRequest(bucket.bucket, key)) } - def getObjectAsString(bucket: Bucket, key: String): Option[String] = { - val content = client.getObject(new GetObjectRequest(bucket, key)) + def getObject(bucket: S3Bucket, obj: S3ObjectSummary): model.S3Object = { + bucket.client.getObject(bucket.bucket, obj.getKey) + } + + def getObjectAsString(bucket: S3Bucket, key: String): Option[String] = { + val content = bucket.client.getObject(new GetObjectRequest(bucket.bucket, key)) val stream = content.getObjectContent try { Some(IOUtils.toString(stream).trim) @@ -100,7 +134,31 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with } } - def store(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) + def getObjectMetadata(bucket: S3Bucket, id: String): ObjectMetadata = { + bucket.client.getObjectMetadata(bucket.bucket, id) + } + + def listObjects(bucket: S3Bucket): ObjectListing = { + bucket.client.listObjects(bucket.bucket) + } + + def listObjects(bucket: S3Bucket, prefix: String): ObjectListing = { + bucket.client.listObjects(bucket.bucket, prefix) + } + + def listObjects(bucket: S3Bucket, request: ListObjectsRequest): ObjectListing = { + bucket.client.listObjects(request) + } + + def listObjectKeys(bucket: S3Bucket): Seq[String] = { + bucket.client.listObjects(bucket.bucket).getObjectSummaries.asScala.map(_.getKey).toSeq + } + + def putObject(bucket: S3Bucket, key: String, content: String): Unit = { + bucket.client.putObject(bucket.bucket, key, content) + } + + def store(bucket: S3Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) (implicit ex: ExecutionContext, logMarker: LogMarker): Future[S3Object] = Future { val metadata = new ObjectMetadata @@ -109,25 +167,26 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with metadata.setUserMetadata(meta.asJava) val fileMarkers = Map( - "bucket" -> bucket, + "bucket" -> bucket.bucket, "fileName" -> id, "mimeType" -> mimeType.getOrElse("none"), ) val markers = logMarker ++ fileMarkers - val req = new PutObjectRequest(bucket, id, file).withMetadata(metadata) + val req = new PutObjectRequest(bucket.bucket, id, file).withMetadata(metadata) Stopwatch(s"S3 client.putObject ($req)"){ + val client = bucket.client client.putObject(req) // once we've completed the PUT read back to ensure that we are returning reality - val metadata = client.getObjectMetadata(bucket, id) + val metadata = client.getObjectMetadata(bucket.bucket, id) S3Object(bucket, id, metadata.getContentLength, S3Metadata(metadata)) }(markers) } - def storeIfNotPresent(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) + def storeIfNotPresent(bucket: S3Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) (implicit ex: ExecutionContext, logMarker: LogMarker): Future[S3Object] = { Future{ - Some(client.getObjectMetadata(bucket, id)) + Some(bucket.client.getObjectMetadata(bucket.bucket, id)) }.recover { // translate this exception into the object not existing case as3e:AmazonS3Exception if as3e.getStatusCode == 404 => None @@ -140,11 +199,11 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with } } - def list(bucket: Bucket, prefixDir: String) + def list(bucket: S3Bucket, prefixDir: String) (implicit ex: ExecutionContext): Future[List[S3Object]] = Future { - val req = new ListObjectsRequest().withBucketName(bucket).withPrefix(s"$prefixDir/") - val listing = client.listObjects(req) + val req = new ListObjectsRequest().withBucketName(bucket.bucket).withPrefix(s"$prefixDir/") + val listing = bucket.client.listObjects(req) val summaries = listing.getObjectSummaries.asScala summaries.map(summary => (summary.getKey, summary)).foldLeft(List[S3Object]()) { case (memo: List[S3Object], (key: String, summary: S3ObjectSummary)) => @@ -152,17 +211,17 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with } } - def getMetadata(bucket: Bucket, key: Key): S3Metadata = { - val meta = client.getObjectMetadata(bucket, key) + def getMetadata(bucket: S3Bucket, key: Key): S3Metadata = { + val meta = bucket.client.getObjectMetadata(bucket.bucket, key) S3Metadata(meta) } - def getUserMetadata(bucket: Bucket, key: Key): Map[Bucket, Bucket] = - client.getObjectMetadata(bucket, key).getUserMetadata.asScala.toMap + def getUserMetadata(bucket: S3Bucket, key: Key): Map[String, String] = + bucket.client.getObjectMetadata(bucket.bucket, key).getUserMetadata.asScala.toMap - def syncFindKey(bucket: Bucket, prefixName: String): Option[Key] = { - val req = new ListObjectsRequest().withBucketName(bucket).withPrefix(s"$prefixName-") - val listing = client.listObjects(req) + def syncFindKey(bucket: S3Bucket, prefixName: String): Option[Key] = { + val req = new ListObjectsRequest().withBucketName(bucket.bucket).withPrefix(s"$prefixName-") + val listing = bucket.client.listObjects(req) val summaries = listing.getObjectSummaries.asScala summaries.headOption.map(_.getKey) } @@ -170,9 +229,44 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with } object S3Ops { - // TODO make this localstack friendly - // TODO: Make this region aware - i.e. RegionUtils.getRegion(region).getServiceEndpoint(AmazonS3.ENDPOINT_PREFIX) - val s3Endpoint = "s3.amazonaws.com" + def buildGoogleS3Client(config: CommonConfig): Option[AmazonS3] = { + config.googleS3AccessKey.flatMap { accessKey => + config.googleS3SecretKey.map { secretKey => + val endpointConfig = new EndpointConfiguration("https://storage.googleapis.com", null) + // create credentials provider + val credentials = new BasicAWSCredentials(accessKey, secretKey) + val credentialsProvider = new AWSStaticCredentialsProvider(credentials) + // create a client config + val clientConfig = new ClientConfiguration() + + val clientBuilder = AmazonS3ClientBuilder.standard() + clientBuilder.setEndpointConfiguration(endpointConfig) + clientBuilder.withCredentials(credentialsProvider) + clientBuilder.withClientConfiguration(clientConfig) + clientBuilder.build() + } + } + } + + def buildLocalS3Client(config: CommonConfig): Option[AmazonS3] = { + config.googleS3AccessKey.flatMap { accessKey => + config.googleS3SecretKey.map { secretKey => + val endpointConfig = new EndpointConfiguration("https://minio.griddev.eelpieconsulting.co.uk", null) + // create credentials provider + val credentials = new BasicAWSCredentials(accessKey, secretKey) + val credentialsProvider = new AWSStaticCredentialsProvider(credentials) + // create a client config + val clientConfig = new ClientConfiguration() + + val clientBuilder = AmazonS3ClientBuilder.standard() + clientBuilder.setEndpointConfiguration(endpointConfig) + clientBuilder.withCredentials(credentialsProvider) + clientBuilder.withClientConfiguration(clientConfig) + clientBuilder.withPathStyleAccessEnabled(true) + clientBuilder.build() + } + } + } def buildS3Client(config: CommonConfig, localstackAware: Boolean = true, maybeRegionOverride: Option[String] = None): AmazonS3 = { val builder = config.awsLocalEndpoint match { @@ -186,4 +280,18 @@ object S3Ops { config.withAWSCredentials(builder, localstackAware, maybeRegionOverride).build() } + + def buildS3ClientV2(config: CommonConfig, localstackAware: Boolean = true, maybeRegionOverride: Option[Region] = None): S3Client = { + val builder = config.awsLocalEndpoint match { + case Some(_) if config.isDev => + S3Client.builder().forcePathStyle(true) + case _ => S3Client.builder() + } + + config.withAWSCredentialsV2(builder, localstackAware, maybeRegionOverride).build() + } +} + +object S3 { + val AmazonAwsS3Endpoint: String = "s3.amazonaws.com" } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala new file mode 100644 index 00000000000..9f217375589 --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala @@ -0,0 +1,29 @@ +package com.gu.mediaservice.lib.aws + +import com.amazonaws.services.s3.AmazonS3 + +import java.net.URI + +case class S3Bucket(bucket: String, endpoint: String, usesPathStyleURLs: Boolean, client: AmazonS3) { + def objectUrl(key: String): URI = { + val bucketBaseURL = bucketURL() + new URI("http", bucketBaseURL.getHost, bucketBaseURL.getPath + key, null) + } + + def keyFromS3URL(url: URI): String = { + if (usesPathStyleURLs) { + url.getPath.drop(bucket.length + 2) + } else { + url.getPath.drop(1) + } + } + + def bucketURL(): URI = { + if (usesPathStyleURLs) { + new URI("https", endpoint, s"/$bucket/", null) + } else { + new URI("https", s"$bucket.$endpoint", "/", null) + } + } + +} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala index 67a6ee77825..0af9faa80f8 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala @@ -16,7 +16,7 @@ class SimpleSqsMessageConsumer (queueUrl: String, config: CommonConfig) { new ReceiveMessageRequest(queueUrl) .withWaitTimeSeconds(20) // Wait for maximum duration (20s) as per doc recommendation: http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-long-polling.html .withMaxNumberOfMessages(1) // Pull 1 message at a time to avoid starvation - .withAttributeNames(attributeNames: _*) + .withMessageSystemAttributeNames(attributeNames: _*) ).getMessages.asScala.headOption def deleteMessage(message: SQSMessage): Unit = diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/ThrallMessageSender.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/ThrallMessageSender.scala index a7fa790454d..fe9d2958cdc 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/ThrallMessageSender.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/ThrallMessageSender.scala @@ -6,7 +6,7 @@ import com.gu.mediaservice.model.leases.MediaLease import com.gu.mediaservice.model.usage.UsageNotice import org.joda.time.{DateTime, DateTimeZone} import play.api.libs.functional.syntax.toFunctionalBuilderOps -import play.api.libs.json.{JodaReads, JodaWrites, Json, OWrites, Reads, Writes, __} +import play.api.libs.json.{JodaReads, JodaWrites, Json, OFormat, OWrites, Reads, Writes, __} // TODO MRB: replace this with the simple Kinesis class once we migrate off SNS class ThrallMessageSender(config: KinesisSenderConfig) { @@ -16,7 +16,11 @@ class ThrallMessageSender(config: KinesisSenderConfig) { kinesis.publish(updateMessage)(UpdateMessage.writes) } - def publish(externalThrallMessage: ExternalThrallMessage) = { + def publish(updateMessages: Seq[UpdateMessage]): Unit = { + kinesis.publish(updateMessages)(UpdateMessage.writes) + } + + def publish(externalThrallMessage: ExternalThrallMessage): Unit = { kinesis.publish(externalThrallMessage) } } @@ -34,6 +38,7 @@ object BulkIndexRequest { object UpdateMessage extends GridLogging { implicit val yourJodaDateReads: Reads[DateTime] = JodaReads.DefaultJodaDateTimeReads.map(d => d.withZone(DateTimeZone.UTC)) implicit val yourJodaDateWrites: Writes[DateTime] = JodaWrites.JodaDateTimeWrites + implicit val instanceFormats: OFormat[Instance] = Json.format[Instance] implicit val unw: OWrites[UsageNotice] = Json.writes[UsageNotice] implicit val unr: Reads[UsageNotice] = Json.reads[UsageNotice] implicit val writes: OWrites[UpdateMessage] = Json.writes[UpdateMessage] @@ -60,7 +65,8 @@ object UpdateMessage extends GridLogging { (__ \ "leases").readNullable[Seq[MediaLease]] ~ (__ \ "syndicationRights").readNullable[SyndicationRights] ~ (__ \ "bulkIndexRequest").readNullable[BulkIndexRequest] ~ - (__ \ "usageId").readNullable[String] + (__ \ "usageId").readNullable[String] ~ + (__ \ "instance").read[Instance] )(UpdateMessage.apply _) } @@ -80,7 +86,8 @@ case class UpdateMessage( leases: Option[Seq[MediaLease]] = None, syndicationRights: Option[SyndicationRights] = None, bulkIndexRequest: Option[BulkIndexRequest] = None, - usageId: Option[String] = None + usageId: Option[String] = None, + instance: Instance ) extends LogMarker { override def markerContents = { val message = Json.stringify(Json.toJson(this)) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessors.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessors.scala index a62276b1618..87b4f28ccd3 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessors.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessors.scala @@ -265,7 +265,7 @@ trait CanonicalisingImageProcessor extends ImageProcessor { object ApParser extends ImageProcessor { val InvisionFor = "^invision for (.+)".r - val PersonInvisionAp = "(.+)\\s*/invision/ap$".r + val IntermediaryAp = "(.+)/ap$".r def getSuppliersReference(image: Image) = { image.fileMetadata.readXmpHeadStringProp("plus:ImageSupplierImageID").orElse(image.metadata.suppliersReference) @@ -275,15 +275,24 @@ object ApParser extends ImageProcessor { } def apply(image: Image): Image = image.metadata.credit.map(_.toLowerCase) match { - case Some("ap") | Some("associated press") => image.copy( + case Some("ap") | Some("ap photo") | Some("associated press") => image.copy( usageRights = Agency("AP"), metadata = image.metadata.copy(credit = Some("AP"), suppliersReference = getSuppliersReference(image)) ) case Some("invision") | Some("invision/ap") | - Some(InvisionFor(_)) | Some(PersonInvisionAp(_)) => image.copy( + Some(InvisionFor(_)) => image.copy( usageRights = Agency("AP", Some("Invision")), metadata = image.metadata.copy(suppliersReference = getSuppliersReference(image)) ) + case Some(IntermediaryAp(_)) => + val collection = image.metadata.credit.map(c => c.replaceAll("(?i)/ap$", "")) + image.copy( + usageRights = Agency("AP", collection), + metadata = image.metadata.copy( + credit = image.metadata.credit.map(c => c.replaceAll("(?i)/ap$", "/AP")), + suppliersReference = getSuppliersReference(image) + ) + ) case _ => image } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/collections/CollectionPaths.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/collections/CollectionPaths.scala new file mode 100644 index 00000000000..c9bb0584b5d --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/collections/CollectionPaths.scala @@ -0,0 +1,18 @@ +package com.gu.mediaservice.lib.collections + +trait CollectionPaths { + + private val delimiter = "/" + private val doublequotes = "\"" + + def stringToPath(s: String): List[String] = s.split(delimiter).toList + + def pathToString(path: Seq[String]): String = path.mkString(delimiter) + + def pathToPathId(path: Seq[String]): String = pathToString(path).toLowerCase + + + // We could use `ValidationNel`s here, but that's overkill + def isValidPathBit(s: String): Boolean = if (s.contains(delimiter) || s.contains(doublequotes)) false else true + +} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/collections/CollectionsManager.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/collections/CollectionsManager.scala index 0e3c6bfc2c3..d8100351c0d 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/collections/CollectionsManager.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/collections/CollectionsManager.scala @@ -4,14 +4,10 @@ import com.gu.mediaservice.lib.net.URI.{encode, decode} import com.gu.mediaservice.model.Collection -object CollectionsManager { - val delimiter = "/" - val doublequotes = "\"" +object CollectionsManager extends CollectionPaths with CssColours { - def stringToPath(s: String) = s.split(delimiter).toList - def pathToString(path: List[String]) = path.mkString(delimiter) - def pathToPathId(path: List[String]) = pathToString(path).toLowerCase def pathToUri(path: List[String]) = pathToString(path.map(encode)) + def uriToPath(uri: String) = stringToPath(decode(uri)) def sortBy(c: Collection) = c.pathId @@ -41,22 +37,4 @@ object CollectionsManager { }} } - // We could use `ValidationNel`s here, but that's overkill - def isValidPathBit(s: String) = if (s.contains(delimiter) || s.contains(doublequotes)) false else true - - // These use Source swatches - val collectionColours = Map( - "australia" -> "#185E36", - "culture" -> "#BB3B80", - "film & music" -> "#6B5840", - "g2" -> "#121212", - "guide" -> "#7D0068", - "observer" -> "#052962", - "sport" -> "#22874D", - "travel" -> "#041F4A" - ) - - def getCollectionColour(s: String) = collectionColours.get(s) - - def getCssColour(path: List[String]) = path.headOption.map(_.toLowerCase).flatMap(getCollectionColour) } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/collections/CssColours.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/collections/CssColours.scala new file mode 100644 index 00000000000..2c0216882b3 --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/collections/CssColours.scala @@ -0,0 +1,35 @@ +package com.gu.mediaservice.lib.collections + +trait CssColours extends CollectionPaths { + + // These use Source swatches + private val collectionColours = Map( + "home/biz & cash" -> "#c98a07", + "home/home news" -> "#022164", + "home/international" -> "#022164", + "home/ofm" -> "#f2327d", + "home/magazine" -> "#5b1e4a", + "home/sensemakers" -> "#3c9bf9", + "home/sport" -> "#00663b", + "home/supplements" -> "#008083", + "home" -> "#052962" + ) + + def getCssColour(path: List[String]): Option[String] = { + def forPath(depth: Int, default: Option[String]): Option[String] = { + if (depth > path.size) { + default + } else { + val pathId = pathToPathId(path.take(depth)) + collectionColours.get(pathId).map { colour => + forPath(depth + 1, Some(colour)) + }.getOrElse { + default + } + } + } + // recurse drop the path return the furthest leaf node + forPath(depth = 1, default = None) + } + +} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala index 2cdf3edea3f..a9e8690e939 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala @@ -1,11 +1,11 @@ package com.gu.mediaservice.lib.config -import com.gu.mediaservice.lib.aws.{AwsClientV1BuilderUtils, AwsClientV2BuilderUtils, KinesisSenderConfig} +import com.amazonaws.services.s3.AmazonS3 +import com.gu.mediaservice.lib.aws._ import com.gu.mediaservice.model.UsageRightsSpec import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import play.api.{ConfigLoader, Configuration} -import scalaz.NonEmptyList import java.net.URI import java.util.UUID @@ -33,6 +33,8 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B val localLogShipping: Boolean = sys.env.getOrElse("LOCAL_LOG_SHIPPING", "false").toBoolean + val thrallAppName = stringOpt("thrall.kinesis.app.name").getOrElse("thrall") + val thrallLowPriorityAppName = stringOpt("thrall.kinesis.lowPriority.app.name").getOrElse("thrall-low-priority") val thrallKinesisStream = string("thrall.kinesis.stream.name") val thrallKinesisLowPriorityStream = string("thrall.kinesis.lowPriorityStream.name") @@ -51,38 +53,64 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B lazy val softDeletedMetadataTable: String = string("dynamo.table.softDelete.metadata") + val googleS3AccessKey: Option[String] = stringOpt("s3.accessKey") + val googleS3SecretKey: Option[String] = stringOpt("s3.secretKey") + + private val amazonS3: AmazonS3 = S3Ops.buildS3Client(this) + private val googleS3: Option[AmazonS3] = S3Ops.buildGoogleS3Client(this) + private val localS3: Option[AmazonS3] = S3Ops.buildLocalS3Client(this) + + def clientFor(bucketEndpoint: String): AmazonS3 = { + (bucketEndpoint match { + case "storage.googleapis.com" => + googleS3 + case "minio.griddev.eelpieconsulting.co.uk" => + localS3 + case _ => + Some(amazonS3) + }).getOrElse { + amazonS3 + } + } + val maybeIngestSqsQueueUrl: Option[String] = stringOpt("sqs.ingest.queue.url") - val maybeIngestBucket: Option[String] = stringOpt("s3.ingest.bucket") - val maybeFailBucket: Option[String] = stringOpt("s3.fail.bucket") + val maybeIngestBucket: Option[S3Bucket] = for { + ingestBucket <- stringOpt("s3.ingest.bucket.name") + ingestBucketEndpoint <- stringOpt("s3.ingest.bucket.endpoint") + } yield { + S3Bucket(ingestBucket, ingestBucketEndpoint, usesPathStyleURLs = booleanOpt("s3.ingest.bucket.pathStyleURLs").getOrElse(false), clientFor(ingestBucketEndpoint)) + } + val maybeFailBucket: Option[S3Bucket] = for { + failBucket <- stringOpt("s3.fail.bucket.name") + failBucketEndpoint <- stringOpt("s3.fail.bucket.endpoint") + } yield { + S3Bucket(failBucket, failBucketEndpoint, usesPathStyleURLs = booleanOpt("s3.fail.bucket.pathStyleURLs").getOrElse(false), clientFor(failBucketEndpoint)) + } + + val maybeQuarantineBucket: Option[S3Bucket] = stringOpt("s3.quarantine.bucket.name").map(S3Bucket(_, S3.AmazonAwsS3Endpoint, booleanOpt("s3.quarantine.bucket.pathStyleURLs").getOrElse(false), clientFor(S3.AmazonAwsS3Endpoint))) - val maybeQuarantineBucket: Option[String] = stringOpt("s3.quarantine.bucket") + val maybeBucketForUIUploads: Option[S3Bucket] = maybeQuarantineBucket orElse maybeIngestBucket - val maybeBucketForUIUploads: Option[String] = maybeQuarantineBucket orElse maybeIngestBucket + val maybeUploadLimitInBytes: Option[Int] = intOpt("upload.limit.mb").map(_ * 1024 * 1024) - val maybeUploadLimitInBytes: Option[Int] = intOpt("upload.limit.mb").map(_ * 1_000_000) + val instancesEndpoint: String = string("instance.service.instances") // Note: had to make these lazy to avoid init order problems ;_; val domainRoot: String = string("domain.root") val domainRootOverride: Option[String] = stringOpt("domain.root-override") val rootAppName: String = stringDefault("app.name.root", "media") - val serviceHosts = ServiceHosts( - stringDefault("hosts.kahunaPrefix", s"$rootAppName."), - stringDefault("hosts.apiPrefix", s"api.$rootAppName."), - stringDefault("hosts.loaderPrefix", s"loader.$rootAppName."), - stringDefault("hosts.projectionPrefix", s"loader-projection.$rootAppName."), - stringDefault("hosts.cropperPrefix", s"cropper.$rootAppName."), - stringDefault("hosts.metadataPrefix", s"$rootAppName-metadata."), - stringDefault("hosts.imgopsPrefix", s"$rootAppName-imgops."), - stringDefault("hosts.usagePrefix", s"$rootAppName-usage."), - stringDefault("hosts.collectionsPrefix", s"$rootAppName-collections."), - stringDefault("hosts.leasesPrefix", s"$rootAppName-leases."), - stringDefault("hosts.authPrefix", s"$rootAppName-auth."), - stringDefault("hosts.thrallPrefix", s"thrall.$rootAppName.") - ) val corsAllowedOrigins: Set[String] = getStringSet("security.cors.allowedOrigins") - val services = new Services(domainRoot, serviceHosts, corsAllowedOrigins, domainRootOverride) + val services = new SingleHostServices(domainRoot) + + private val imageBucketEndpoint = string("s3.image.bucket.endpoint") + val imageBucket: S3Bucket = S3Bucket(string("s3.image.bucket.name"), imageBucketEndpoint, usesPathStyleURLs = booleanOpt("s3.image.bucket.pathStyleURLs").getOrElse(false), clientFor(imageBucketEndpoint)) + private val thumbBucketEndpoint = string("s3.thumb.bucket.endpoint") + val thumbnailBucket: S3Bucket = S3Bucket(string("s3.thumb.bucket.name"), thumbBucketEndpoint, usesPathStyleURLs = booleanOpt("s3.thumb.bucket.pathStyleURLs").getOrElse(false), clientFor(thumbBucketEndpoint)) + + private val embeddingSourceBucketEndpoint = string("s3.embedding.bucket.endpoint") + val embeddingSourceBucket: S3Bucket = S3Bucket(string("s3.embedding.bucket.name"), embeddingSourceBucketEndpoint, usesPathStyleURLs = booleanOpt("s3.embedding.bucket.pathStyleURLs").getOrElse(false), clientFor(embeddingSourceBucketEndpoint)) /** * Load in a list of domain metadata specifications from configuration. For example: @@ -125,6 +153,9 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B val recordDownloadAsUsage: Boolean = boolean("image.record.download") val shortenDownloadFilename: Boolean = boolean("image.download.shorten") + val myInstancesEndpoint: String = string("instance.service.my") + + val usageEventsQueueName: String = string("usageEvents.queue.name") /** * Load in a list of external staff photographers, internal staff photographers, contracted photographers, @@ -257,4 +288,5 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B private def missing(key: String, type_ : String): Nothing = sys.error(s"Required $type_ configuration property missing: $key") + } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfigWithElastic.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfigWithElastic.scala index ceb44bf61dd..9d948c2046c 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfigWithElastic.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfigWithElastic.scala @@ -16,14 +16,11 @@ class CommonConfigWithElastic(resources: GridConfigResources) extends CommonConf replicas = string("es6.replicas").toInt ) - private val persistenceIdentifier = string("persistence.identifier") val persistenceIdentifiers = NonEmptyList( - persistenceIdentifier, ImageStorageProps.derivativeOfMediaIdsIdentifierKey, ImageStorageProps.replacesMediaIdIdentifierKey ) val queriableIdentifiers = Seq( - persistenceIdentifier, ImageStorageProps.derivativeOfMediaIdsIdentifierKey, ) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/GridConfigLoader.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/GridConfigLoader.scala index 7484ccf8103..1e027209b95 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/GridConfigLoader.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/GridConfigLoader.scala @@ -57,7 +57,10 @@ object GridConfigLoader extends StrictLogging { if (file.getPath.endsWith(".properties")) { logger.warn(s"Configuring the Grid with Java properties files is deprecated as of #3011, please switch to .conf files. See #3037 for a conversion utility.") } - Configuration(ConfigFactory.parseFile(file)) + val parsed = ConfigFactory.parseFile(file) + logger.info(s"Resolving config parsed from file: $file") + val resolved = parsed.resolve() + Configuration(resolved) } else { logger.info(s"Skipping config file $file as it doesn't exist") Configuration.empty diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/InstanceForRequest.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/InstanceForRequest.scala new file mode 100644 index 00000000000..45da79a9c95 --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/InstanceForRequest.scala @@ -0,0 +1,13 @@ +package com.gu.mediaservice.lib.config + +import com.gu.mediaservice.model.Instance +import play.api.mvc.RequestHeader + +trait InstanceForRequest { + + def instanceOf(request: RequestHeader): Instance = { + // TODO some sort of filter supplied attribute + Instance(request.host.split("\\.").head) + } + +} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/Services.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/Services.scala index 896acf8549f..57f7ffc96d9 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/Services.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/Services.scala @@ -1,91 +1,80 @@ package com.gu.mediaservice.lib.config -case class ServiceHosts( - kahunaPrefix: String, - apiPrefix: String, - loaderPrefix: String, - projectionPrefix: String, - cropperPrefix: String, - metadataPrefix: String, - imgopsPrefix: String, - usagePrefix: String, - collectionsPrefix: String, - leasesPrefix: String, - authPrefix: String, - thrallPrefix: String -) - -object ServiceHosts { - // this is tightly coupled to the Guardian's deployment. - // TODO make more generic but w/out relying on Play config - def guardianPrefixes: ServiceHosts = { - val rootAppName: String = "media" - - ServiceHosts( - kahunaPrefix = s"$rootAppName.", - apiPrefix = s"api.$rootAppName.", - loaderPrefix = s"loader.$rootAppName.", - projectionPrefix = s"loader-projection.$rootAppName", - cropperPrefix = s"cropper.$rootAppName.", - metadataPrefix = s"$rootAppName-metadata.", - imgopsPrefix = s"$rootAppName-imgops.", - usagePrefix = s"$rootAppName-usage.", - collectionsPrefix = s"$rootAppName-collections.", - leasesPrefix = s"$rootAppName-leases.", - authPrefix = s"$rootAppName-auth.", - thrallPrefix = s"thrall.$rootAppName." - ) - } +import com.gu.mediaservice.model.Instance + +trait Services { + + def kahunaBaseUri(instance: Instance): String + + def apiBaseUri(instance: Instance): String + + def loaderBaseUri(instance: Instance): String + + def projectionBaseUri(instance: Instance): String + + def cropperBaseUri(instance: Instance): String + + def metadataBaseUri(instance: Instance): String + + def imgopsBaseUri(instance: Instance): String + + def usageBaseUri(instance: Instance): String + + def collectionsBaseUri(instance: Instance): String + + def leasesBaseUri(instance: Instance): String + + def authBaseUri(instance: Instance): String + def authBaseInstanceUri(instance: Instance): String + + def guardianWitnessBaseUri: String + + def corsAllowedDomains(instance: Instance): Set[String] + + def redirectUriParam: String + + def redirectUriPlaceholder: String + + def loginUriTemplate(instance: Instance): String } -class Services(val domainRoot: String, hosts: ServiceHosts, corsAllowedOrigins: Set[String], domainRootOverride: Option[String] = None) { - val kahunaHost: String = s"${hosts.kahunaPrefix}$domainRoot" - val apiHost: String = s"${hosts.apiPrefix}$domainRoot" - val loaderHost: String = s"${hosts.loaderPrefix}${domainRootOverride.getOrElse(domainRoot)}" - val cropperHost: String = s"${hosts.cropperPrefix}${domainRootOverride.getOrElse(domainRoot)}" - val metadataHost: String = s"${hosts.metadataPrefix}${domainRootOverride.getOrElse(domainRoot)}" - val imgopsHost: String = s"${hosts.imgopsPrefix}${domainRootOverride.getOrElse(domainRoot)}" - val usageHost: String = s"${hosts.usagePrefix}${domainRootOverride.getOrElse(domainRoot)}" - val collectionsHost: String = s"${hosts.collectionsPrefix}${domainRootOverride.getOrElse(domainRoot)}" - val leasesHost: String = s"${hosts.leasesPrefix}${domainRootOverride.getOrElse(domainRoot)}" - val authHost: String = s"${hosts.authPrefix}$domainRoot" - val projectionHost: String = s"${hosts.projectionPrefix}${domainRootOverride.getOrElse(domainRoot)}" - val thrallHost: String = s"${hosts.thrallPrefix}${domainRootOverride.getOrElse(domainRoot)}" - - - val kahunaBaseUri = baseUri(kahunaHost) - val apiBaseUri = baseUri(apiHost) - val loaderBaseUri = baseUri(loaderHost) - val projectionBaseUri = baseUri(projectionHost) - val cropperBaseUri = baseUri(cropperHost) - val metadataBaseUri = baseUri(metadataHost) - val imgopsBaseUri = baseUri(imgopsHost) - val usageBaseUri = baseUri(usageHost) - val collectionsBaseUri = baseUri(collectionsHost) - val leasesBaseUri = baseUri(leasesHost) - val authBaseUri = baseUri(authHost) - val thrallBaseUri = baseUri(thrallHost) - - val allInternalUris = Seq( - kahunaBaseUri, - apiBaseUri, - loaderBaseUri, - cropperBaseUri, - metadataBaseUri, - usageBaseUri, - collectionsBaseUri, - leasesBaseUri, - authBaseUri, - thrallBaseUri - ) +protected class SingleHostServices(val domain: String) extends Services { + override def kahunaBaseUri(instance: Instance): String = vhostServiceName("", instance) + + override def apiBaseUri(instance: Instance): String = vhostServiceName("media-api", instance) + + override def loaderBaseUri(instance: Instance): String = vhostServiceName("image-loader", instance) + + override def projectionBaseUri(instance: Instance): String = vhostServiceName("image-loader", instance) + + override def cropperBaseUri(instance: Instance): String = vhostServiceName("cropper", instance) + + override def metadataBaseUri(instance: Instance): String = vhostServiceName("metadata-editor", instance) + + override def imgopsBaseUri(instance: Instance): String= vhostServiceName("imgproxy", instance) + + override def usageBaseUri(instance: Instance): String = vhostServiceName("usage", instance) + + override def collectionsBaseUri(instance: Instance): String = vhostServiceName("collections", instance) + + override def leasesBaseUri(instance: Instance): String = vhostServiceName("leases", instance) + + override def authBaseUri(instance: Instance): String = s"https://$domain/auth" + override def authBaseInstanceUri(instance: Instance): String = vhostServiceName("auth", instance) + + private def thrallBaseUri(instance: Instance): String = vhostServiceName("thrall", instance) val guardianWitnessBaseUri: String = "https://n0ticeapis.com" - val corsAllowedDomains: Set[String] = corsAllowedOrigins.map(baseUri) + kahunaBaseUri + apiBaseUri + thrallBaseUri + override def corsAllowedDomains(instance: Instance): Set[String] = Set(kahunaBaseUri(instance), apiBaseUri(instance), thrallBaseUri(instance)) val redirectUriParam = "redirectUri" val redirectUriPlaceholder = s"{?$redirectUriParam}" - val loginUriTemplate = s"$authBaseUri/login$redirectUriPlaceholder" + def loginUriTemplate(instance: Instance): String = s"${authBaseUri(instance)}/login$redirectUriPlaceholder" - def baseUri(host: String) = s"https://$host" + private def vhostServiceName(serviceName: String, instance: Instance): String = { + val vhost = instance.id + s"https://$vhost.$domain" + (if (serviceName.nonEmpty) "/" + serviceName else "") + } } + diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchClient.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchClient.scala index ee6ca13fbd4..fb8444b4c7b 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchClient.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchClient.scala @@ -7,11 +7,14 @@ import com.sksamuel.elastic4s.http.JavaClient import com.sksamuel.elastic4s.requests.common.HealthStatus import com.sksamuel.elastic4s.requests.indexes.CreateIndexResponse import com.sksamuel.elastic4s.requests.indexes.admin.IndexExistsResponse +import com.sksamuel.elastic4s.requests.searches.SearchHit import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration._ import scala.concurrent.{Await, Future} +case class ScrolledSearchResults(hits: List[SearchHit], scrollId: Option[String]) + case class ElasticSearchImageCounts( catCount: Long, searchResponseCount: Long, @@ -23,18 +26,13 @@ trait ElasticSearchClient extends ElasticSearchExecutions with GridLogging { private val tenSeconds = Duration(10, SECONDS) private val thirtySeconds = Duration(30, SECONDS) + protected val scrollKeepAlive = 5.minutes def url: String - def imagesCurrentAlias: String - def imagesMigrationAlias: String - lazy val imagesHistoricalAlias: String = "Images_Historical" - protected val imagesIndexPrefix = "images" protected val imageType = "image" - val initialImagesIndex = "images" - def shards: Int def replicas: Int @@ -45,12 +43,12 @@ trait ElasticSearchClient extends ElasticSearchExecutions with GridLogging { } //TODO: this function should fail and cause healthcheck fails - def ensureIndexExistsAndAliasAssigned(): Unit = { - logger.info(s"Checking alias $imagesCurrentAlias is assigned to index…") - val indexForCurrentAlias = Await.result(getIndexForAlias(imagesCurrentAlias), tenSeconds) + def ensureIndexExistsAndAliasAssigned(alias: String, index: String): Unit = { + logger.info(s"Checking alias $alias is assigned to index $index") + val indexForCurrentAlias = Await.result(getIndexForAlias(alias), tenSeconds) if (indexForCurrentAlias.isEmpty) { - createIndexIfMissing(initialImagesIndex) - assignAliasTo(initialImagesIndex, imagesCurrentAlias) + createIndexIfMissing(index) + assignAliasTo(index, alias) waitUntilHealthy() } } @@ -65,9 +63,7 @@ trait ElasticSearchClient extends ElasticSearchExecutions with GridLogging { } def healthCheck(): Future[Boolean] = { - implicit val logMarker: MarkerMap = MarkerMap() - val request = search(imagesCurrentAlias) limit 0 - executeAndLog(request, "Healthcheck").map { _ => true}.recover { case _ => false} + Future.successful(true) // TODO reimplement } case class IndexWithAliases(name: String, aliases: Seq[String]) @@ -80,7 +76,7 @@ trait ElasticSearchClient extends ElasticSearchExecutions with GridLogging { }) } - def countImages(indexName: String = imagesCurrentAlias): Future[ElasticSearchImageCounts] = { + def countImages(indexName: String): Future[ElasticSearchImageCounts] = { implicit val logMarker: MarkerMap = MarkerMap() val queryCatCount = catCount(indexName) // document count only of index including live documents, not deleted documents which have not yet been removed by the merge process val queryImageSearch = search(indexName) trackTotalHits true limit 0 // hits that match the query defined in the request @@ -181,7 +177,7 @@ trait ElasticSearchClient extends ElasticSearchExecutions with GridLogging { } } - def changeAliasTo(newIndex: String, oldIndex: String, alias: String = imagesCurrentAlias): Unit = { + def changeAliasTo(newIndex: String, oldIndex: String, alias: String): Unit = { logger.info(s"Assigning alias $alias to $newIndex") val aliasActionResponse = Await.result(client.execute { aliases( @@ -192,6 +188,20 @@ trait ElasticSearchClient extends ElasticSearchExecutions with GridLogging { logger.info("Got alias action response: " + aliasActionResponse) } + def continueScrolling(scrollId: String)(implicit logMarker: LogMarker = MarkerMap()) = { + val query = searchScroll(scrollId).keepAlive(scrollKeepAlive) + executeAndLog(query, "retrieving next batch of image ids to migrate, continuation of scroll").map { response => + ScrolledSearchResults(response.result.hits.hits.toList, response.result.scrollId) + } + } + + def closeScroll(scrollId: String)(implicit logMarker: LogMarker = MarkerMap()) = { + val close = clearScroll(scrollId) + executeAndLog(close, s"Closing unwanted scroll").failed.foreach { e => + logger.error(logMarker, "ES closeScroll request failed", e) + } + } + def removeAliasFrom(index: String, alias: String) = { logger.info(s"Removing alias $alias from $index") val removeAliasResponse = Await.result(client.execute { diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MappingTest.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MappingTest.scala index f4935c04c53..0f93e7e0eba 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MappingTest.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MappingTest.scala @@ -73,7 +73,8 @@ object MappingTest { lastModified = Some(imageModified), identifiers = Map("id1" -> "value1"), uploadInfo = UploadInfo( - filename = Some("filename.jpg") + filename = Some("filename.jpg"), + isFeedUpload = Some(true) ), source = testAsset, thumbnail = Some(Asset( @@ -183,8 +184,8 @@ object MappingTest { ), digitalUsageMetadata = Some(DigitalUsageMetadata( webUrl = new URI("https://gu.com/12345"), - webTitle = "Article title", - sectionId = "uk/news", + webTitle = Some("Article title"), + sectionId = Some("uk/news"), composerUrl = Some(new URI("https://composer/api/2345678987654321345678")) )), syndicationUsageMetadata = Some(SyndicationUsageMetadata( @@ -256,7 +257,8 @@ object MappingTest { embedding = Some( Embedding( cohereEmbedEnglishV3 = Some(CohereV3Embedding(image = (0 until 1024).map(_ * 0.001).toList)), - cohereEmbedV4 = Some(CohereV4Embedding(image = (0 until 256).map(_ * 0.001).toList)) + cohereEmbedV4 = Some(CohereV4Embedding(image = (0 until 256).map(_ * 0.001).toList)), + geminiEmbedding2 = Some(GeminiEmbedding2(image = (0 until 768).map(_ * 0.001).toList)), ) ) ) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/Mappings.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/Mappings.scala index 8a87e9124db..c49837a05c7 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/Mappings.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/Mappings.scala @@ -106,6 +106,19 @@ object Mappings { efConstruction = Some(100) )) ) + )), + nonDynamicObjectField("geminiEmbedding2").copy(properties = Seq( + new DenseVectorField( + name = "image", + dims = Some(768), + index = Some(true), + similarity = Some(Cosine), + indexOptions = Some(DenseVectorIndexOptions( + `type` = DenseVectorField.Int8Hnsw, + m = Some(16), + efConstruction = Some(100) + )) + ) )) )) @@ -272,7 +285,8 @@ object Mappings { )) def uploadInfoMapping(name: String): ObjectField = nonDynamicObjectField(name).copy(properties = Seq( - keywordField("filename") + keywordField("filename"), + booleanField("isFeedUpload"), )) def usageReference(name: String): ObjectField = { diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MigrationStatusProvider.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MigrationStatusProvider.scala index 0622115303e..3080df636f4 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MigrationStatusProvider.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MigrationStatusProvider.scala @@ -1,12 +1,14 @@ package com.gu.mediaservice.lib.elasticsearch +import com.gu.mediaservice.lib.instances.InstancesClient +import com.gu.mediaservice.model.Instance import org.apache.pekko.actor.Scheduler -import com.sksamuel.elastic4s.Index +import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicReference import scala.concurrent.Await -import scala.concurrent.duration.DurationInt import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.duration.{Duration, DurationInt, SECONDS} sealed trait MigrationStatus @@ -36,12 +38,24 @@ object MigrationStatusProvider { trait MigrationStatusProvider { self: ElasticSearchClient => + def elasticSearchConfig: ElasticSearchConfig + + def imagesCurrentAlias(instance: Instance): String = instance.id + "_" + elasticSearchConfig.aliases.current + def imagesMigrationAlias(instance: Instance): String = instance.id + "_" + elasticSearchConfig.aliases.migration + def imagesHistoricalAlias(instance: Instance): String = instance.id + "_" + "Images_Historical" + def scheduler: Scheduler - private val migrationStatusRef = new AtomicReference[MigrationStatus](fetchMigrationStatus(bubbleErrors = true)) + def instancesClient: InstancesClient + + private val migrationStatues: ConcurrentHashMap[String, AtomicReference[MigrationStatus]] = new ConcurrentHashMap() + + private def migrationStatusRef(instance: Instance): AtomicReference[MigrationStatus] = { + migrationStatues.getOrDefault(instance.id, new AtomicReference(fetchMigrationStatus(bubbleErrors = true, instance = instance))) + } - private def fetchMigrationStatus(bubbleErrors: Boolean): MigrationStatus = { - val statusFuture = getIndexForAlias(imagesMigrationAlias) + private def fetchMigrationStatus(bubbleErrors: Boolean, instance: Instance): MigrationStatus = { + val statusFuture = getIndexForAlias(imagesMigrationAlias(instance)) .map { case Some(index) if index.aliases.contains(MigrationStatusProvider.COMPLETION_PREVIEW_ALIAS) => CompletionPreview(index.name) case Some(index) if index.aliases.contains(MigrationStatusProvider.PAUSED_ALIAS) => Paused(index.name) @@ -54,30 +68,32 @@ trait MigrationStatusProvider { } catch { case e if !bubbleErrors => logger.error("Failed to get name of index for ongoing migration", e) - StatusRefreshError(cause = e, preErrorStatus = migrationStatusRef.get()) + StatusRefreshError(cause = e, preErrorStatus = migrationStatusRef(instance).get) } } - private def refreshMigrationStatus(): Unit = { - migrationStatusRef.set( - fetchMigrationStatus(bubbleErrors = false) - ) + private def refreshMigrationStatus(instance: Instance): Unit = { + migrationStatues.put(instance.id, new AtomicReference(fetchMigrationStatus(bubbleErrors = false, instance = instance))) } private val migrationStatusRefresher = scheduler.scheduleAtFixedRate( initialDelay = 0.seconds, - interval = 5.seconds - ) { () => refreshMigrationStatus() } - - def migrationStatus: MigrationStatus = migrationStatusRef.get() - def migrationIsInProgress: Boolean = migrationStatus.isInstanceOf[InProgress] - def refreshAndRetrieveMigrationStatus(): MigrationStatus = { - refreshMigrationStatus() - migrationStatus + interval = 1.minutes + ) { () => { + val instances = Await.result(instancesClient.getInstances(), Duration(10, SECONDS)) + instances.foreach(refreshMigrationStatus) + } + } + + def migrationStatus(implicit instance: Instance): MigrationStatus = migrationStatusRef(instance).get() + def migrationIsInProgress(implicit instance: Instance): Boolean = migrationStatus.isInstanceOf[InProgress] + def refreshAndRetrieveMigrationStatus(instance: Instance): MigrationStatus = { + refreshMigrationStatus(instance) + migrationStatus(instance) } - def migrationStatusRefresherHealth: Option[String] = { - migrationStatusRef.get() match { + def migrationStatusRefresherHealth(implicit instance: Instance): Option[String] = { + migrationStatusRef(instance).get match { case StatusRefreshError(_, _) => Some("Could not determine status of migration") case _ => None } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ReapableEligibility.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ReapableEligibility.scala index 56cc1a2eec0..d8dc2e76fbb 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ReapableEligibility.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ReapableEligibility.scala @@ -18,7 +18,7 @@ trait ReapableEligibility extends Provider{ val persistenceIdentifiers: NonEmptyList[String] // typically from config private def moreThanTwentyDaysOld = - filters.date("uploadTime", None, Some(DateTime.now().minusDays(20))).getOrElse(matchAllQuery()) + filters.date("uploadTime", None, Some(DateTime.now().minusDays(ReapableEligibility.ReapableAfterMoreThanDaysOld))).getOrElse(matchAllQuery()) private lazy val persistedQueries = filters.or( PersistedQueries.hasCrops, @@ -35,8 +35,22 @@ trait ReapableEligibility extends Provider{ PersistedQueries.isInPersistedCollection(maybePersistOnlyTheseCollections) ) + private def isFeedUpload = + filters.boolTerm("uploadInfo.isFeedUpload", value = true) + def query: Query = filters.and( moreThanTwentyDaysOld, + isFeedUpload, + filters.not(persistedQueries) + ) + + def preview: Query = filters.and( + isFeedUpload, filters.not(persistedQueries) ) + +} + +object ReapableEligibility { + val ReapableAfterMoreThanDaysOld: Int = 20 } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/embeddings/EmbeddingImplementation.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/embeddings/EmbeddingImplementation.scala new file mode 100644 index 00000000000..31a66fe6c0b --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/embeddings/EmbeddingImplementation.scala @@ -0,0 +1,13 @@ +package com.gu.mediaservice.lib.embeddings + +import com.gu.mediaservice.lib.aws.EmbeddingSourceImageFormat +import com.gu.mediaservice.lib.logging.LogMarker +import com.gu.mediaservice.model.{Embedding, ImageMetadata} + +import scala.concurrent.{ExecutionContext, Future} + +trait EmbeddingImplementation { + def createImageEmbeddings(source: Array[Byte], maybeMetadata: Option[ImageMetadata])(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Embedding] + def createTextEmbedding(query: String)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[List[Float]] + def embeddingSourceImageFormat(): EmbeddingSourceImageFormat +} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/embeddings/GoogleCloudEmbedding.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/embeddings/GoogleCloudEmbedding.scala new file mode 100644 index 00000000000..a724d1b2448 --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/embeddings/GoogleCloudEmbedding.scala @@ -0,0 +1,57 @@ +package com.gu.mediaservice.lib.embeddings + +import com.google.genai.Client +import com.google.genai.types._ +import com.gu.mediaservice.lib.aws.EmbeddingSourceImageFormat +import com.gu.mediaservice.lib.logging.LogMarker +import com.gu.mediaservice.model.{Embedding, GeminiEmbedding2, ImageMetadata, Png} + +import scala.compat.java8.OptionConverters.RichOptionalGeneric +import scala.concurrent.{ExecutionContext, Future} +import scala.jdk.CollectionConverters._ + +class GoogleCloudEmbedding(projectId: String, location: String) extends EmbeddingImplementation { + private val client = Client.builder().vertexAI(true).project(projectId).location(location).build() + + private val modelId = "gemini-embedding-2" + + private val embedContentConfig = EmbedContentConfig.builder() + .outputDimensionality(768) + .build() + + def createImageEmbeddings(source: Array[Byte], maybeMetadata: Option[ImageMetadata])(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Embedding] = { + Future { + val imagePart = Some(Part.fromBytes(source, embeddingSourceImageFormat().format.name)) + val titlePart = maybeMetadata.flatMap(_.title.map(Part.fromText)) + val descriptionPart = maybeMetadata.flatMap(_.description.map(Part.fromText)) + + val parts = List(imagePart, titlePart, descriptionPart).flatten.asJava + + val content = Content.builder(). + parts(parts). + build() + + val response = client.models.embedContent(modelId, content, embedContentConfig) + + val embeddings = firstEmbeddingFromResponse(response) + Embedding(geminiEmbedding2 = Some(GeminiEmbedding2(embeddings.map(_.toDouble)))) + } + } + + def createTextEmbedding(query: String)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[List[Float]] = { + Future { + val q = query + val response = client.models.embedContent(modelId, q, embedContentConfig) + firstEmbeddingFromResponse(response) + } + } + + def embeddingSourceImageFormat(): EmbeddingSourceImageFormat = EmbeddingSourceImageFormat(longestAxis = 768, format = Png, letterBox = true) + + private def firstEmbeddingFromResponse(response: EmbedContentResponse): List[Float] = { + val a: Seq[ContentEmbedding] = response.embeddings().asScala.map(_.asScala.toSeq).getOrElse(Seq.empty) + val v = a.head.values().asScala.map(_.asScala).getOrElse(Seq.empty).toSeq + v.map(_.floatValue()).toList + } + +} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/events/UsageEvents.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/events/UsageEvents.scala new file mode 100644 index 00000000000..6b4265ee654 --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/events/UsageEvents.scala @@ -0,0 +1,97 @@ +package com.gu.mediaservice.lib.events + +import org.apache.pekko.actor.{Actor, ActorSystem, Props} +import org.apache.pekko.pattern.ask +import org.apache.pekko.util.Timeout +import com.gu.mediaservice.lib.logging.GridLogging +import com.gu.mediaservice.model.Instance +import org.joda.time.DateTime +import play.api.inject.ApplicationLifecycle +import play.api.libs.json.{JodaWrites, Json, OWrites} +import software.amazon.awssdk.services.sqs.SqsClient +import software.amazon.awssdk.services.sqs.model.SendMessageRequest + +import scala.concurrent.duration.DurationInt +import scala.util.Random + +class UsageEvents(actorSystem: ActorSystem, applicationLifecycle: ApplicationLifecycle, sqsClient: SqsClient, queueUrl: String) { + + private val random = new Random() + private val usageEventsActor = actorSystem.actorOf(UsageEventsActor.props(sqsClient, queueUrl), s"usageeventsactor-${random.alphanumeric.take(8).mkString}") + + applicationLifecycle.addStopHook(() => (usageEventsActor ? UsageEventsActor.Shutdown)(Timeout(5.seconds))) + + def successfulIngestFromQueue(instance: Instance, image: String, filesize: Long): Unit = { + usageEventsActor ! UsageEvent(`type` = "imageIngest", instance = instance.id, image = Some(image), filesize = Some(filesize)) + } + + def prepareUpload(instance: Instance, image: String, apiKey: Option[String], user: Option[String]): Unit = { + usageEventsActor ! UsageEvent(`type` = "prepareUpload", instance = instance.id, image = Some(image), apiKey = apiKey, user = user) + } + + def uploadImage(instance: Instance, image: String, filesize: Long, apiKey: Option[String], user: Option[String]): Unit = { + usageEventsActor ! UsageEvent(`type` = "imageUpload", instance = instance.id, image = Some(image), filesize = Some(filesize), apiKey = apiKey, user = user) + } + + def downloadOriginal(instance: Instance, image: String, filesize: Option[Long], apiKey: Option[String], user: Option[String]): Unit = { + usageEventsActor ! UsageEvent(`type` = "downloadOriginal", instance = instance.id, image = Some(image), filesize = filesize, apiKey = apiKey, user = user) + } + + def softDelete(instance: Instance, image: String): Unit = { + usageEventsActor ! UsageEvent(`type` = "softDelete", instance = instance.id, image = Some(image), filesize = None, apiKey = None, user = None) + } + + def unsoftDelete(instance: Instance, image: String): Unit = { + usageEventsActor ! UsageEvent(`type` = "unsoftDelete", instance = instance.id, image = Some(image), filesize = None, apiKey = None, user = None) + } + + def deleteImage(instance: Instance, image: String): Unit = { + usageEventsActor ! UsageEvent(`type` = "deleteImage", instance = instance.id, image = Some(image), filesize = None, apiKey = None, user = None) + } + + def hardDeleteImage(instance: Instance, image: String) = { + usageEventsActor ! UsageEvent(`type` = "hardDeleteImage", instance = instance.id, image = Some(image), filesize = None, apiKey = None, user = None) + } + + def apiKeyUsed(instance: Instance, apiKey: String) = { + usageEventsActor ! UsageEvent(`type` = "apiKeyUsed", instance = instance.id, apiKey = Some(apiKey)) + } + + def userAuthed(instance: Instance, user: String) = { + usageEventsActor ! UsageEvent(`type` = "userAuthed", instance = instance.id, user = Some(user)) + } +} + +case class UsageEvent(`type`: String, instance: String, + image: Option[String] = None, + filesize: Option[Long] = None, + date: DateTime = DateTime.now, + apiKey: Option[String] = None, + user: Option[String] = None) + +object UsageEvent extends JodaWrites { + implicit val uew: OWrites[UsageEvent] = Json.writes[UsageEvent] +} + + +object UsageEventsActor { + def props(sqsClient: SqsClient, queueUrl: String): Props = + Props(new UsageEventsActor(sqsClient, queueUrl)) + + final case object Shutdown +} + + +private class UsageEventsActor(sqsClient: SqsClient, queueUrl: String) extends Actor with GridLogging { + override def receive: Receive = { + case usageEvent: UsageEvent => + logger.info("Got usageEvent: " + usageEvent) + send(usageEvent) + } + + private def send(usageEvent: UsageEvent): Unit = { + import play.api.libs.json._ + sqsClient.sendMessage(SendMessageRequest.builder.queueUrl(queueUrl).messageBody(Json.stringify(Json.toJson(usageEvent))).build) + } + +} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/guardian/GuardianUsageRightsConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/guardian/GuardianUsageRightsConfig.scala index 3bdeb73b3dd..d96cca5d610 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/guardian/GuardianUsageRightsConfig.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/guardian/GuardianUsageRightsConfig.scala @@ -130,7 +130,6 @@ object GuardianUsageRightsConfig extends UsageRightsConfigProvider { PublicationPhotographer("Steve Bell"), )), PublicationPhotographers(ObserverPublication, List( - PublicationPhotographer("Chris Riddell"), PublicationPhotographer("David Foldvari"), PublicationPhotographer("David Simonds"), )) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/imaging/ImageOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/imaging/ImageOperations.scala index d07fe814fa4..4d09d7a86a0 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/imaging/ImageOperations.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/imaging/ImageOperations.scala @@ -1,24 +1,28 @@ package com.gu.mediaservice.lib.imaging -import java.io._ -import org.im4java.core.IMOperation -import com.gu.mediaservice.lib.Files._ -import com.gu.mediaservice.lib.{BrowserViewableImage, StorableThumbImage} -import com.gu.mediaservice.lib.imaging.ImageOperations.{optimisedMimeType, thumbMimeType} -import com.gu.mediaservice.lib.imaging.im4jwrapper.ImageMagick.{addDestImage, addImage, format, runIdentifyCmd} -import com.gu.mediaservice.lib.imaging.im4jwrapper.{ExifTool, ImageMagick} +import app.photofox.vipsffm.enums.{VipsCompassDirection, VipsIntent, VipsInterpretation} +import app.photofox.vipsffm.jextract.VipsRaw +import app.photofox.vipsffm.{VBlob, VImage, VipsHelper, VipsOption} +import com.adobe.internal.xmp.options.SerializeOptions +import com.adobe.internal.xmp.{XMPConst, XMPMetaFactory} +import com.gu.mediaservice.lib.BrowserViewableImage +import com.gu.mediaservice.lib.aws.EmbeddingSourceImageFormat +import com.gu.mediaservice.lib.imaging.ImageOperations.thumbMimeType +import com.gu.mediaservice.lib.imaging.im4jwrapper.ImageMagick import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch, addLogMarkers} import com.gu.mediaservice.model._ +import org.im4java.core.IMOperation +import java.io._ +import java.lang.foreign.Arena +import java.nio.charset.StandardCharsets import scala.concurrent.{ExecutionContext, Future} -import scala.sys.process._ case class ExportResult(id: String, masterCrop: Asset, othersizings: List[Asset]) class UnsupportedCropOutputTypeException extends Exception class ImageOperations(playPath: String) extends GridLogging { - import ExifTool._ import ImageMagick._ private def profilePath(fileName: String): String = s"$playPath/$fileName" @@ -36,97 +40,108 @@ class ImageOperations(playPath: String) extends GridLogging { "Greyscale" -> profilePath("grayscale.icc") ) - private def tagFilter(metadata: ImageMetadata) = { - Map[String, Option[String]]( - "Copyright" -> metadata.copyright, - "Credit" -> metadata.credit, - "OriginalTransmissionReference" -> metadata.suppliersReference - ).collect { case (key, Some(value)) => (key, value) } - } + def cropImageVips( + sourceFile: File, + bounds: Bounds, + metadata: ImageMetadata, + orientationMetadata: Option[OrientationMetadata] + )(implicit logMarker: LogMarker, arena: Arena): VImage = { + // Read source image + val image = VImage.newFromFile(arena, sourceFile.getAbsolutePath) + + // Orient + val rotated = orientationMetadata.map(_.orientationCorrection()).map { angle => + image.rotate(angle) + }.getOrElse { + image + } - private def applyOutputProfile(base: IMOperation, optimised: Boolean = false) = profile(base)(rgbProfileLocation(optimised)) - - // Optionally apply transforms to the base operation if the colour space - // in the ICC profile doesn't match the colour model of the image data - private def correctColour(base: IMOperation)(iccColourSpace: Option[String], colourModel: Option[String], isTransformedFromSource: Boolean)(implicit logMarker: LogMarker): IMOperation = { - (iccColourSpace, colourModel, isTransformedFromSource) match { - // If matching, all is well, just pass through - case (icc, model, _) if icc == model => base - // If no colour model detected, we can't do anything anyway so just hope all is well - case (_, None, _) => base - // Do not correct colour if file has already been transformed (ie. source file was TIFF) as correctColour has already been run - case (_, _, true) => base - // If mismatching, strip any (incorrect) ICC profile and inject a profile matching the model - // Note: Strip both ICC and ICM (Windows variant?) to be safe - case (icc, Some(model), _) => - profileLocations.get(model) match { - // If this is a supported model, strip profile from base and add profile for model - case Some(location) => profile(stripProfile(base)("icm,icc"))(location) - // Do not attempt to correct colour if we don't support that colour model - case None => - logger.warn( - logMarker, - s"Wanted to update colour model where iccColourSpace=$icc and colourModel=$model but we don't have a profile file for that model" - ) - base - } + val cropped = rotated.extractArea(bounds.x, bounds.y, bounds.width, bounds.height) + + // If we saw and ICC profile than we will need to transform + val needsICCTransform = VipsHelper.image_get_typeof(arena, image.getUnsafeStructAddress, "icc-profile-data") != 0 + val correctedForICCProfile = if (needsICCTransform) { + cropped.iccTransform("srgb", + VipsOption.Enum("intent", VipsIntent.INTENT_PERCEPTUAL), // Helps with CMYK; see https://github.com/libvips/libvips/issues/1110 + ) + } else { + // LAB gets corrupted by a needless icc_transform + cropped } + + // Apply crop metadata + // https://developers.google.com/search/docs/appearance/structured-data/image-license-metadata#iptc-photo-metadata + makeXmpBlog(metadata).foreach { xmpBlob => + logger.info("Tagging master crop with XMP metadata: " + new String(xmpBlob)) + VipsHelper.image_set_blob_copy(arena, correctedForICCProfile.getUnsafeStructAddress, "xmp-data", VBlob.newFromBytes(arena, xmpBlob).getUnsafeDataAddress, xmpBlob.length) + } + + correctedForICCProfile } - def cropImage( - sourceFile: File, - sourceMimeType: Option[MimeType], - bounds: Bounds, - qual: Double = 100d, - tempDir: File, - iccColourSpace: Option[String], - colourModel: Option[String], - fileType: MimeType, - isTransformedFromSource: Boolean, - orientationMetadata: Option[OrientationMetadata] - )(implicit logMarker: LogMarker): Future[File] = Stopwatch.async("magick crop image") { - for { - outputFile <- createTempFile(s"crop-", s"${fileType.fileExtension}", tempDir) - cropSource = addImage(sourceFile) - oriented = orient(cropSource, orientationMetadata) - qualified = quality(oriented)(qual) - corrected = correctColour(qualified)(iccColourSpace, colourModel, isTransformedFromSource) - converted = applyOutputProfile(corrected) - stripped = stripMeta(converted) - profiled = applyOutputProfile(stripped) - cropped = crop(profiled)(bounds) - depthAdjusted = depth(cropped)(8) - addOutput = addDestImage(depthAdjusted)(outputFile) - _ <- runConvertCmd(addOutput, useImageMagick = sourceMimeType.contains(Tiff)) - _ <- checkForOutputFileChange(outputFile) + private def makeXmpBlog(metadata: ImageMetadata): Option[Array[Byte]] = { + val mappings: Seq[(String, String, String)] = Seq(metadata.byline.map { creator => + (XMPConst.NS_DC, "creator", creator) + }, + metadata.credit.map { credit => + (XMPConst.NS_PHOTOSHOP, "Credit", credit) + }, + metadata.copyright.map { copyright => + (XMPConst.NS_DC, "rights", copyright) + }, + metadata.suppliersReference.map { suppliersReference => + (XMPConst.NS_PHOTOSHOP, "TransmissionReference", suppliersReference) + }).flatten + + mappings.headOption.map { _ => + val xmpMeta = XMPMetaFactory.create() + mappings.foreach { mapping => + xmpMeta.setProperty(mapping._1, mapping._2, mapping._3) + } + + val serializeOptions = new SerializeOptions() + serializeOptions.setUseCompactFormat(true) + serializeOptions.setUseCanonicalFormat(false) + val xmpXml = XMPMetaFactory.serializeToString(xmpMeta, serializeOptions) + xmpXml.getBytes(StandardCharsets.UTF_8) } - yield outputFile } - // Updates metadata on existing file - def appendMetadata(sourceFile: File, metadata: ImageMetadata): Future[File] = { - runExiftoolCmd( - setTags(tagSource(sourceFile))(tagFilter(metadata)) - ).map(_ => sourceFile) + def createCrops(sourceImage: VImage, dimensionList: List[Dimensions], imageId: String, bounds: Bounds, cropType: MimeType, tempDir: File, cropQuality: Int + )(implicit logMarker: LogMarker, instance: Instance, arena: Arena): Future[Seq[(File, String, Dimensions)]] = { + Stopwatch(s"Resizing crops for $imageId") { + logger.info("Starting resizes") + val resizes = dimensionList.map { dimensions => + val outputFile = File.createTempFile(s"resize-", s"${cropType.fileExtension}", tempDir) // TODO function for this + + resizeImageVips(sourceImage, dimensions, cropQuality, outputFile, cropType).map { f => + def outputFilename(imageId: String, bounds: Bounds, outputWidth: Int, fileType: MimeType, isMaster: Boolean = false, instance: Instance): String = { // TODO push back to Crops + val masterString: String = if (isMaster) "master/" else "" + instance.id + "/" + s"$imageId/${Crop.getCropId(bounds)}/$masterString$outputWidth${fileType.fileExtension}" + } + + val filename = outputFilename(imageId, bounds, dimensions.width, cropType, instance = instance) + (f, filename, dimensions) + } + } + logger.info("Done resizes") + Future.sequence(resizes) + } } - def resizeImage( - sourceFile: File, - sourceMimeType: Option[MimeType], - dimensions: Dimensions, - qual: Double = 100d, - tempDir: File, - fileType: MimeType - )(implicit logMarker: LogMarker): Future[File] = Stopwatch.async("magick resize image") { - for { - outputFile <- createTempFile(s"resize-", s".${fileType.fileExtension}", tempDir) - resizeSource = addImage(sourceFile) - qualified = quality(resizeSource)(qual) - resized = scale(qualified)(dimensions) - addOutput = addDestImage(resized)(outputFile) - _ <- runConvertCmd(addOutput, useImageMagick = sourceMimeType.contains(Tiff)) + def resizeImageVips( + sourceImage: VImage, + dimensions: Dimensions, + quality: Int = 100, + outputFile: File, + fileType: MimeType + )(implicit logMarker: LogMarker, arena: Arena): Future[File] = { + Future { + val scale = dimensions.width.toDouble / sourceImage.getWidth.toDouble + val resized = sourceImage.resize(scale) + + saveImageToFile(resized, fileType, quality, outputFile, quantise = true, keep = Some(VipsRaw.VIPS_FOREIGN_KEEP_XMP)) } - yield outputFile } private def orient(op: IMOperation, orientationMetadata: Option[OrientationMetadata]): IMOperation = { @@ -137,192 +152,235 @@ class ImageOperations(playPath: String) extends GridLogging { } } - def optimiseImage(resizedFile: File, mediaType: MimeType)(implicit logMarker: LogMarker): File = mediaType match { - case Png => - val fileName: String = resizedFile.getAbsolutePath + val interlacedHow = "Line" + val backgroundColour = "#333333" - val optimisedImageName: String = fileName.split('.')(0) + "optimised.png" - Stopwatch("pngquant") { - Seq("pngquant", "-s10", "--quality", "1-85", fileName, "--output", optimisedImageName).! - } + /** + * Given a source file containing an image (the 'browser viewable' file), + * construct a thumbnail file in the provided temp directory, and return + * the file with metadata about it. + * + * @param browserViewableImage + * @param width Desired with of thumbnail + * @param qual Desired quality of thumbnail + * @param outputFile Location to create thumbnail file + * @param orientationMetadata OrientationMetadata for rotation correction + * @return The file created and the mimetype of the content of that file and it's dimensions, in a future. + */ + def createThumbnailVips(browserViewableImage: BrowserViewableImage, + width: Int, + qual: Double = 100d, + outputFile: File, + orientationMetadata: Option[OrientationMetadata] + )(implicit logMarker: LogMarker): Future[(File, MimeType, Option[Dimensions])] = { + Future { + val stopwatch = Stopwatch.start + val arena = Arena.ofConfined + + try { + val thumbnail = VImage.thumbnail(arena, browserViewableImage.file.getAbsolutePath, width, + VipsOption.Boolean("auto-rotate", false), + VipsOption.Enum("intent", VipsIntent.INTENT_PERCEPTUAL), + VipsOption.String("export-profile", "srgb") + ) + val rotated = orientationMetadata.map(_.orientationCorrection()).map { angle => + logger.info("Rotating thumbnail: " + angle) + thumbnail.rotate(angle) + }.getOrElse { + thumbnail + } + logger.info("Created thumbnail: " + rotated.getWidth + "x" + rotated.getHeight) + saveImageToFile(rotated, Jpeg, qual.toInt, outputFile) - new File(optimisedImageName) - case Jpeg => resizedFile + val thumbDimensions = Some(Dimensions(rotated.getWidth, rotated.getHeight)) + arena.close() + + logger.info(addLogMarkers(stopwatch.elapsed), "Finished creating thumbnail") + (outputFile, thumbMimeType, thumbDimensions) + + } catch { + case e: Throwable => + arena.close() + throw e + } - // This should never happen as we only ever crop as PNG or JPEG. See `Crops.cropType` and `CropsTest` - // TODO We should create a `CroppingMimeType` to enforce this at the type level. - // However we'd need to change the `Asset` model as source image and crop use this model - // and a source can legally be a `Tiff`. It's not a small change... - case Tiff => - logger.error("Attempting to optimize a Tiff crop. Cropping as Tiff is not supported.") - throw new UnsupportedCropOutputTypeException + }.recoverWith { + case e: Throwable => + logger.error("Error creating thumbnail", e) + Future.failed(e) + } } - val thumbUnsharpRadius = 0.5d - val thumbUnsharpSigma = 0.5d - val thumbUnsharpAmount = 0.8d - val interlacedHow = "Line" - val backgroundColour = "#333333" + // Given the path to an original image return a rendering of it which + // can be ingested by an embedding prediction end point. + def createEmbeddingSource(originalImageFile: File, + orientationMetadata: Option[OrientationMetadata], + embeddingSourceImageFormat: EmbeddingSourceImageFormat + ): Future[Option[Array[Byte]]] = { + Future { + val arena = Arena.ofConfined + + val embeddingLongestAxis = embeddingSourceImageFormat.longestAxis + val embeddingFormat = embeddingSourceImageFormat.format + + try { + val thumbnail = VImage.thumbnail(arena, originalImageFile.getAbsolutePath, embeddingLongestAxis, + VipsOption.Boolean("auto-rotate", false), + VipsOption.Enum("intent", VipsIntent.INTENT_PERCEPTUAL), + VipsOption.String("export-profile", "srgb") + ) + val rotated = orientationMetadata.map(_.orientationCorrection()).map { angle => + logger.info("Rotating thumbnail: " + angle) + thumbnail.rotate(angle) + }.getOrElse { + thumbnail + } + logger.info("Created embedding source: " + rotated.getWidth + "x" + rotated.getHeight) + + // Letter box to preserve aspect ratio of subjects + val letterBoxed = if (embeddingSourceImageFormat.letterBox) { + rotated.gravity( + VipsCompassDirection.COMPASS_DIRECTION_CENTRE, + embeddingLongestAxis, + embeddingLongestAxis, + ) + } else { + rotated + } - /** - * Given a source file containing an image (the 'browser viewable' file), - * construct a thumbnail file in the provided temp directory, and return - * the file with metadata about it. - * @param browserViewableImage - * @param width Desired with of thumbnail - * @param qual Desired quality of thumbnail - * @param outputFile Location to create thumbnail file - * @param iccColourSpace (Approximately) number of colours to use - * @param colourModel Colour model - eg RGB or CMYK - * @return The file created and the mimetype of the content of that file, in a future. - */ - def createThumbnail(browserViewableImage: BrowserViewableImage, - width: Int, - qual: Double = 100d, - outputFile: File, - iccColourSpace: Option[String], - colourModel: Option[String], - orientationMetadata: Option[OrientationMetadata] - )(implicit logMarker: LogMarker): Future[(File, MimeType)] = { - val stopwatch = Stopwatch.start + // Extract to image bytes + val buffer = new ByteArrayOutputStream() + letterBoxed.writeToStream(buffer, embeddingFormat.fileExtension, VipsOption.Boolean("strip", true)) - val cropSource = addImage(browserViewableImage.file) - val orientated = orient(cropSource, orientationMetadata) - val thumbnailed = thumbnail(orientated)(width) - val corrected = correctColour(thumbnailed)(iccColourSpace, colourModel, browserViewableImage.isTransformedFromSource) - val converted = applyOutputProfile(corrected, optimised = true) - val stripped = stripMeta(converted) - val profiled = applyOutputProfile(stripped, optimised = true) - val withBackground = setBackgroundColour(profiled)(backgroundColour) - val flattened = flatten(withBackground) - val unsharpened = unsharp(flattened)(thumbUnsharpRadius, thumbUnsharpSigma, thumbUnsharpAmount) - val qualified = quality(unsharpened)(qual) - val interlaced = interlace(qualified)(interlacedHow) - val addOutput = {file:File => addDestImage(interlaced)(file)} - for { - _ <- runConvertCmd(addOutput(outputFile), useImageMagick = browserViewableImage.mimeType == Tiff) - _ = logger.info(addLogMarkers(stopwatch.elapsed), "Finished creating thumbnail") - } yield (outputFile, thumbMimeType) - } + val bytes = buffer.toByteArray + val embeddingSource = bytes + logger.info("Created embedding source with length: " + embeddingSource.length) + Some(embeddingSource) - /** - * Given a source file containing a file which requires optimising to make it suitable for viewing in - * a browser, construct a new image file in the provided temp directory, and return - * * the file with metadata about it. - * @param sourceFile File containing browser viewable (ie not too big or colourful) image - * @param sourceMimeType Mime time of browser viewable file - * @param tempDir Location to create optimised file - * @return The file created and the mimetype of the content of that file, in a future. - */ - def transformImage(sourceFile: File, sourceMimeType: Option[MimeType], tempDir: File)(implicit logMarker: LogMarker): Future[(File, MimeType)] = { - val stopwatch = Stopwatch.start - for { - // png suffix is used by imagemagick to infer the required type - outputFile <- createTempFile(s"transformed-", optimisedMimeType.fileExtension, tempDir) - transformSource = addImage(sourceFile) - converted = applyOutputProfile(transformSource, optimised = true) - stripped = stripMeta(converted) - profiled = applyOutputProfile(stripped, optimised = true) - depthAdjusted = depth(profiled)(8) - addOutput = addDestImage(depthAdjusted)(outputFile) - _ <- runConvertCmd(addOutput, useImageMagick = sourceMimeType.contains(Tiff)) - _ <- checkForOutputFileChange(outputFile) - _ = logger.info(addLogMarkers(stopwatch.elapsed), "Finished creating browser-viewable image") - } yield (outputFile, optimisedMimeType) - } + } catch { + case e: Throwable => + arena.close() + throw e + } - // When a layered tiff is unpacked, the temp file (blah.something) is moved - // to blah-0.something and contains the composite layer (which is what we want). - // Other layers are then saved as blah-1.something etc. - // As the file has been renamed, the file object still exists, but has the wrong name - // We will need to put it back where it is expected to be found, and clean up the other - // files. - private def checkForOutputFileChange(f: File): Future[Unit] = Future { - val fileBits = f.getAbsolutePath.split("\\.").toList - val mainPart = fileBits.dropRight(1).mkString(".") - val extension = fileBits.last - - // f2 is the blah-0 name that gets created from a layered tiff. - val f2 = new File(List(s"$mainPart-0", extension).mkString(".")) - if (f2.exists()) { - // f HAS been renamed to blah-0. Rename it right back! - f2.renameTo(f) - // Tidy up any other files (blah-1,2,3 etc will be created for each subsequent layer) - cleanUpLayerFiles(mainPart, extension, 1) + }.recoverWith { + case e: Throwable => + logger.error("Error creating embedding source", e) + Future.failed(e) } } - @scala.annotation.tailrec - private def cleanUpLayerFiles(mainPart: String, extension: String, index: Int):Unit = { - val newFile = List(s"$mainPart-$index", extension).mkString(".") - val f3 = new File(newFile) - if (f3.exists()) { - f3.delete() - cleanUpLayerFiles(mainPart, extension, index+1) - } - } + def saveImageToFile(image: VImage, mimeType: MimeType, quality: Int, outputFile: File, quantise: Boolean = false, keep: Option[Int] = None): File = { + val k = keep.getOrElse(VipsRaw.VIPS_FOREIGN_KEEP_NONE) + mimeType match { + case Jpeg => + image.jpegsave(outputFile.getAbsolutePath, + VipsOption.Int("Q", quality), + //VipsOption.Boolean("optimize-scans", true), + VipsOption.Boolean("optimize-coding", true), + //VipsOption.Boolean("interlace", true), + //VipsOption.Boolean("trellis-quant", true), + // VipsOption.Int("quant-table", 3), + VipsOption.Boolean("strip", true), + VipsOption.Int("keep", k) + ) + outputFile + case Png => + // We are allowed to quantise PNG crops but not the master + if (quantise) { + image.pngsave(outputFile.getAbsolutePath, + VipsOption.Boolean("palette", true), + VipsOption.Int("Q", quality), + VipsOption.Int("effort", 1), + //VipsOption.Int("compression", 6), + VipsOption.Boolean("strip", true), + VipsOption.Int("keep", k) + ) + } else { + image.pngsave(outputFile.getAbsolutePath, + //VipsOption.Int("compression", 6), + VipsOption.Boolean("strip", true), + VipsOption.Int("keep", k) + ) + } + outputFile + + case _ => + logger.error(s"Save to $mimeType is not supported.") + throw new UnsupportedCropOutputTypeException + } + } } -object ImageOperations { +object ImageOperations extends GridLogging { val thumbMimeType = Jpeg val optimisedMimeType = Png - def identifyColourModel(sourceFile: File, mimeType: MimeType)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[String]] = { - // TODO: use mimeType to lookup other properties once we support other formats - mimeType match { - case Jpeg => - val source = addImage(sourceFile) - val formatter = format(source)("%[JPEG-Colorspace-Name]") - - for { - output <- runIdentifyCmd(formatter, false) - colourModel = output.headOption - } yield colourModel match { - case Some("GRAYSCALE") => Some("Greyscale") - case Some("CMYK") => Some("CMYK") - case _ => Some("RGB") - } - case Tiff => - val op = new IMOperation() - val formatter = format(op)("%[colorspace]") - val withSource = addDestImage(formatter)(sourceFile) - - for { - output <- runIdentifyCmd(withSource, true) - colourModel = output.headOption - } yield colourModel match { - case Some("sRGB") => Some("RGB") - case Some("Gray") => Some("Greyscale") - case Some("CIELab") => Some("LAB") - // IM returns doubles for TIFFs with transparency… - case Some("sRGBsRGB") => Some("RGB") - case Some("GrayGray") => Some("Greyscale") - case Some("CIELabCIELab") => Some("LAB") - case Some("CMYKCMYK") => Some("CMYK") - // …and triples for TIFFs with transparency and alpha channel(s). I think. - case Some("sRGBsRGBsRGB") => Some("RGB") - case Some("GrayGrayGray") => Some("Greyscale") - case Some("CIELabCIELabCIELab") => Some("LAB") - case Some("CMYKCMYKCMYK") => Some("CMYK") - case _ => colourModel + def getImageInformation(sourceFile: File)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[(Option[Dimensions], Option[OrientationMetadata], Option[String], Map[String, String])] = { + val stopwatch = Stopwatch.start + Future { + var dimensions: Option[Dimensions] = None + var maybeExifOrientationWhichTransformsImage: Option[OrientationMetadata] = None + var colourModel: Option[String] = None + var colourModelInformation: Map[String, String] = Map.empty + + implicit val arena: Arena = Arena.ofConfined + try { + val image = VImage.newFromFile(arena, sourceFile.getAbsolutePath) + + dimensions = Some(Dimensions(width = image.getWidth, height = image.getHeight)) + + val exifOrientation = VipsHelper.image_get_orientation(image.getUnsafeStructAddress) + val orientation = Some(OrientationMetadata( + exifOrientation = Some(exifOrientation) + )) + maybeExifOrientationWhichTransformsImage = Seq(orientation).flatten.find(_.transformsImage()) + + val interpretationRawValue = VipsHelper.image_get_interpretation(image.getUnsafeStructAddress) + // TODO better way to go straight from int to enum? + val maybeInterpretation = VipsInterpretation.values().toSeq.find(_.getRawValue == interpretationRawValue) + colourModel = maybeInterpretation match { + case Some(VipsInterpretation.INTERPRETATION_B_W) => Some("Greyscale") + case Some(VipsInterpretation.INTERPRETATION_CMYK) => Some("CMYK") + case Some(VipsInterpretation.INTERPRETATION_LAB) => Some("LAB") + case Some(VipsInterpretation.INTERPRETATION_LABS) => Some("LAB") + case Some(VipsInterpretation.INTERPRETATION_RGB16) => Some("RGB") + case Some(VipsInterpretation.INTERPRETATION_sRGB) => Some("RGB") + case _ => None } - case Png => - val op = new IMOperation() - val formatter = format(op)("%[colorspace]") - val withSource = addDestImage(formatter)(sourceFile) - - for { - output <- runIdentifyCmd(withSource, true) - colourModel = output.headOption - } yield colourModel match { - case Some("sRGB") => Some("RGB") - case Some("Gray") => Some("Greyscale") - case _ => Some("RGB") + + colourModelInformation = Map { + "hasAlpha" -> hasAlpha(image).toString } - case _ => - // assume that the colour model is RGB for other image types - Future.successful(Some("RGB")) + } catch { + case e: Exception => + logger.error("Error during getImageInformation", e) + arena.close() + throw e + } + arena.close() + + (dimensions, maybeExifOrientationWhichTransformsImage, colourModel, colourModelInformation) + }.map { result => + logger.info(addLogMarkers(stopwatch.elapsed), "Finished getImageInformation") + result } } + + def hasAlpha(image: VImage)(implicit arena: Arena): Boolean = image.hasAlpha + + def isGraphicVips(image: VImage)(implicit arena: Arena): Boolean = { + val numberOfBands = VipsHelper.image_get_bands(image.getUnsafeStructAddress) + logger.info("Number of bands: " + numberOfBands) + // Indexed plus alpha would be 2 bands + + val format = VipsHelper.image_get_format(image.getUnsafeStructAddress) + logger.info("Format: " + format) + + val paletteType = VipsHelper.image_get_typeof(arena, image.getUnsafeStructAddress, "palette") + logger.info("Palette type: " + paletteType) + + paletteType > 0 || numberOfBands < 3 + } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/imaging/im4jwrapper/ExifTool.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/imaging/im4jwrapper/ExifTool.scala deleted file mode 100644 index 88ea75dbdd5..00000000000 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/imaging/im4jwrapper/ExifTool.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.gu.mediaservice.lib.imaging.im4jwrapper - -import java.util.concurrent.Executors - -import java.io.File -import scala.concurrent.{Future, ExecutionContext} -import org.im4java.core.{ETOperation, ExiftoolCmd} - - -object ExifTool { - private implicit val ctx: ExecutionContext = - ExecutionContext.fromExecutor(Executors.newFixedThreadPool(Config.imagingThreadPoolSize)) - - def tagSource(source: File): ETOperation = { - val op = new ETOperation() - op.addImage(source.getAbsolutePath) - op - } - - def setTags(ops: ETOperation)(tags: Map[String, String]): ETOperation = { - tags.foldLeft(ops) { case (ops, (key, value)) => - ops.setTags(s"$key=$value") - } - } - - def overwriteOriginal(ops: ETOperation): ETOperation = { - ops.overwrite_original() - ops - } - - def runExiftoolCmd(ops: ETOperation): Future[Unit] = { - // Set overwrite original to ensure temporary file deletion - overwriteOriginal(ops) - Future((new ExiftoolCmd).run(ops)) - } -} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/imaging/im4jwrapper/ImageMagick.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/imaging/im4jwrapper/ImageMagick.scala index 302eb189f00..72a164730dc 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/imaging/im4jwrapper/ImageMagick.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/imaging/im4jwrapper/ImageMagick.scala @@ -87,14 +87,6 @@ object ImageMagick extends GridLogging { op } - def runConvertCmd(op: IMOperation, useImageMagick: Boolean)(implicit logMarker: LogMarker): Future[Unit] = { - Stopwatch.async(s"Using ${if(useImageMagick) "imagemagick" else "graphicsmagick"} for imaging conversion operation '$op'") { - Future { - new ConvertCmd(!useImageMagick).run(op) - } - } - } - def runIdentifyCmd(op: IMOperation, useImageMagick: Boolean)(implicit logMarker: LogMarker): Future[List[String]] = { Stopwatch.async(s"Using ${if (useImageMagick) "imagemagick" else "graphicsmagick"} for imaging identification operation '$op'") { Future { diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/instances/Instances.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/instances/Instances.scala new file mode 100644 index 00000000000..8f37d67710a --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/instances/Instances.scala @@ -0,0 +1,42 @@ +package com.gu.mediaservice.lib.instances + +import com.gu.mediaservice.lib.config.CommonConfig +import com.gu.mediaservice.model.Instance +import com.typesafe.scalalogging.StrictLogging +import play.api.libs.json.{Json, Reads} +import play.api.libs.ws.{WSClient, WSResponse} + +import scala.concurrent.{ExecutionContext, Future} + +trait Instances extends StrictLogging { + def config: CommonConfig + + def wsClient: WSClient + + def getInstances()(implicit ec: ExecutionContext): Future[Seq[Instance]] = { + wsClient.url(config.instancesEndpoint).get().map { r => + handleInstancesResponse(r) + } + } + + def getMyInstances(owner: String)(implicit ec: ExecutionContext): Future[Seq[Instance]] = { + wsClient.url(config.myInstancesEndpoint).withQueryStringParameters("owner" -> owner).get().map { r => + handleInstancesResponse(r) + } + } + + private def handleInstancesResponse(r: WSResponse): Seq[Instance] = { + r.status match { + case 200 => + implicit val ir: Reads[Instance] = Json.reads[Instance] + Json.parse(r.body).as[Seq[Instance]] + case 404 => + logger.warn("Got 404 status for instances call; returning no permissions") + Seq.empty + case _ => + logger.error("Got non 200 status for instances call: " + r.status) + throw new RuntimeException("Could not load instances") + } + } + +} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/instances/InstancesClient.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/instances/InstancesClient.scala new file mode 100644 index 00000000000..ada7fc7df04 --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/instances/InstancesClient.scala @@ -0,0 +1,6 @@ +package com.gu.mediaservice.lib.instances + +import com.gu.mediaservice.lib.config.CommonConfig +import play.api.libs.ws.WSClient + +class InstancesClient(val config: CommonConfig, val wsClient: WSClient) extends Instances diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/metadata/ImageMetadataConverter.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/metadata/ImageMetadataConverter.scala index 0e73246752c..a8b13ec6b99 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/metadata/ImageMetadataConverter.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/metadata/ImageMetadataConverter.scala @@ -144,13 +144,13 @@ object ImageMetadataConverter extends GridLogging { DateTimeFormat.forPattern("E MMM dd HH:mm:ss.SSS 'BST' yyyy").withZone(DateTimeZone.forOffsetHours(1)), DateTimeFormat.forPattern("E MMM dd HH:mm:ss 'BST' yyyy").withZone(DateTimeZone.forOffsetHours(1)), + // TODO these group of formatters are locale dependent. Is this intentional? DateTimeFormat.forPattern("yyyyMMdd"), DateTimeFormat.forPattern("yyyyMM"), DateTimeFormat.forPattern("yyyyddMM"), DateTimeFormat.forPattern("yyyy"), DateTimeFormat.forPattern("yyyy-MM"), DateTimeFormat.forPattern("yyyy:MM:dd"), - DateTimeFormat.forPattern("yyyy-MM-dd"), // 2014-12-16 - Maybe it's just a date // no timezone provided so force UTC rather than use the machine's timezone diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/metadata/SoftDeletedMetadataTable.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/metadata/SoftDeletedMetadataTable.scala index e20edf694a5..5315a5844f3 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/metadata/SoftDeletedMetadataTable.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/metadata/SoftDeletedMetadataTable.scala @@ -1,11 +1,10 @@ package com.gu.mediaservice.lib.metadata -import com.amazonaws.services.dynamodbv2.model.BatchWriteItemResult import com.gu.mediaservice.lib.config.CommonConfig -import com.gu.mediaservice.model.ImageStatusRecord +import com.gu.mediaservice.model.{ImageStatusRecord, Instance} import org.scanamo._ -import org.scanamo.syntax._ import org.scanamo.generic.auto._ +import org.scanamo.syntax._ import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient import scala.concurrent.{ExecutionContext, Future} @@ -15,8 +14,8 @@ class SoftDeletedMetadataTable(config: CommonConfig) { private val softDeletedMetadataTable = Table[ImageStatusRecord](config.softDeletedMetadataTable) - def getStatus(imageId: String)(implicit ex: ExecutionContext) = { - ScanamoAsync(client).exec(softDeletedMetadataTable.get("id" === imageId)) + def getStatus(imageId: String)(implicit ex: ExecutionContext, instance: Instance) = { + ScanamoAsync(client).exec(softDeletedMetadataTable.get("id" === imageId and "instance" === instance.id)) } def setStatus(imageStatus: ImageStatusRecord)(implicit ex: ExecutionContext) = { @@ -28,18 +27,22 @@ class SoftDeletedMetadataTable(config: CommonConfig) { else ScanamoAsync(client).exec(softDeletedMetadataTable.putAll(imageStatuses)) } - def clearStatuses(imageIds: Set[String])(implicit ex: ExecutionContext) = { + def clearStatuses(imageIds: Set[String])(implicit ex: ExecutionContext, instance: Instance) = { if (imageIds.isEmpty) Future.successful(List.empty) - else ScanamoAsync(client).exec(softDeletedMetadataTable.deleteAll("id" in imageIds)) - } - - def updateStatus(imageId: String, isDeleted: Boolean)(implicit ex: ExecutionContext) = { + else { + Future.sequence(imageIds.map { id => + // Scanomo batch can't do composite keys? DSL is too confusing + ScanamoAsync(client).exec(softDeletedMetadataTable.delete("id" === id and "instance" === instance.id)) + }).map(_ => List.empty) + } } + + def updateStatus(imageId: String, isDeleted: Boolean)(implicit ex: ExecutionContext, instance: Instance) = { val updateExpression = set("isDeleted", isDeleted) ScanamoAsync(client).exec( softDeletedMetadataTable .when(attributeExists("id")) .update( - key = "id" === imageId, + "id" === imageId and "instance" === instance.id, update = updateExpression ) ) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/metrics/CloudWatchMetrics.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/metrics/CloudWatchMetrics.scala index da56dd2b823..0cbd552972f 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/metrics/CloudWatchMetrics.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/metrics/CloudWatchMetrics.scala @@ -107,11 +107,13 @@ private class MetricsActor(namespace: String, client: CloudWatchClient) extends .toSeq aggregatedMetrics.grouped(maxGroupSize).foreach(chunkedMetrics => { //can only send max 20 metrics to CW at a time + /* Yeah nah client.putMetricData(PutMetricDataRequest.builder() .namespace(namespace) .metricData(chunkedMetrics.asJava) .build() ) + */ }) logger.info(s"Put ${data.size} metric data points (aggregated to ${aggregatedMetrics.size} points) to namespace $namespace") diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/tortoise/TortoiseUsageRightsConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/tortoise/TortoiseUsageRightsConfig.scala new file mode 100644 index 00000000000..e3d17043ba0 --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/tortoise/TortoiseUsageRightsConfig.scala @@ -0,0 +1,555 @@ +package com.gu.mediaservice.lib.tortoise + +import com.gu.mediaservice.lib.config.{PublicationPhotographer, PublicationPhotographers, UsageRightsConfigProvider} +import org.joda.time.LocalDate + +object TortoiseUsageRightsConfig extends UsageRightsConfigProvider { + private val ObserverPublication = "The Observer" + + val externalStaffPhotographers: List[PublicationPhotographers] = List( + PublicationPhotographers(ObserverPublication, List( + )) + ) + + // these are people who aren't photographers by trade, but have taken photographs for us. + // This is mainly used so when we ingest photos from Picdar, we make sure we categorise + // them correctly. + // TODO: Think about removin these once Picdar is dead. + val internalStaffPhotographers = List( + PublicationPhotographers(ObserverPublication, List( + )) + ) + + val contractedPhotographers = List( + PublicationPhotographers(ObserverPublication, List( + PublicationPhotographer("Andy Hall", from = Some(LocalDate.parse("2025-04-22"))), + PublicationPhotographer("Gary Calton", from = Some(LocalDate.parse("2025-04-22"))), + PublicationPhotographer("Suki Dhanda", from = Some(LocalDate.parse("2025-04-22"))), + PublicationPhotographer("Richard Saker", from = Some(LocalDate.parse("2025-04-22"))), + PublicationPhotographer("Karen Robinson", from = Some(LocalDate.parse("2025-04-22"))), + PublicationPhotographer("Sophia Evans", from = Some(LocalDate.parse("2025-04-22"))), + PublicationPhotographer("Katherine Anne Rose", from = Some(LocalDate.parse("2025-04-22"))), + PublicationPhotographer("Antonio Olmos", from = Some(LocalDate.parse("2025-04-22"))), + PublicationPhotographer("Jonathan Lovekin", from = Some(LocalDate.parse("2025-04-22"))) + )) + ) + + val staffIllustrators = List( + ) + + val contractIllustrators = List( + PublicationPhotographers(ObserverPublication, List( + PublicationPhotographer("Chris Riddell", from = Some(LocalDate.parse("2025-04-22"))) + )) + ) + + val creativeCommonsLicense = List( + "CC BY-4.0", "CC BY-SA-4.0", "CC BY-ND-4.0" + ) + + /* These are currently hardcoded */ + val payGettySourceList = List( + "ABC News", + "AFPTV", + "Alinari", + "Arnold Newman Collection", + "Baim Collection", + "Barrett-Jackson", + "Bob Thomas Sports Photography", + "Catwalking", + "CBS Television Stations Group RR", + "Contour", + "Contour RA", + "Corbis Premium Historical", + "Editorial Specials", + "Ercole Colombo", + "Extreme E", + "First Freedom", + "Formula E", + "Gamma-Legends", + "Getty Images Sport Classic", + "Icon Sport", + "J.LEAGUE", + "KBC - Japan", + "Klaud9", + "Kyodo News", + "Kyodo News Stills", + "LAT Image", + "Lichfield Studios Limited", + "Lonely Planet RF", + "Maggi & Maggi", + "Major League Baseball Platinum", + "Manchester City FC", + "Mondadori Portfolio Premium", + "NBA Classic", + "NBC News Archives Clips", + "Neil Leifer Collection", + "Newspix", + "NHK Video Bank Creative", + "NHK Video Bank Editorial", + "NHK Video Bank Premium", + "PA Images", + "Papixs", + "Paris Match Archive", + "Pele 10", + "Popperfoto", + "Premium Archive", + "Premium Archive Films Editorial", + "Rainer Schlegelmilch", + "Reportage Archive", + "SAMURAI JAPAN", + "SNS Group", + "Sports Illustrated", + "Sports Illustrated Classic", + "Storyful", + "Sutton Images", + "Sygma Premium", + "The Asahi Shimbun Premium", + "The Asahi Shimbun Video", + "Tottenham Hotspur FC", + "UEFA Exclusive", + "ullstein bild Premium", + "Ulrich Baumgarten", + "Vision Media", + // Here goes the list of inactive collections too + "#girlgaze", + "2VISTA", + "360cities.net Editorial", + "360cities.net RM", + "3D4Medical.com", + "3DClinic", + "40260 RF", + "ABC News", + "ABSODELS RM", + "ACP", + "Action Plus", + "Addictive Stock", + "Aflo Foto Agency RM", + "AFP Creative", + "age fotostock RM", + "Alaska Stock Images RF", + "Alaska Stock Images RR", + "Alaskan Express RF", + "All Canada Photos RM", + "Allsport Concepts", + "Altrendo", + "Altrendo RR", + "amana images RM", + "America 24-7", + "arabianEye RM", + "Arcaid Images", + "Arcaid RR", + "Arcangel Images RR", + "Archive Photos Creative", + "Arena Football League", + "Aridi", + "Art Images", + "Artville", + "ASAblanca", + "Asia Images RF", + "Asia Images RM", + "Astrakan", + "Aurora", + "Aurora Plus", + "Author's Image RF", + "AWL Images RM", + "Axiom Photographic Agency", + "Barcroft", + "Barcroft Media", + "Bettmann Creative", + "bilderlounge RR", + "Biosphoto RM", + "Black Box", + "Blend Images RM", + "Blend Images RR", + "Bloomberg Creative Photos RM", + "Boost", + "Botanica", + "Broadway.com RM", + "Built.Images RF", + "BuzzFoto", + "Caiaimage", + "Canopy RM", + "Car Culture", + "Cavan Images RM", + "CBS Watch Magazine", + "CCN Images RR", + "CGIBackgrounds", + "Check Six", + "Chic Sketch Editorial", + "Chic Sketch RF", + "China Span RM", + "Christian Science Monitor", + "CI BuzzFoto", + "CI Europa Press", + "CI FilmMagic", + "CI FilmMagic, Inc", + "CI FM Europa Press", + "CI Getty Images Entertainment", + "CI Getty Images Sport", + "CI News Feature", + "CI WI Europa Press", + "CI WireImage", + "Citizen Stock RM", + "Clerkenwell", + "clipart.com", + "Code Red", + "Codex", + "Collection Mix Subjects RM", + "Collection Vogue Paris", + "Collegiate Images", + "Colorsport", + "Comstock Images", + "Conde Nast Collection Editorial", + "Conde Nast Collection RM", + "Conde Nast Lifestyle Collection", + "Construction Photography RF", + "Contour Style", + "Contour Style Creative", + "Corbis Documentary", + "Corbis Historical Creative", + "Corbis NX", + "Corbis RF", + "Corbis RM Stills", + "Cote", + "Country Music Hall of Fame and Museum", + "CSA Images RM", + "Cuboimages RM", + "Cultura Exclusive", + "Cultura RF", + "Cultura RM", + "Cusp RM", + "Custom Medical Stock Photo RF", + "Custom Medical Stock Photo RM", + "Cut and Deal RF", + "Da Vinci Codex Atlanticus", + "Daily Express", + "DAJ", + "DAJ RM", + "De Agostini RM", + "Denkou RF", + "DigitalGlobe", + "Discovery Channel Images RM", + "DK Stock", + "Dorling Kindersley", + "Eastphoto RF", + "Eastphoto RM", + "Ecoscene RR", + "El Universal", + "Emotive Images RF", + "ESTADÃO CONTEÚDO", + "Everyday Projects", + "Eye Ubiquitous RR", + "EyeEm", + "EyeEm Premium", + "EyeEm RM", + "Eyewire", + "EyeWire Other", + "F1online RM", + "Fame Flynet Stills", + "Fancy RF", + "Federugby", + "Fever Images RF", + "Feyenoord", + "Finanzen Verlag", + "First Light", + "Flickr Flash", + "Flickr Prime", + "Flickr State", + "Flirt RF", + "FM Europa Press", + "FogStock", + "Folio Images RF", + "Folio Images RM", + "FoodPix", + "FoodShapes RF", + "Fototrove", + "Fox Entertainment Group", + "Gallo Images", + "Gamma-Features", + "GAP Photos RM", + "Garden Picture Library RM", + "Genuine Japan Creative Stills", + "Genuine Japan Editorial Stills", + "George Steinmetz", + "Getty Images - NASCAR Partners", + "Getty Images Multimedia Footage", + "Getty Images Special Access", + "Global Cricket Ventures - BCCI", + "Globe Photos", + "Globo", + "Glow RM", + "Glowimages RM", + "GoGo Images RF", + "Golden Boy Promotions", + "GoodSalt RR", + "GraphEast RF", + "GraphEast RM", + "Gulf Images RM", + "Hemera", + "Hemis.fr RM", + "Her Og Nu", + "Hero Images", + "Hero Images Corbis", + "HillCreek Pictures RF", + "Historic Map Works", + "Hoberman Collection UK RR", + "Hola Images RM", + "Hoxton", + "I Love Images RF", + "Iconic Images", + "Iconica", + "Iconotec RF", + "Ikon Images", + "Illustration Works", + "Image Farm RF", + "Image Ideas RF", + "Image Partner Media", + "Image100", + "imageBROKER RM", + "ImageDJ RF", + "Imagemore", + "ImageRite RF", + "Images Bazaar", + "Images.com RF", + "imageshop RF", + "imagesouk", + "ImageState RF", + "ImageState RM", + "Imagezoo RM", + "ImaZinS RM", + "Index Stock Images RR", + "Indian Premier League", + "Ingram Publishing RF", + "Inmagineasia", + "InsideOutPix RF", + "Inspirestock RF", + "Interact Images", + "International Speedway Corp.", + "Iromaya RF", + "IS Stock RF", + "iStock Exclusive RF", + "iStock Main", + "iStock Signature", + "iStock Signature Plus", + "iStock Vectors Plus", + "ItaliaStock", + "JLPGA", + "John Warburton-Lee RR", + "Johner Images", + "Jon Arnold Images RF", + "JTB Photo RM", + "Juice Images RF", + "Juniors Bildarchiv RM", + "Kablonk RF", + "Kallista Images", + "Keith Levit Photography RF", + "Keystone RF", + "Kobal Collection", + "Las Vegas Stock RR", + "LAT", + "LatinContent RM", + "Laughing Stock RM", + "Lifesize", + "Link Image RM", + "London Stills RR", + "Lonely Planet Images", + "LOOK", + "LuckyPix RR", + "Luxy", + "Map Resources", + "Mary Evans Picture Library RM", + "Masterfile", + "Masters", + "mauritius images RM", + "Mayo Clinic Collection", + "MedioImages", + "Melba Photo Agency RF", + "Mike King", + "Minden Pictures II", + "Minden Pictures RM", + "Mint Images RM", + "Mise En Beaute RR", + "MLBPA - The Players Choice", + "Moment RM", + "Moment Select", + "Moment Unreleased", + "Mondadori Portfolio", + "National Geographic", + "National Geographic Magazines", + "National Geographic RF", + "Nativestock", + "Nature Picture Library", + "Neovision RM", + "Nettavisen", + "New York Cosmos", + "newstockimages RF", + "NFL", + "Nordic Life", + "Nordic Photos", + "NucleusMedicalArt.com RM", + "NYonAir", + "Oceans-Image RR", + "Offside Live", + "OJO Images RM", + "OJO Plus RF", + "Old Visuals RF", + "Olive Images RF", + "Open Door Images RF", + "Open Mike Productions", + "Oxford Scientific RM", + "Pacific Stock RM", + "PANAPRESS", + "PanoramaStock RF", + "Panoramic Images RM", + "Panoramic Images RR", + "Panther Media RF", + "Paris Match Collection", + "Passage RM", + "Perspectives", + "Peter Arnold", + "Photo Exchange Bank Germany", + "PhotoAlto Agency RM", + "Photodisc", + "Photographer's Choice", + "Photographer's Choice RR", + "Photolibrary RF", + "Photolibrary RM", + "Photonica", + "Photonica World", + "Photononstop RM", + "Phototake RM", + "Phovoir RF", + "Picture Press RM", + "Pixmann RF", + "Pixta", + "Popperfoto Creative", + "Popstar Pictures", + "Popular Science", + "Portsmouth FC", + "Premium Ent", + "Private Label", + "Publisher Mix RM", + "Queerstock", + "QuickImage RF", + "QuickImage RR", + "Radius Images RF", + "Rainer Schlegelmilch", + "Real Latino RF", + "Realistic Reflections", + "Red Cover RM", + "Redlink RM", + "Refinery29 RM", + "relaximages", + "Reportage by Getty Images", + "Retrofile", + "Reunion Images", + "Riser", + "Robert Harding World Imagery", + "RooM RM", + "SAKIstyle RM", + "SambaPhoto", + "Science Faction", + "Science Faction Jewels", + "Science Photo Library RM", + "Science Source", + "ScienceFoto RM", + "Scoopt", + "Sebun", + "simple stock shots RF", + "Sites & Photos", + "Smart.MAGNA RF", + "Snapwi.re", + "SodaStyle", + "SoFood Collection RF", + "Solus", + "Somos RF", + "Sony BMG Music Entertainment", + "SPL Creative RM", + "Sport Plus", + "Starface Image Collection", + "Stock Illustration RF", + "Stock Illustration Source", + "Stock4B", + "stockbrokerXtra RF", + "Stockbyte", + "Stockbyte Global", + "stockbyway RF", + "Stockdisc", + "StockFood Creative RM", + "StockFood Creative RR", + "StockImage", + "Stone", + "Studio Harcourt", + "SuperStock RF", + "SuperStock RM", + "Swimwear by Popstar", + "Tango Stock RM", + "TAO Images RM", + "TASS", + "Taxi", + "Taxi Japan RM", + "Televisa", + "Terry O'Neill", + "TF-Images", + "the Agency Collection", + "The Axel Springer Collection", + "The England Collection", + "The Gruner & Jahr Collection", + "The Image Bank", + "The LIFE Images Collection", + "The LIFE Picture Collection", + "The LIFE Premium Collection", + "The New York Post", + "The Stock Connection RR", + "The StockPile Collection RF", + "Thinkstock", + "Tim de Waele", + "Tohoku Colour Agency RM", + "TongRo Images RF", + "Topic Images", + "Triangle", + "Trond Tandberg", + "Twenty20 RF", + "Universal Images Group", + "Untitled X-Ray", + "UpperCut RF", + "Uppercut RM", + "Urban CGI RF", + "View Stock RM", + "VII", + "VII Premium", + "VisitBritain RF", + "VisitBritain RM", + "Visual China Group Video", + "Visual Language RF", + "Visuals Unlimited", + "Warner Bros. Entertainment", + "WaterFrame RM", + "Wembley National Stadium Ltd", + "West Ham United FC", + "Westend61 RM", + "WI Europa Press", + "WIN-Initiative RM", + "WNET Collection RF", + "Workbook Stock", + "World Kabbadi League", + "World Sport Group", + "Yann Arthus-Bertrand", + "Zefa RF", + "Zen Shui RF" + ) + + val freeSuppliers = List( + "Alamy", + "AP", + "Getty Images", + "GNM", + "PA" + ) + + val suppliersCollectionExcl = Map( + "Getty Images" -> payGettySourceList + ) + +} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala index eb3194bfe5b..117d407e35d 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala @@ -1,46 +1,48 @@ package com.gu.mediaservice.lib.usage import java.net.URI -import com.amazonaws.services.dynamodbv2.document.Item import com.gu.mediaservice.model.usage._ import org.joda.time.DateTime import org.joda.time.format.ISODateTimeFormat +import software.amazon.awssdk.services.dynamodb.model.{AttributeValue => AttributeValueV2} import scala.jdk.CollectionConverters._ import scala.util.Try object ItemToMediaUsage { - def transform(item: Item): MediaUsage = { + def transformV2(attrs: java.util.Map[String, AttributeValueV2]): MediaUsage = { + val m = attrs.asScala MediaUsage( - UsageId(item.getString("usage_id")), - item.getString("grouping"), - item.getString("media_id"), - UsageType(item.getString("usage_type")), - item.getString("media_type"), - UsageStatus(item.getString("usage_status")), - Option(item.getMap[Any]("print_metadata")) - .map(_.asScala.toMap).flatMap(buildPrint), - Option(item.getMap[Any]("digital_metadata")) - .map(_.asScala.toMap).flatMap(buildDigital), - Option(item.getMap[Any]("syndication_metadata")) - .map(_.asScala.toMap).flatMap(buildSyndication), - Option(item.getMap[Any]("front_metadata")) - .map(_.asScala.toMap).flatMap(buildFront), - Option(item.getMap[Any]("download_metadata")) - .map(_.asScala.toMap).flatMap(buildDownload), - Option(item.getMap[Any]("child_metadata")) - .map(_.asScala.toMap).flatMap(buildChild), - new DateTime(item.getLong("last_modified")), - Try { - item.getLong("date_added") - }.toOption.map(new DateTime(_)), - Try { - item.getLong("date_removed") - }.toOption.map(new DateTime(_)) + UsageId(m("usage_id").s()), + m("grouping").s(), + m("media_id").s(), + UsageType(m("usage_type").s()), + m("media_type").s(), + UsageStatus(m("usage_status").s()), + m.get("print_metadata").map(_.m().asScala.view.mapValues(attrToAny).toMap).flatMap(buildPrint), + m.get("digital_metadata").map(_.m().asScala.view.mapValues(attrToAny).toMap).flatMap(buildDigital), + m.get("syndication_metadata").map(_.m().asScala.view.mapValues(attrToAny).toMap).flatMap(buildSyndication), + m.get("front_metadata").map(_.m().asScala.view.mapValues(attrToAny).toMap).flatMap(buildFront), + m.get("download_metadata").map(_.m().asScala.view.mapValues(attrToAny).toMap).flatMap(buildDownload), + m.get("child_metadata").map(_.m().asScala.view.mapValues(attrToAny).toMap).flatMap(buildChild), + new DateTime(m("last_modified").n().toLong), + Try(m("date_added").n().toLong).toOption.map(new DateTime(_)), + Try(m("date_removed").n().toLong).toOption.map(new DateTime(_)) ) } + private def attrToAny(av: AttributeValueV2): Any = { + if (av.s() != null) av.s() + else if (av.n() != null) new java.math.BigDecimal(av.n()) + else if (av.hasM) { + val linkedMap = new java.util.LinkedHashMap[String, Any]() + av.m().forEach((k, v) => linkedMap.put(k, attrToAny(v))) + linkedMap + } + else null + } + private def buildFront(metadataMap: Map[String, Any]): Option[FrontUsageMetadata] = { Try { FrontUsageMetadata( @@ -63,8 +65,8 @@ object ItemToMediaUsage { Try { DigitalUsageMetadata( URI.create(metadataMap("webUrl").asInstanceOf[String]), - metadataMap("webTitle").asInstanceOf[String], - metadataMap("sectionId").asInstanceOf[String], + metadataMap.get("webTitle").map(x => x.asInstanceOf[String]), + metadataMap.get("sectionId").map(x => x.asInstanceOf[String]), metadataMap.get("composerUrl").map(x => URI.create(x.asInstanceOf[String])) ) }.toOption diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/UsageBuilder.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/UsageBuilder.scala index eae0587c2d6..06386d6575d 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/UsageBuilder.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/UsageBuilder.scala @@ -57,7 +57,7 @@ object UsageBuilder { private def buildDigitalUsageReference(usage: MediaUsage): List[UsageReference] = { (usage.digitalUsageMetadata, usage.frontUsageMetadata) match { case (Some(metadata), None) => List( - UsageReference(FrontendUsageReference, Some(metadata.webUrl), Some(metadata.webTitle)) + UsageReference(FrontendUsageReference, Some(metadata.webUrl), metadata.webTitle) ) ++ metadata.composerUrl.map(url => UsageReference(ComposerUsageReference, Some(url))) case (None, Some(metadata)) => List( UsageReference(FrontUsageReference, None, name = Some(metadata.front)) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/Edits.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/Edits.scala index eff2b51188a..85a8ef5fe1b 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/Edits.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/Edits.scala @@ -2,11 +2,13 @@ package com.gu.mediaservice.model import java.net.{URI, URLEncoder} import com.gu.mediaservice.lib.argo.model.{Action, EmbeddedEntity} +import com.gu.mediaservice.lib.config.InstanceForRequest import org.joda.time.DateTime import play.api.libs.json._ import play.api.libs.json.JodaReads._ import play.api.libs.json.JodaWrites._ import play.api.libs.functional.syntax._ +import play.api.mvc.Request case class Edits( @@ -55,7 +57,7 @@ object Edits { } trait EditsResponse { - val metadataBaseUri: String + val metadataBaseUri: Instance => String type ArchivedEntity = EmbeddedEntity[Boolean] type SetEntity = EmbeddedEntity[Seq[EmbeddedEntity[String]]] @@ -63,11 +65,11 @@ trait EditsResponse { type UsageRightsEntity = EmbeddedEntity[UsageRights] type PhotoshootEntity = EmbeddedEntity[Photoshoot] - def editsEmbeddedEntity(id: String, edits: Edits) = + def editsEmbeddedEntity(id: String, edits: Edits)(implicit instance: Instance) = EmbeddedEntity(entityUri(id), Some(Json.toJson(edits)(editsEntity(id)))) // the types are in the arguments because of a whining scala compiler - def editsEntity(id: String): Writes[Edits] = ( + def editsEntity(id: String)(implicit instance: Instance): Writes[Edits] = ( (__ \ Edits.Archived).write[ArchivedEntity].contramap(archivedEntity(id, _: Boolean)) ~ (__ \ Edits.Labels).write[SetEntity].contramap(setEntity(id, "labels", _: List[String])) ~ (__ \ Edits.Metadata).write[MetadataEntity].contramap(metadataEntity(id, _: ImageMetadata)) ~ @@ -76,31 +78,31 @@ trait EditsResponse { (__ \ Edits.LastModified).writeNullable[DateTime] )(unlift(Edits.unapply)) - def photoshootEntity(id: String, photoshoot: Option[Photoshoot]): PhotoshootEntity = + def photoshootEntity(id: String, photoshoot: Option[Photoshoot])(implicit instance: Instance): PhotoshootEntity = EmbeddedEntity(entityUri(id, "/photoshoot"), photoshoot) - def archivedEntity(id: String, a: Boolean): ArchivedEntity = + def archivedEntity(id: String, a: Boolean)(implicit instance: Instance): ArchivedEntity = EmbeddedEntity(entityUri(id, "/archived"), Some(a)) - def metadataEntity(id: String, m: ImageMetadata): MetadataEntity = + def metadataEntity(id: String, m: ImageMetadata)(implicit instance: Instance): MetadataEntity = EmbeddedEntity(entityUri(id, "/metadata"), Some(m), actions = List( Action("set-from-usage-rights", entityUri(id, "/metadata/set-from-usage-rights"), "POST") )) - def usageRightsEntity(id: String, u: Option[UsageRights]): UsageRightsEntity = + def usageRightsEntity(id: String, u: Option[UsageRights])(implicit instance: Instance): UsageRightsEntity = u.map(i => EmbeddedEntity(entityUri(id, "/usage-rights"), Some(i))) .getOrElse(EmbeddedEntity(entityUri(id, "/usage-rights"), None)) - def setEntity(id: String, setName: String, labels: List[String]): SetEntity = + def setEntity(id: String, setName: String, labels: List[String])(implicit instance: Instance): SetEntity = EmbeddedEntity(entityUri(id, s"/$setName"), Some(labels.map(setUnitEntity(id, setName, _)))) - def setUnitEntity(id: String, setName: String, name: String): EmbeddedEntity[String] = + def setUnitEntity(id: String, setName: String, name: String)(implicit instance: Instance): EmbeddedEntity[String] = EmbeddedEntity(entityUri(id, s"/$setName/${URLEncoder.encode(name, "UTF-8")}"), Some(name)) - private def entityUri(id: String, endpoint: String = ""): URI = - URI.create(s"$metadataBaseUri/metadata/$id$endpoint") + private def entityUri(id: String, endpoint: String = "")(implicit instance: Instance): URI = + URI.create(s"${metadataBaseUri(instance)}/metadata/$id$endpoint") - def labelsUri(id: String) = entityUri(id, "/labels") + def labelsUri(id: String)(implicit instance: Instance) = entityUri(id, "/labels") - def metadataUri(id: String) = entityUri(id, "/metadata") + def metadataUri(id: String)(implicit instance: Instance) = entityUri(id, "/metadata") } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/Embedding.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/Embedding.scala index 562ca0dc871..0e9bc300d2e 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/Embedding.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/Embedding.scala @@ -7,7 +7,8 @@ import play.api.libs.json.OFormat // We currently only write V4 to ES, see the Embedding type in image-embedder-lambda case class Embedding( cohereEmbedEnglishV3: Option[CohereV3Embedding] = None, - cohereEmbedV4: Option[CohereV4Embedding] = None + cohereEmbedV4: Option[CohereV4Embedding] = None, + geminiEmbedding2: Option[GeminiEmbedding2] = None, ) case class CohereV3Embedding( @@ -26,6 +27,14 @@ object CohereV4Embedding { implicit val format: OFormat[CohereV4Embedding] = Json.format[CohereV4Embedding] } +case class GeminiEmbedding2( + image: List[Double] +) + +object GeminiEmbedding2 { + implicit val format: OFormat[GeminiEmbedding2] = Json.format[GeminiEmbedding2] +} + object Embedding { implicit val format: OFormat[Embedding] = Json.format[Embedding] } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/Image.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/Image.scala index a55929cc979..79de137fc47 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/Image.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/Image.scala @@ -44,9 +44,13 @@ case class Image( def hasNonInferredRights: Boolean = !hasInferredSyndicationRightsOrNoRights def syndicationStatus: SyndicationStatus = { - val isRightsAcquired: Boolean = syndicationRights.exists(_.isRightsAcquired) + // TODO Deduplicate this with syndicationFilter + val isOwned: Boolean = usageRights match { + case _: Photographer => true + case _ => false + } - if (!isRightsAcquired) { + if (!isOwned) { UnsuitableForSyndication } else { val hasSyndicationUsage = usages.exists(_.platform == SyndicationUsage) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/ImageStatusRecord.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/ImageStatusRecord.scala index b58ebddb9f0..c942e813211 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/ImageStatusRecord.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/ImageStatusRecord.scala @@ -6,7 +6,8 @@ case class ImageStatusRecord( id: String, deletedBy: String, deleteTime: String, - isDeleted: Boolean + isDeleted: Boolean, + instance: String ) object ImageStatusRecord { diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/Instance.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/Instance.scala new file mode 100644 index 00000000000..ba631d5bf0c --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/Instance.scala @@ -0,0 +1,5 @@ +package com.gu.mediaservice.model + +case class Instance(id: String) { + override def toString: String = id // TODO need to visit all the urls builders an make them use .id +} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/MimeType.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/MimeType.scala index 13e527625f8..14df82982d7 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/MimeType.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/MimeType.scala @@ -10,6 +10,7 @@ sealed trait MimeType { case Jpeg => "image/jpeg" case Png => "image/png" case Tiff => "image/tiff" + case Heif => "image/heif" } def fileExtension: String = s".${name.split('/').reverse.head}" @@ -22,6 +23,8 @@ object MimeType extends GridLogging { case "image/jpeg" => Jpeg case "image/png" => Png case "image/tiff" => Tiff + case "image/heif" => Heif + case "image/heic" => Heif // Support crops created in the early years of Grid (~2016) which state mime type w/out an 'image/' prefix // TODO correct these values in a reindex @@ -51,3 +54,4 @@ object Jpeg extends MimeType { object Png extends MimeType object Tiff extends MimeType +object Heif extends MimeType diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/ThrallMessage.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/ThrallMessage.scala index a80012a5d8c..434f51ef0fb 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/ThrallMessage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/ThrallMessage.scala @@ -25,13 +25,13 @@ sealed trait InternalThrallMessage extends ThrallMessage {} sealed trait MigrationMessage extends InternalThrallMessage {} -case class MigrateImageMessage(id: String, maybeImageWithVersion: Either[String, (Image, Long)]) extends MigrationMessage +case class MigrateImageMessage(id: String, maybeImageWithVersion: Either[String, (Image, Long)], instance: Instance) extends MigrationMessage object MigrateImageMessage { - def apply(imageId: String, maybeProjection: Option[Image], maybeVersion: Option[Long]): MigrateImageMessage = (maybeProjection, maybeVersion) match { - case (Some(projection), Some(version)) => MigrateImageMessage(imageId, scala.Right((projection, version))) - case (None, _) => MigrateImageMessage(imageId, Left("There was no projection returned")) - case _ => MigrateImageMessage(imageId, Left("There was no version returned")) + def apply(imageId: String, maybeProjection: Option[Image], maybeVersion: Option[Long], instance: Instance): MigrateImageMessage = (maybeProjection, maybeVersion) match { + case (Some(projection), Some(version)) => MigrateImageMessage(imageId, scala.Right((projection, version)), instance) + case (None, _) => MigrateImageMessage(imageId, Left("There was no projection returned"), instance) + case _ => MigrateImageMessage(imageId, Left("There was no version returned"), instance) } } @@ -45,6 +45,7 @@ sealed trait ExternalThrallMessage extends ThrallMessage { implicit val yourJodaDateWrites: Writes[DateTime] = JodaWrites.JodaDateTimeWrites val id: String val lastModified: DateTime + val instance: Instance def toJson: JsValue = Json.toJson(this)(ExternalThrallMessage.writes) override def markerContents: Map[String, Any] = { @@ -61,6 +62,8 @@ object ExternalThrallMessage{ implicit val yourJodaDateReads: Reads[DateTime] = JodaReads.DefaultJodaDateTimeReads.map(d => d.withZone(DateTimeZone.UTC)) implicit val yourJodaDateWrites: Writes[DateTime] = JodaWrites.JodaDateTimeWrites + implicit val instanceMessageFormat: OFormat[Instance] = Json.format[Instance] + implicit val usageNoticeFormat: OFormat[UsageNotice] = Json.format[UsageNotice] implicit val replaceImageLeasesMessageFormat: OFormat[ReplaceImageLeasesMessage] = Json.format[ReplaceImageLeasesMessage] @@ -86,58 +89,64 @@ object ExternalThrallMessage{ implicit val completeMigrationMessage: OFormat[CompleteMigrationMessage] = Json.format[CompleteMigrationMessage] implicit val upsertFromProjectionMessage: OFormat[UpsertFromProjectionMessage] = Json.format[UpsertFromProjectionMessage] + implicit val createInstanceMessage: OFormat[CreateInstanceMessage] = Json.format[CreateInstanceMessage] + implicit val reindexImageMessage: OFormat[ReindexImageMessage] = Json.format[ReindexImageMessage] + implicit val writes: OWrites[ExternalThrallMessage] = Json.writes[ExternalThrallMessage] implicit val reads: Reads[ExternalThrallMessage] = Json.reads[ExternalThrallMessage] } -case class ImageMessage(lastModified: DateTime, image: Image) extends ExternalThrallMessage { +case class ImageMessage(lastModified: DateTime, image: Image, instance: Instance) extends ExternalThrallMessage { override def additionalMarkers: () => Map[String, Any] = ()=> Map("fileName" -> image.source.file.toString) override val id: String = image.id } -case class DeleteImageMessage(id: String, lastModified: DateTime) extends ExternalThrallMessage +case class DeleteImageMessage(id: String, lastModified: DateTime, instance: Instance) extends ExternalThrallMessage -case class SoftDeleteImageMessage(id: String, lastModified: DateTime, softDeletedMetadata: SoftDeletedMetadata) extends ExternalThrallMessage +case class SoftDeleteImageMessage(id: String, lastModified: DateTime, softDeletedMetadata: SoftDeletedMetadata, instance: Instance) extends ExternalThrallMessage -case class UnSoftDeleteImageMessage(id: String, lastModified: DateTime) extends ExternalThrallMessage +case class UnSoftDeleteImageMessage(id: String, lastModified: DateTime, instance: Instance) extends ExternalThrallMessage -case class DeleteImageExportsMessage(id: String, lastModified: DateTime) extends ExternalThrallMessage +case class DeleteImageExportsMessage(id: String, lastModified: DateTime, instance: Instance) extends ExternalThrallMessage -case class UpdateImageExportsMessage(id: String, lastModified: DateTime, crops: Seq[Crop]) extends ExternalThrallMessage +case class UpdateImageExportsMessage(id: String, lastModified: DateTime, crops: Seq[Crop], instance: Instance) extends ExternalThrallMessage -case class UpdateImageUserMetadataMessage(id: String, lastModified: DateTime, edits: Edits) extends ExternalThrallMessage +case class UpdateImageUserMetadataMessage(id: String, lastModified: DateTime, edits: Edits, instance: Instance) extends ExternalThrallMessage -case class UpdateImageUsagesMessage(id: String, lastModified: DateTime, usageNotice: UsageNotice) extends ExternalThrallMessage +case class UpdateImageUsagesMessage(id: String, lastModified: DateTime, usageNotice: UsageNotice, instance: Instance) extends ExternalThrallMessage -case class ReplaceImageLeasesMessage(id: String, lastModified: DateTime, leases: Seq[MediaLease]) extends ExternalThrallMessage +case class ReplaceImageLeasesMessage(id: String, lastModified: DateTime, leases: Seq[MediaLease], instance: Instance) extends ExternalThrallMessage -case class AddImageLeaseMessage(id: String, lastModified: DateTime, lease: MediaLease) extends ExternalThrallMessage +case class AddImageLeaseMessage(id: String, lastModified: DateTime, lease: MediaLease, instance: Instance) extends ExternalThrallMessage -case class RemoveImageLeaseMessage(id: String, lastModified: DateTime, leaseId: String) extends ExternalThrallMessage +case class RemoveImageLeaseMessage(id: String, lastModified: DateTime, leaseId: String, instance: Instance) extends ExternalThrallMessage -case class SetImageCollectionsMessage(id: String, lastModified: DateTime, collections: Seq[Collection]) extends ExternalThrallMessage +case class SetImageCollectionsMessage(id: String, lastModified: DateTime, collections: Seq[Collection], instance: Instance) extends ExternalThrallMessage -case class DeleteSingleUsageMessage(id: String, lastModified: DateTime, usageId: String) extends ExternalThrallMessage +case class DeleteSingleUsageMessage(id: String, lastModified: DateTime, usageId: String, instance: Instance) extends ExternalThrallMessage -case class DeleteUsagesMessage(id: String, lastModified: DateTime) extends ExternalThrallMessage +case class DeleteUsagesMessage(id: String, lastModified: DateTime, instance: Instance) extends ExternalThrallMessage -case class UpdateUsageStatusMessage(id: String, usageNotice: UsageNotice, lastModified: DateTime) extends ExternalThrallMessage +case class UpdateUsageStatusMessage(id: String, usageNotice: UsageNotice, lastModified: DateTime, instance: Instance) extends ExternalThrallMessage -case class UpdateEmbeddingMessage(id: String, lastModified: DateTime, embedding: Embedding) extends ExternalThrallMessage +case class UpdateEmbeddingMessage(id: String, lastModified: DateTime, embedding: Embedding, instance: Instance) extends ExternalThrallMessage object DeleteUsagesMessage { implicit val yourJodaDateReads: Reads[DateTime] = JodaReads.DefaultJodaDateTimeReads.map(d => d.withZone(DateTimeZone.UTC)) implicit val yourJodaDateWrites: Writes[DateTime] = JodaWrites.JodaDateTimeWrites + + implicit val instanceMessageFormat: OFormat[Instance] = Json.format[Instance] + implicit val what: OFormat[DeleteUsagesMessage] = Json.format[DeleteUsagesMessage] } -case class UpdateImageSyndicationMetadataMessage(id: String, lastModified: DateTime, maybeSyndicationRights: Option[SyndicationRights]) extends ExternalThrallMessage +case class UpdateImageSyndicationMetadataMessage(id: String, lastModified: DateTime, maybeSyndicationRights: Option[SyndicationRights], instance: Instance) extends ExternalThrallMessage -case class UpdateImagePhotoshootMetadataMessage(id: String, lastModified: DateTime, edits: Edits) extends ExternalThrallMessage +case class UpdateImagePhotoshootMetadataMessage(id: String, lastModified: DateTime, edits: Edits, instance: Instance) extends ExternalThrallMessage /** * Message to start a new 'migration' (for re-index, re-ingestion etc.) @@ -146,7 +155,8 @@ case class UpdateImagePhotoshootMetadataMessage(id: String, lastModified: DateTi */ case class CreateMigrationIndexMessage( migrationStart: DateTime, - gitHash: String + gitHash: String, + instance: Instance ) extends ExternalThrallMessage { val id: String = "N/A" val lastModified: DateTime = migrationStart @@ -155,8 +165,12 @@ case class CreateMigrationIndexMessage( s"images_${migrationStart.toString(DateTimeFormat.forPattern("yyyy-MM-dd_HH-mm-ss").withZoneUTC())}_${gitHash.take(7)}" } -case class UpsertFromProjectionMessage(id: String, image: Image, lastModified: DateTime) extends ExternalThrallMessage +case class UpsertFromProjectionMessage(id: String, image: Image, lastModified: DateTime, instance: Instance) extends ExternalThrallMessage -case class CompleteMigrationMessage(lastModified: DateTime) extends ExternalThrallMessage { +case class CompleteMigrationMessage(lastModified: DateTime, instance: Instance) extends ExternalThrallMessage { val id: String = "N/A" } + +case class CreateInstanceMessage(id: String, lastModified: DateTime, instance: Instance) extends ExternalThrallMessage + +case class ReindexImageMessage(id: String, lastModified: DateTime, instance: Instance) extends ExternalThrallMessage diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/UploadInfo.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/UploadInfo.scala index 5b8beb65455..b4c10e0c51c 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/UploadInfo.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/UploadInfo.scala @@ -2,7 +2,7 @@ package com.gu.mediaservice.model import play.api.libs.json.{Json, OWrites, Reads} -case class UploadInfo(filename: Option[String] = None) +case class UploadInfo(filename: Option[String] = None, isFeedUpload: Option[Boolean] = None) object UploadInfo { implicit val jsonWrites: OWrites[UploadInfo] = Json.writes[UploadInfo] diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/UsageRights.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/UsageRights.scala index 2db517e51d9..3921d948d45 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/UsageRights.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/UsageRights.scala @@ -182,9 +182,9 @@ object Chargeable extends UsageRightsSpec { val defaultCost = Some(Pay) def name(commonConfig: CommonConfig) = "Chargeable supplied / on spec" def description(commonConfig: CommonConfig) = - s"Images acquired by or supplied to ${commonConfig.staffPhotographerOrganisation} that do not fit other categories in ${commonConfig.systemName} and " + + s"Images acquired or supplied that do not fit other categories in ${commonConfig.systemName} and " + "therefore fees will be payable per use. Unless negotiated otherwise, fees should be based on " + - s"standard published ${commonConfig.staffPhotographerOrganisation} rates for stock and speculative images." + s"standard published rates for stock and speculative images." implicit val formats: Format[Chargeable] = UsageRights.subtypeFormat(Chargeable.category)(Json.format[Chargeable]) @@ -289,7 +289,7 @@ object Screengrab extends UsageRightsSpec { val defaultCost = Some(Free) def name(commonConfig: CommonConfig) = "Screengrab" def description(commonConfig: CommonConfig) = - s"Stills created by ${commonConfig.staffPhotographerOrganisation} from moving footage in television broadcasts usually in relation to " + + s"Stills created by us from moving footage in television broadcasts usually in relation to " + "breaking news stories." implicit val formats: Format[Screengrab] = diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/usage/DigitalUsageMetadata.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/usage/DigitalUsageMetadata.scala index df6337a5f89..117a6e36959 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/usage/DigitalUsageMetadata.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/usage/DigitalUsageMetadata.scala @@ -2,22 +2,21 @@ package com.gu.mediaservice.model.usage import java.net.URI import play.api.libs.json._ -import com.gu.mediaservice.syntax._ +import org.joda.time.DateTime case class DigitalUsageMetadata ( webUrl: URI, - webTitle: String, - sectionId: String, + webTitle: Option[String], + sectionId: Option[String], composerUrl: Option[URI] = None ) extends UsageMetadata { private val placeholderWebTitle = "No title given" - private val dynamoSafeWebTitle = if(webTitle.isEmpty) placeholderWebTitle else webTitle + private val dynamoSafeWebTitle = webTitle.find(_.nonEmpty).getOrElse(placeholderWebTitle) override def toMap: Map[String, String] = Map( "webUrl" -> webUrl.toString, - "webTitle" -> dynamoSafeWebTitle, - "sectionId" -> sectionId - ) ++ composerUrl.map("composerUrl" -> _.toString) + "webTitle" -> dynamoSafeWebTitle + ) ++ sectionId.filter(_.nonEmpty).map("sectionId" -> _) ++ composerUrl.map("composerUrl" -> _.toString) } object DigitalUsageMetadata { diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/usage/UsageNotice.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/usage/UsageNotice.scala index 27abc9ec487..a1e57b0cd38 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/usage/UsageNotice.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/usage/UsageNotice.scala @@ -1,10 +1,11 @@ package com.gu.mediaservice.model.usage import com.gu.mediaservice.lib.formatting.printDateTime +import com.gu.mediaservice.model.Instance import org.joda.time.DateTime import play.api.libs.json.{JodaWrites, JsArray, JsObject, Json} -case class UsageNotice(mediaId: String, usageJson: JsArray) { +case class UsageNotice(mediaId: String, usageJson: JsArray, instance: Instance) { def toJson = Json.obj( "id" -> mediaId, "data" -> usageJson, diff --git a/common-lib/src/main/scala/com/gu/mediaservice/syntax/MessageSubjects.scala b/common-lib/src/main/scala/com/gu/mediaservice/syntax/MessageSubjects.scala index a490f427f5c..b0a2f32c6ee 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/syntax/MessageSubjects.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/syntax/MessageSubjects.scala @@ -21,7 +21,8 @@ trait MessageSubjects { val DeleteSingleUsage = "delete-single-usage" val UpdateImageSyndicationMetadata = "update-image-syndication-metadata" val UpdateImagePhotoshootMetadata = "update-image-photoshoot-metadata" - + val CreateInstance = "create-instance" + val ReindexImage = "reindex-image" } object MessageSubjects extends MessageSubjects diff --git a/common-lib/src/test/resources/CMYK-with-profile.jpg b/common-lib/src/test/resources/CMYK-with-profile.jpg new file mode 100644 index 00000000000..6c78dff467a Binary files /dev/null and b/common-lib/src/test/resources/CMYK-with-profile.jpg differ diff --git a/common-lib/src/test/resources/IMG_0128.HEIC b/common-lib/src/test/resources/IMG_0128.HEIC new file mode 100644 index 00000000000..a5fa30b48a5 Binary files /dev/null and b/common-lib/src/test/resources/IMG_0128.HEIC differ diff --git a/common-lib/src/test/resources/IMG_4403.JPG b/common-lib/src/test/resources/IMG_4403.JPG new file mode 100755 index 00000000000..da8ed3230d9 Binary files /dev/null and b/common-lib/src/test/resources/IMG_4403.JPG differ diff --git a/common-lib/src/test/resources/application.conf b/common-lib/src/test/resources/application.conf index 962a7678896..b5c3e1aee23 100644 --- a/common-lib/src/test/resources/application.conf +++ b/common-lib/src/test/resources/application.conf @@ -56,3 +56,15 @@ usageRightsConfigProvider = { suppliersCollectionExcl {} } } + +instance.service.my="" +instance.service.instances="" + +usageEvents.queue.name="" + +s3.image.bucket.name="images" +s3.image.bucket.endpoint="some-providers-s3-endpoint" + +s3.thumb.bucket.name="thumbs" +s3.thumb.bucket.endpoint="some-providers-s3-endpoint" + diff --git a/rest-lib/src/test/resources/cmyk.jpg b/common-lib/src/test/resources/cmyk.jpg similarity index 100% rename from rest-lib/src/test/resources/cmyk.jpg rename to common-lib/src/test/resources/cmyk.jpg diff --git a/common-lib/src/test/resources/cs-black-000.png b/common-lib/src/test/resources/cs-black-000.png new file mode 100644 index 00000000000..01f409774ad Binary files /dev/null and b/common-lib/src/test/resources/cs-black-000.png differ diff --git a/common-lib/src/test/resources/exif-orientated-no-rotation.jpg b/common-lib/src/test/resources/exif-orientated-no-rotation.jpg new file mode 100755 index 00000000000..0f37a7c3dbd Binary files /dev/null and b/common-lib/src/test/resources/exif-orientated-no-rotation.jpg differ diff --git a/common-lib/src/test/resources/exif-orientated.jpg b/common-lib/src/test/resources/exif-orientated.jpg new file mode 100755 index 00000000000..d7e45067170 Binary files /dev/null and b/common-lib/src/test/resources/exif-orientated.jpg differ diff --git a/common-lib/src/test/resources/flower.tif b/common-lib/src/test/resources/flower.tif new file mode 100644 index 00000000000..2278aff1822 Binary files /dev/null and b/common-lib/src/test/resources/flower.tif differ diff --git a/rest-lib/src/test/resources/grayscale-with-profile.jpg b/common-lib/src/test/resources/grayscale-with-profile.jpg similarity index 100% rename from rest-lib/src/test/resources/grayscale-with-profile.jpg rename to common-lib/src/test/resources/grayscale-with-profile.jpg diff --git a/rest-lib/src/test/resources/grayscale-wo-profile.jpg b/common-lib/src/test/resources/grayscale-wo-profile.jpg similarity index 100% rename from rest-lib/src/test/resources/grayscale-wo-profile.jpg rename to common-lib/src/test/resources/grayscale-wo-profile.jpg diff --git a/common-lib/src/test/resources/halfdome_LAB.tif b/common-lib/src/test/resources/halfdome_LAB.tif new file mode 100644 index 00000000000..564562e5ea6 Binary files /dev/null and b/common-lib/src/test/resources/halfdome_LAB.tif differ diff --git a/common-lib/src/test/resources/halfdome_LAB16.tif b/common-lib/src/test/resources/halfdome_LAB16.tif new file mode 100644 index 00000000000..0b54c644b30 Binary files /dev/null and b/common-lib/src/test/resources/halfdome_LAB16.tif differ diff --git a/common-lib/src/test/resources/lab8-with-alpha.tif b/common-lib/src/test/resources/lab8-with-alpha.tif new file mode 100644 index 00000000000..f63cc44b309 Binary files /dev/null and b/common-lib/src/test/resources/lab8-with-alpha.tif differ diff --git a/rest-lib/src/test/resources/rgb-with-cmyk-profile.jpg b/common-lib/src/test/resources/rgb-with-cmyk-profile.jpg similarity index 100% rename from rest-lib/src/test/resources/rgb-with-cmyk-profile.jpg rename to common-lib/src/test/resources/rgb-with-cmyk-profile.jpg diff --git a/rest-lib/src/test/resources/rgb-with-rgb-profile.jpg b/common-lib/src/test/resources/rgb-with-rgb-profile.jpg similarity index 100% rename from rest-lib/src/test/resources/rgb-with-rgb-profile.jpg rename to common-lib/src/test/resources/rgb-with-rgb-profile.jpg diff --git a/rest-lib/src/test/resources/rgb-wo-profile.jpg b/common-lib/src/test/resources/rgb-wo-profile.jpg similarity index 100% rename from rest-lib/src/test/resources/rgb-wo-profile.jpg rename to common-lib/src/test/resources/rgb-wo-profile.jpg diff --git a/common-lib/src/test/resources/schaik.com_pngsuite/README.md b/common-lib/src/test/resources/schaik.com_pngsuite/README.md new file mode 100644 index 00000000000..3854ba52a73 --- /dev/null +++ b/common-lib/src/test/resources/schaik.com_pngsuite/README.md @@ -0,0 +1,25 @@ +Test PNG images +=============== + +These images are taken from http://www.schaik.com/pngsuite/pngsuite_bas_png.html + + basn0g08 - 8 bit (256 level) grayscale + basn2c08 - 3x8 bits rgb color + basn3p08 - 8 bit (256 color) paletted + basn6a08 - 3x8 bits rgb color + 8 bit alpha-channel + +LICENCE +------- + +At the time of downloading these images the licence file at http://www.schaik.com/pngsuite/PngSuite.LICENSE contained the following text: + +``` +PngSuite +-------- + +Permission to use, copy, modify and distribute these images for any +purpose and without fee is hereby granted. + + +(c) Willem van Schaik, 1996, 2011 +``` diff --git a/common-lib/src/test/resources/schaik.com_pngsuite/basi2c16.png b/common-lib/src/test/resources/schaik.com_pngsuite/basi2c16.png new file mode 100644 index 00000000000..cd7e50f9140 Binary files /dev/null and b/common-lib/src/test/resources/schaik.com_pngsuite/basi2c16.png differ diff --git a/common-lib/src/test/resources/schaik.com_pngsuite/basn0g08.png b/common-lib/src/test/resources/schaik.com_pngsuite/basn0g08.png new file mode 100644 index 00000000000..23c82379a29 Binary files /dev/null and b/common-lib/src/test/resources/schaik.com_pngsuite/basn0g08.png differ diff --git a/common-lib/src/test/resources/schaik.com_pngsuite/basn3p08.png b/common-lib/src/test/resources/schaik.com_pngsuite/basn3p08.png new file mode 100644 index 00000000000..0ddad07e5f5 Binary files /dev/null and b/common-lib/src/test/resources/schaik.com_pngsuite/basn3p08.png differ diff --git a/common-lib/src/test/resources/schaik.com_pngsuite/tbbn0g04.png b/common-lib/src/test/resources/schaik.com_pngsuite/tbbn0g04.png new file mode 100644 index 00000000000..39a7050d27a Binary files /dev/null and b/common-lib/src/test/resources/schaik.com_pngsuite/tbbn0g04.png differ diff --git a/common-lib/src/test/resources/with-alpha.png b/common-lib/src/test/resources/with-alpha.png new file mode 100644 index 00000000000..2e765a1ba2c Binary files /dev/null and b/common-lib/src/test/resources/with-alpha.png differ diff --git a/common-lib/src/test/resources/with-alpha.tif b/common-lib/src/test/resources/with-alpha.tif new file mode 100644 index 00000000000..a4efd9cc37b Binary files /dev/null and b/common-lib/src/test/resources/with-alpha.tif differ diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/ContentDispositionTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/ContentDispositionTest.scala index 90b5142e8af..efccf8d7b59 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/ContentDispositionTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/ContentDispositionTest.scala @@ -25,17 +25,17 @@ class ContentDispositionTest extends AnyFunSuiteLike with ContentDisposition { header shouldBe """attachment; filename="abcdef1234567890.jpg"; filename*=UTF-8''%C2%A9House%20of%20Commons_240508_MU_PMQs-09_42668%20%28abcdef1234567890%29.jpg""" } - test("include crop id and dimensions in main crop asset filename") { + test("crop asset filename is the image uploaded filename with the correct file extension for the crop mime type") { val image = withFilename(MappingTest.testImage, "imagefilename.jpg") val crop = image.exports.head val cropAsset = crop.assets.head - val header = getContentDisposition(image, crop, cropAsset, shortenDownloadFilename = false) + val header = getContentDisposition(image, crop, cropAsset) // Latin1 fallback wants to remain simple header.contains("""filename="abcdef1234567890.jpg";""") shouldBe true val decoded = URLDecoder.decode(header, "UTF-8") - decoded shouldBe """attachment; filename="abcdef1234567890.jpg"; filename*=UTF-8''imagefilename (abcdef1234567890)(1234567890987654321)(1000 x 2000).jpg""" + decoded shouldBe """attachment; filename="abcdef1234567890.jpg"; filename*=UTF-8''imagefilename.jpg""" } test("use just the generated filename suffix as filename if short filenames are requested") { diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/DynamoDBTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/DynamoDBTest.scala index 88c1686b9b1..755491e9a1d 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/DynamoDBTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/DynamoDBTest.scala @@ -1,121 +1,11 @@ package com.gu.mediaservice.lib.aws -import com.amazonaws.services.dynamodbv2.document.spec.UpdateItemSpec -import com.amazonaws.services.dynamodbv2.document.utils.ValueMap -import com.amazonaws.services.dynamodbv2.model.ReturnValue -import com.gu.mediaservice.model.{ActionData, Collection} -import org.joda.time.DateTime import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers -import play.api.libs.json.{Format, JsObject, Json} - -import scala.jdk.CollectionConverters._ +import play.api.libs.json.{Format, Json} class DynamoDBTest extends AnyFunSpec with Matchers { - describe("jsonToValueMap") { - it ("should convert a simple JsObject to a valueMap") { - val json = Json.toJson(SimpleDynamoDBObj("this is a string", 100, true, List("list"))).as[JsObject] - val valueMap = DynamoDB.jsonToValueMap(json) - - - // This is the only way to get stuff type safely out of the valueMap - // It's not a problem as we shoulnd't be doing this anywhere else - val s: String = valueMap.get("s").asInstanceOf[String] - val d: BigDecimal = valueMap.get("d").asInstanceOf[java.math.BigDecimal] - val b: Boolean = valueMap.get("b").asInstanceOf[Boolean] - val a: List[String] = valueMap.get("a").asInstanceOf[java.util.ArrayList[String]].toArray().toList.map(_.asInstanceOf[String]) - - - s should be ("this is a string") - d should be (100) - b should equal(true) - a should equal(List("list")) - } - - it ("should convert a nested JsObject to a valueMap") { - val nestedObj = NestedDynamoDBObj("string", 100, false, SimpleDynamoDBObj("strang", 500, true, List("list"))) - val json = Json.toJson(nestedObj).as[JsObject] - val valueMap = DynamoDB.jsonToValueMap(json) - - val ss: String = valueMap.get("ss").asInstanceOf[String] - val dd: BigDecimal = valueMap.get("dd").asInstanceOf[java.math.BigDecimal] - val bb: Boolean = valueMap.get("bb").asInstanceOf[Boolean] - - val simpleMap = valueMap.get("simple").asInstanceOf[ValueMap] - - val s: String = simpleMap.get("s").asInstanceOf[String] - val d: BigDecimal = simpleMap.get("d").asInstanceOf[java.math.BigDecimal] - val b: Boolean = simpleMap.get("b").asInstanceOf[Boolean] - val a: List[String] = simpleMap.get("a").asInstanceOf[java.util.ArrayList[String]].toArray().toList.map(_.asInstanceOf[String]) - - - ss should be ("string") - dd should be (100) - bb should equal(false) - - s should be ("strang") - d should be (500) - b should equal(true) - a should equal(List("list")) - } - - it ("should convert a Collection to ValueMap") { - val collection = Collection.build(List("g2", "art", "batik"), ActionData("mighty.mouse@guardian.co.uk", DateTime.now)) - val json = Json.toJson(collection).as[JsObject] - val valueMap = DynamoDB.jsonToValueMap(json) - - val path = valueMap.get("path").asInstanceOf[java.util.ArrayList[String]].toArray.toList.asInstanceOf[List[String]] - val pathId = valueMap.get("pathId").asInstanceOf[String] - val actionData = valueMap.get("actionData").asInstanceOf[ValueMap] - val author = actionData.get("author").asInstanceOf[String] - - pathId should be (collection.pathId) - author should be (collection.actionData.author) - path should be (collection.path) - } - } - - describe("addLastModifiedUpdate") { - val testTime = new DateTime(2021, 3, 22, 14, 58) - - it ("should prefix a lastModified update to existing SET expression") { - val input = new UpdateItemSpec() - .withUpdateExpression("SET anotherKey = :anotherValue") - val output = DynamoDB.addLastModifiedUpdate(input, "lm", testTime) - output.getUpdateExpression shouldBe "SET lm = :lm, anotherKey = :anotherValue" - } - - it ("should prefix a lastModified SET clause to existing ADD expression") { - val input = new UpdateItemSpec() - .withUpdateExpression("ADD anotherKey :anotherValue") - val output = DynamoDB.addLastModifiedUpdate(input, "lm", testTime) - output.getUpdateExpression shouldBe "SET lm = :lm ADD anotherKey :anotherValue" - } - - it ("should add a date time to an existing value map") { - val valueMap = new ValueMap() - valueMap.put(":anotherValue", "anotherValue") - val input = new UpdateItemSpec() - .withUpdateExpression("banana") - .withValueMap(valueMap) - val output = DynamoDB.addLastModifiedUpdate(input, "lm", testTime) - output.getValueMap.asScala shouldBe Map( - ":anotherValue" -> "anotherValue", - ":lm" -> testTime.toString - ) - } - - it ("should add a date time when there is no value map") { - val input = new UpdateItemSpec() - .withUpdateExpression("banana") - val output = DynamoDB.addLastModifiedUpdate(input, "lm", testTime) - output.getValueMap.asScala shouldBe Map( - ":lm" -> testTime.toString - ) - } - } - describe("removeExpr") { it("should generate the correction expression when the lastModified is set") { DynamoDB.removeExpr("key", Some("lastModifiedKey")) shouldBe "REMOVE key SET lastModifiedKey = :lastModifiedKey" diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/ThrallMessageSenderTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/ThrallMessageSenderTest.scala index 0b505673211..f126f7be964 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/ThrallMessageSenderTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/ThrallMessageSenderTest.scala @@ -1,5 +1,6 @@ package com.gu.mediaservice.lib.aws +import com.gu.mediaservice.model.Instance import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers import play.api.libs.json.Json @@ -12,7 +13,7 @@ class ThrallMessageSenderTest extends AnyFunSpec with Matchers { describe("json to message and back") { // This is most interested for ensuring time zone correctness it ("should convert a message to json and back again") { - val m = UpdateMessage(subject = "test") + val m = UpdateMessage(subject = "test", instance = Instance("an-instance")) val j = Json.toJson(m).toString() val m2 = Json.parse(j).as[UpdateMessage] m2 shouldEqual m @@ -20,7 +21,7 @@ class ThrallMessageSenderTest extends AnyFunSpec with Matchers { it ("should convert a message from an external source which does not have last modified") { val subject = "test" - val j = s"""{"subject":"$subject"}""" + val j = s"""{"subject":"$subject", "instance": {"id": "an-instance"}}""" val m = Json.parse(j).as[UpdateMessage] m.lastModified.getZone.toString should be ("UTC") } @@ -28,7 +29,7 @@ class ThrallMessageSenderTest extends AnyFunSpec with Matchers { it ("should convert a message last modified with an offset timezone to UTC") { val now = DateTime.now(DateTimeZone.forOffsetHours(9)) val nowUtc = new DateTime(now.getMillis()).toDateTime(DateTimeZone.UTC) - val m = UpdateMessage(subject = "test", lastModified = now) + val m = UpdateMessage(subject = "test", lastModified = now, instance = Instance("an-instance")) val j = Json.toJson(m).toString() val m2 = Json.parse(j).as[UpdateMessage] m2 shouldEqual m.copy(lastModified = nowUtc) diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/cleanup/MetadataHelper.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/cleanup/MetadataHelper.scala index b78542f7deb..1ac9e3381ee 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/cleanup/MetadataHelper.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/cleanup/MetadataHelper.scala @@ -19,7 +19,7 @@ trait MetadataHelper { softDeletedMetadata = None, lastModified = None, identifiers = Map(), - uploadInfo = UploadInfo(), + uploadInfo = UploadInfo(isFeedUpload = Some(true)), source = Asset(URI.create("http://example.com/image.jpg"), Some(0), None, None, None, None), thumbnail = None, optimisedPng = None, diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessorsTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessorsTest.scala index 8a6da2cc22e..3cb7da390a6 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessorsTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessorsTest.scala @@ -294,11 +294,46 @@ class SupplierProcessorsTest extends AnyFunSpec with Matchers with MetadataHelpe processedImage.metadata.credit should be(Some("Invision for Quaker")) } - it("should match __/Invision/AP credit") { - val image = createImageFromMetadata("credit" -> "Andy Kropa /Invision/AP") + it("should match intermediary/AP credit") { + val image = createImageFromMetadata("credit" -> "Lehtikuva/AP") val processedImage = applyProcessors(image) - processedImage.usageRights should be(Agency("AP", Some("Invision"))) - processedImage.metadata.credit should be(Some("Andy Kropa /Invision/AP")) + processedImage.usageRights should be(Agency("AP", Some("Lehtikuva"))) + processedImage.metadata.credit should be(Some("Lehtikuva/AP")) + } + + it("should match intermediary/ap credit, capitalising AP") { + val image = createImageFromMetadata("credit" -> "Lehtikuva/ap") + val processedImage = applyProcessors(image) + processedImage.usageRights should be(Agency("AP", Some("Lehtikuva"))) + processedImage.metadata.credit should be(Some("Lehtikuva/AP")) + } + + it("should match ABC/NYT/AP credit") { + val image = createImageFromMetadata("credit" -> "ABC/NYT/AP") + val processedImage = applyProcessors(image) + processedImage.usageRights should be(Agency("AP", Some("ABC/NYT"))) + processedImage.metadata.credit should be(Some("ABC/NYT/AP")) + } + + it("should not match 'ABC/NYT/AP AP' credit") { + val image = createImageFromMetadata("credit" -> "ABC/NYT/AP AP") + val processedImage = applyProcessors(image) + processedImage.usageRights should be(NoRights) + processedImage.metadata.credit should be(Some("ABC/NYT/AP AP")) + } + + it("should not match 'ABC/Associated Press', because we don't think we get those") { + val image = createImageFromMetadata("credit" -> "ABC/Associated Press") + val processedImage = applyProcessors(image) + processedImage.usageRights should be(NoRights) + processedImage.metadata.credit should be(Some("ABC/Associated Press")) + } + + it("should still cover the case of 'Photographer /Invision/AP', which had its own special case before") { + val image = createImageFromMetadata("credit" -> "Photographer /Invision/AP") + val processedImage = applyProcessors(image) + processedImage.usageRights should be(Agency("AP", Some("Photographer /Invision"))) + processedImage.metadata.credit should be(Some("Photographer /Invision/AP")) } } diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/collections/CssColoursTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/collections/CssColoursTest.scala new file mode 100644 index 00000000000..314fd4dd191 --- /dev/null +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/collections/CssColoursTest.scala @@ -0,0 +1,25 @@ +package com.gu.mediaservice.lib.collections + +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers + +class CssColoursTest extends AnyFunSpec with Matchers with CssColours { + + describe("CssColours") { + describe("getCssColour") { + it("should return exact match for specific collections") { + getCssColour(List("Home", "Supplements")) shouldBe Some("#008083") + } + it("should default of none for collections with no specific colour preferences or parents with a colour") { + getCssColour(List("Unknown")) shouldBe None + } + it("should return colour of parent for collection with no specific colour") { + getCssColour(List("Home", "Something")) shouldBe Some("#052962") + } + it("should return colour of closet parent for collection with no specific colour") { + getCssColour(List("Home", "Supplements", "Something", "Something else")) shouldBe Some("#008083") + } + } + } + +} diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/imaging/ImageOperationsTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/imaging/ImageOperationsTest.scala index 8f78918f2db..02fbe0b3838 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/imaging/ImageOperationsTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/imaging/ImageOperationsTest.scala @@ -1,67 +1,459 @@ package com.gu.mediaservice.lib.imaging +import app.photofox.vipsffm.jextract.VipsRaw +import app.photofox.vipsffm.{VImage, Vips} +import com.gu.mediaservice.lib.BrowserViewableImage +import com.gu.mediaservice.lib.aws.EmbeddingSourceImageFormat import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} - -import java.io.File -import com.gu.mediaservice.model.Jpeg -import org.scalatest.time.{Millis, Span} -import org.scalatest.Ignore +import com.gu.mediaservice.model._ +import org.apache.commons.io.FileUtils import org.scalatest.concurrent.ScalaFutures import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers +import org.scalatest.time.{Millis, Span} +import java.io.File +import java.lang.foreign.Arena import scala.concurrent.ExecutionContext.Implicits.global // This test is disabled for now as it doesn't run on our CI environment, because GraphicsMagick is not present... -@Ignore class ImageOperationsTest extends AnyFunSpec with Matchers with ScalaFutures { + Vips.init() + implicit override val patienceConfig: PatienceConfig = PatienceConfig(timeout = Span(1000, Millis), interval = Span(25, Millis)) implicit val logMarker: LogMarker = MarkerMap() + private val metadata = ImageMetadata( + credit = Some("Tony McCrae"), + copyright = Some("Eel Pie Consulting Ltd"), + suppliersReference = Some("eelpie-123") + ) + + describe("thumbnail") { + it("should write thumbnail to output file") { + val image = fileAt("IMG_4403.jpg") + + val outputFile = new File("/Users/tony/Desktop/thumbnail.jpg") + val browserViewableImageImage = BrowserViewableImage("TODO", image, Tiff, Map.empty, false, Instance("TODO")) + + val eventualThumbnail = new ImageOperations("").createThumbnailVips(browserViewableImageImage, 240, 95, outputFile, None) + whenReady(eventualThumbnail) { r => + r._1.isFile should be(true) + } + } + + it("render LAB colour spaces correctly in sRGB") { + val image = fileAt("halfdome_LAB.tif") + + val outputFile = new File("/Users/tony/Desktop/out2.jpg") + val browserViewableImageImage = BrowserViewableImage("TODO", image, Tiff, Map.empty, false, Instance("TODO")) + + val eventualThumbnail = new ImageOperations("").createThumbnailVips(browserViewableImageImage, 1000, 95, outputFile, None) + whenReady(eventualThumbnail) { r => + r._1.isFile should be(true) + } + } + + it("render LAB 16 bits colour spaces correctly in 8 bit sRGB") { + val image = fileAt("halfdome_LAB16.tif") + + val outputFile = new File("/Users/tony/Desktop/out3.jpg") + val browserViewableImageImage = BrowserViewableImage("TODO", image, Tiff, Map.empty, false, Instance("TODO")) + + val eventualThumbnail = new ImageOperations("").createThumbnailVips(browserViewableImageImage, 1000, 95, outputFile, None) + whenReady(eventualThumbnail) { r => + r._1.isFile should be(true) + } + } + + it("render PNG with alpha correctly") { + val image = fileAt("with-alpha.png") + + val outputFile = new File("/Users/tony/Desktop/thumbnail-png-with-alpha.jpg") + val browserViewableImageImage = BrowserViewableImage("TODO", image, Tiff, Map.empty, false, Instance("TODO")) + + val eventualThumbnail = new ImageOperations("").createThumbnailVips(browserViewableImageImage, 1000, 95, outputFile, None) + whenReady(eventualThumbnail) { r => + r._1.isFile should be(true) + } + } + + it("render TIF with alpha correctly") { + val image = fileAt("with-alpha.tif") + + val outputFile = new File("/Users/tony/Desktop/thumbnail-tif-with-alpha.jpg") + val browserViewableImageImage = BrowserViewableImage("TODO", image, Tiff, Map.empty, false, Instance("TODO")) + + val eventualThumbnail = new ImageOperations("").createThumbnailVips(browserViewableImageImage, 1000, 95, outputFile, None) + whenReady(eventualThumbnail) { r => + r._1.isFile should be(true) + } + } + + it("render Heif correctly") { + val image = fileAt("IMG_0128.HEIC") + + val outputFile = new File("/Users/tony/Desktop/thumbnail-heic.jpg") + val browserViewableImageImage = BrowserViewableImage("TODO", image, Jpeg, Map.empty, false, Instance("TODO")) + + val eventualThumbnail = new ImageOperations("").createThumbnailVips(browserViewableImageImage, 1000, 95, outputFile, None) + whenReady(eventualThumbnail) { r => + r._1.isFile should be(true) + } + } + } + + describe("embeddings") { + it("should produce embedding sources from original images") { + implicit val arena: Arena = Arena.ofShared() + val fullSizedImage = fileAt("exif-orientated.jpg") + val imageOperations = new ImageOperations("") + + val format = EmbeddingSourceImageFormat( + longestAxis = 1000, format = Jpeg, letterBox = false + ) + + val eventualEmbeddingSource = imageOperations.createEmbeddingSource(fullSizedImage, orientationMetadata = Some(OrientationMetadata(exifOrientation = Some(6))), embeddingSourceImageFormat = format) + + whenReady(eventualEmbeddingSource) { source => + arena.close() + val outputFile = new File("/Users/tony/Desktop/embedding-source.jpg") + FileUtils.writeByteArrayToFile(outputFile, source) + + source.length > 100 should be(true) + } + } + } + + describe("resize") { + it("should output resized image to file in chosen format") { + implicit val arena: Arena = Arena.ofShared() + val fullSizedImage = VImage.newFromFile(arena, fileAt("IMG_4403.jpg").getAbsolutePath) + val imageOperations = new ImageOperations("") + + val outputFile = new File("/Users/tony/Desktop/out5.jpg") + + val eventuallyResized = imageOperations.resizeImageVips(fullSizedImage, Dimensions(1000, 800), 95, outputFile, Jpeg) + + whenReady(eventuallyResized) { resized => + arena.close() + resized.isFile should be(true) + } + } + + it("render LAB colour spaces correctly in sRGB") { + implicit val arena: Arena = Arena.ofShared + val imageOperations = new ImageOperations("") + + val fullSizedImage = VImage.newFromFile(arena, fileAt("halfdome_LAB.tif").getAbsolutePath) + val outputFile = new File("/Users/tony/Desktop/out6.jpg") + + val eventuallyResized = imageOperations.resizeImageVips(fullSizedImage, Dimensions(800, 600), 95, outputFile, Jpeg) + + whenReady(eventuallyResized) { resized => + arena.close() + resized.isFile should be(true) + } + } + + it("render LAB colour spaces correctly as PNG") { + implicit val arena: Arena = Arena.ofShared + val imageOperations = new ImageOperations("") + + val fullSizedImage = VImage.newFromFile(arena, fileAt("halfdome_LAB.tif").getAbsolutePath) + val outputFile = new File("/Users/tony/Desktop/out7.png") + + val eventuallyResized = imageOperations.resizeImageVips(fullSizedImage, Dimensions(800, 600), 95, outputFile, Png) + + whenReady(eventuallyResized) { resized => + arena.close() + resized.isFile should be(true) + } + } + + it("render LAB 16 bit colour spaces correctly") { + implicit val arena: Arena = Arena.ofShared + val imageOperations = new ImageOperations("") + + val fullSizedImage = VImage.newFromFile(arena, fileAt("halfdome_LAB16.tif").getAbsolutePath) + val outputFile = new File("/Users/tony/Desktop/out8.jpg") + + val eventuallyResized = imageOperations.resizeImageVips(fullSizedImage, Dimensions(800, 600), 95, outputFile, Jpeg) + + whenReady(eventuallyResized) { resized => + arena.close() + resized.isFile should be(true) + } + } + + it("render PNG with alpha correctly") { + implicit val arena: Arena = Arena.ofShared + val imageOperations = new ImageOperations("") + + val image = fileAt("with-alpha.png") + val fullSizedImage = VImage.newFromFile(arena, image.getAbsolutePath) + val outputFile = new File("/Users/tony/Desktop/resized-png-with-alpha.png") + + val eventuallyResized = imageOperations.resizeImageVips(fullSizedImage, Dimensions(800, 600), 95, outputFile, Png) + + whenReady(eventuallyResized) { resized => + arena.close() + resized.isFile should be(true) + } + } + + it("render LAB TIFF with alpha correctly") { + implicit val arena: Arena = Arena.ofShared + val imageOperations = new ImageOperations("") + + val image = fileAt("lab8-with-alpha.tif") + val fullSizedImage = VImage.newFromFile(arena, image.getAbsolutePath) + val outputFile = new File("/Users/tony/Desktop/out13.jpg") + + val eventuallyResized = imageOperations.resizeImageVips(fullSizedImage, Dimensions(800, 600), 95, outputFile, Jpeg) + + whenReady(eventuallyResized) { resized => + arena.close() + resized.isFile should be(true) + } + } + } + + describe("alpha") { + it("should return false for RGB for a Jpeg with no alpha") { + implicit val arena: Arena = Arena.ofShared + val image = VImage.newFromFile(arena, fileAt("rgb-wo-profile.jpg").getAbsolutePath) + val hasAlpha = ImageOperations.hasAlpha(image) + arena.close() + hasAlpha should be(false) + } + + it("should return true for PNG with alpha") { + implicit val arena: Arena = Arena.ofShared + val image = VImage.newFromFile(arena, fileAt("with-alpha.png").getAbsolutePath) + val hasAlpha = ImageOperations.hasAlpha(image) + arena.close() + hasAlpha should be(true) + } + } + describe("identifyColourModel") { it("should return RGB for a JPG image with RGB image data and no embedded profile") { val image = fileAt("rgb-wo-profile.jpg") - val colourModelFuture = ImageOperations.identifyColourModel(image, Jpeg) + val colourModelFuture = ImageOperations.getImageInformation(image) whenReady(colourModelFuture) { colourModel => - colourModel should be (Some("RGB")) + colourModel._3 should be(Some("RGB")) } } it("should return RGB for a JPG image with RGB image data and an RGB embedded profile") { val image = fileAt("rgb-with-rgb-profile.jpg") - val colourModelFuture = ImageOperations.identifyColourModel(image, Jpeg) + val colourModelFuture = ImageOperations.getImageInformation(image) whenReady(colourModelFuture) { colourModel => - colourModel should be (Some("RGB")) + colourModel._3 should be(Some("RGB")) + } + } + + it("should return RGB for a PNG image with RGB image data and an embedded profile") { + val image = fileAt("cs-black-000.png") + val colourModelFuture = ImageOperations.getImageInformation(image) + whenReady(colourModelFuture) { colourModel => + colourModel._3 should be(Some("RGB")) } } it("should return RGB for a JPG image with RGB image data and an incorrect CMYK embedded profile") { val image = fileAt("rgb-with-cmyk-profile.jpg") - val colourModelFuture = ImageOperations.identifyColourModel(image, Jpeg) + val colourModelFuture = ImageOperations.getImageInformation(image) whenReady(colourModelFuture) { colourModel => - colourModel should be (Some("RGB")) + colourModel._3 should be(Some("RGB")) } } it("should return CMYK for a JPG image with CMYK image data") { val image = fileAt("cmyk.jpg") - val colourModelFuture = ImageOperations.identifyColourModel(image, Jpeg) + val colourModelFuture = ImageOperations.getImageInformation(image) whenReady(colourModelFuture) { colourModel => - colourModel should be (Some("CMYK")) + colourModel._3 should be(Some("CMYK")) } } it("should return Greyscale for a JPG image with greyscale image data and no embedded profile") { val image = fileAt("grayscale-wo-profile.jpg") - val colourModelFuture = ImageOperations.identifyColourModel(image, Jpeg) + val colourModelFuture = ImageOperations.getImageInformation(image) whenReady(colourModelFuture) { colourModel => - colourModel should be (Some("Greyscale")) + colourModel._3 should be(Some("Greyscale")) + } + } + + it("should return RGB for a PNG image with 16 bit RGB image data") { + val image = fileAt("schaik.com_pngsuite/basi2c16.png") + val colourModelFuture = ImageOperations.getImageInformation(image) + whenReady(colourModelFuture) { colourModel => + colourModel._3 should be(Some("RGB")) + } + } + + it("should return LAB for a TIFF image with LAB16 image data") { + val image = fileAt("halfdome_LAB16.tif") + val colourModelFuture = ImageOperations.getImageInformation(image) + whenReady(colourModelFuture) { colourModel => + colourModel._3 should be(Some("LAB")) + } + } + + it("should return CMYK for a TIFF image with CMYK image data") { + val image = fileAt("CMYK-with-profile.jpg") + val colourModelFuture = ImageOperations.getImageInformation(image) + whenReady(colourModelFuture) { colourModel => + colourModel._3 should be(Some("CMYK")) + } + } + } + + describe("dimensions") { + it("should return dimensions of horizontal image") { + val inputFile = fileAt("exif-orientated-no-rotation.jpg") + val dimsFuture = ImageOperations.getImageInformation(inputFile) + whenReady(dimsFuture) { dims => + dims._1.get shouldBe new Dimensions(3456, 2304) + } + } + + it("should return uncorrected dimensions for exif oriented images") { + val inputFile = fileAt("exif-orientated.jpg") + val dimsFuture = ImageOperations.getImageInformation(inputFile) + whenReady(dimsFuture) { dims => + dims._1.get shouldBe new Dimensions(3456, 2304) + } + } + + it("should read the correct dimensions for a tiff image") { + val inputFile = fileAt("flower.tif") + val dimsFuture = ImageOperations.getImageInformation(inputFile) + whenReady(dimsFuture) { dimOpt => + dimOpt._1 should be(Symbol("defined")) + dimOpt._1.get.width should be(73) + dimOpt._1.get.height should be(43) + } + } + + it("should read the correct dimensions for a png image") { + val inputFile = fileAt("schaik.com_pngsuite/basn0g08.png") + val dimsFuture = ImageOperations.getImageInformation(inputFile) + whenReady(dimsFuture) { dimOpt => + dimOpt._1 should be(Symbol("defined")) + dimOpt._1.get.width should be(32) + dimOpt._1.get.height should be(32) } } } - // TODO: test cropImage and its conversions + describe("orientation") { + it("should capture exif orientation tag from JPG images") { + val image = fileAt("exif-orientated.jpg") + val orientationFuture = ImageOperations.getImageInformation(image) + whenReady(orientationFuture) { orientationOpt => + orientationOpt._2 should be(defined) + orientationOpt._2.get.exifOrientation should be(Some(6)) + } + } + + it("should ignore 0 degree exif orientation tag as it has no material effect") { + val image = fileAt("exif-orientated-no-rotation.jpg") + val orientationFuture = ImageOperations.getImageInformation(image) + whenReady(orientationFuture) { orientationOpt => + orientationOpt._2 should be(None) + } + } + } + + describe("graphic detection") { + it("should return not graphic for true colour jpeg") { + val arena = Arena.ofConfined + val image = VImage.newFromFile(arena, fileAt("exif-orientated-no-rotation.jpg").getAbsolutePath) + ImageOperations.isGraphicVips(image)(arena) should be(false) + arena.close() + } + + it("should return is graphic for depth 2 tiff") { + val arena = Arena.ofConfined + val image = VImage.newFromFile(arena, fileAt("flower.tif").getAbsolutePath) + ImageOperations.isGraphicVips(image)(arena) should be(true) + arena.close() + } + + it("should return is graphic for depth 4 png with alpha") { + val arena = Arena.ofConfined + val image = VImage.newFromFile(arena, fileAt("schaik.com_pngsuite/tbbn0g04.png").getAbsolutePath) + ImageOperations.isGraphicVips(image)(arena) should be(true) + arena.close() + } + + it("should return is graphic for depth 8 indexed png") { + val arena = Arena.ofConfined + val image = VImage.newFromFile(arena, fileAt("schaik.com_pngsuite/basn3p08.png").getAbsolutePath) + ImageOperations.isGraphicVips(image)(arena) should be(true) + arena.close() + } + + } + + describe("cropping") { + val operations = new ImageOperations("") + + it("should create unscaled master crop to resize from full sized images") { + implicit val arena: Arena = Arena.ofConfined + //val fullsizedImage = fileAt("Lab 16bpc (7d0b7c7b8e890d7e5d369093aa437bd833e20f71).tiff") + val fullsizedImage = fileAt("IMG_4403.jpg") + val metadata = ImageMetadata() + + val masterCrop = operations.cropImageVips(fullsizedImage, Bounds(100, 100, 2000, 2400), metadata, None) + + val outputFile = new File("/Users/tony/Desktop/master.jpg") + operations.saveImageToFile(masterCrop, Jpeg, 95, outputFile, keep = Some(VipsRaw.VIPS_FOREIGN_KEEP_XMP)) + arena.close() + } + + it("should create unscaled master crop from CMYK full sized image") { + implicit val arena: Arena = Arena.ofConfined + val fullsizedImage = fileAt("CMYK-with-profile.jpg") + val metadata = ImageMetadata() + + val masterCrop = operations.cropImageVips(fullsizedImage, Bounds(100, 100, 2000, 2400), metadata, None) + + val outputFile = new File("/Users/tony/Desktop/master-from-cmyk.jpg") + operations.saveImageToFile(masterCrop, Jpeg, 95, outputFile, keep = Some(VipsRaw.VIPS_FOREIGN_KEEP_XMP)) + + arena.close() + } + + it("should create files foreach crop size") { + implicit val arena: Arena = Arena.ofShared() + val fullsizedImage = fileAt("CMYK-with-profile.jpg") + val metadata = ImageMetadata() + + val masterCrop = operations.cropImageVips(fullsizedImage, Bounds(100, 100, 3000, 2000), metadata, None) + val landscapeCropSizingWidths = Seq( + Dimensions(140, 100), + Dimensions(320, 200), + Dimensions(800, 600), + Dimensions(1000, 1200), + Dimensions(2000, 3000), + ) + implicit val i: Instance = Instance("id") + + val crops = operations.createCrops(masterCrop, landscapeCropSizingWidths.toList, "test-image-id", + Bounds(0, 0, 1000, 1200), + Jpeg, + new File("/Users/tony/tmp/crops"), + 75 + ) + + arena.close() + } + } def fileAt(resourcePath: String): File = { new File(getClass.getResource(s"/$resourcePath").toURI) diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/json/JsonOrderingTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/json/JsonOrderingTest.scala index c076bedb257..93cd5423c64 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/json/JsonOrderingTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/json/JsonOrderingTest.scala @@ -26,7 +26,7 @@ class JsonOrderingTest extends AnyFreeSpec with Matchers { uploadedBy = "Biden", softDeletedMetadata = None, lastModified = None, - uploadInfo = UploadInfo(None), + uploadInfo = UploadInfo(None, None), source = Asset(new URI("fileUri"), None, None, None), optimisedPng = None, originalUsageRights = Handout(None), diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/metadata/ImageMetadataConverterTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/metadata/ImageMetadataConverterTest.scala index 792f38879ca..8f0e6d2142a 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/metadata/ImageMetadataConverterTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/metadata/ImageMetadataConverterTest.scala @@ -340,6 +340,7 @@ class ImageMetadataConverterTest extends AnyFunSpec with Matchers { it("should clean up 'just date' dates into iso format") { ImageMetadataConverter.cleanDate("2014-12-16") shouldBe "2014-12-16T00:00:00.000Z" + ImageMetadataConverter.cleanDate("2014-08-20") shouldBe "2014-08-20T00:00:00.000Z" } it("should clean up iso dates with seconds into iso format") { diff --git a/common-lib/src/test/scala/com/gu/mediaservice/model/ExternalThrallMessageTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/model/ExternalThrallMessageTest.scala index 44a8d4f169e..ee1ad991dd8 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/model/ExternalThrallMessageTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/model/ExternalThrallMessageTest.scala @@ -24,59 +24,59 @@ class ExternalThrallMessageTest extends AnyFreeSpec with Matchers with TableDriv "Make some JSON" - { "from an imageMessage" in { val image = ImageTest.createImage("hello") - val message = ImageMessage(nowUtc, image.copy(uploadTime = nowUtc)) + val message = ImageMessage(nowUtc, image.copy(uploadTime = nowUtc), instance = Instance("an-instance")) //Manually set the image time, because the time zone data is lost in //conversion roundTrip(message) } "from a DeleteImageMessage" in { - val dim = DeleteImageMessage("hey", nowUtc) + val dim = DeleteImageMessage("hey", nowUtc, instance = Instance("an-instance")) roundTrip(dim) } "from a DeleteImageExportsMessage" in { - val diem = DeleteImageExportsMessage("carpe", nowUtc) + val diem = DeleteImageExportsMessage("carpe", nowUtc, instance = Instance("an-instance")) roundTrip(diem) } "from a UpdateImageExportsMessage" in { - val uiem = UpdateImageExportsMessage("id", nowUtc, Seq()) + val uiem = UpdateImageExportsMessage("id", nowUtc, Seq(), instance = Instance("an-instance")) roundTrip(uiem) } "from a UpdateImageUserMetadataMessage" in { - val msg = UpdateImageUserMetadataMessage("hello", nowUtc, Edits(metadata = ImageMetadata())) + val msg = UpdateImageUserMetadataMessage("hello", nowUtc, Edits(metadata = ImageMetadata()), instance = Instance("an-instance")) roundTrip(msg) } "from a UpdateImageUsagesMessage" in { - val msg = UpdateImageUsagesMessage("hello", nowUtc, UsageNotice("hello", JsArray())) + val msg = UpdateImageUsagesMessage("hello", nowUtc, UsageNotice("hello", JsArray(), instance = Instance("an-instance")), instance = Instance("an-instance")) roundTrip(msg) } "from a ReplaceImageLeasesMessage" in { - val msg = ReplaceImageLeasesMessage("hello", nowUtc, Seq()) + val msg = ReplaceImageLeasesMessage("hello", nowUtc, Seq(), instance = Instance("an-instance")) roundTrip(msg) } "from a AddImageLeaseMessage" in { - val msg = AddImageLeaseMessage("hello", nowUtc, MediaLease(None, None, notes = None, mediaId = "")) + val msg = AddImageLeaseMessage("hello", nowUtc, MediaLease(None, None, notes = None, mediaId = ""), instance = Instance("an-instance")) roundTrip(msg) } "from a RemoveImageLeaseMessage" in { - val msg = RemoveImageLeaseMessage("hello", nowUtc, "bye") + val msg = RemoveImageLeaseMessage("hello", nowUtc, "bye", instance = Instance("an-instance")) roundTrip(msg) } "from a SetImageCollectionsMessage" in { - val msg = SetImageCollectionsMessage("hello", nowUtc, Seq()) + val msg = SetImageCollectionsMessage("hello", nowUtc, Seq(), instance = Instance("an-instance")) roundTrip(msg) } "from a DeleteUsagesMessage" in { - val msg = DeleteUsagesMessage("hello", nowUtc) + val msg = DeleteUsagesMessage("hello", nowUtc, instance = Instance("an-instance")) roundTrip(msg) } "from a UpdateImageSyndicationMetadataMessage" in { - val msg = UpdateImageSyndicationMetadataMessage("hello", nowUtc, None) + val msg = UpdateImageSyndicationMetadataMessage("hello", nowUtc, None, instance = Instance("an-instance")) roundTrip(msg) } "from a UpdateImagePhotoshootMetadataMessage" in { - val msg = UpdateImagePhotoshootMetadataMessage("hello", nowUtc, Edits(metadata = ImageMetadata())) + val msg = UpdateImagePhotoshootMetadataMessage("hello", nowUtc, Edits(metadata = ImageMetadata()), instance = Instance("an-instance")) roundTrip(msg) } } diff --git a/common-lib/src/test/scala/com/gu/mediaservice/model/ImageTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/model/ImageTest.scala index 4b5e1c7effa..54ba392fff4 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/model/ImageTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/model/ImageTest.scala @@ -14,7 +14,9 @@ class ImageTest extends AnyFunSpec with Matchers { describe("Image syndication status") { it("should be UnsuitableForSyndication by default") { - val image = createImage() + val image = createImage( + usageRights = NoRights + ) image.usages.length shouldBe 0 image.syndicationRights shouldBe None @@ -23,23 +25,21 @@ class ImageTest extends AnyFunSpec with Matchers { image.syndicationStatus shouldBe UnsuitableForSyndication } - it("should be AwaitingReviewForSyndication if syndication rights are acquired") { - val image = createImage( - syndicationRights = Some(rightsAcquired) - ) + it("should be AwaitingReviewForSyndication if owned") { + val image = createImage() image.syndicationStatus shouldBe AwaitingReviewForSyndication } - it("should be UnsuitableForSyndication if syndication rights are not acquired") { + it("should be UnsuitableForSyndication if not owned") { val image = createImage( - syndicationRights = Some(noRightsAcquired) + usageRights = Agency("An agency") ) image.syndicationStatus shouldBe UnsuitableForSyndication } - it("should be UnsuitableForSyndication if there is no syndication rights") { + it("should be UnsuitableForSyndication if not owned even if has syndication usages") { val imageId = UUID.randomUUID().toString val usages = List( @@ -60,7 +60,8 @@ class ImageTest extends AnyFunSpec with Matchers { val image = createImage( id = imageId, usages = usages, - leases = Some(leaseByMedia) + leases = Some(leaseByMedia), + usageRights = NoRights ) image.syndicationStatus shouldBe UnsuitableForSyndication @@ -135,7 +136,7 @@ class ImageTest extends AnyFunSpec with Matchers { } object ImageTest { - def createImage(id: String = UUID.randomUUID().toString, usages: List[Usage] = List(), leases: Option[LeasesByMedia] = None, syndicationRights: Option[SyndicationRights] = None): Image = { + def createImage(id: String = UUID.randomUUID().toString, usages: List[Usage] = List(), leases: Option[LeasesByMedia] = None, syndicationRights: Option[SyndicationRights] = None, usageRights: UsageRights = StaffPhotographer("T. Hanks", "The Guardian")): Image = { Image( id = id, uploadTime = DateTime.now(), @@ -143,7 +144,7 @@ object ImageTest { softDeletedMetadata = None, lastModified = None, identifiers = Map.empty, - uploadInfo = UploadInfo(filename = Some(s"test_$id.jpeg")), + uploadInfo = UploadInfo(filename = Some(s"test_$id.jpeg"), isFeedUpload = Some(true)), source = Asset( file = new URI(s"https://file/$id"), size = Some(1L), @@ -157,8 +158,8 @@ object ImageTest { userMetadata = None, metadata = ImageMetadata(dateTaken = None, title = Some(s"Test image $id"), keywords = None), originalMetadata = ImageMetadata(), - usageRights = StaffPhotographer("T. Hanks", "The Guardian"), - originalUsageRights = StaffPhotographer("T. Hanks", "The Guardian"), + usageRights = usageRights, + originalUsageRights = usageRights, exports = Nil, syndicationRights = syndicationRights, diff --git a/container-images/jdk-vips/Dockerfile b/container-images/jdk-vips/Dockerfile new file mode 100644 index 00000000000..6c51c523c59 --- /dev/null +++ b/container-images/jdk-vips/Dockerfile @@ -0,0 +1,44 @@ +FROM eclipse-temurin:25-noble +RUN apt-get update +RUN apt-get -y --no-install-suggests install -f build-essential ninja-build python3-pip bc wget +RUN apt-get -y --no-install-suggests install meson +RUN apt -y --no-install-suggests install \ + libfftw3-dev \ + libopenexr-dev \ + libgsf-1-dev \ + libglib2.0-dev \ + liborc-dev \ + libopenslide-dev \ + libmatio-dev \ + libwebp-dev \ + libjpeg-turbo8-dev \ + libexpat1-dev \ + libexif-dev \ + libtiff5-dev \ + libcfitsio-dev \ + libpoppler-glib-dev \ + librsvg2-dev \ + libpango1.0-dev \ + libopenjp2-7-dev \ + liblcms2-dev \ + libimagequant-dev \ + libheif-dev + +WORKDIR /tmp +RUN wget https://github.com/libvips/libvips/releases/download/v8.18.3/vips-8.18.3.tar.xz +RUN tar xf vips-8.18.3.tar.xz +WORKDIR /tmp/vips-8.18.3 +RUN meson setup build +WORKDIR /tmp/vips-8.18.3/build +RUN meson compile +RUN meson test +RUN meson install +RUN ldconfig + +RUN rm /tmp/vips-8.18.3.tar.xz +RUN rm -r /tmp/vips-8.18.3/ + +RUN apt -y --no-install-suggests install \ + libjemalloc-dev \ + graphicsmagick \ + graphicsmagick-imagemagick-compat diff --git a/container-images/jdk-vips/cloudbuild.yaml b/container-images/jdk-vips/cloudbuild.yaml new file mode 100644 index 00000000000..a59ea8c39b4 --- /dev/null +++ b/container-images/jdk-vips/cloudbuild.yaml @@ -0,0 +1,6 @@ +steps: + - name: 'gcr.io/cloud-builders/docker' + args: ['build', '-t', 'eu.gcr.io/$PROJECT_ID/jdk-vips:25-8.18.3', '.'] + - name: 'gcr.io/cloud-builders/docker' + args: ['push', 'eu.gcr.io/$PROJECT_ID/jdk-vips:25-8.18.3'] + diff --git a/cropper/app/CropperComponents.scala b/cropper/app/CropperComponents.scala index c51f554d364..0bd730b3606 100644 --- a/cropper/app/CropperComponents.scala +++ b/cropper/app/CropperComponents.scala @@ -1,5 +1,8 @@ +import app.photofox.vipsffm.{Vips, VipsHelper} +import com.gu.mediaservice.GridClient +import com.gu.mediaservice.lib.aws.S3 import com.gu.mediaservice.lib.imaging.ImageOperations -import com.gu.mediaservice.lib.management.{InnerServiceStatusCheckController, Management} +import com.gu.mediaservice.lib.management.Management import com.gu.mediaservice.lib.play.GridComponents import controllers.CropperController import lib.{CropStore, CropperConfig, Crops, Notifications} @@ -10,15 +13,21 @@ class CropperComponents(context: Context) extends GridComponents(context, new Cr final override val buildInfo = utils.buildinfo.BuildInfo val store = new CropStore(config) - val imageOperations = new ImageOperations(context.environment.rootPath.getAbsolutePath) + val imageOperations = { + Vips.init() + VipsHelper.cache_set_max(0) + new ImageOperations(context.environment.rootPath.getAbsolutePath) + } + val s3 = new S3(config) - val crops = new Crops(config, store, imageOperations) + val crops = new Crops(config, store, imageOperations, config.imageBucket, s3) val notifications = new Notifications(config) - val controller = new CropperController(auth, crops, store, notifications, config, controllerComponents, wsClient, authorisation) + private val gridClient = GridClient(config.services)(wsClient) + + val controller = new CropperController(auth, crops, store, notifications, config, controllerComponents, authorisation, gridClient) val permissionsAwareManagement = new Management(controllerComponents, buildInfo) - val InnerServiceStatusCheckController = new InnerServiceStatusCheckController(auth, controllerComponents, config.services, wsClient) - override lazy val router = new Routes(httpErrorHandler, controller, permissionsAwareManagement, InnerServiceStatusCheckController) + override lazy val router = new Routes(httpErrorHandler, controller, permissionsAwareManagement) } diff --git a/cropper/app/controllers/CropperController.scala b/cropper/app/controllers/CropperController.scala index dc2f05aff7b..8b52017bfe5 100644 --- a/cropper/app/controllers/CropperController.scala +++ b/cropper/app/controllers/CropperController.scala @@ -2,12 +2,14 @@ package controllers import _root_.play.api.libs.json._ import _root_.play.api.mvc.{BaseController, ControllerComponents} +import com.gu.mediaservice.GridClient import com.gu.mediaservice.lib.argo.ArgoHelpers import com.gu.mediaservice.lib.argo.model.Link import com.gu.mediaservice.lib.auth.Authentication.Principal import com.gu.mediaservice.lib.auth.Permissions.{DeleteCropsOrUsages, PrincipalFilter} import com.gu.mediaservice.lib.auth._ import com.gu.mediaservice.lib.aws.UpdateMessage +import com.gu.mediaservice.lib.config.InstanceForRequest import com.gu.mediaservice.lib.imaging.ExportResult import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} import com.gu.mediaservice.lib.play.RequestLoggingFilter @@ -16,7 +18,6 @@ import com.gu.mediaservice.syntax.MessageSubjects import lib._ import model._ import org.joda.time.DateTime -import play.api.libs.ws.WSClient import java.net.URI import scala.concurrent.{ExecutionContext, Future} @@ -30,23 +31,26 @@ case object ApiRequestFailed extends Exception("Failed to fetch the source") class CropperController(auth: Authentication, crops: Crops, store: CropStore, notifications: Notifications, config: CropperConfig, override val controllerComponents: ControllerComponents, - ws: WSClient, authorisation: Authorisation)(implicit val ec: ExecutionContext) - extends BaseController with MessageSubjects with ArgoHelpers { + authorisation: Authorisation, + gridClient: GridClient)(implicit val ec: ExecutionContext) + extends BaseController with MessageSubjects with ArgoHelpers with MediaApiUrls with InstanceForRequest { // Stupid name clash between Argo and Play import com.gu.mediaservice.lib.argo.model.{Action => ArgoAction} val AuthenticatedAndAuthorisedToDeleteCrops = auth andThen authorisation.CommonActionFilters.authorisedForDeleteCropsOrUsages - val indexResponse = { + private def indexResponse(instance: Instance) = { val indexData = Map("description" -> "This is the Cropper Service") val indexLinks = List( - Link("crop", s"${config.rootUri}/crops") + Link("crop", s"${config.rootUri(instance)}/crops") ) respond(indexData, indexLinks) } - def index = auth { indexResponse } + def index = auth { request => + indexResponse(instanceOf(request)) + } def addExport = auth.async(parse.json) { httpRequest => httpRequest.body.validate[ExportRequest] map { exportRequest => @@ -58,10 +62,10 @@ class CropperController(auth: Authentication, crops: Crops, store: CropStore, no val user = httpRequest.user val onBehalfOfPrincipal = auth.getOnBehalfOfPrincipal(user) - executeRequest(exportRequest, user, onBehalfOfPrincipal).map { case (imageId, export) => + executeRequest(exportRequest, user, onBehalfOfPrincipal, httpRequest).map { case (imageId, export) => val cropJson = Json.toJson(export).as[JsObject] - val updateMessage = UpdateMessage(subject = UpdateImageExports, id = Some(imageId), crops = Some(Seq(export))) + val updateMessage = UpdateMessage(subject = UpdateImageExports, id = Some(imageId), crops = Some(Seq(export)), instance = instanceOf(httpRequest)) notifications.publish(updateMessage) Ok(cropJson).as(ArgoMediaType) @@ -76,9 +80,6 @@ class CropperController(auth: Authentication, crops: Crops, store: CropStore, no case InvalidImage => logger.error(logMarker, InvalidImage.getMessage) respondError(BadRequest, "invalid-image", InvalidImage.getMessage) - case MissingSecureSourceUrl => - logger.error(logMarker, MissingSecureSourceUrl.getMessage) - respondError(BadRequest, "no-source-image", MissingSecureSourceUrl.getMessage) case InvalidCropRequest => logger.error(logMarker, InvalidCropRequest.getMessage) respondError(BadRequest, "invalid-crop", InvalidCropRequest.getMessage) @@ -102,10 +103,12 @@ class CropperController(auth: Authentication, crops: Crops, store: CropStore, no private val canDeleteCrops: PrincipalFilter = authorisation.hasPermissionTo(DeleteCropsOrUsages) - private def downloadExportLink(imageId: String, exportId: String, width: Int) = Link(s"crop-download-$exportId-$width", s"${config.apiUri}/images/$imageId/export/$exportId/asset/$width/download") + private def downloadExportMasterLink(imageId: String, exportId: String)(implicit instance: Instance) = Link(s"crop-download-$exportId-master", s"${config.apiUri(instance)}/images/$imageId/export/$exportId/master/download") - def getCrops(id: String) = auth.async { httpRequest => + private def downloadExportLink(imageId: String, exportId: String, width: Int)(implicit instance: Instance) = Link(s"crop-download-$exportId-$width", s"${config.apiUri(instance)}/images/$imageId/export/$exportId/asset/$width/download") + def getCrops(id: String) = auth.async { httpRequest => + implicit val instance: Instance = instanceOf(httpRequest) implicit val logMarker: LogMarker = MarkerMap( "requestType" -> "getCrops", "requestId" -> RequestLoggingFilter.getRequestId(httpRequest), @@ -114,24 +117,31 @@ class CropperController(auth: Authentication, crops: Crops, store: CropStore, no logger.info(logMarker, s"getting crops for $id") - store.listCrops(id) map (_.toList) map { crops => + store.listCrops(id, instance) map (_.toList) map { crops => val deleteCropsAction = - ArgoAction("delete-crops", URI.create(s"${config.rootUri}/crops/$id"), "DELETE") - - lazy val cropDownloadLinks = for { - crop <- crops - asset <- crop.assets - dimensions <- asset.dimensions - width = dimensions.width - cropId <- crop.id - } yield downloadExportLink(id, cropId, width) + ArgoAction("delete-crops", URI.create(s"${config.rootUri(instance)}/crops/$id"), "DELETE") + + lazy val cropDownloadAssetsLinks = + for { + crop <- crops + asset <- crop.assets + dimensions <- asset.dimensions + width = dimensions.width + cropId <- crop.id + } yield downloadExportLink(id, cropId, width) + + lazy val cropDownloadMasterLinks = + for { + crop <- crops + cropId <- crop.id + } yield downloadExportMasterLink(id, cropId) val links = (for { crop <- crops.headOption link = Link("image", crop.specification.uri) } yield { if (config.canDownloadCrop) { - link :: cropDownloadLinks + link :: cropDownloadAssetsLinks ++ cropDownloadMasterLinks } else List(link) }) getOrElse List() @@ -144,13 +154,14 @@ class CropperController(auth: Authentication, crops: Crops, store: CropStore, no } def deleteCrops(id: String) = AuthenticatedAndAuthorisedToDeleteCrops.async { httpRequest => + implicit val instance: Instance = instanceOf(httpRequest) implicit val logMarker: LogMarker = MarkerMap( "requestType" -> "deleteCrops", "requestId" -> RequestLoggingFilter.getRequestId(httpRequest), "imageId" -> id ) store.deleteCrops(id).map { _ => - val updateMessage = UpdateMessage(subject = DeleteImageExports, id = Some(id)) + val updateMessage = UpdateMessage(subject = DeleteImageExports, id = Some(id), instance = instance) notifications.publish(updateMessage) Accepted } recover { @@ -159,11 +170,12 @@ class CropperController(auth: Authentication, crops: Crops, store: CropStore, no } def executeRequest( - exportRequest: ExportRequest, user: Principal, onBehalfOfPrincipal: Authentication.OnBehalfOfPrincipal + exportRequest: ExportRequest, user: Principal, onBehalfOfPrincipal: Authentication.OnBehalfOfPrincipal, + request: Authentication.Request[JsValue] )(implicit logMarker: LogMarker): Future[(String, Crop)] = { - + implicit val instance: Instance = instanceOf(request) for { - _ <- verify(isMediaApiUri(exportRequest.uri), InvalidSource) + _ <- verify(isMediaApiImageUri(exportRequest.uri, config.apiUri(instance)), InvalidSource) apiImage <- fetchSourceFromApi(exportRequest.uri, onBehalfOfPrincipal) _ <- verify(apiImage.valid, InvalidImage) // Image should always have dimensions, but we want to safely extract the Option @@ -178,44 +190,13 @@ class CropperController(auth: Authentication, crops: Crops, store: CropStore, no specification = cropSpec ) markersWithCropDetails = logMarker ++ Map("imageId" -> apiImage.id, "cropId" -> Crop.getCropId(cropSpec.bounds)) - ExportResult(id, masterSizing, sizings) <- crops.makeExport(apiImage, crop)(markersWithCropDetails) + ExportResult(id, masterSizing, sizings) <- crops.makeExport(apiImage, crop)(markersWithCropDetails, instance) finalCrop = Crop.createFromCrop(crop, masterSizing, sizings) } yield (id, finalCrop) } - // TODO: lame, parse into URI object and compare host instead - def isMediaApiUri(uri: String): Boolean = uri.startsWith(config.apiUri) - - def fetchSourceFromApi(uri: String, onBehalfOfPrincipal: Authentication.OnBehalfOfPrincipal): Future[SourceImage] = { - - case class HttpClientResponse(status: Int, statusText: String, json: JsValue) - - val baseRequest = ws.url(uri) - .withQueryStringParameters("include" -> "fileMetadata") - - val request = onBehalfOfPrincipal(baseRequest) - - val responseFuture = request.get().map { r => - HttpClientResponse(r.status, r.statusText, Json.parse(r.body)) - } - - responseFuture recoverWith { - case NonFatal(e) => - logger.warn(s"HTTP request to fetch source failed: $e") - Future.failed(ApiRequestFailed) - } - - for (resp <- responseFuture) - yield { - if (resp.status == 404) { - throw ImageNotFound - } else if (resp.status != 200) { - logger.warn(s"HTTP status ${resp.status} ${resp.statusText} from $uri") - throw ApiRequestFailed - } else { - resp.json.as[SourceImage] - } - } + private def fetchSourceFromApi(uri: String, onBehalfOfPrincipal: Authentication.OnBehalfOfPrincipal)(implicit instance: Instance): Future[SourceImage] = { + gridClient.getSourceImage(imageIdFrom(uri), onBehalfOfPrincipal) } def verify(cond: => Boolean, error: Throwable): Future[Unit] = diff --git a/cropper/app/controllers/MediaApiUrls.scala b/cropper/app/controllers/MediaApiUrls.scala new file mode 100644 index 00000000000..5ccb8f4161e --- /dev/null +++ b/cropper/app/controllers/MediaApiUrls.scala @@ -0,0 +1,16 @@ +package controllers + +trait MediaApiUrls { + + def isMediaApiImageUri(uri: String, apiUri: String): Boolean = { + val hasMediaApiPrefix = uri.startsWith(apiUri) + val suffix = uri.drop(apiUri.length) + val suffixComponents = suffix.split("/") + val hasImageSuffix = suffixComponents.length == 3 && suffixComponents(1) == "images" + hasMediaApiPrefix && hasImageSuffix + } + + def imageIdFrom(uri: String): String = { + uri.split("/").last + } +} diff --git a/cropper/app/lib/AspectRatio.scala b/cropper/app/lib/AspectRatio.scala index b093f1b5875..28161afe301 100644 --- a/cropper/app/lib/AspectRatio.scala +++ b/cropper/app/lib/AspectRatio.scala @@ -9,7 +9,8 @@ object AspectRatio { Ratio("5:3", 5, 3), Ratio("2:3", 2, 3), Ratio("16:9", 16, 9), - Ratio("1:1", 1, 1) + Ratio("1:1", 1, 1), + Ratio("3:2", 3, 2) ) def clean(aspect: String): Option[Float] = knownRatios diff --git a/cropper/app/lib/CropStore.scala b/cropper/app/lib/CropStore.scala index a81b141793c..034790420f6 100644 --- a/cropper/app/lib/CropStore.scala +++ b/cropper/app/lib/CropStore.scala @@ -26,8 +26,8 @@ class CropStore(config: CropperConfig) extends S3ImageStorage(config) with CropS } } - def listCrops(id: String): Future[List[Crop]] = { - list(config.imgPublishingBucket, id).map { crops => + def listCrops(id: String, instance: Instance): Future[List[Crop]] = { + list(config.imgPublishingBucket, folderForImagesCrops(id, instance)).map { crops => // TODO crops layout want to be pull up crops.foldLeft(Map[String, Crop]()) { case (map, (s3Object)) => { val filename::containingFolder::_ = s3Object.uri.getPath.split("/").reverse.toList @@ -49,9 +49,11 @@ class CropStore(config: CropperConfig) extends S3ImageStorage(config) with CropS date = userMetadata.get("date").flatMap(parseDateTime) dimensions = Dimensions(width, height) + key = config.imgPublishingBucket.keyFromS3URL(s3Object.uri) + sizing = Asset( - translateImgHost(s3Object.uri), + signedCropAssetUrl(key), Some(s3Object.size), objectMetadata.contentType, Some(dimensions), @@ -70,11 +72,20 @@ class CropStore(config: CropperConfig) extends S3ImageStorage(config) with CropS } } - def deleteCrops(id: String)(implicit logMarker: LogMarker) = { - deleteFolder(config.imgPublishingBucket, id) + def deleteCrops(id: String)(implicit logMarker: LogMarker, instance: Instance) = { + deleteFolder(config.imgPublishingBucket, folderForImagesCrops(id, instance)) } // FIXME: this doesn't really belong here def translateImgHost(uri: URI): URI = new URI("https", config.imgPublishingHost, uri.getPath, uri.getFragment) + + private def folderForImagesCrops(id: String, instance: Instance) = { + instance.id + "/" + id + } + + private def signedCropAssetUrl(key: String): URI = { + signUrlTony(config.imgPublishingBucket, key).toURI + } + } diff --git a/cropper/app/lib/CropperConfig.scala b/cropper/app/lib/CropperConfig.scala index 25f78a5dacc..5e082585bbb 100644 --- a/cropper/app/lib/CropperConfig.scala +++ b/cropper/app/lib/CropperConfig.scala @@ -1,21 +1,28 @@ package lib +import com.gu.mediaservice.lib.aws.{S3, S3Bucket} import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources} +import com.gu.mediaservice.model.Instance import java.io.File class CropperConfig(resources: GridConfigResources) extends CommonConfig(resources) { - val imgPublishingBucket = string("publishing.image.bucket") - + // TODO this is common with media-api download exports + val imgPublishingBucket: S3Bucket = S3Bucket( + string("publishing.image.bucket.name"), + string("publishing.image.bucket.endpoint"), + boolean("publishing.image.bucket.pathStyleURLs"), + clientFor(string("publishing.image.bucket.endpoint")) + ) val canDownloadCrop: Boolean = boolean("canDownloadCrop") val imgPublishingHost = string("publishing.image.host") // Note: work around CloudFormation not allowing optional parameters val imgPublishingSecureHost = stringOpt("publishing.image.secure.host").filterNot(_.isEmpty) - val rootUri = services.cropperBaseUri - val apiUri = services.apiBaseUri + val rootUri: Instance => String = services.cropperBaseUri + val apiUri: Instance => String = services.apiBaseUri val tempDir: File = new File(stringDefault("crop.output.tmp.dir", "/tmp")) diff --git a/cropper/app/lib/Crops.scala b/cropper/app/lib/Crops.scala index a38771aa000..631d2ac1727 100644 --- a/cropper/app/lib/Crops.scala +++ b/cropper/app/lib/Crops.scala @@ -1,95 +1,66 @@ package lib -import java.io.File -import com.gu.mediaservice.lib.metadata.FileMetadataHelper +import app.photofox.vipsffm.VImage +import app.photofox.vipsffm.jextract.VipsRaw import com.gu.mediaservice.lib.Files +import com.gu.mediaservice.lib.aws.{S3, S3Bucket} import com.gu.mediaservice.lib.imaging.{ExportResult, ImageOperations} import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch} import com.gu.mediaservice.model._ +import java.io.File +import java.lang.foreign.Arena import scala.concurrent.{ExecutionContext, Future} import scala.util.Try case object InvalidImage extends Exception("Invalid image cannot be cropped") case object MissingMimeType extends Exception("Missing mimeType from source API") -case object MissingSecureSourceUrl extends Exception("Missing secureUrl from source API") case object InvalidCropRequest extends Exception("Crop request invalid for image dimensions") -case class MasterCrop(sizing: Future[Asset], file: File, dimensions: Dimensions, aspectRatio: Float) +case class MasterCrop(image: VImage, dimensions: Dimensions, aspectRatio: Float) -class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOperations)(implicit ec: ExecutionContext) extends GridLogging { +class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOperations, imageBucket: S3Bucket, s3: S3)(implicit ec: ExecutionContext) extends GridLogging { import Files._ - private val cropQuality = 75d - private val masterCropQuality = 95d + private val cropQuality = 75 + private val jpegMasterCropQuality = 95 // For PNGs, Magick considers "quality" parameter as effort spent on compression - 1 meaning none, 100 meaning max. // We don't overly care about output crop file sizes here, but prefer a fast output, so turn it right down. - private val pngCropQuality = 1d + private val pngMasterCropQuality = 1 // No effort spend compressing the PNG master - def outputFilename(source: SourceImage, bounds: Bounds, outputWidth: Int, fileType: MimeType, isMaster: Boolean = false): String = { + def outputFilename(imageId: String, bounds: Bounds, outputWidth: Int, fileType: MimeType, isMaster: Boolean = false)(implicit instance: Instance): String = { val masterString: String = if (isMaster) "master/" else "" - s"${source.id}/${Crop.getCropId(bounds)}/$masterString$outputWidth${fileType.fileExtension}" + instance.id + "/" + s"$imageId/${Crop.getCropId(bounds)}/$masterString$outputWidth${fileType.fileExtension}" } - def createMasterCrop( + private def createMasterCrop( apiImage: SourceImage, sourceFile: File, crop: Crop, - mediaType: MimeType, - colourModel: Option[String], - orientationMetadata: Option[OrientationMetadata], - )(implicit logMarker: LogMarker): Future[MasterCrop] = { + metadata: ImageMetadata, + orientationMetadata: Option[OrientationMetadata] + )(implicit logMarker: LogMarker, arena: Arena): MasterCrop = { - Stopwatch.async(s"creating master crop for ${apiImage.id}") { + Stopwatch(s"creating master crop for ${apiImage.id}") { val source = crop.specification - val metadata = apiImage.metadata - val iccColourSpace = FileMetadataHelper.normalisedIccColourSpace(apiImage.fileMetadata) - // pngs are always lossless, so quality only means effort spent compressing them. We don't - // care too much about filesize of master crops, so skip expensive compression to get faster cropping - val quality = if (mediaType == Png) pngCropQuality else masterCropQuality - - for { - strip <- imageOperations.cropImage( - sourceFile, apiImage.source.mimeType, source.bounds, quality, config.tempDir, - iccColourSpace, colourModel, mediaType, isTransformedFromSource = false, - orientationMetadata = orientationMetadata - ) - file: File <- imageOperations.appendMetadata(strip, metadata) - dimensions = Dimensions(source.bounds.width, source.bounds.height) - filename = outputFilename(apiImage, source.bounds, dimensions.width, mediaType, isMaster = true) - sizing = store.storeCropSizing(file, filename, mediaType, crop, dimensions) - dirtyAspect = source.bounds.width.toFloat / source.bounds.height - aspect = crop.specification.aspectRatio.flatMap(AspectRatio.clean).getOrElse(dirtyAspect) - } - yield MasterCrop(sizing, file, dimensions, aspect) - } - } - - def createCrops(sourceFile: File, dimensionList: List[Dimensions], apiImage: SourceImage, crop: Crop, cropType: MimeType)(implicit logMarker: LogMarker): Future[List[Asset]] = { - val quality = if (cropType == Png) pngCropQuality else cropQuality - - Stopwatch.async(s"creating crops for ${apiImage.id}") { - Future.sequence(dimensionList.map { dimensions => - val cropLogMarker = logMarker ++ Map("crop-dimensions" -> s"${dimensions.width}x${dimensions.height}") - for { - file <- imageOperations.resizeImage(sourceFile, - apiImage.source.mimeType, - dimensions, - quality, - config.tempDir, - cropType)(cropLogMarker) - optimisedFile = imageOperations.optimiseImage(file, cropType)(cropLogMarker) - filename = outputFilename(apiImage, crop.specification.bounds, dimensions.width, cropType) - sizing <- store.storeCropSizing(optimisedFile, filename, cropType, crop, dimensions)(cropLogMarker) - _ <- delete(file) - _ <- delete(optimisedFile) - } - yield sizing - }) + logger.info(logMarker, s"creating master crop for ${apiImage.id}") + val masterImage = imageOperations.cropImageVips( + sourceFile, + source.bounds, + metadata, + orientationMetadata = orientationMetadata + ) + + //file: File <- imageOperations.appendMetadata(strip, metadata) + val dimensions = Dimensions(source.bounds.width, source.bounds.height) + val dirtyAspect = source.bounds.width.toFloat / source.bounds.height + val aspect = crop.specification.aspectRatio.flatMap(AspectRatio.clean).getOrElse(dirtyAspect) + + MasterCrop(masterImage, dimensions, aspect) } } - def deleteCrops(id: String)(implicit logMarker: LogMarker): Future[Unit] = store.deleteCrops(id) + def deleteCrops(id: String)(implicit logMarker: LogMarker, instance: Instance): Future[Unit] = store.deleteCrops(id) private def dimensionsFromConfig(bounds: Bounds, aspectRatio: Float): List[Dimensions] = if (bounds.isPortrait) config.portraitCropSizingHeights.filter(_ <= bounds.height).map(h => Dimensions(math.round(h * aspectRatio), h)) @@ -106,45 +77,99 @@ class Crops(config: CropperConfig, store: CropStore, imageOperations: ImageOpera positiveCoords && strictlyPositiveSize && withinBounds } - def makeExport(apiImage: SourceImage, crop: Crop)(implicit logMarker: LogMarker): Future[ExportResult] = { - val source = crop.specification + def makeExport(apiImage: SourceImage, crop: Crop)(implicit logMarker: LogMarker, instance: Instance): Future[ExportResult] = { + val source = crop.specification val mimeType = apiImage.source.mimeType.getOrElse(throw MissingMimeType) - val secureUrl = apiImage.source.secureUrl.getOrElse(throw MissingSecureSourceUrl) - val colourType = apiImage.fileMetadata.colourModelInformation.getOrElse("colorType", "") - val hasAlpha = apiImage.fileMetadata.colourModelInformation.get("hasAlpha").flatMap(a => Try(a.toBoolean).toOption).getOrElse(true) - val cropType = Crops.cropType(mimeType, colourType, hasAlpha) + val secureFile = apiImage.source.file + + val key = imageBucket.keyFromS3URL(secureFile) + val secureUrl = s3.signUrlTony(imageBucket, key) - Stopwatch.async(s"making crop assets for ${apiImage.id} ${Crop.getCropId(source.bounds)}") { - for { - sourceFile <- tempFileFromURL(secureUrl, "cropSource", "", config.tempDir) - colourModel <- ImageOperations.identifyColourModel(sourceFile, mimeType) - masterCrop <- createMasterCrop(apiImage, sourceFile, crop, cropType, colourModel, apiImage.source.orientationMetadata) + //val eventualResult = Stopwatch(s"making crop assets for ${apiImage.id} ${Crop.getCropId(source.bounds)}") { + tempFileFromURL(secureUrl, "cropSource", "", config.tempDir).flatMap { sourceFile => + logger.info("Starting vips operations") + implicit val arena: Arena = Arena.ofShared() + val masterCrop = createMasterCrop(apiImage, sourceFile, crop, apiImage.metadata, apiImage.source.orientationMetadata) - outputDims = dimensionsFromConfig(source.bounds, masterCrop.aspectRatio) :+ masterCrop.dimensions + val isGraphic = ImageOperations.isGraphicVips(masterCrop.image) + val hasAlpha = ImageOperations.hasAlpha(masterCrop.image) + val cropType = Crops.cropType(mimeType, isGraphic = isGraphic, hasAlpha = hasAlpha) - sizes <- createCrops(masterCrop.file, outputDims, apiImage, crop, cropType) - masterSize <- masterCrop.sizing - _ <- Future.sequence(List(masterCrop.file, sourceFile).map(delete)) + // pngs are always lossless, so quality only means effort spent compressing them. We don't + // care too much about filesize of master crops, so skip expensive compression to get faster cropping + val masterQuality = if (mimeType == Png) pngMasterCropQuality else jpegMasterCropQuality + + // High quality rendering with minimal compression which will be used as the CDN resizer origin + logger.info("Requesting master file save") + val eventualMasterSaved = Future { + val masterCropFile = File.createTempFile(s"crop-", s"${cropType.fileExtension}", config.tempDir) // TODO function for this + imageOperations.saveImageToFile(masterCrop.image, cropType, masterQuality, masterCropFile, keep = Some(VipsRaw.VIPS_FOREIGN_KEEP_XMP)) + masterCropFile + } + + // Static crops; higher compression + logger.info("Requesting resize file saves") + val outputDims = dimensionsFromConfig(source.bounds, masterCrop.aspectRatio) :+ masterCrop.dimensions + val eventualResizes = imageOperations.createCrops(masterCrop.image, outputDims, apiImage.id, crop.specification.bounds, cropType, config.tempDir, cropQuality) + + // Store assets after master and resize file generation completes; to avoid sending a partial set on failure + // Delete the stored files and return the export result + eventualMasterSaved.flatMap { masterCropFile => + eventualResizes.flatMap { resizes => + // All vips operations have completed; we can close the arena + arena.close() + logger.info("Finished vips operations") + + val eventuallyStoredMasterAsset = store.storeCropSizing(masterCropFile, outputFilename(apiImage.id, source.bounds, masterCrop.dimensions.width, cropType, isMaster = true), + cropType, crop, masterCrop.dimensions).map { masterAsset => + logger.info("Master Crop stored") + delete(masterCropFile) + masterAsset + } + + val eventualStoredResizeAssets = Future.sequence(resizes.map { resize: (File, String, Dimensions) => + val file = resize._1 + val filename = resize._2 + val dimensions = resize._3 + logger.info(s"Storing crop for: $file, $filename, $cropType") + for { + resizedAsset: Asset <- store.storeCropSizing(file, filename, cropType, crop, dimensions) + _ <- delete(file) + } + yield resizedAsset + }) + + for { + masterAsset: Asset <- eventuallyStoredMasterAsset + resizedAssets: Seq[Asset] <- eventualStoredResizeAssets + } yield { + logger.info("Store assets completed") + delete(sourceFile) + ExportResult(apiImage.id, masterAsset, resizedAssets.toList) + } + } } - yield ExportResult(apiImage.id, masterSize, sizes) } } } -object Crops { +object Crops extends GridLogging { /** * The aim here is to decide whether the crops should be JPEG or PNGs depending on a predicted quality/size trade-off. * - If the image has transparency then it should always be a PNG as the transparency is not available in JPEG * - If the image is not true colour then we assume it is a graphic that should be retained as a PNG */ - def cropType(mediaType: MimeType, colourType: String, hasAlpha: Boolean): MimeType = { - val isGraphic = !colourType.matches("True[ ]?Color.*") + def cropType(mediaType: MimeType, isGraphic: Boolean, hasAlpha: Boolean): MimeType = { val outputAsPng = hasAlpha || isGraphic - mediaType match { - case Png | Tiff if outputAsPng => Png + val decision = mediaType match { + case Png if outputAsPng => Png + case Tiff if outputAsPng => Png case _ => Jpeg } + + logger.info(s"Choose crop type for $mediaType, $isGraphic, $hasAlpha: " + decision) + decision } } diff --git a/cropper/conf/routes b/cropper/conf/routes index c4aad4e4ee0..207c5df4b17 100644 --- a/cropper/conf/routes +++ b/cropper/conf/routes @@ -8,7 +8,6 @@ DELETE /crops/:id controllers.CropperControl # Management GET /management/healthcheck com.gu.mediaservice.lib.management.Management.healthCheck GET /management/manifest com.gu.mediaservice.lib.management.Management.manifest -GET /management/whoAmI com.gu.mediaservice.lib.management.InnerServiceStatusCheckController.whoAmI(depth: Int) # Shoo robots away GET /robots.txt com.gu.mediaservice.lib.management.Management.disallowRobots diff --git a/cropper/test/controllers/MediaApiUrlsTest.scala b/cropper/test/controllers/MediaApiUrlsTest.scala new file mode 100644 index 00000000000..6033c6b8ab4 --- /dev/null +++ b/cropper/test/controllers/MediaApiUrlsTest.scala @@ -0,0 +1,22 @@ +package controllers + +import com.gu.mediaservice.lib.imaging.ImageOperations +import com.gu.mediaservice.model._ +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.mockito.MockitoSugar +import org.scalatest._ +import flatspec._ +import matchers._ + +class MediaApiUrlsTest extends AnyFlatSpec with Matchers with MediaApiUrls { + + "media api urls" should + "identify media API image urls" in { + isMediaApiImageUri("https://media.api.test.com/images/cb5b8c05b690db2d034457b5461ef32abf29eff8", "https://media.api.test.com") shouldBe true + isMediaApiImageUri("https://media.api.test.com/images", "https://media.api.test.com") shouldBe false + isMediaApiImageUri("images/cb5b8c05b690db2d034457b5461ef32abf29eff8", "https://media.api.test.com") shouldBe false + isMediaApiImageUri("cb5b8c05b690db2d034457b5461ef32abf29eff8", "https://media.api.test.com") shouldBe false + } + +} diff --git a/cropper/test/lib/CropsTest.scala b/cropper/test/lib/CropsTest.scala index a0d0ba8a62c..cf66d76899d 100644 --- a/cropper/test/lib/CropsTest.scala +++ b/cropper/test/lib/CropsTest.scala @@ -1,7 +1,10 @@ package lib +import com.amazonaws.services.s3.AmazonS3 +import com.gu.mediaservice.lib.aws.{S3, S3Bucket} import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.model._ +import org.mockito.Mockito.when import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers import org.scalatestplus.mockito.MockitoSugar @@ -9,64 +12,75 @@ import org.scalatestplus.mockito.MockitoSugar class CropsTest extends AnyFunSpec with Matchers with MockitoSugar { import scala.concurrent.ExecutionContext.Implicits.global + private implicit val instance: Instance = Instance(id = "an-instance") + it("should return JPEG when the input type is a JPEG") { - Crops.cropType(Jpeg, "True Color", hasAlpha = false) shouldBe Jpeg - Crops.cropType(Jpeg, "Monkey", hasAlpha = false) shouldBe Jpeg + Crops.cropType(Jpeg, isGraphic = false, hasAlpha = false) shouldBe Jpeg + Crops.cropType(Jpeg, isGraphic = true, hasAlpha = false) shouldBe Jpeg } it("should return PNG when the input type is PNG and it has alpha") { - Crops.cropType(Png, "Monkey", hasAlpha = true) shouldBe Png + Crops.cropType(Png, isGraphic = true, hasAlpha = true) shouldBe Png } it("should return PNG when the input type is PNG and it has alpha even if it is True Color") { - Crops.cropType(Png, "True Color", hasAlpha = true) shouldBe Png + Crops.cropType(Png, isGraphic = false, hasAlpha = true) shouldBe Png } it("should return PNG when the input type is PNG and it is NOT true color (a graphic)") { - Crops.cropType(Png, "Monkey", hasAlpha = false) shouldBe Png + Crops.cropType(Png, isGraphic = true, hasAlpha = false) shouldBe Png } it("should return JPEG when the input type is PNG and it is true color") { - Crops.cropType(Png, "True Color", hasAlpha = false) shouldBe Jpeg + Crops.cropType(Png, isGraphic = false, hasAlpha = false) shouldBe Jpeg } it("should return PNG when the input type is TIFF and it has alpha") { - Crops.cropType(Tiff, "Monkey", hasAlpha = true) shouldBe Png + Crops.cropType(Tiff, isGraphic = false, hasAlpha = true) shouldBe Png } it("should return PNG when the input type is TIFF and it doesn't have alpha or is true color") { - Crops.cropType(Tiff, "Monkey", hasAlpha = false) shouldBe Png + Crops.cropType(Tiff, isGraphic = true, hasAlpha = false) shouldBe Png } it("should return JPEG when the input type is TIFF and it doesn't have alpha and it is true color") { - Crops.cropType(Tiff, "TrueColor", hasAlpha = false) shouldBe Jpeg + Crops.cropType(Tiff, isGraphic = false, hasAlpha = false) shouldBe Jpeg } - private val config = mock[CropperConfig] + private val config = { + val mockConfig = mock[CropperConfig] + when(mockConfig.awsRegion).thenReturn("eu-west-1") + when(mockConfig.googleS3AccessKey).thenReturn(None) + when(mockConfig.googleS3SecretKey).thenReturn(None) + mockConfig + } private val store = mock[CropStore] private val imageOperations: ImageOperations = mock[ImageOperations] private val source: SourceImage = SourceImage("test", mock[Asset], valid = true, mock[ImageMetadata], mock[FileMetadata]) private val bounds: Bounds = Bounds(10, 20, 30, 40) private val outputWidth = 1234 + private val mockS3Client = mock[AmazonS3] + private val imageBucket = S3Bucket("crops-bucket", S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false, mockS3Client) + private val s3 = new S3(config) it("should should construct a correct address for a master jpg") { - val outputFilename = new Crops(config, store, imageOperations) - .outputFilename(source, bounds, outputWidth, Jpeg, isMaster = true) - outputFilename shouldBe "test/10_20_30_40/master/1234.jpg" + val outputFilename = new Crops(config, store, imageOperations, imageBucket, s3) + .outputFilename(source.id, bounds, outputWidth, Jpeg, isMaster = true) + outputFilename shouldBe "an-instance/test/10_20_30_40/master/1234.jpg" } it("should should construct a correct address for a non-master jpg") { - val outputFilename = new Crops(config, store, imageOperations) - .outputFilename(source, bounds, outputWidth, Jpeg) - outputFilename shouldBe "test/10_20_30_40/1234.jpg" + val outputFilename = new Crops(config, store, imageOperations, imageBucket, s3) + .outputFilename(source.id, bounds, outputWidth, Jpeg) + outputFilename shouldBe "an-instance/test/10_20_30_40/1234.jpg" } it("should should construct a correct address for a non-master tiff") { - val outputFilename = new Crops(config, store, imageOperations) - .outputFilename(source, bounds, outputWidth, Tiff) - outputFilename shouldBe "test/10_20_30_40/1234.tiff" + val outputFilename = new Crops(config, store, imageOperations, imageBucket, s3) + .outputFilename(source.id, bounds, outputWidth, Tiff) + outputFilename shouldBe "an-instance/test/10_20_30_40/1234.tiff" } it("should should construct a correct address for a non-master png") { - val outputFilename = new Crops(config, store, imageOperations) - .outputFilename(source, bounds, outputWidth, Png) - outputFilename shouldBe "test/10_20_30_40/1234.png" + val outputFilename = new Crops(config, store, imageOperations, imageBucket, s3) + .outputFilename(source.id, bounds, outputWidth, Png) + outputFilename shouldBe "an-instance/test/10_20_30_40/1234.png" } } diff --git a/dev/imgops/nginx.conf b/dev/imgops/nginx.conf index 7c9d5ee8796..d8402e31687 100644 --- a/dev/imgops/nginx.conf +++ b/dev/imgops/nginx.conf @@ -17,8 +17,10 @@ http { map $no_q $no_w { "~*^(.*)(?:(?:^|&)w=[^&]*)(.*)$" $1$2; default $no_q; } map $no_w $no_h { "~*^(.*)(?:(?:^|&)h=[^&]*)(.*)$" $1$2; default $no_w; } map $no_h $no_r { "~*^(.*)(?:(?:^|&)r=[^&]*)(.*)$" $1$2; default $no_h; } + # timestamp param is added by cropperjs for crop previews + map $no_r $no_timestamp { "~*^(.*)(?:(?:^|&)r=[^&]*)(.*)$" $1$2; default $no_r; } # Then another pass to remove any leading, trailing or doubled '&'s - map $no_r $clean_leading { "~^&+(.*)" $1; default $no_r; } + map $no_timestamp $clean_leading { "~^&+(.*)" $1; default $no_timestamp; } map $clean_leading $clean_trailing { "~(.*)&+$" $1; default $clean_leading; } map $clean_trailing $final_clean_args { "~(?
.*)&&+(?.*)" "$pre&$post"; default $clean_trailing; }
 
@@ -48,7 +50,7 @@ http {
       image_filter rotate $arg_r;
 
       # connect to the localstack docker container
-      proxy_pass http://localstack:4566$uri?final_clean_args;
+      proxy_pass http://localstack:4566$uri?$final_clean_args;
     }
   }
 }
diff --git a/dev/oidc-provider/Dockerfile b/dev/oidc-provider/Dockerfile
index 24ae5ae3a39..51efc744a10 100644
--- a/dev/oidc-provider/Dockerfile
+++ b/dev/oidc-provider/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:16
+FROM node:24
 
 WORKDIR /usr/src/app
 
diff --git a/dev/oidc-provider/app.js b/dev/oidc-provider/app.js
index e4fd510a8b7..f2753772af2 100644
--- a/dev/oidc-provider/app.js
+++ b/dev/oidc-provider/app.js
@@ -2,7 +2,7 @@
 import { findAccountFunc } from "./find-account.js";
 import { makeProvider } from "./make-provider.js";
 // relative path to the users file within the localstack box ( to /etc/grid/users.json)
-import USER_JSON from "../../../etc/grid/users.json" assert { type: "json" };
+import USER_JSON from "../../../etc/grid/users.json" with { type: "json" };
 
 const { DOMAIN, EMAIL_DOMAIN, OIDC_CLIENT_ID, OIDC_CLIENT_SECRET } =
   process.env;
diff --git a/dev/oidc-provider/package-lock.json b/dev/oidc-provider/package-lock.json
index d14276de07e..5a792a9a130 100644
--- a/dev/oidc-provider/package-lock.json
+++ b/dev/oidc-provider/package-lock.json
@@ -9,7 +9,7 @@
       "version": "1.0.0",
       "license": "ISC",
       "dependencies": {
-        "oidc-provider": "^8.5.1"
+        "oidc-provider": "^9.8.3"
       },
       "devDependencies": {
         "jest": "^29.7.0"
@@ -959,18 +959,26 @@
       }
     },
     "node_modules/@koa/router": {
-      "version": "12.0.1",
-      "resolved": "https://registry.npmjs.org/@koa/router/-/router-12.0.1.tgz",
-      "integrity": "sha512-ribfPYfHb+Uw3b27Eiw6NPqjhIhTpVFzEWLwyc/1Xp+DCdwRRyIlAUODX+9bPARF6aQtUu1+/PHzdNvRzcs/+Q==",
+      "version": "15.5.0",
+      "resolved": "https://registry.npmjs.org/@koa/router/-/router-15.5.0.tgz",
+      "integrity": "sha512-KSC0oG/5t6ITu5wqX4lJseA/dngoj14hEaohrLZEXtlUT2RRyJvwaJ0KV+5uQoaWrY3A8ClHOrBEU4g8dujn8Q==",
+      "license": "MIT",
       "dependencies": {
-        "debug": "^4.3.4",
-        "http-errors": "^2.0.0",
+        "debug": "^4.4.3",
+        "http-errors": "^2.0.1",
         "koa-compose": "^4.1.0",
-        "methods": "^1.1.2",
-        "path-to-regexp": "^6.2.1"
+        "path-to-regexp": "^8.4.2"
       },
       "engines": {
-        "node": ">= 12"
+        "node": ">= 20"
+      },
+      "peerDependencies": {
+        "koa": "^2.0.0 || ^3.0.0"
+      },
+      "peerDependenciesMeta": {
+        "koa": {
+          "optional": false
+        }
       }
     },
     "node_modules/@sinclair/typebox": {
@@ -979,17 +987,6 @@
       "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==",
       "dev": true
     },
-    "node_modules/@sindresorhus/is": {
-      "version": "5.6.0",
-      "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.6.0.tgz",
-      "integrity": "sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==",
-      "engines": {
-        "node": ">=14.16"
-      },
-      "funding": {
-        "url": "https://github.com/sindresorhus/is?sponsor=1"
-      }
-    },
     "node_modules/@sinonjs/commons": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz",
@@ -1008,17 +1005,6 @@
         "@sinonjs/commons": "^3.0.0"
       }
     },
-    "node_modules/@szmarczak/http-timer": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-5.0.1.tgz",
-      "integrity": "sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==",
-      "dependencies": {
-        "defer-to-connect": "^2.0.1"
-      },
-      "engines": {
-        "node": ">=14.16"
-      }
-    },
     "node_modules/@types/babel__core": {
       "version": "7.20.5",
       "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz",
@@ -1069,11 +1055,6 @@
         "@types/node": "*"
       }
     },
-    "node_modules/@types/http-cache-semantics": {
-      "version": "4.0.4",
-      "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz",
-      "integrity": "sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA=="
-    },
     "node_modules/@types/istanbul-lib-coverage": {
       "version": "2.0.6",
       "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz",
@@ -1129,6 +1110,7 @@
       "version": "1.3.8",
       "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
       "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==",
+      "license": "MIT",
       "dependencies": {
         "mime-types": "~2.1.34",
         "negotiator": "0.6.3"
@@ -1137,6 +1119,27 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/accepts/node_modules/mime-db": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/accepts/node_modules/mime-types": {
+      "version": "2.1.35",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+      "license": "MIT",
+      "dependencies": {
+        "mime-db": "1.52.0"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
     "node_modules/ansi-escapes": {
       "version": "4.3.2",
       "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz",
@@ -1374,47 +1377,11 @@
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
       "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==",
+      "license": "MIT",
       "engines": {
         "node": ">= 0.8"
       }
     },
-    "node_modules/cache-content-type": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/cache-content-type/-/cache-content-type-1.0.1.tgz",
-      "integrity": "sha512-IKufZ1o4Ut42YUrZSo8+qnMTrFuKkvyoLXUywKz9GJ5BrhOFGhLdkx9sG4KAnVvbY6kEcSFjLQul+DVmBm2bgA==",
-      "dependencies": {
-        "mime-types": "^2.1.18",
-        "ylru": "^1.2.0"
-      },
-      "engines": {
-        "node": ">= 6.0.0"
-      }
-    },
-    "node_modules/cacheable-lookup": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-7.0.0.tgz",
-      "integrity": "sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==",
-      "engines": {
-        "node": ">=14.16"
-      }
-    },
-    "node_modules/cacheable-request": {
-      "version": "10.2.14",
-      "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-10.2.14.tgz",
-      "integrity": "sha512-zkDT5WAF4hSSoUgyfg5tFIxz8XQK+25W/TLVojJTMKBaxevLBBtLxgqguAuVQB8PVW79FVjHcU+GJ9tVbDZ9mQ==",
-      "dependencies": {
-        "@types/http-cache-semantics": "^4.0.2",
-        "get-stream": "^6.0.1",
-        "http-cache-semantics": "^4.1.1",
-        "keyv": "^4.5.3",
-        "mimic-response": "^4.0.0",
-        "normalize-url": "^8.0.0",
-        "responselike": "^3.0.0"
-      },
-      "engines": {
-        "node": ">=14.16"
-      }
-    },
     "node_modules/callsites": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
@@ -1517,6 +1484,7 @@
       "version": "4.6.0",
       "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
       "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==",
+      "dev": true,
       "engines": {
         "iojs": ">= 1.0.0",
         "node": ">= 0.12.0"
@@ -1553,20 +1521,23 @@
       "dev": true
     },
     "node_modules/content-disposition": {
-      "version": "0.5.4",
-      "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
-      "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==",
-      "dependencies": {
-        "safe-buffer": "5.2.1"
-      },
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz",
+      "integrity": "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==",
+      "license": "MIT",
       "engines": {
-        "node": ">= 0.6"
+        "node": ">=18"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/express"
       }
     },
     "node_modules/content-type": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz",
       "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==",
+      "license": "MIT",
       "engines": {
         "node": ">= 0.6"
       }
@@ -1581,6 +1552,7 @@
       "version": "0.9.1",
       "resolved": "https://registry.npmjs.org/cookies/-/cookies-0.9.1.tgz",
       "integrity": "sha512-TG2hpqe4ELx54QER/S3HQ9SRVnQnGBtKUz5bLQWtYAQ+o6GpgMs6sYUvaiJjVxb+UXwhRhAEP3m7LbsIZ77Hmw==",
+      "license": "MIT",
       "dependencies": {
         "depd": "~2.0.0",
         "keygrip": "~1.1.0"
@@ -1625,11 +1597,12 @@
       }
     },
     "node_modules/debug": {
-      "version": "4.3.5",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz",
-      "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==",
+      "version": "4.4.3",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
+      "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
+      "license": "MIT",
       "dependencies": {
-        "ms": "2.1.2"
+        "ms": "^2.1.3"
       },
       "engines": {
         "node": ">=6.0"
@@ -1640,31 +1613,6 @@
         }
       }
     },
-    "node_modules/decompress-response": {
-      "version": "6.0.0",
-      "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
-      "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
-      "dependencies": {
-        "mimic-response": "^3.1.0"
-      },
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
-    "node_modules/decompress-response/node_modules/mimic-response": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
-      "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==",
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
     "node_modules/dedent": {
       "version": "1.5.3",
       "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.5.3.tgz",
@@ -1682,7 +1630,8 @@
     "node_modules/deep-equal": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz",
-      "integrity": "sha512-bHtC0iYvWhyaTzvV3CZgPeZQqCOBGyGsVV7v4eevpdkLHfiSrXUdBG+qAuSz4RI70sszvjQ1QSZ98An1yNwpSw=="
+      "integrity": "sha512-bHtC0iYvWhyaTzvV3CZgPeZQqCOBGyGsVV7v4eevpdkLHfiSrXUdBG+qAuSz4RI70sszvjQ1QSZ98An1yNwpSw==",
+      "license": "MIT"
     },
     "node_modules/deepmerge": {
       "version": "4.3.1",
@@ -1693,23 +1642,17 @@
         "node": ">=0.10.0"
       }
     },
-    "node_modules/defer-to-connect": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz",
-      "integrity": "sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==",
-      "engines": {
-        "node": ">=10"
-      }
-    },
     "node_modules/delegates": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
-      "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ=="
+      "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==",
+      "license": "MIT"
     },
     "node_modules/depd": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
       "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==",
+      "license": "MIT",
       "engines": {
         "node": ">= 0.8"
       }
@@ -1718,6 +1661,7 @@
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz",
       "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==",
+      "license": "MIT",
       "engines": {
         "node": ">= 0.8",
         "npm": "1.2.8000 || >= 1.4.16"
@@ -1744,7 +1688,8 @@
     "node_modules/ee-first": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
-      "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="
+      "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==",
+      "license": "MIT"
     },
     "node_modules/electron-to-chromium": {
       "version": "1.5.0",
@@ -1771,9 +1716,10 @@
       "dev": true
     },
     "node_modules/encodeurl": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
-      "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==",
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz",
+      "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==",
+      "license": "MIT",
       "engines": {
         "node": ">= 0.8"
       }
@@ -1799,7 +1745,8 @@
     "node_modules/escape-html": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
-      "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="
+      "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==",
+      "license": "MIT"
     },
     "node_modules/escape-string-regexp": {
       "version": "2.0.0",
@@ -1824,14 +1771,15 @@
       }
     },
     "node_modules/eta": {
-      "version": "3.4.0",
-      "resolved": "https://registry.npmjs.org/eta/-/eta-3.4.0.tgz",
-      "integrity": "sha512-tCsc7WXTjrTx4ZjYLplcqrI3o4mYJ+Z6YspeuGL8tbt/hHoMchwBwtKfwM09svEY86iRapY93vUqQttcNuIO5Q==",
+      "version": "4.6.0",
+      "resolved": "https://registry.npmjs.org/eta/-/eta-4.6.0.tgz",
+      "integrity": "sha512-lW6is4T1NFOYnmqGZIfvixqj7A7sSvScF+DN8EK6K58xI5MZ5UvYe0GjopxOXQtZvUn4eDdVuZ8XSoYWTMEKwA==",
+      "license": "MIT",
       "engines": {
-        "node": ">=6.0.0"
+        "node": ">=20"
       },
       "funding": {
-        "url": "https://github.com/eta-dev/eta?sponsor=1"
+        "url": "https://github.com/bgub/eta?sponsor=1"
       }
     },
     "node_modules/execa": {
@@ -1922,18 +1870,11 @@
         "node": ">=8"
       }
     },
-    "node_modules/form-data-encoder": {
-      "version": "2.1.4",
-      "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-2.1.4.tgz",
-      "integrity": "sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==",
-      "engines": {
-        "node": ">= 14.17"
-      }
-    },
     "node_modules/fresh": {
       "version": "0.5.2",
       "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
       "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==",
+      "license": "MIT",
       "engines": {
         "node": ">= 0.6"
       }
@@ -1998,6 +1939,7 @@
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
       "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==",
+      "dev": true,
       "engines": {
         "node": ">=10"
       },
@@ -2035,30 +1977,6 @@
         "node": ">=4"
       }
     },
-    "node_modules/got": {
-      "version": "13.0.0",
-      "resolved": "https://registry.npmjs.org/got/-/got-13.0.0.tgz",
-      "integrity": "sha512-XfBk1CxOOScDcMr9O1yKkNaQyy865NbYs+F7dr4H0LZMVgCj2Le59k6PqbNHoL5ToeaEQUYh6c6yMfVcc6SJxA==",
-      "dependencies": {
-        "@sindresorhus/is": "^5.2.0",
-        "@szmarczak/http-timer": "^5.0.1",
-        "cacheable-lookup": "^7.0.0",
-        "cacheable-request": "^10.2.8",
-        "decompress-response": "^6.0.0",
-        "form-data-encoder": "^2.1.2",
-        "get-stream": "^6.0.1",
-        "http2-wrapper": "^2.1.10",
-        "lowercase-keys": "^3.0.0",
-        "p-cancelable": "^3.0.0",
-        "responselike": "^3.0.0"
-      },
-      "engines": {
-        "node": ">=16"
-      },
-      "funding": {
-        "url": "https://github.com/sindresorhus/got?sponsor=1"
-      }
-    },
     "node_modules/graceful-fs": {
       "version": "4.2.11",
       "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
@@ -2074,31 +1992,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/has-symbols": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
-      "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==",
-      "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
-    "node_modules/has-tostringtag": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
-      "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
-      "dependencies": {
-        "has-symbols": "^1.0.3"
-      },
-      "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
     "node_modules/hasown": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
@@ -2121,6 +2014,7 @@
       "version": "1.5.0",
       "resolved": "https://registry.npmjs.org/http-assert/-/http-assert-1.5.0.tgz",
       "integrity": "sha512-uPpH7OKX4H25hBmU6G1jWNaqJGpTXxey+YOUizJUAgu0AjLUeC8D73hTrhvDS5D+GJN1DN1+hhc/eF/wpxtp0w==",
+      "license": "MIT",
       "dependencies": {
         "deep-equal": "~1.0.1",
         "http-errors": "~1.8.0"
@@ -2133,6 +2027,7 @@
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
       "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==",
+      "license": "MIT",
       "engines": {
         "node": ">= 0.6"
       }
@@ -2141,6 +2036,7 @@
       "version": "1.8.1",
       "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz",
       "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==",
+      "license": "MIT",
       "dependencies": {
         "depd": "~1.1.2",
         "inherits": "2.0.4",
@@ -2156,51 +2052,29 @@
       "version": "1.5.0",
       "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
       "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==",
+      "license": "MIT",
       "engines": {
         "node": ">= 0.6"
       }
     },
-    "node_modules/http-cache-semantics": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz",
-      "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ=="
-    },
     "node_modules/http-errors": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
-      "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==",
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz",
+      "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==",
+      "license": "MIT",
       "dependencies": {
-        "depd": "2.0.0",
-        "inherits": "2.0.4",
-        "setprototypeof": "1.2.0",
-        "statuses": "2.0.1",
-        "toidentifier": "1.0.1"
+        "depd": "~2.0.0",
+        "inherits": "~2.0.4",
+        "setprototypeof": "~1.2.0",
+        "statuses": "~2.0.2",
+        "toidentifier": "~1.0.1"
       },
       "engines": {
         "node": ">= 0.8"
-      }
-    },
-    "node_modules/http2-wrapper": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-2.2.1.tgz",
-      "integrity": "sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==",
-      "dependencies": {
-        "quick-lru": "^5.1.1",
-        "resolve-alpn": "^1.2.0"
-      },
-      "engines": {
-        "node": ">=10.19.0"
-      }
-    },
-    "node_modules/http2-wrapper/node_modules/quick-lru": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz",
-      "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==",
-      "engines": {
-        "node": ">=10"
       },
       "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "type": "opencollective",
+        "url": "https://opencollective.com/express"
       }
     },
     "node_modules/human-signals": {
@@ -2213,14 +2087,19 @@
       }
     },
     "node_modules/iconv-lite": {
-      "version": "0.4.24",
-      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
-      "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
+      "version": "0.7.2",
+      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz",
+      "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==",
+      "license": "MIT",
       "dependencies": {
-        "safer-buffer": ">= 2.1.2 < 3"
+        "safer-buffer": ">= 2.1.2 < 3.0.0"
       },
       "engines": {
         "node": ">=0.10.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/express"
       }
     },
     "node_modules/import-local": {
@@ -2306,20 +2185,6 @@
         "node": ">=6"
       }
     },
-    "node_modules/is-generator-function": {
-      "version": "1.0.10",
-      "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz",
-      "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==",
-      "dependencies": {
-        "has-tostringtag": "^1.0.0"
-      },
-      "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
     "node_modules/is-number": {
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
@@ -2993,9 +2858,10 @@
       }
     },
     "node_modules/jose": {
-      "version": "5.6.3",
-      "resolved": "https://registry.npmjs.org/jose/-/jose-5.6.3.tgz",
-      "integrity": "sha512-1Jh//hEEwMhNYPDDLwXHa2ePWgWiFNNUadVmguAAw2IJ6sj9mNxV5tGXJNqlMkJAybF6Lgw1mISDxTePP/187g==",
+      "version": "6.2.3",
+      "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.3.tgz",
+      "integrity": "sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==",
+      "license": "MIT",
       "funding": {
         "url": "https://github.com/sponsors/panva"
       }
@@ -3020,9 +2886,10 @@
       }
     },
     "node_modules/jsesc": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.0.2.tgz",
-      "integrity": "sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g==",
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz",
+      "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==",
+      "license": "MIT",
       "bin": {
         "jsesc": "bin/jsesc"
       },
@@ -3030,11 +2897,6 @@
         "node": ">=6"
       }
     },
-    "node_modules/json-buffer": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
-      "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="
-    },
     "node_modules/json-parse-even-better-errors": {
       "version": "2.3.1",
       "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz",
@@ -3057,6 +2919,7 @@
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/keygrip/-/keygrip-1.1.0.tgz",
       "integrity": "sha512-iYSchDJ+liQ8iwbSI2QqsQOvqv58eJCEanyJPJi+Khyu8smkcKSFUCbPwzFcL7YVtZ6eONjqRX/38caJ7QjRAQ==",
+      "license": "MIT",
       "dependencies": {
         "tsscmp": "1.0.6"
       },
@@ -3064,14 +2927,6 @@
         "node": ">= 0.6"
       }
     },
-    "node_modules/keyv": {
-      "version": "4.5.4",
-      "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz",
-      "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==",
-      "dependencies": {
-        "json-buffer": "3.0.1"
-      }
-    },
     "node_modules/kleur": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz",
@@ -3082,86 +2937,39 @@
       }
     },
     "node_modules/koa": {
-      "version": "2.15.4",
-      "resolved": "https://registry.npmjs.org/koa/-/koa-2.15.4.tgz",
-      "integrity": "sha512-7fNBIdrU2PEgLljXoPWoyY4r1e+ToWCmzS/wwMPbUNs7X+5MMET1ObhJBlUkF5uZG9B6QhM2zS1TsH6adegkiQ==",
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/koa/-/koa-3.2.1.tgz",
+      "integrity": "sha512-e7IpWJrnanNUroVK2taAgMxoEZvHLXdQiNjeExSu/DEIWm83jaKGBgb7tLmu2rMYpA027qFB3iLR/k3AVpFRnA==",
       "license": "MIT",
       "dependencies": {
-        "accepts": "^1.3.5",
-        "cache-content-type": "^1.0.0",
-        "content-disposition": "~0.5.2",
-        "content-type": "^1.0.4",
-        "cookies": "~0.9.0",
-        "debug": "^4.3.2",
+        "accepts": "^1.3.8",
+        "content-disposition": "~1.0.1",
+        "content-type": "^1.0.5",
+        "cookies": "~0.9.1",
         "delegates": "^1.0.0",
-        "depd": "^2.0.0",
-        "destroy": "^1.0.4",
-        "encodeurl": "^1.0.2",
+        "destroy": "^1.2.0",
+        "encodeurl": "^2.0.0",
         "escape-html": "^1.0.3",
         "fresh": "~0.5.2",
-        "http-assert": "^1.3.0",
-        "http-errors": "^1.6.3",
-        "is-generator-function": "^1.0.7",
+        "http-assert": "^1.5.0",
+        "http-errors": "^2.0.0",
         "koa-compose": "^4.1.0",
-        "koa-convert": "^2.0.0",
-        "on-finished": "^2.3.0",
-        "only": "~0.0.2",
-        "parseurl": "^1.3.2",
-        "statuses": "^1.5.0",
-        "type-is": "^1.6.16",
+        "mime-types": "^3.0.1",
+        "on-finished": "^2.4.1",
+        "parseurl": "^1.3.3",
+        "statuses": "^2.0.1",
+        "type-is": "^2.0.1",
         "vary": "^1.1.2"
       },
       "engines": {
-        "node": "^4.8.4 || ^6.10.1 || ^7.10.1 || >= 8.1.4"
+        "node": ">= 18"
       }
     },
     "node_modules/koa-compose": {
       "version": "4.1.0",
       "resolved": "https://registry.npmjs.org/koa-compose/-/koa-compose-4.1.0.tgz",
-      "integrity": "sha512-8ODW8TrDuMYvXRwra/Kh7/rJo9BtOfPc6qO8eAfC80CnCvSjSl0bkRM24X6/XBBEyj0v1nRUQ1LyOy3dbqOWXw=="
-    },
-    "node_modules/koa-convert": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/koa-convert/-/koa-convert-2.0.0.tgz",
-      "integrity": "sha512-asOvN6bFlSnxewce2e/DK3p4tltyfC4VM7ZwuTuepI7dEQVcvpyFuBcEARu1+Hxg8DIwytce2n7jrZtRlPrARA==",
-      "dependencies": {
-        "co": "^4.6.0",
-        "koa-compose": "^4.1.0"
-      },
-      "engines": {
-        "node": ">= 10"
-      }
-    },
-    "node_modules/koa/node_modules/http-errors": {
-      "version": "1.8.1",
-      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz",
-      "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==",
-      "dependencies": {
-        "depd": "~1.1.2",
-        "inherits": "2.0.4",
-        "setprototypeof": "1.2.0",
-        "statuses": ">= 1.5.0 < 2",
-        "toidentifier": "1.0.1"
-      },
-      "engines": {
-        "node": ">= 0.6"
-      }
-    },
-    "node_modules/koa/node_modules/http-errors/node_modules/depd": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
-      "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==",
-      "engines": {
-        "node": ">= 0.6"
-      }
-    },
-    "node_modules/koa/node_modules/statuses": {
-      "version": "1.5.0",
-      "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
-      "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==",
-      "engines": {
-        "node": ">= 0.6"
-      }
+      "integrity": "sha512-8ODW8TrDuMYvXRwra/Kh7/rJo9BtOfPc6qO8eAfC80CnCvSjSl0bkRM24X6/XBBEyj0v1nRUQ1LyOy3dbqOWXw==",
+      "license": "MIT"
     },
     "node_modules/leven": {
       "version": "3.1.0",
@@ -3190,17 +2998,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/lowercase-keys": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz",
-      "integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==",
-      "engines": {
-        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
     "node_modules/lru-cache": {
       "version": "5.1.1",
       "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
@@ -3247,11 +3044,12 @@
       }
     },
     "node_modules/media-typer": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
-      "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz",
+      "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==",
+      "license": "MIT",
       "engines": {
-        "node": ">= 0.6"
+        "node": ">= 0.8"
       }
     },
     "node_modules/merge-stream": {
@@ -3260,14 +3058,6 @@
       "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==",
       "dev": true
     },
-    "node_modules/methods": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
-      "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==",
-      "engines": {
-        "node": ">= 0.6"
-      }
-    },
     "node_modules/micromatch": {
       "version": "4.0.7",
       "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz",
@@ -3282,22 +3072,28 @@
       }
     },
     "node_modules/mime-db": {
-      "version": "1.52.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
-      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+      "version": "1.54.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz",
+      "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==",
+      "license": "MIT",
       "engines": {
         "node": ">= 0.6"
       }
     },
     "node_modules/mime-types": {
-      "version": "2.1.35",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
-      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz",
+      "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==",
+      "license": "MIT",
       "dependencies": {
-        "mime-db": "1.52.0"
+        "mime-db": "^1.54.0"
       },
       "engines": {
-        "node": ">= 0.6"
+        "node": ">=18"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/express"
       }
     },
     "node_modules/mimic-fn": {
@@ -3309,17 +3105,6 @@
         "node": ">=6"
       }
     },
-    "node_modules/mimic-response": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-4.0.0.tgz",
-      "integrity": "sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==",
-      "engines": {
-        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
     "node_modules/minimatch": {
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
@@ -3343,20 +3128,22 @@
       }
     },
     "node_modules/ms": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
+      "license": "MIT"
     },
     "node_modules/nanoid": {
-      "version": "5.0.7",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.7.tgz",
-      "integrity": "sha512-oLxFY2gd2IqnjcYyOXD8XGCftpGtZP2AbHbOkthDkvRywH5ayNtPVy9YlOPcHckXzbLTCHpkb7FB+yuxKV13pQ==",
+      "version": "5.1.11",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.11.tgz",
+      "integrity": "sha512-v+KEsUv2ps74PaSKv0gHTxTCgMXOIfBEbaqa6w6ISIGC7ZsvHN4N9oJ8d4cmf0n5oTzQz2SLmThbQWhjd/8eKg==",
       "funding": [
         {
           "type": "github",
           "url": "https://github.com/sponsors/ai"
         }
       ],
+      "license": "MIT",
       "bin": {
         "nanoid": "bin/nanoid.js"
       },
@@ -3374,6 +3161,7 @@
       "version": "0.6.3",
       "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
       "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==",
+      "license": "MIT",
       "engines": {
         "node": ">= 0.6"
       }
@@ -3399,17 +3187,6 @@
         "node": ">=0.10.0"
       }
     },
-    "node_modules/normalize-url": {
-      "version": "8.0.1",
-      "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-8.0.1.tgz",
-      "integrity": "sha512-IO9QvjUMWxPQQhs60oOu10CRkWCiZzSUkzbXGGV9pviYl1fXYcvkzQ5jV9z8Y6un8ARoVRl4EtC6v6jNqbaJ/w==",
-      "engines": {
-        "node": ">=14.16"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
     "node_modules/npm-run-path": {
       "version": "4.0.1",
       "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz",
@@ -3422,49 +3199,32 @@
         "node": ">=8"
       }
     },
-    "node_modules/object-hash": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz",
-      "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==",
-      "engines": {
-        "node": ">= 6"
-      }
-    },
     "node_modules/oidc-provider": {
-      "version": "8.5.1",
-      "resolved": "https://registry.npmjs.org/oidc-provider/-/oidc-provider-8.5.1.tgz",
-      "integrity": "sha512-Bm3EyxN68/KS76IlciJ3+4pnVtfdRWL+NghWpIF0XQbiRT1gzc6Qf/cyFmpL9yieko/jXYZ/uLHUv77jD00qww==",
+      "version": "9.8.3",
+      "resolved": "https://registry.npmjs.org/oidc-provider/-/oidc-provider-9.8.3.tgz",
+      "integrity": "sha512-YkchaAyVAZbsn/l7IQhcEMdeDL3lwSo/PNUtnsXSqPqT7EG8DRko0EAWzHd/n9VfCtKVkxGjYOY4h4UwFcWnUA==",
+      "license": "MIT",
       "dependencies": {
         "@koa/cors": "^5.0.0",
-        "@koa/router": "^12.0.1",
-        "debug": "^4.3.5",
-        "eta": "^3.4.0",
-        "got": "^13.0.0",
-        "jose": "^5.6.2",
-        "jsesc": "^3.0.2",
-        "koa": "^2.15.3",
-        "nanoid": "^5.0.7",
-        "object-hash": "^3.0.0",
-        "oidc-token-hash": "^5.0.3",
-        "quick-lru": "^7.0.0",
-        "raw-body": "^2.5.2"
+        "@koa/router": "^15.4.0",
+        "debug": "^4.4.3",
+        "eta": "^4.5.1",
+        "jose": "^6.2.2",
+        "jsesc": "^3.1.0",
+        "koa": "^3.2.0",
+        "nanoid": "^5.1.7",
+        "quick-lru": "^7.3.0",
+        "raw-body": "^3.0.2"
       },
       "funding": {
         "url": "https://github.com/sponsors/panva"
       }
     },
-    "node_modules/oidc-token-hash": {
-      "version": "5.0.3",
-      "resolved": "https://registry.npmjs.org/oidc-token-hash/-/oidc-token-hash-5.0.3.tgz",
-      "integrity": "sha512-IF4PcGgzAr6XXSff26Sk/+P4KZFJVuHAJZj3wgO3vX2bMdNVp/QXTP3P7CEm9V1IdG8lDLY3HhiqpsE/nOwpPw==",
-      "engines": {
-        "node": "^10.13.0 || >=12.0.0"
-      }
-    },
     "node_modules/on-finished": {
       "version": "2.4.1",
       "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
       "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==",
+      "license": "MIT",
       "dependencies": {
         "ee-first": "1.1.1"
       },
@@ -3496,19 +3256,6 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/only": {
-      "version": "0.0.2",
-      "resolved": "https://registry.npmjs.org/only/-/only-0.0.2.tgz",
-      "integrity": "sha512-Fvw+Jemq5fjjyWz6CpKx6w9s7xxqo3+JCyM0WXWeCSOboZ8ABkyvP8ID4CZuChA/wxSx+XSJmdOm8rGVyJ1hdQ=="
-    },
-    "node_modules/p-cancelable": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-3.0.0.tgz",
-      "integrity": "sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==",
-      "engines": {
-        "node": ">=12.20"
-      }
-    },
     "node_modules/p-limit": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
@@ -3582,6 +3329,7 @@
       "version": "1.3.3",
       "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
       "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==",
+      "license": "MIT",
       "engines": {
         "node": ">= 0.8"
       }
@@ -3620,9 +3368,14 @@
       "dev": true
     },
     "node_modules/path-to-regexp": {
-      "version": "6.3.0",
-      "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz",
-      "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ=="
+      "version": "8.4.2",
+      "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz",
+      "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==",
+      "license": "MIT",
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/express"
+      }
     },
     "node_modules/picocolors": {
       "version": "1.0.1",
@@ -3719,9 +3472,10 @@
       ]
     },
     "node_modules/quick-lru": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-7.0.0.tgz",
-      "integrity": "sha512-MX8gB7cVYTrYcFfAnfLlhRd0+Toyl8yX8uBx1MrX7K0jegiz9TumwOK27ldXrgDlHRdVi+MqU9Ssw6dr4BNreg==",
+      "version": "7.3.0",
+      "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-7.3.0.tgz",
+      "integrity": "sha512-k9lSsjl36EJdK7I06v7APZCbyGT2vMTsYSRX1Q2nbYmnkBqgUhRkAuzH08Ciotteu/PLJmIF2+tti7o3C/ts2g==",
+      "license": "MIT",
       "engines": {
         "node": ">=18"
       },
@@ -3730,17 +3484,18 @@
       }
     },
     "node_modules/raw-body": {
-      "version": "2.5.2",
-      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz",
-      "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==",
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz",
+      "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==",
+      "license": "MIT",
       "dependencies": {
-        "bytes": "3.1.2",
-        "http-errors": "2.0.0",
-        "iconv-lite": "0.4.24",
-        "unpipe": "1.0.0"
+        "bytes": "~3.1.2",
+        "http-errors": "~2.0.1",
+        "iconv-lite": "~0.7.0",
+        "unpipe": "~1.0.0"
       },
       "engines": {
-        "node": ">= 0.8"
+        "node": ">= 0.10"
       }
     },
     "node_modules/react-is": {
@@ -3775,11 +3530,6 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/resolve-alpn": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz",
-      "integrity": "sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g=="
-    },
     "node_modules/resolve-cwd": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz",
@@ -3810,43 +3560,11 @@
         "node": ">=10"
       }
     },
-    "node_modules/responselike": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/responselike/-/responselike-3.0.0.tgz",
-      "integrity": "sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==",
-      "dependencies": {
-        "lowercase-keys": "^3.0.0"
-      },
-      "engines": {
-        "node": ">=14.16"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
-    "node_modules/safe-buffer": {
-      "version": "5.2.1",
-      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
-      "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ]
-    },
     "node_modules/safer-buffer": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
-      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
+      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
+      "license": "MIT"
     },
     "node_modules/semver": {
       "version": "6.3.1",
@@ -3860,7 +3578,8 @@
     "node_modules/setprototypeof": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
-      "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="
+      "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==",
+      "license": "ISC"
     },
     "node_modules/shebang-command": {
       "version": "2.0.0",
@@ -3942,9 +3661,10 @@
       }
     },
     "node_modules/statuses": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
-      "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==",
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",
+      "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==",
+      "license": "MIT",
       "engines": {
         "node": ">= 0.8"
       }
@@ -4087,6 +3807,7 @@
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
       "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==",
+      "license": "MIT",
       "engines": {
         "node": ">=0.6"
       }
@@ -4095,6 +3816,7 @@
       "version": "1.0.6",
       "resolved": "https://registry.npmjs.org/tsscmp/-/tsscmp-1.0.6.tgz",
       "integrity": "sha512-LxhtAkPDTkVCMQjt2h6eBVY28KCjikZqZfMcC15YBeNjkgUpdCfBu5HoiOTDu86v6smE8yOjyEktJ8hlbANHQA==",
+      "license": "MIT",
       "engines": {
         "node": ">=0.6.x"
       }
@@ -4121,21 +3843,41 @@
       }
     },
     "node_modules/type-is": {
-      "version": "1.6.18",
-      "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
-      "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==",
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz",
+      "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==",
+      "license": "MIT",
       "dependencies": {
-        "media-typer": "0.3.0",
-        "mime-types": "~2.1.24"
+        "content-type": "^2.0.0",
+        "media-typer": "^1.1.0",
+        "mime-types": "^3.0.0"
       },
       "engines": {
-        "node": ">= 0.6"
+        "node": ">= 18"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/express"
+      }
+    },
+    "node_modules/type-is/node_modules/content-type": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz",
+      "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=18"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/express"
       }
     },
     "node_modules/unpipe": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
       "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==",
+      "license": "MIT",
       "engines": {
         "node": ">= 0.8"
       }
@@ -4294,14 +4036,6 @@
         "node": ">=12"
       }
     },
-    "node_modules/ylru": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/ylru/-/ylru-1.4.0.tgz",
-      "integrity": "sha512-2OQsPNEmBCvXuFlIni/a+Rn+R2pHW9INm0BxXJ4hVDA8TirqMj+J/Rp9ItLatT/5pZqWwefVrTQcHpixsxnVlA==",
-      "engines": {
-        "node": ">= 4.0.0"
-      }
-    },
     "node_modules/yocto-queue": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",
@@ -5037,15 +4771,14 @@
       }
     },
     "@koa/router": {
-      "version": "12.0.1",
-      "resolved": "https://registry.npmjs.org/@koa/router/-/router-12.0.1.tgz",
-      "integrity": "sha512-ribfPYfHb+Uw3b27Eiw6NPqjhIhTpVFzEWLwyc/1Xp+DCdwRRyIlAUODX+9bPARF6aQtUu1+/PHzdNvRzcs/+Q==",
+      "version": "15.5.0",
+      "resolved": "https://registry.npmjs.org/@koa/router/-/router-15.5.0.tgz",
+      "integrity": "sha512-KSC0oG/5t6ITu5wqX4lJseA/dngoj14hEaohrLZEXtlUT2RRyJvwaJ0KV+5uQoaWrY3A8ClHOrBEU4g8dujn8Q==",
       "requires": {
-        "debug": "^4.3.4",
-        "http-errors": "^2.0.0",
+        "debug": "^4.4.3",
+        "http-errors": "^2.0.1",
         "koa-compose": "^4.1.0",
-        "methods": "^1.1.2",
-        "path-to-regexp": "^6.2.1"
+        "path-to-regexp": "^8.4.2"
       }
     },
     "@sinclair/typebox": {
@@ -5054,11 +4787,6 @@
       "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==",
       "dev": true
     },
-    "@sindresorhus/is": {
-      "version": "5.6.0",
-      "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.6.0.tgz",
-      "integrity": "sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g=="
-    },
     "@sinonjs/commons": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz",
@@ -5077,14 +4805,6 @@
         "@sinonjs/commons": "^3.0.0"
       }
     },
-    "@szmarczak/http-timer": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-5.0.1.tgz",
-      "integrity": "sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==",
-      "requires": {
-        "defer-to-connect": "^2.0.1"
-      }
-    },
     "@types/babel__core": {
       "version": "7.20.5",
       "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz",
@@ -5135,11 +4855,6 @@
         "@types/node": "*"
       }
     },
-    "@types/http-cache-semantics": {
-      "version": "4.0.4",
-      "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz",
-      "integrity": "sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA=="
-    },
     "@types/istanbul-lib-coverage": {
       "version": "2.0.6",
       "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz",
@@ -5198,6 +4913,21 @@
       "requires": {
         "mime-types": "~2.1.34",
         "negotiator": "0.6.3"
+      },
+      "dependencies": {
+        "mime-db": {
+          "version": "1.52.0",
+          "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+          "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="
+        },
+        "mime-types": {
+          "version": "2.1.35",
+          "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+          "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+          "requires": {
+            "mime-db": "1.52.0"
+          }
+        }
       }
     },
     "ansi-escapes": {
@@ -5375,34 +5105,6 @@
       "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
       "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="
     },
-    "cache-content-type": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/cache-content-type/-/cache-content-type-1.0.1.tgz",
-      "integrity": "sha512-IKufZ1o4Ut42YUrZSo8+qnMTrFuKkvyoLXUywKz9GJ5BrhOFGhLdkx9sG4KAnVvbY6kEcSFjLQul+DVmBm2bgA==",
-      "requires": {
-        "mime-types": "^2.1.18",
-        "ylru": "^1.2.0"
-      }
-    },
-    "cacheable-lookup": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-7.0.0.tgz",
-      "integrity": "sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w=="
-    },
-    "cacheable-request": {
-      "version": "10.2.14",
-      "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-10.2.14.tgz",
-      "integrity": "sha512-zkDT5WAF4hSSoUgyfg5tFIxz8XQK+25W/TLVojJTMKBaxevLBBtLxgqguAuVQB8PVW79FVjHcU+GJ9tVbDZ9mQ==",
-      "requires": {
-        "@types/http-cache-semantics": "^4.0.2",
-        "get-stream": "^6.0.1",
-        "http-cache-semantics": "^4.1.1",
-        "keyv": "^4.5.3",
-        "mimic-response": "^4.0.0",
-        "normalize-url": "^8.0.0",
-        "responselike": "^3.0.0"
-      }
-    },
     "callsites": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
@@ -5463,7 +5165,8 @@
     "co": {
       "version": "4.6.0",
       "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
-      "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ=="
+      "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==",
+      "dev": true
     },
     "collect-v8-coverage": {
       "version": "1.0.2",
@@ -5493,12 +5196,9 @@
       "dev": true
     },
     "content-disposition": {
-      "version": "0.5.4",
-      "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
-      "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==",
-      "requires": {
-        "safe-buffer": "5.2.1"
-      }
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz",
+      "integrity": "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q=="
     },
     "content-type": {
       "version": "1.0.5",
@@ -5547,26 +5247,11 @@
       }
     },
     "debug": {
-      "version": "4.3.5",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz",
-      "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==",
-      "requires": {
-        "ms": "2.1.2"
-      }
-    },
-    "decompress-response": {
-      "version": "6.0.0",
-      "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
-      "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
+      "version": "4.4.3",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
+      "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
       "requires": {
-        "mimic-response": "^3.1.0"
-      },
-      "dependencies": {
-        "mimic-response": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
-          "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ=="
-        }
+        "ms": "^2.1.3"
       }
     },
     "dedent": {
@@ -5587,11 +5272,6 @@
       "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==",
       "dev": true
     },
-    "defer-to-connect": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz",
-      "integrity": "sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg=="
-    },
     "delegates": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
@@ -5643,9 +5323,9 @@
       "dev": true
     },
     "encodeurl": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
-      "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w=="
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz",
+      "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="
     },
     "error-ex": {
       "version": "1.3.2",
@@ -5680,9 +5360,9 @@
       "dev": true
     },
     "eta": {
-      "version": "3.4.0",
-      "resolved": "https://registry.npmjs.org/eta/-/eta-3.4.0.tgz",
-      "integrity": "sha512-tCsc7WXTjrTx4ZjYLplcqrI3o4mYJ+Z6YspeuGL8tbt/hHoMchwBwtKfwM09svEY86iRapY93vUqQttcNuIO5Q=="
+      "version": "4.6.0",
+      "resolved": "https://registry.npmjs.org/eta/-/eta-4.6.0.tgz",
+      "integrity": "sha512-lW6is4T1NFOYnmqGZIfvixqj7A7sSvScF+DN8EK6K58xI5MZ5UvYe0GjopxOXQtZvUn4eDdVuZ8XSoYWTMEKwA=="
     },
     "execa": {
       "version": "5.1.1",
@@ -5754,11 +5434,6 @@
         "path-exists": "^4.0.0"
       }
     },
-    "form-data-encoder": {
-      "version": "2.1.4",
-      "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-2.1.4.tgz",
-      "integrity": "sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw=="
-    },
     "fresh": {
       "version": "0.5.2",
       "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
@@ -5804,7 +5479,8 @@
     "get-stream": {
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
-      "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg=="
+      "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==",
+      "dev": true
     },
     "glob": {
       "version": "7.2.3",
@@ -5826,24 +5502,6 @@
       "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==",
       "dev": true
     },
-    "got": {
-      "version": "13.0.0",
-      "resolved": "https://registry.npmjs.org/got/-/got-13.0.0.tgz",
-      "integrity": "sha512-XfBk1CxOOScDcMr9O1yKkNaQyy865NbYs+F7dr4H0LZMVgCj2Le59k6PqbNHoL5ToeaEQUYh6c6yMfVcc6SJxA==",
-      "requires": {
-        "@sindresorhus/is": "^5.2.0",
-        "@szmarczak/http-timer": "^5.0.1",
-        "cacheable-lookup": "^7.0.0",
-        "cacheable-request": "^10.2.8",
-        "decompress-response": "^6.0.0",
-        "form-data-encoder": "^2.1.2",
-        "get-stream": "^6.0.1",
-        "http2-wrapper": "^2.1.10",
-        "lowercase-keys": "^3.0.0",
-        "p-cancelable": "^3.0.0",
-        "responselike": "^3.0.0"
-      }
-    },
     "graceful-fs": {
       "version": "4.2.11",
       "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
@@ -5856,19 +5514,6 @@
       "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true
     },
-    "has-symbols": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
-      "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A=="
-    },
-    "has-tostringtag": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
-      "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
-      "requires": {
-        "has-symbols": "^1.0.3"
-      }
-    },
     "hasown": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
@@ -5917,37 +5562,16 @@
         }
       }
     },
-    "http-cache-semantics": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz",
-      "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ=="
-    },
     "http-errors": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
-      "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==",
-      "requires": {
-        "depd": "2.0.0",
-        "inherits": "2.0.4",
-        "setprototypeof": "1.2.0",
-        "statuses": "2.0.1",
-        "toidentifier": "1.0.1"
-      }
-    },
-    "http2-wrapper": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-2.2.1.tgz",
-      "integrity": "sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==",
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz",
+      "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==",
       "requires": {
-        "quick-lru": "^5.1.1",
-        "resolve-alpn": "^1.2.0"
-      },
-      "dependencies": {
-        "quick-lru": {
-          "version": "5.1.1",
-          "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz",
-          "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA=="
-        }
+        "depd": "~2.0.0",
+        "inherits": "~2.0.4",
+        "setprototypeof": "~1.2.0",
+        "statuses": "~2.0.2",
+        "toidentifier": "~1.0.1"
       }
     },
     "human-signals": {
@@ -5957,11 +5581,11 @@
       "dev": true
     },
     "iconv-lite": {
-      "version": "0.4.24",
-      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
-      "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
+      "version": "0.7.2",
+      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz",
+      "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==",
       "requires": {
-        "safer-buffer": ">= 2.1.2 < 3"
+        "safer-buffer": ">= 2.1.2 < 3.0.0"
       }
     },
     "import-local": {
@@ -6022,14 +5646,6 @@
       "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==",
       "dev": true
     },
-    "is-generator-function": {
-      "version": "1.0.10",
-      "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz",
-      "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==",
-      "requires": {
-        "has-tostringtag": "^1.0.0"
-      }
-    },
     "is-number": {
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
@@ -6542,9 +6158,9 @@
       }
     },
     "jose": {
-      "version": "5.6.3",
-      "resolved": "https://registry.npmjs.org/jose/-/jose-5.6.3.tgz",
-      "integrity": "sha512-1Jh//hEEwMhNYPDDLwXHa2ePWgWiFNNUadVmguAAw2IJ6sj9mNxV5tGXJNqlMkJAybF6Lgw1mISDxTePP/187g=="
+      "version": "6.2.3",
+      "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.3.tgz",
+      "integrity": "sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw=="
     },
     "js-tokens": {
       "version": "4.0.0",
@@ -6563,14 +6179,9 @@
       }
     },
     "jsesc": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.0.2.tgz",
-      "integrity": "sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g=="
-    },
-    "json-buffer": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
-      "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz",
+      "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="
     },
     "json-parse-even-better-errors": {
       "version": "2.3.1",
@@ -6592,14 +6203,6 @@
         "tsscmp": "1.0.6"
       }
     },
-    "keyv": {
-      "version": "4.5.4",
-      "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz",
-      "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==",
-      "requires": {
-        "json-buffer": "3.0.1"
-      }
-    },
     "kleur": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz",
@@ -6607,59 +6210,28 @@
       "dev": true
     },
     "koa": {
-      "version": "2.15.4",
-      "resolved": "https://registry.npmjs.org/koa/-/koa-2.15.4.tgz",
-      "integrity": "sha512-7fNBIdrU2PEgLljXoPWoyY4r1e+ToWCmzS/wwMPbUNs7X+5MMET1ObhJBlUkF5uZG9B6QhM2zS1TsH6adegkiQ==",
-      "requires": {
-        "accepts": "^1.3.5",
-        "cache-content-type": "^1.0.0",
-        "content-disposition": "~0.5.2",
-        "content-type": "^1.0.4",
-        "cookies": "~0.9.0",
-        "debug": "^4.3.2",
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/koa/-/koa-3.2.1.tgz",
+      "integrity": "sha512-e7IpWJrnanNUroVK2taAgMxoEZvHLXdQiNjeExSu/DEIWm83jaKGBgb7tLmu2rMYpA027qFB3iLR/k3AVpFRnA==",
+      "requires": {
+        "accepts": "^1.3.8",
+        "content-disposition": "~1.0.1",
+        "content-type": "^1.0.5",
+        "cookies": "~0.9.1",
         "delegates": "^1.0.0",
-        "depd": "^2.0.0",
-        "destroy": "^1.0.4",
-        "encodeurl": "^1.0.2",
+        "destroy": "^1.2.0",
+        "encodeurl": "^2.0.0",
         "escape-html": "^1.0.3",
         "fresh": "~0.5.2",
-        "http-assert": "^1.3.0",
-        "http-errors": "^1.6.3",
-        "is-generator-function": "^1.0.7",
+        "http-assert": "^1.5.0",
+        "http-errors": "^2.0.0",
         "koa-compose": "^4.1.0",
-        "koa-convert": "^2.0.0",
-        "on-finished": "^2.3.0",
-        "only": "~0.0.2",
-        "parseurl": "^1.3.2",
-        "statuses": "^1.5.0",
-        "type-is": "^1.6.16",
+        "mime-types": "^3.0.1",
+        "on-finished": "^2.4.1",
+        "parseurl": "^1.3.3",
+        "statuses": "^2.0.1",
+        "type-is": "^2.0.1",
         "vary": "^1.1.2"
-      },
-      "dependencies": {
-        "http-errors": {
-          "version": "1.8.1",
-          "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz",
-          "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==",
-          "requires": {
-            "depd": "~1.1.2",
-            "inherits": "2.0.4",
-            "setprototypeof": "1.2.0",
-            "statuses": ">= 1.5.0 < 2",
-            "toidentifier": "1.0.1"
-          },
-          "dependencies": {
-            "depd": {
-              "version": "1.1.2",
-              "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
-              "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ=="
-            }
-          }
-        },
-        "statuses": {
-          "version": "1.5.0",
-          "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
-          "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA=="
-        }
       }
     },
     "koa-compose": {
@@ -6667,15 +6239,6 @@
       "resolved": "https://registry.npmjs.org/koa-compose/-/koa-compose-4.1.0.tgz",
       "integrity": "sha512-8ODW8TrDuMYvXRwra/Kh7/rJo9BtOfPc6qO8eAfC80CnCvSjSl0bkRM24X6/XBBEyj0v1nRUQ1LyOy3dbqOWXw=="
     },
-    "koa-convert": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/koa-convert/-/koa-convert-2.0.0.tgz",
-      "integrity": "sha512-asOvN6bFlSnxewce2e/DK3p4tltyfC4VM7ZwuTuepI7dEQVcvpyFuBcEARu1+Hxg8DIwytce2n7jrZtRlPrARA==",
-      "requires": {
-        "co": "^4.6.0",
-        "koa-compose": "^4.1.0"
-      }
-    },
     "leven": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz",
@@ -6697,11 +6260,6 @@
         "p-locate": "^4.1.0"
       }
     },
-    "lowercase-keys": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz",
-      "integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ=="
-    },
     "lru-cache": {
       "version": "5.1.1",
       "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
@@ -6738,9 +6296,9 @@
       }
     },
     "media-typer": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
-      "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ=="
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz",
+      "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw=="
     },
     "merge-stream": {
       "version": "2.0.0",
@@ -6748,11 +6306,6 @@
       "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==",
       "dev": true
     },
-    "methods": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
-      "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w=="
-    },
     "micromatch": {
       "version": "4.0.7",
       "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz",
@@ -6764,16 +6317,16 @@
       }
     },
     "mime-db": {
-      "version": "1.52.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
-      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="
+      "version": "1.54.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz",
+      "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ=="
     },
     "mime-types": {
-      "version": "2.1.35",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
-      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz",
+      "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==",
       "requires": {
-        "mime-db": "1.52.0"
+        "mime-db": "^1.54.0"
       }
     },
     "mimic-fn": {
@@ -6782,11 +6335,6 @@
       "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==",
       "dev": true
     },
-    "mimic-response": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-4.0.0.tgz",
-      "integrity": "sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg=="
-    },
     "minimatch": {
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
@@ -6809,14 +6357,14 @@
       }
     },
     "ms": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
     },
     "nanoid": {
-      "version": "5.0.7",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.7.tgz",
-      "integrity": "sha512-oLxFY2gd2IqnjcYyOXD8XGCftpGtZP2AbHbOkthDkvRywH5ayNtPVy9YlOPcHckXzbLTCHpkb7FB+yuxKV13pQ=="
+      "version": "5.1.11",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.11.tgz",
+      "integrity": "sha512-v+KEsUv2ps74PaSKv0gHTxTCgMXOIfBEbaqa6w6ISIGC7ZsvHN4N9oJ8d4cmf0n5oTzQz2SLmThbQWhjd/8eKg=="
     },
     "natural-compare": {
       "version": "1.4.0",
@@ -6847,11 +6395,6 @@
       "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==",
       "dev": true
     },
-    "normalize-url": {
-      "version": "8.0.1",
-      "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-8.0.1.tgz",
-      "integrity": "sha512-IO9QvjUMWxPQQhs60oOu10CRkWCiZzSUkzbXGGV9pviYl1fXYcvkzQ5jV9z8Y6un8ARoVRl4EtC6v6jNqbaJ/w=="
-    },
     "npm-run-path": {
       "version": "4.0.1",
       "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz",
@@ -6861,35 +6404,22 @@
         "path-key": "^3.0.0"
       }
     },
-    "object-hash": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz",
-      "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw=="
-    },
     "oidc-provider": {
-      "version": "8.5.1",
-      "resolved": "https://registry.npmjs.org/oidc-provider/-/oidc-provider-8.5.1.tgz",
-      "integrity": "sha512-Bm3EyxN68/KS76IlciJ3+4pnVtfdRWL+NghWpIF0XQbiRT1gzc6Qf/cyFmpL9yieko/jXYZ/uLHUv77jD00qww==",
+      "version": "9.8.3",
+      "resolved": "https://registry.npmjs.org/oidc-provider/-/oidc-provider-9.8.3.tgz",
+      "integrity": "sha512-YkchaAyVAZbsn/l7IQhcEMdeDL3lwSo/PNUtnsXSqPqT7EG8DRko0EAWzHd/n9VfCtKVkxGjYOY4h4UwFcWnUA==",
       "requires": {
         "@koa/cors": "^5.0.0",
-        "@koa/router": "^12.0.1",
-        "debug": "^4.3.5",
-        "eta": "^3.4.0",
-        "got": "^13.0.0",
-        "jose": "^5.6.2",
-        "jsesc": "^3.0.2",
-        "koa": "^2.15.3",
-        "nanoid": "^5.0.7",
-        "object-hash": "^3.0.0",
-        "oidc-token-hash": "^5.0.3",
-        "quick-lru": "^7.0.0",
-        "raw-body": "^2.5.2"
-      }
-    },
-    "oidc-token-hash": {
-      "version": "5.0.3",
-      "resolved": "https://registry.npmjs.org/oidc-token-hash/-/oidc-token-hash-5.0.3.tgz",
-      "integrity": "sha512-IF4PcGgzAr6XXSff26Sk/+P4KZFJVuHAJZj3wgO3vX2bMdNVp/QXTP3P7CEm9V1IdG8lDLY3HhiqpsE/nOwpPw=="
+        "@koa/router": "^15.4.0",
+        "debug": "^4.4.3",
+        "eta": "^4.5.1",
+        "jose": "^6.2.2",
+        "jsesc": "^3.1.0",
+        "koa": "^3.2.0",
+        "nanoid": "^5.1.7",
+        "quick-lru": "^7.3.0",
+        "raw-body": "^3.0.2"
+      }
     },
     "on-finished": {
       "version": "2.4.1",
@@ -6917,16 +6447,6 @@
         "mimic-fn": "^2.1.0"
       }
     },
-    "only": {
-      "version": "0.0.2",
-      "resolved": "https://registry.npmjs.org/only/-/only-0.0.2.tgz",
-      "integrity": "sha512-Fvw+Jemq5fjjyWz6CpKx6w9s7xxqo3+JCyM0WXWeCSOboZ8ABkyvP8ID4CZuChA/wxSx+XSJmdOm8rGVyJ1hdQ=="
-    },
-    "p-cancelable": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-3.0.0.tgz",
-      "integrity": "sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw=="
-    },
     "p-limit": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
@@ -7004,9 +6524,9 @@
       "dev": true
     },
     "path-to-regexp": {
-      "version": "6.3.0",
-      "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz",
-      "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ=="
+      "version": "8.4.2",
+      "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz",
+      "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA=="
     },
     "picocolors": {
       "version": "1.0.1",
@@ -7071,19 +6591,19 @@
       "dev": true
     },
     "quick-lru": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-7.0.0.tgz",
-      "integrity": "sha512-MX8gB7cVYTrYcFfAnfLlhRd0+Toyl8yX8uBx1MrX7K0jegiz9TumwOK27ldXrgDlHRdVi+MqU9Ssw6dr4BNreg=="
+      "version": "7.3.0",
+      "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-7.3.0.tgz",
+      "integrity": "sha512-k9lSsjl36EJdK7I06v7APZCbyGT2vMTsYSRX1Q2nbYmnkBqgUhRkAuzH08Ciotteu/PLJmIF2+tti7o3C/ts2g=="
     },
     "raw-body": {
-      "version": "2.5.2",
-      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz",
-      "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==",
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz",
+      "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==",
       "requires": {
-        "bytes": "3.1.2",
-        "http-errors": "2.0.0",
-        "iconv-lite": "0.4.24",
-        "unpipe": "1.0.0"
+        "bytes": "~3.1.2",
+        "http-errors": "~2.0.1",
+        "iconv-lite": "~0.7.0",
+        "unpipe": "~1.0.0"
       }
     },
     "react-is": {
@@ -7109,11 +6629,6 @@
         "supports-preserve-symlinks-flag": "^1.0.0"
       }
     },
-    "resolve-alpn": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz",
-      "integrity": "sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g=="
-    },
     "resolve-cwd": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz",
@@ -7135,19 +6650,6 @@
       "integrity": "sha512-X2UW6Nw3n/aMgDVy+0rSqgHlv39WZAlZrXCdnbyEiKm17DSqHX4MmQMaST3FbeWR5FTuRcUwYAziZajji0Y7mg==",
       "dev": true
     },
-    "responselike": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/responselike/-/responselike-3.0.0.tgz",
-      "integrity": "sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==",
-      "requires": {
-        "lowercase-keys": "^3.0.0"
-      }
-    },
-    "safe-buffer": {
-      "version": "5.2.1",
-      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
-      "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
-    },
     "safer-buffer": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
@@ -7229,9 +6731,9 @@
       }
     },
     "statuses": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
-      "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ=="
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",
+      "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw=="
     },
     "string-length": {
       "version": "4.0.2",
@@ -7351,12 +6853,20 @@
       "dev": true
     },
     "type-is": {
-      "version": "1.6.18",
-      "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
-      "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==",
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz",
+      "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==",
       "requires": {
-        "media-typer": "0.3.0",
-        "mime-types": "~2.1.24"
+        "content-type": "^2.0.0",
+        "media-typer": "^1.1.0",
+        "mime-types": "^3.0.0"
+      },
+      "dependencies": {
+        "content-type": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz",
+          "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ=="
+        }
       }
     },
     "unpipe": {
@@ -7468,11 +6978,6 @@
       "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==",
       "dev": true
     },
-    "ylru": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/ylru/-/ylru-1.4.0.tgz",
-      "integrity": "sha512-2OQsPNEmBCvXuFlIni/a+Rn+R2pHW9INm0BxXJ4hVDA8TirqMj+J/Rp9ItLatT/5pZqWwefVrTQcHpixsxnVlA=="
-    },
     "yocto-queue": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",
diff --git a/dev/oidc-provider/package.json b/dev/oidc-provider/package.json
index f9e00a8d852..52b22cabc14 100644
--- a/dev/oidc-provider/package.json
+++ b/dev/oidc-provider/package.json
@@ -9,7 +9,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "oidc-provider": "^8.5.1"
+    "oidc-provider": "^9.8.3"
   },
   "devDependencies": {
     "jest": "^29.7.0"
diff --git a/dev/script/generate-config/service-config.js b/dev/script/generate-config/service-config.js
index dc208300b46..061d5954126 100644
--- a/dev/script/generate-config/service-config.js
+++ b/dev/script/generate-config/service-config.js
@@ -152,9 +152,7 @@ function getKahunaConfig(config){
 
     return stripMargin`${getCommonConfig(config)}
         |aws.region="${config.AWS_DEFAULT_REGION}"
-        |origin.full="images.media.${config.DOMAIN}"
         |origin.thumb="localstack.media.${config.DOMAIN}"
-        |origin.images="images.media.${config.DOMAIN}"
         |origin.crops="public.media.${config.DOMAIN}"
         |google.tracking.id="${config.google.tracking.id}"
         |links.feedbackForm="${config.links.feedbackForm}"
@@ -187,7 +185,6 @@ function getMediaApiConfig(config) {
         |s3.thumb.bucket="${config.coreStackProps.ThumbBucket}"
         |s3.config.bucket="${config.coreStackProps.ConfigBucket}"
         |s3.usagemail.bucket="${config.coreStackProps.UsageMailBucket}"
-        |persistence.identifier="picdarUrn"
         |es6.url="${config.es6.url}"
         |es6.shards=${config.es6.shards}
         |es6.replicas=${config.es6.replicas}
@@ -220,7 +217,6 @@ function getThrallConfig(config) {
         |s3.image.bucket="${config.coreStackProps.ImageBucket}"
         |s3.thumb.bucket="${config.coreStackProps.ThumbBucket}"
         |s3.reaper.bucket="${config.coreStackProps.ReaperBucket}"
-        |persistence.identifier="picdarUrn"
         |indexed.image.sns.topic.arn="${config.coreStackProps.IndexedImageTopic}"
         |es6.url="${config.es6.url}"
         |es6.shards=${config.es6.shards}
diff --git a/docs/06-objects-of-interest/02-config.md b/docs/06-objects-of-interest/02-config.md
index bbb968a096c..7b8d78d4524 100644
--- a/docs/06-objects-of-interest/02-config.md
+++ b/docs/06-objects-of-interest/02-config.md
@@ -86,12 +86,6 @@ Service-specific configs. These will override all other config files.
     Json Object Array
     []
   
-  
-    persistence.identifier
Used by the reaper to retain images which have a particular identifier (e.g. picdarUrn for Guardian)
-    True
-    string
-    
-  
   
     persistence.onlyTheseCollections
Used by the reaper…
       
    
diff --git a/image-embedder-lambda/package-lock.json b/image-embedder-lambda/package-lock.json
index 7c5430305dc..8bdb0a1d16f 100644
--- a/image-embedder-lambda/package-lock.json
+++ b/image-embedder-lambda/package-lock.json
@@ -6396,9 +6396,9 @@
 			"license": "MIT"
 		},
 		"node_modules/fast-xml-builder": {
-			"version": "1.1.4",
-			"resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.4.tgz",
-			"integrity": "sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg==",
+			"version": "1.2.0",
+			"resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.2.0.tgz",
+			"integrity": "sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==",
 			"funding": [
 				{
 					"type": "github",
@@ -6407,7 +6407,8 @@
 			],
 			"license": "MIT",
 			"dependencies": {
-				"path-expression-matcher": "^1.1.3"
+				"path-expression-matcher": "^1.5.0",
+				"xml-naming": "^0.1.0"
 			}
 		},
 		"node_modules/fast-xml-parser": {
@@ -11676,6 +11677,21 @@
 				"node": "^14.17.0 || ^16.13.0 || >=18.0.0"
 			}
 		},
+		"node_modules/xml-naming": {
+			"version": "0.1.0",
+			"resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.1.0.tgz",
+			"integrity": "sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==",
+			"funding": [
+				{
+					"type": "github",
+					"url": "https://github.com/sponsors/NaturalIntelligence"
+				}
+			],
+			"license": "MIT",
+			"engines": {
+				"node": ">=16.0.0"
+			}
+		},
 		"node_modules/y18n": {
 			"version": "5.0.8",
 			"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
diff --git a/image-loader/app/ImageLoaderComponents.scala b/image-loader/app/ImageLoaderComponents.scala
index db2b9a059e3..fcc73eb01aa 100644
--- a/image-loader/app/ImageLoaderComponents.scala
+++ b/image-loader/app/ImageLoaderComponents.scala
@@ -1,13 +1,14 @@
+import app.photofox.vipsffm.{Vips, VipsHelper}
 import com.gu.mediaservice.GridClient
-import com.gu.mediaservice.lib.aws.{Bedrock, S3Vectors, SimpleSqsMessageConsumer, Embedder}
-import com.gu.mediaservice.lib.config.Services
+import com.gu.mediaservice.lib.aws._
+import com.gu.mediaservice.lib.embeddings.GoogleCloudEmbedding
 import com.gu.mediaservice.lib.imaging.ImageOperations
 import com.gu.mediaservice.lib.logging.GridLogging
-import com.gu.mediaservice.lib.management.InnerServiceStatusCheckController
 import com.gu.mediaservice.lib.play.GridComponents
 import controllers.{ImageLoaderController, ImageLoaderManagement, UploadStatusController}
 import lib._
 import lib.storage.{ImageLoaderStore, QuarantineStore}
+import model.upload.OptimiseWithPngQuant
 import model.{Projector, QuarantineUploader, Uploader}
 import play.api.ApplicationLoader.Context
 import router.Routes
@@ -26,18 +27,29 @@ class ImageLoaderComponents(context: Context) extends GridComponents(context, ne
   val store = new ImageLoaderStore(config)
   val maybeIngestQueue = config.maybeIngestSqsQueueUrl.map(queueUrl => new SimpleSqsMessageConsumer(queueUrl, config))
   val uploadStatusTable = new UploadStatusTable(config)
-  val imageOperations = new ImageOperations(context.environment.rootPath.getAbsolutePath)
+  val imageOperations = {
+    Vips.init()
+    VipsHelper.cache_set_max(0)
+    new ImageOperations(context.environment.rootPath.getAbsolutePath)
+  }
   val notifications = new Notifications(config)
   val downloader = new Downloader()(ec,wsClient)
 
+  private val gcpProjectId = "eelpie-cloud-registry"
+  private val vertexApiLocation = "eu"
+  private val googleCloudEmbedding = new GoogleCloudEmbedding(projectId = gcpProjectId, location = vertexApiLocation)
+
   val maybeEmbedder: Option[Embedder] = config.maybeImageEmbedderQueueUrl
     .filter(_ => config.shouldEmbed)
     .map {queueUrl =>
-      new Embedder(new Bedrock(config), new SimpleSqsMessageConsumer(queueUrl, config))
+      logger.info("Image loader is configured to queue embedding requests to: " + queueUrl)
+      new Embedder(googleCloudEmbedding, new SimpleSqsMessageConsumer(queueUrl, config))
     }
 
-  val uploader = new Uploader(store, config, imageOperations, notifications, maybeEmbedder, imageProcessor, gridClient, auth)
-  val projector = Projector(config, imageOperations, imageProcessor, auth, maybeEmbedder)
+  val optimiseOps = new OptimiseWithPngQuant(imageOperations)
+  val uploader = new Uploader(store, config, imageOperations, notifications, maybeEmbedder, imageProcessor, gridClient, auth, optimiseOps)
+  val s3 = new S3(config)
+  val projector = Projector(config, imageOperations, imageProcessor, auth, maybeEmbedder, s3, optimiseOps)
   val quarantineUploader: Option[QuarantineUploader] = config.maybeQuarantineBucket.map(_ =>
     new QuarantineUploader(new QuarantineStore(config), config)
   )
@@ -45,10 +57,9 @@ class ImageLoaderComponents(context: Context) extends GridComponents(context, ne
   val metrics = new ImageLoaderMetrics(config, actorSystem, applicationLifecycle)
 
   val controller = new ImageLoaderController(
-    auth, downloader, store, maybeIngestQueue, uploadStatusTable, notifications, config, uploader, quarantineUploader, projector, controllerComponents, gridClient, authorisation, metrics, applicationLifecycle)
+    auth, downloader, store, maybeIngestQueue, uploadStatusTable, config, uploader, quarantineUploader, projector, controllerComponents, gridClient, authorisation, metrics, usageEvents, wsClient, applicationLifecycle)
   val uploadStatusController = new UploadStatusController(auth, uploadStatusTable, config, controllerComponents, authorisation)
-  val InnerServiceStatusCheckController = new InnerServiceStatusCheckController(auth, controllerComponents, config.services, wsClient)
   val imageLoaderManagement = new ImageLoaderManagement(controllerComponents, buildInfo, controller.maybeIngestQueueAndProcessor)
 
-  override lazy val router = new Routes(httpErrorHandler, controller, uploadStatusController, imageLoaderManagement, InnerServiceStatusCheckController)
+  override lazy val router = new Routes(httpErrorHandler, controller, uploadStatusController, imageLoaderManagement)
 }
diff --git a/image-loader/app/controllers/ImageLoaderController.scala b/image-loader/app/controllers/ImageLoaderController.scala
index ab44b57a88f..e2c546da4cb 100644
--- a/image-loader/app/controllers/ImageLoaderController.scala
+++ b/image-loader/app/controllers/ImageLoaderController.scala
@@ -1,8 +1,5 @@
 package controllers
 
-import org.apache.pekko.Done
-import org.apache.pekko.stream.Materializer
-import org.apache.pekko.stream.scaladsl.Source
 import com.amazonaws.services.s3.AmazonS3
 import com.amazonaws.services.sqs.model.{Message => SQSMessage}
 import com.amazonaws.util.IOUtils
@@ -11,25 +8,34 @@ import com.gu.mediaservice.GridClient
 import com.gu.mediaservice.lib.ImageIngestOperations.fileKeyFromId
 import com.gu.mediaservice.lib.argo.ArgoHelpers
 import com.gu.mediaservice.lib.argo.model.Link
-import com.gu.mediaservice.lib.auth.Authentication.OnBehalfOfPrincipal
+import com.gu.mediaservice.lib.auth.Authentication.{MachinePrincipal, OnBehalfOfPrincipal, UserPrincipal}
 import com.gu.mediaservice.lib.auth._
+import com.gu.mediaservice.lib.auth.provider.ApiKeyAuthenticationProvider
 import com.gu.mediaservice.lib.aws.{S3Ops, SimpleSqsMessageConsumer, SqsHelpers}
+import com.gu.mediaservice.lib.config.InstanceForRequest
+import com.gu.mediaservice.lib.events.UsageEvents
 import com.gu.mediaservice.lib.formatting.printDateTime
+import com.gu.mediaservice.lib.instances.Instances
 import com.gu.mediaservice.lib.logging.{FALLBACK, LogMarker, MarkerMap}
 import com.gu.mediaservice.lib.play.RequestLoggingFilter
 import com.gu.mediaservice.lib.{DateTimeUtils, ImageIngestOperations, ImageStorageProps}
-import com.gu.mediaservice.model.{UnsupportedMimeTypeException, UploadInfo}
-import org.scanamo.{ConditionNotMet, ScanamoError}
+import com.gu.mediaservice.model.{Instance, UnsupportedMimeTypeException, UploadInfo}
 import lib.FailureResponse.Response
+import lib._
 import lib.imaging.{MimeTypeDetection, NoSuchImageExistsInS3, UserImageLoaderException}
 import lib.storage.{ImageLoaderStore, S3FileDoesNotExistException}
-import lib._
 import model.upload.UploadRequest
 import model.{Projector, QuarantineUploader, S3FileExtractedMetadata, S3IngestObject, StatusType, UploadStatus, UploadStatusRecord, UploadStatusUri, Uploader}
+import org.apache.pekko.Done
+import org.apache.pekko.stream.Materializer
+import org.apache.pekko.stream.scaladsl.Source
+import org.joda.time.{DateTime, Duration}
+import org.scanamo.{ConditionNotMet, ScanamoError}
 import play.api.data.Form
 import play.api.data.Forms._
 import play.api.inject.ApplicationLifecycle
 import play.api.libs.json.Json
+import play.api.libs.ws.WSClient
 import play.api.mvc._
 import software.amazon.awssdk.services.cloudwatch.model.Dimension
 
@@ -46,8 +52,7 @@ class ImageLoaderController(auth: Authentication,
                             store: ImageLoaderStore,
                             maybeIngestQueue: Option[SimpleSqsMessageConsumer],
                             uploadStatusTable: UploadStatusTable,
-                            notifications: Notifications,
-                            config: ImageLoaderConfig,
+                            val config: ImageLoaderConfig,
                             uploader: Uploader,
                             quarantineUploader: Option[QuarantineUploader],
                             projector: Projector,
@@ -55,9 +60,11 @@ class ImageLoaderController(auth: Authentication,
                             gridClient: GridClient,
                             authorisation: Authorisation,
                             metrics: ImageLoaderMetrics,
+                            usageEvents: UsageEvents,
+                            val wsClient: WSClient,
                             applicationLifecycle: ApplicationLifecycle)
                            (implicit val ec: ExecutionContext, materializer: Materializer)
-  extends BaseController with ArgoHelpers with SqsHelpers {
+  extends BaseController with ArgoHelpers with SqsHelpers with InstanceForRequest with Instances {
 
   private val AuthenticatedAndAuthorised = auth andThen authorisation.CommonActionFilters.authorisedForUpload
 
@@ -113,37 +120,49 @@ class ImageLoaderController(auth: Authentication,
     (ingestQueue, processor)
   }
 
-  private lazy val indexResponse: Result = {
+  private def indexResponse(instance: Instance): Result = {
     val indexData = Map("description" -> "This is the Loader Service")
     val indexLinks = List(
-      Link("prepare", s"${config.rootUri}/prepare"),
-      Link("uploadStatus", s"${config.rootUri}/uploadStatus/{id}"),
-      Link("uploadStatuses", s"${config.rootUri}/uploadStatuses"),
-      Link("load", s"${config.rootUri}/images{?uploadedBy,identifiers,uploadTime,filename}"),
-      Link("import", s"${config.rootUri}/imports{?uri,uploadedBy,identifiers,uploadTime,filename}")
+      Link("prepare", s"${config.rootUri(instance)}/prepare"),
+      Link("uploadStatus", s"${config.rootUri(instance)}/uploadStatus/{id}"),
+      Link("uploadStatuses", s"${config.rootUri(instance)}/uploadStatuses"),
+      Link("load", s"${config.rootUri(instance)}/images{?uploadedBy,identifiers,uploadTime,filename}"),
+      Link("import", s"${config.rootUri(instance)}/imports{?uri,uploadedBy,identifiers,uploadTime,filename}")
     )
     respond(indexData, indexLinks)
   }
 
-  def index: Action[AnyContent] = AuthenticatedAndAuthorised { indexResponse }
-
-  private def quarantineOrStoreImage(uploadRequest: UploadRequest)(implicit logMarker: LogMarker) = {
-    quarantineUploader.map(_.quarantineFile(uploadRequest)).getOrElse(for { uploadStatusUri <- uploader.storeFile(uploadRequest)} yield{uploadStatusUri.toJsObject})
+  def index: Action[AnyContent] = AuthenticatedAndAuthorised { request =>
+    indexResponse(instanceOf(request))
   }
 
-  private def handleMessageFromIngestBucket(sqsMessage:SQSMessage)(basicLogMarker: LogMarker): Future[Unit] = Future[Future[Unit]]{
-
-    logger.info(basicLogMarker, sqsMessage.toString)
+  private def quarantineOrStoreImage(uploadRequest: UploadRequest)(implicit logMarker: LogMarker, instance: Instance) = {
+    quarantineUploader.map(_.quarantineFile(uploadRequest)(instance)).getOrElse(for { uploadStatusUri <- uploader.storeFile(uploadRequest)} yield{uploadStatusUri.toJsObject})
+  }
 
+  private def handleMessageFromIngestBucket(sqsMessage: SQSMessage)(basicLogMarker: LogMarker): Future[Unit] = {
     extractS3KeyFromSqsMessage(sqsMessage) match {
       case Failure(exception) =>
         metrics.failedIngestsFromQueue.increment()
         logger.error(basicLogMarker, s"Failed to parse s3 data from SQS message", exception)
         Future.unit
       case Success(key) =>
-        val s3IngestObject = S3IngestObject(key, store)(basicLogMarker)
+        val pathComponents = key.split("/")
+        val instanceId = pathComponents.head
+        val path = pathComponents.drop(1).mkString("/")
+        logger.info(s"Instance and key: $instanceId / $path")
+
+        val eventualIsValidInstanceId = getInstances().map { instances =>
+          instances.map(_.id).contains(instanceId)
+        }
 
-        val isUiUpload = s3IngestObject.maybeMediaIdFromUiUpload.isDefined
+        eventualIsValidInstanceId.flatMap { isValidInstanceSpecificKey =>
+          if (isValidInstanceSpecificKey) {
+            try {
+              implicit val instance: Instance = Instance(id = instanceId)
+
+              val s3IngestObject = S3IngestObject(key, store)(basicLogMarker)
+              val isUiUpload = s3IngestObject.maybeMediaIdFromUiUpload.isDefined
 
         implicit val logMarker: LogMarker = basicLogMarker ++ Map(
           "uploadedBy" -> s3IngestObject.uploadedBy,
@@ -157,51 +176,62 @@ class ImageLoaderController(auth: Authentication,
           Dimension.builder().name("IsUiUpload").value(isUiUpload.toString).build(),
         )
 
-        val approximateReceiveCount = getApproximateReceiveCount(sqsMessage)
-
-        if(config.maybeUploadLimitInBytes.exists(_ < s3IngestObject.contentLength)){
-          val errorMessage = s"File size exceeds the maximum allowed size (${config.maybeUploadLimitInBytes.get / 1_000_000}MB). Moving to fail bucket."
-          logger.warn(logMarker, errorMessage)
-          store.moveObjectToFailedBucket(s3IngestObject.key)
-          s3IngestObject.maybeMediaIdFromUiUpload foreach { imageId =>
-            uploadStatusTable.updateStatus( // fire & forget, since there's nothing else we can do
-              imageId, UploadStatus(StatusType.Failed, Some(errorMessage))
-            )
-          }
-          metrics.failedIngestsFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
-          Future.unit
-        }
-        else if (approximateReceiveCount > 2) {
-          metrics.abandonedMessagesFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
-          val errorMessage = s"File processing has been attempted $approximateReceiveCount times. Moving to fail bucket."
-          logger.warn(logMarker, errorMessage)
-          store.moveObjectToFailedBucket(s3IngestObject.key)
-          s3IngestObject.maybeMediaIdFromUiUpload foreach { imageId =>
-            uploadStatusTable.updateStatus( // fire & forget, since there's nothing else we can do
-              imageId, UploadStatus(StatusType.Failed, Some(errorMessage))
-            )
-          }
-          Future.unit
-        } else {
-          attemptToProcessIngestedFile(s3IngestObject, isUiUpload)(logMarker) map { digestedFile =>
-            metrics.successfulIngestsFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
-            logger.info(logMarker, s"Successfully processed image ${digestedFile.file.getName}")
-            store.deleteObjectFromIngestBucket(s3IngestObject.key)
-          } recover {
-            case _: UnsupportedMimeTypeException =>
-              metrics.failedIngestsFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
-              logger.info(logMarker, s"Unsupported mime type. Moving straight to fail bucket.")
-              store.moveObjectToFailedBucket(s3IngestObject.key)
-            case t: Throwable =>
-              metrics.failedIngestsFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
-              logger.error(logMarker, s"Failed to process file. Moving to fail bucket.", t)
-              store.moveObjectToFailedBucket(s3IngestObject.key)
+              val approximateReceiveCount = getApproximateReceiveCount(sqsMessage)
+
+              if (config.maybeUploadLimitInBytes.exists(_ < s3IngestObject.contentLength)) {
+                val errorMessage = s"File size exceeds the maximum allowed size (${config.maybeUploadLimitInBytes.get / 1_000_000}MB). Moving to fail bucket."
+                logger.warn(logMarker, errorMessage)
+                store.moveObjectToFailedBucket(s3IngestObject.key)
+                s3IngestObject.maybeMediaIdFromUiUpload foreach { imageId =>
+                  uploadStatusTable.updateStatus( // fire & forget, since there's nothing else we can do
+                    imageId, UploadStatus(StatusType.Failed, Some(errorMessage))
+                  )
+                }
+                metrics.failedIngestsFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
+                Future.unit
+              }
+              else if (approximateReceiveCount > 2) {
+                metrics.abandonedMessagesFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
+                val errorMessage = s"File processing has been attempted $approximateReceiveCount times. Moving to fail bucket."
+                logger.warn(logMarker, errorMessage)
+                store.moveObjectToFailedBucket(s3IngestObject.key)
+                s3IngestObject.maybeMediaIdFromUiUpload foreach { imageId =>
+                  uploadStatusTable.updateStatus( // fire & forget, since there's nothing else we can do
+                    imageId, UploadStatus(StatusType.Failed, Some(errorMessage))
+                  )
+                }
+                Future.unit
+              } else {
+                attemptToProcessIngestedFile(s3IngestObject, isUiUpload)(logMarker)(instance) map { digestedFile =>
+                  metrics.successfulIngestsFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
+                  usageEvents.successfulIngestFromQueue(instance = instance, image = digestedFile.digest, filesize = s3IngestObject.contentLength )
+                  logger.info(logMarker, s"Successfully processed image ${digestedFile.file.getName}")
+                  store.deleteObjectFromIngestBucket(s3IngestObject.key)
+                } recover {
+                  case _: UnsupportedMimeTypeException =>
+                    metrics.failedIngestsFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
+                    logger.info(logMarker, s"Unsupported mime type. Moving straight to fail bucket.")
+                    store.moveObjectToFailedBucket(s3IngestObject.key)
+                  case t: Throwable =>
+                    metrics.failedIngestsFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
+                    logger.error(logMarker, s"Failed to process file. Moving to fail bucket.", t)
+                    store.moveObjectToFailedBucket(s3IngestObject.key)
+                }
+              }
+            }
+            catch {
+              case t: Throwable =>
+                logger.error("Uncaught throw:", t)
+                Future.unit
+            }
+          } else {
+            Future.unit
           }
         }
     }
-  }.flatten
+  }
 
-  private def attemptToProcessIngestedFile(s3IngestObject:S3IngestObject, isUiUpload: Boolean)(initialLogMarker:LogMarker): Future[DigestedFile] = {
+  private def attemptToProcessIngestedFile(s3IngestObject:S3IngestObject, isUiUpload: Boolean)(initialLogMarker:LogMarker)(implicit instance: Instance): Future[DigestedFile] = {
 
     logger.info(initialLogMarker, "Attempting to process file")
     val tempFile = createTempFile("s3IngestBucketFile")(initialLogMarker)
@@ -215,12 +245,14 @@ class ImageLoaderController(auth: Authentication,
       "mediaId" -> digestedFile.digest
     )
 
+    val filename = s3IngestObject.filename
     val futureUploadStatusUri = uploadDigestedFileToStore(
         digestedFileFuture = Future(digestedFile),
         uploadedBy = s3IngestObject.uploadedBy,
         identifiers =  s3IngestObject.identifiers,
         uploadTime = Some(s3IngestObject.uploadTime.toString) , // upload time as iso string - uploader uses DateTimeUtils.fromValueOrNow
-        filename = Some(s3IngestObject.filename)
+        filename = Some(filename),
+        isFeedUpload = s3IngestObject.isFeedUpload,
     )
 
     // under all circumstances, remove the temp files
@@ -236,6 +268,8 @@ class ImageLoaderController(auth: Authentication,
   }
 
   def getPreSignedUploadUrlsAndTrack: Action[AnyContent] = AuthenticatedAndAuthorised.async { request =>
+    implicit val instance: Instance = instanceOf(request)
+
     val expiration = DateTimeUtils.now().plusHours(1)
 
     val mediaIdToFilenameMap = request.body.asJson.get.as[Map[String, String]]
@@ -245,6 +279,7 @@ class ImageLoaderController(auth: Authentication,
     Future.sequence(
 
       mediaIdToFilenameMap.map{case (mediaId, filename) =>
+        logger.info(s"Preparing file upload for instance $instance: $mediaId / $filename")
 
         val preSignedUrl = store.generatePreSignedUploadUrl(filename, expiration, uploadedBy, mediaId)
 
@@ -257,9 +292,19 @@ class ImageLoaderController(auth: Authentication,
           StatusType.Prepared,
           errorMessage = None,
           expires = expiration.toEpochSecond, // TTL in case upload is never completed by client
-        )).map(_ =>
+          instance = instance.id
+        )).map { _ =>
+          val user = request.user match {
+            case u: UserPrincipal => u.attributes.get(ApiKeyAuthenticationProvider.KindeIdKey)
+            case _ => None
+          }
+          val apiKey = request.user match {
+            case m: MachinePrincipal => Some(m.accessor.identity)
+            case _ => None
+          }
+          usageEvents.prepareUpload(instance = instance, image = mediaId, user = user, apiKey = apiKey)
           mediaId -> preSignedUrl
-        )
+        }
       }
     )
     .map(_.toMap)
@@ -267,7 +312,7 @@ class ImageLoaderController(auth: Authentication,
     .map(Ok(_))
   }
 
-  def loadImage(uploadedBy: Option[String], identifiers: Option[String], uploadTime: Option[String], filename: Option[String]): Action[DigestedFile] =  {
+  def loadImage(uploadedBy: Option[String], identifiers: Option[String], uploadTime: Option[String], filename: Option[String]): Action[DigestedFile] = {
     val uploadTimeToRecord = DateTimeUtils.fromValueOrNow(uploadTime)
 
     val initialContext = MarkerMap(
@@ -284,7 +329,8 @@ class ImageLoaderController(auth: Authentication,
     logger.info(initialContext, "body parsed")
     val bodyParser = DigestBodyParser.create(tempFile)
 
-    AuthenticatedAndAuthorised.async(bodyParser) { req =>
+    AuthenticatedAndAuthorised.async(bodyParser) { req: Authentication.Request[DigestedFile] =>
+      implicit val instance: Instance = instanceOf(req)
       val uploadedByToRecord = uploadedBy.getOrElse(Authentication.getIdentity(req.user))
 
       implicit val context: LogMarker =
@@ -295,23 +341,38 @@ class ImageLoaderController(auth: Authentication,
 
       val uploadStatus = if(config.maybeQuarantineBucket.isDefined) StatusType.Pending else StatusType.Completed
       val uploadExpiry = Instant.now.getEpochSecond + config.uploadStatusExpiry.toSeconds
-      val record = UploadStatusRecord(req.body.digest, filename, uploadedByToRecord, printDateTime(uploadTimeToRecord), identifiers, uploadStatus, None, uploadExpiry)
+      val record = UploadStatusRecord(req.body.digest, filename, uploadedByToRecord, printDateTime(uploadTimeToRecord), identifiers, uploadStatus, None, uploadExpiry, instance.id)
+      logger.info(s"Loading image for instance ${instance.id}: record ${record.id} / $filename")
+
       val result = for {
         uploadRequest <- uploader.loadFile(
           req.body,
           uploadedByToRecord,
           identifiers.map(Json.parse(_).as[Map[String, String]]).getOrElse(Map.empty),
           uploadTimeToRecord,
-          filename.flatMap(_.trim.nonEmptyOpt)
+          filename.flatMap(_.trim.nonEmptyOpt),
+          instance,
+          isFeedUpload = false,
         )
         _ <- uploadStatusTable.setStatus(record)
-        result <- quarantineOrStoreImage(uploadRequest)
+
+        result <- quarantineOrStoreImage(uploadRequest)(context, instance)
+
       } yield result
       result.onComplete( _ => Try { deleteTempFile(tempFile) } )
 
       result map { r =>
         val result = Accepted(r).as(ArgoMediaType)
         logger.info(context, "loadImage request end")
+        val user = req.user match {
+          case u: UserPrincipal => u.attributes.get(ApiKeyAuthenticationProvider.KindeIdKey)
+          case _ => None
+        }
+        val apiKey = req.user match {
+          case m: MachinePrincipal => Some(m.accessor.identity)
+          case _ => None
+        }
+        usageEvents.uploadImage(instance = instance, image = req.body.digest, filesize = req.body.file.length(), apiKey = apiKey, user = user)
         result
       } recover {
         case NonFatal(e) =>
@@ -343,6 +404,7 @@ class ImageLoaderController(auth: Authentication,
     val bodyParser = DigestBodyParser.create(tempFile)
 
     AuthenticatedAndAuthorised.async(bodyParser) { req =>
+      implicit val instance: Instance = instanceOf(req)
 
       val allIdentifiers = identifiers.map(Json.parse(_).as[Map[String, String]]).getOrElse(Map.empty) ++ Map(
         ImageStorageProps.derivativeOfMediaIdsIdentifierKey -> derivativeOfMediaIds
@@ -368,12 +430,14 @@ class ImageLoaderController(auth: Authentication,
 
   // Fetch
   def projectImageBy(imageId: String): Action[AnyContent] = {
+
     val initialContext = MarkerMap(
       "imageId" -> imageId,
       "requestType" -> "image-projection"
     )
     val tempFile = createTempFile(s"projection-$imageId")(initialContext)
     auth.async { req =>
+      implicit val instance: Instance = instanceOf(req)
       implicit val context: LogMarker = initialContext ++ Map(
         "requestId" -> RequestLoggingFilter.getRequestId(req)
       )
@@ -404,9 +468,11 @@ class ImageLoaderController(auth: Authentication,
                    uploadedBy: Option[String],
                    identifiers: Option[String],
                    uploadTime: Option[String],
-                   filename: Option[String]
+                   filename: Option[String],
                  ): Action[AnyContent] = {
     AuthenticatedAndAuthorised.async { request =>
+      implicit val instance: Instance = instanceOf(request)
+
       implicit val context: MarkerMap = MarkerMap(
         "requestType" -> "import-image",
         "key-tier" -> request.user.accessor.tier.toString,
@@ -414,7 +480,7 @@ class ImageLoaderController(auth: Authentication,
         "requestId" -> RequestLoggingFilter.getRequestId(request)
       )
 
-      logger.info(context, "importImage request start")
+      logger.info(context, "importImage request start for $uri into instance $instance")
 
       val tempFile = createTempFile("download")
       val digestedFileFuture = for {
@@ -427,7 +493,8 @@ class ImageLoaderController(auth: Authentication,
           uploadedBy.getOrElse(Authentication.getIdentity(request.user)),
           identifiers.map(Json.parse(_).as[Map[String, String]]).getOrElse(Map.empty),
           uploadTime,
-          filename
+          filename,
+          isFeedUpload = false,
       )
 
       // under all circumstances, remove the temp files
@@ -454,9 +521,10 @@ class ImageLoaderController(auth: Authentication,
     uploadedBy: String,
     identifiers: Map[String, String],
     uploadTime: Option[String],
-    filename: Option[String]
-  )(implicit logMarker:LogMarker): Future[UploadStatusUri] = {
-
+    filename: Option[String],
+    isFeedUpload: Boolean
+  )(implicit logMarker:LogMarker, instance: Instance): Future[UploadStatusUri] = {
+    val start = DateTime.now()
     for {
         digestedFile <- digestedFileFuture
         uploadStatusResult <- uploadStatusTable.getStatus(digestedFile.digest)
@@ -469,10 +537,13 @@ class ImageLoaderController(auth: Authentication,
           ).getOrElse(identifiers),
           uploadTime =  DateTimeUtils.fromValueOrNow(maybeStatus.map(_.uploadTime).orElse(uploadTime)),
           filename =  maybeStatus.flatMap(_.fileName).orElse(filename).flatMap(_.trim.nonEmptyOpt),
+          instance,
+          isFeedUpload = isFeedUpload
         )
         result <- uploader.storeFile(uploadRequest)
       } yield {
-        logger.info(logMarker, "importImage request end")
+        val duration = new Duration(start, DateTime.now())
+        logger.info(logMarker, s"importImage request end; took ${duration.getMillis} ms")
         result
       }
   }
@@ -480,7 +551,7 @@ class ImageLoaderController(auth: Authentication,
   private def resolveUploadAndUpdateStatus (
    uploadResultFuture: Future[UploadStatusUri],
    digestedFileFuture: Future[DigestedFile],
-  )(implicit logMarker:LogMarker):Future[Either[Response,UploadStatusUri]] = {
+  )(implicit logMarker:LogMarker, instance: Instance):Future[Either[Response,UploadStatusUri]] = {
     // combine the import result and digest file together into a single future
     uploadResultFuture.transformWith { // note that we use transformWith instead of zip here as we are still interested in value of digestedFile even if the import fails
       maybeImportResult =>
@@ -519,7 +590,7 @@ class ImageLoaderController(auth: Authentication,
   private def updateUploadStatusTable(
     uploadAttempt: Future[UploadStatusUri],
     digestedFile: DigestedFile
-  )(implicit logMarker: LogMarker): Future[Unit] = {
+  )(implicit logMarker: LogMarker, instance: Instance): Future[Unit] = {
 
     def reportFailure(error: Throwable): Unit = {
       val errorMessage = s"an error occurred while updating image upload status, error:$error"
@@ -562,6 +633,7 @@ class ImageLoaderController(auth: Authentication,
 
   private case class RestoreFromReplicaForm(imageId: String)
   def restoreFromReplica: Action[AnyContent] = AuthenticatedAndAuthorised.async { implicit request =>
+    implicit val instance: Instance = instanceOf(request)
 
     val imageId = Form(
       mapping(
@@ -608,7 +680,8 @@ class ImageLoaderController(auth: Authentication,
               metadata.uploadTime,
               metadata.uploadedBy,
               metadata.identifiers,
-              UploadInfo(metadata.uploadFileName)
+              UploadInfo(metadata.uploadFileName, metadata.isFeedUpload),
+              instance,
             ),
             gridClient,
             auth.getOnBehalfOfPrincipal(request.user)
@@ -620,7 +693,7 @@ class ImageLoaderController(auth: Authentication,
 
           future.map { _ =>
             logger.info(logMarker, s"Restored image $imageId from replica bucket $replicaBucket (key: $s3Key)")
-            Redirect(s"${config.kahunaUri}/images/$imageId")
+            Redirect(s"${config.kahunaUri(instance)}/images/$imageId")
           }
         case _ =>
           Future.successful(NotFound("Image not found in replica bucket"))
diff --git a/image-loader/app/controllers/UploadStatusController.scala b/image-loader/app/controllers/UploadStatusController.scala
index 4bde0c8b3be..c449f8d2ffb 100644
--- a/image-loader/app/controllers/UploadStatusController.scala
+++ b/image-loader/app/controllers/UploadStatusController.scala
@@ -3,8 +3,10 @@ package controllers
 
 import com.gu.mediaservice.lib.argo.ArgoHelpers
 import com.gu.mediaservice.lib.auth._
+import com.gu.mediaservice.lib.config.InstanceForRequest
 import lib._
 import model.{StatusType, UploadStatus}
+import com.gu.mediaservice.model.Instance
 import org.scanamo.{ConditionNotMet, ScanamoError}
 import play.api.mvc._
 
@@ -18,13 +20,14 @@ class UploadStatusController(auth: Authentication,
                              authorisation: Authorisation
                             )
                             (implicit val ec: ExecutionContext)
-  extends BaseController with ArgoHelpers {
+  extends BaseController with ArgoHelpers with InstanceForRequest {
 
-  def getUploadStatus(imageId: String) = auth.async {
+  def getUploadStatus(imageId: String) = auth.async { request =>
+    implicit val instance: Instance = instanceOf(request)
     store.getStatus(imageId)
       .map {
         case Some(Right(record)) => respond(UploadStatus(record.status, record.errorMessage),
-          uri = Some(URI.create(s"${config.apiUri}/images/${imageId}")))
+          uri = Some(URI.create(s"${config.apiUri(instance)}/images/$imageId")))
         case Some(Left(error)) => respondError(BadRequest, "cannot-get", s"Cannot get upload status ${error}")
         case None => respondNotFound(s"No upload status found for image id: ${imageId}")
       }
@@ -32,6 +35,7 @@ class UploadStatusController(auth: Authentication,
   }
 
   def updateUploadStatus(imageId: String) = (auth andThen authorisation.CommonActionFilters.authorisedForUpload).async(parse.json[UploadStatus]) { request =>
+    implicit val instance: Instance = instanceOf(request)
     request.body match {
       case UploadStatus(StatusType.Failed, None) =>
         Future.successful(respondError(
@@ -55,6 +59,7 @@ class UploadStatusController(auth: Authentication,
   def getUploadsBy(user:String): Action[AnyContent] = getUploads(Some(user))
 
   private def getUploads(maybeUser: Option[String]): Action[AnyContent] = auth.async { req =>
+    implicit val instance: Instance = instanceOf(req)
     store.queryByUser(maybeUser.getOrElse(req.user.accessor.identity))
       .map(list => respond(list))
   }
diff --git a/image-loader/app/lib/ImageLoaderConfig.scala b/image-loader/app/lib/ImageLoaderConfig.scala
index 9417585a471..1ad67fe2fe0 100644
--- a/image-loader/app/lib/ImageLoaderConfig.scala
+++ b/image-loader/app/lib/ImageLoaderConfig.scala
@@ -1,5 +1,7 @@
 package lib
 
+import com.gu.mediaservice.lib.aws.{S3, S3Bucket}
+
 import java.io.File
 import com.gu.mediaservice.lib.cleanup.{ComposedImageProcessor, ImageProcessor, ImageProcessorResources}
 import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources, ImageProcessorLoader}
@@ -10,11 +12,13 @@ import play.api.inject.ApplicationLifecycle
 import scala.concurrent.duration.FiniteDuration
 
 class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(resources) with StrictLogging {
-  val imageBucket: String = string("s3.image.bucket")
 
   val maybeImageReplicaBucket: Option[String] = stringOpt("s3.image.replicaBucket")
 
-  val thumbnailBucket: String = string("s3.thumb.bucket")
+  private val quarantineBucketEndpoint = S3.AmazonAwsS3Endpoint
+  val quarantineBucket: Option[S3Bucket] = stringOpt("s3.quarantine.bucket").map { bucket =>
+    S3Bucket(bucket, quarantineBucketEndpoint, usesPathStyleURLs = false, clientFor(quarantineBucketEndpoint))
+  }
 
   val lowerEnvironmentSamplingPercentageAsDecimal = intOpt("s3.sampling.percentage").getOrElse(1) / 100.0
   val maybeLowerEnvironmentQueueBucketToSampleInto = stringOpt("s3.sampling.targetBucket")
@@ -24,18 +28,18 @@ class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(res
   val thumbWidth: Int = 256
   val thumbQuality: Double = 85d // out of 100
 
-  val rootUri: String = services.loaderBaseUri
-  val apiUri: String = services.apiBaseUri
-  val kahunaUri: String = services.kahunaBaseUri
+  val rootUri: Instance => String = services.loaderBaseUri
+  val apiUri: Instance => String = services.apiBaseUri
+  val kahunaUri: Instance => String = services.kahunaBaseUri
 
-  val transcodedMimeTypes: List[MimeType] = getStringSet("transcoded.mime.types").toList.map(MimeType(_))
-  val supportedMimeTypes: List[MimeType] = List(Jpeg, Png) ::: transcodedMimeTypes
+  val supportedMimeTypes: List[MimeType] = List(Jpeg, Png)
 
   val uploadStatusTable: String = string("dynamo.table.upload.status")
   val uploadStatusExpiry: FiniteDuration = configuration.get[FiniteDuration]("uploadStatus.recordExpiry")
 
   val shouldEmbed: Boolean = boolean("s3.vectors.shouldEmbed")
   val maybeImageEmbedderQueueUrl: Option[String] = stringOpt("sqs.image.embedder.queue.url")
+
   /**
     * Load in the chain of image processors from config. This can be a list of
     * companion objects, class names, both with and without config.
diff --git a/image-loader/app/lib/ImageLoaderStore.scala b/image-loader/app/lib/ImageLoaderStore.scala
index fefb71177c7..6435a1e6281 100644
--- a/image-loader/app/lib/ImageLoaderStore.scala
+++ b/image-loader/app/lib/ImageLoaderStore.scala
@@ -4,47 +4,52 @@ import com.amazonaws.HttpMethod
 import com.amazonaws.services.s3.model.{AmazonS3Exception, GeneratePresignedUrlRequest, S3Object}
 import lib.ImageLoaderConfig
 import com.gu.mediaservice.lib
-import com.gu.mediaservice.lib.logging.LogMarker
+import com.gu.mediaservice.lib.aws
+import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker}
+import com.gu.mediaservice.model.Instance
 
 import java.io.File
 import java.time.ZonedDateTime
 import java.util.Date
+import scala.concurrent.Future
 
 class S3FileDoesNotExistException extends Exception()
 
-class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperations(config.imageBucket, config.thumbnailBucket, config) {
+class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperations(config.imageBucket, config.thumbnailBucket, config.embeddingSourceBucket, config) with GridLogging {
 
   private def handleNotFound[T](key: String)(doWork: => T)(loggingIfNotFound: => Unit): T = {
     try {
       doWork
     } catch {
-      case e: AmazonS3Exception if e.getStatusCode == 404 || e.getStatusCode == 403 =>
+      case e: AmazonS3Exception if e.getStatusCode == 404 || e.getStatusCode == 403 => {
+        logger.warn(s"AmazonS3Exception ${e.getStatusCode} for key '$key'")
         loggingIfNotFound
         throw new S3FileDoesNotExistException
+      }
       case other: Throwable => throw other
     }
   }
 
   def getS3Object(key: String)(implicit logMarker: LogMarker): S3Object = handleNotFound(key) {
-    client.getObject(config.maybeIngestBucket.get, key)
+    getObject(config.maybeIngestBucket.get, key)
   } {
     logger.error(logMarker, s"Attempted to read $key from ingest bucket, but it does not exist.")
   }
 
-  def queueS3Object(uploader: String, filename: String, s3Meta: Map[String, String], file: File)(implicit logMarker: LogMarker) = {
+  def queueS3Object(uploader: String, filename: String, s3Meta: Map[String, String], file: File)(implicit logMarker: LogMarker, instance: Instance): Future[aws.S3Object] = {
     store(
         config.maybeIngestBucket.get,
-        s"$uploader/$filename",
+        s"${instance.id}/$uploader/$filename",
         file,
         mimeType = None, // we don't care as this is just the queue bucket
-        meta = s3Meta,
+        meta = s3Meta
       )
   }
 
-  def generatePreSignedUploadUrl(filename: String, expiration: ZonedDateTime, uploadedBy: String, mediaId: String): String = {
+  def generatePreSignedUploadUrl(filename: String, expiration: ZonedDateTime, uploadedBy: String, mediaId: String)(implicit instance: Instance): String = {
     val request = new GeneratePresignedUrlRequest(
-      config.maybeBucketForUIUploads.get, // bucket
-      s"$uploadedBy/$filename", // key
+      config.maybeBucketForUIUploads.get.bucket, // bucket
+      s"${instance.id}/$uploadedBy/$filename", // key
     )
       .withMethod(HttpMethod.PUT)
       .withExpiration(Date.from(expiration.toInstant));
@@ -52,20 +57,21 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati
     // sent by the client in manager.js
     request.putCustomRequestHeader("x-amz-meta-media-id", mediaId)
 
-    client.generatePresignedUrl(request).toString
+    generatePresignedRequest(request, config.maybeIngestBucket.get).toString
   }
 
   def moveObjectToFailedBucket(key: String)(implicit logMarker: LogMarker) = handleNotFound(key){
-    client.copyObject(config.maybeIngestBucket.get, key, config.maybeFailBucket.get, key)
+    copyObject(config.maybeIngestBucket.get, config.maybeFailBucket.get, key) // TODO Naked get - make optional
     deleteObjectFromIngestBucket(key)
   } {
     logger.warn(logMarker, s"Attempted to copy $key from ingest bucket to fail bucket, but it does not exist.")
   }
 
   def deleteObjectFromIngestBucket(key: String)(implicit logMarker: LogMarker) = handleNotFound(key) {
-    client.deleteObject(config.maybeIngestBucket.get,key)
+    deleteObject(config.maybeIngestBucket.get, key)
   } {
     logger.warn(logMarker, s"Attempted to delete $key from ingest bucket, but it does not exist.")
   }
+
 }
 
diff --git a/image-loader/app/lib/QuarantineStore.scala b/image-loader/app/lib/QuarantineStore.scala
index 4fc8581672a..0c7e5f515f3 100644
--- a/image-loader/app/lib/QuarantineStore.scala
+++ b/image-loader/app/lib/QuarantineStore.scala
@@ -3,4 +3,4 @@ package lib.storage
 import lib.ImageLoaderConfig
 import com.gu.mediaservice.lib
 
-class QuarantineStore(config: ImageLoaderConfig) extends lib.ImageQuarantineOperations(config.maybeQuarantineBucket.get, config)
+class QuarantineStore(config: ImageLoaderConfig) extends lib.ImageQuarantineOperations(config.quarantineBucket.get, config)
diff --git a/image-loader/app/lib/UploadStatusTable.scala b/image-loader/app/lib/UploadStatusTable.scala
index c545eb045a9..008f42e8139 100644
--- a/image-loader/app/lib/UploadStatusTable.scala
+++ b/image-loader/app/lib/UploadStatusTable.scala
@@ -1,51 +1,56 @@
 package lib
 
+import com.gu.mediaservice.lib.logging.GridLogging
+import com.gu.mediaservice.model.Instance
 import org.scanamo._
 import org.scanamo.syntax._
 import org.scanamo.generic.auto._
 import model.StatusType.{Prepared, Queued}
-import model.{UploadStatus, UploadStatusRecord}
+import model.{StatusType, UploadStatus, UploadStatusRecord}
 import software.amazon.awssdk.services.dynamodb.{DynamoDbAsyncClient, DynamoDbAsyncClientBuilder}
 
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
 
-class UploadStatusTable(config: ImageLoaderConfig) {
+class UploadStatusTable(config: ImageLoaderConfig) extends GridLogging {
+
+  implicit val statusTypeFormat: Typeclass[StatusType] =
+    DynamoFormat.coercedXmap[StatusType, String, IllegalArgumentException](StatusType.apply, _.name)
 
   val client = config.withAWSCredentialsV2(DynamoDbAsyncClient.builder()).build()
   val scanamo = ScanamoAsync(client)
   private val uploadStatusTable = Table[UploadStatusRecord](config.uploadStatusTable)
 
-  def getStatus(imageId: String) = {
-    scanamo.exec(uploadStatusTable.get("id" === imageId))
+  def getStatus(imageId: String)(implicit instance: Instance) = {
+    scanamo.exec(uploadStatusTable.get("instance" === instance.id and "id" === imageId))
   }
 
   def setStatus(uploadStatus: UploadStatusRecord) = {
     scanamo.exec(uploadStatusTable.put(uploadStatus))
   }
 
-  def updateStatus(imageId: String, updateRequest: UploadStatus) = {
-    val updateExpression = updateRequest.errorMessage match {
-      case Some(error) => set("status", updateRequest.status) and set("errorMessage", error)
-      case None => set("status", updateRequest.status)
+  def updateStatus(imageId: String, updateStatus: UploadStatus)(implicit instance: Instance) = {
+    val updateExpression = updateStatus.errorMessage match {
+      case Some(error) => set("status", updateStatus.status) and set("errorMessage", error)
+      case None => set("status", updateStatus.status)
     }
     val uploadStatusTableWithCondition =
-      if(updateRequest.status == Queued) // can only transition to Queued status from Prepared status
-        uploadStatusTable.when(attributeExists("id") and ("status" === Prepared.toString))
+      if(updateStatus.status == Queued) // can only transition to Queued status from Prepared status
+        uploadStatusTable.when(attributeExists("id") and attributeExists("instance") and "status" === Prepared.name)
       else
-        uploadStatusTable.when(attributeExists("id"))
+        uploadStatusTable.when(attributeExists("id") and attributeExists("instance"))
 
     scanamo.exec(
       uploadStatusTableWithCondition
         .update(
-          key = "id" === imageId,
+          "id" === imageId and "instance" === instance.id,
           update = updateExpression
         )
     )
   }
 
-  def queryByUser(user: String): Future[List[UploadStatusRecord]] = {
-    scanamo.exec(uploadStatusTable.scan()).map {
+  def queryByUser(user: String)(implicit instance: Instance): Future[List[UploadStatusRecord]] = {
+    scanamo.exec(uploadStatusTable.query("instance" === instance.id)).map {
       case Nil => List.empty[UploadStatusRecord]
       case recordsAndErrors => {
         recordsAndErrors
diff --git a/image-loader/app/lib/imaging/FileMetadataReader.scala b/image-loader/app/lib/imaging/FileMetadataReader.scala
index f81b9ce3a7f..ca2c78d712e 100644
--- a/image-loader/app/lib/imaging/FileMetadataReader.scala
+++ b/image-loader/app/lib/imaging/FileMetadataReader.scala
@@ -13,7 +13,7 @@ import com.drew.metadata.xmp.XmpDirectory
 import com.drew.metadata.{Directory, Metadata}
 import com.gu.mediaservice.lib.{ImageWrapper, StorableImage}
 import com.gu.mediaservice.lib.imaging.im4jwrapper.ImageMagick._
-import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker}
+import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch, addLogMarkers}
 import com.gu.mediaservice.lib.metadata.ImageMetadataConverter
 import com.gu.mediaservice.model._
 import model.upload.UploadRequest
@@ -53,7 +53,7 @@ object FileMetadataReader extends GridLogging {
   private implicit val ctx: ExecutionContext =
     ExecutionContext.fromExecutor(Executors.newCachedThreadPool)
 
-  def fromIPTCHeaders(image: File, imageId:String): Future[FileMetadata] =
+  def fromIPTCHeaders(image: File, imageId:String)(implicit logMarker: LogMarker): Future[FileMetadata] =
     for {
       metadata <- readMetadata(image)
     }
@@ -125,7 +125,7 @@ object FileMetadataReader extends GridLogging {
   val redactionReplacementValue = s"REDACTED (value longer than $redactionThreshold characters, please refer to the metadata stored in the file itself)"
   private def redactLongFieldValues(imageId: String, metadataType: String, exceptions: List[String] = Nil)(props: Map[String, String]) = props.map {
     case (fieldName, value) if value.length > redactionThreshold && !exceptions.exists(fieldName.contains) =>
-      logger.warn(s"Redacting '$fieldName' $metadataType field for image $imageId, as it's problematically long (longer than $redactionThreshold characters")
+      logger.debug(s"Redacting '$fieldName' $metadataType field for image $imageId, as it's problematically long (longer than $redactionThreshold characters")
       fieldName -> redactionReplacementValue
     case keyValuePair => keyValuePair
   }
@@ -183,67 +183,17 @@ object FileMetadataReader extends GridLogging {
 
   private def dateToUTCString(date: DateTime): String = ISODateTimeFormat.dateTime.print(date.withZone(DateTimeZone.UTC))
 
-
-  def orientation(image: File): Future[Option[OrientationMetadata]] = {
-    for {
-      metadata <- readMetadata(image)
-    } yield {
-
-      for {
-        exifDirectory <- Option(metadata.getFirstDirectoryOfType(classOf[ExifIFD0Directory]))
-        exifOrientation <- Option(exifDirectory.getInteger(ExifDirectoryBase.TAG_ORIENTATION))
-        orientation = OrientationMetadata(exifOrientation = Some(exifOrientation))
-        orientationWhichTransformsImage <- Seq(orientation).find(_.transformsImage())
-      } yield {
-        orientationWhichTransformsImage
-      }
-    }
-  }
-
-  def dimensions(image: File, mimeType: Option[MimeType]): Future[Option[Dimensions]] =
-    for {
-      metadata <- readMetadata(image)
-    }
-    yield {
-
-      mimeType match {
-
-        case Some(Jpeg) => for {
-          jpegDir <- Option(metadata.getFirstDirectoryOfType(classOf[JpegDirectory]))
-
-        } yield Dimensions(jpegDir.getImageWidth, jpegDir.getImageHeight)
-
-        case Some(Png) => for {
-          pngDir <- Option(metadata.getFirstDirectoryOfType(classOf[PngDirectory]))
-
-        } yield {
-          val width = pngDir.getInt(PngDirectory.TAG_IMAGE_WIDTH)
-          val height = pngDir.getInt(PngDirectory.TAG_IMAGE_HEIGHT)
-          Dimensions(width, height)
-        }
-
-        case Some(Tiff) => for {
-          exifDir <- Option(metadata.getFirstDirectoryOfType(classOf[ExifIFD0Directory]))
-
-        } yield {
-          val width = exifDir.getInt(ExifDirectoryBase.TAG_IMAGE_WIDTH)
-          val height = exifDir.getInt(ExifDirectoryBase.TAG_IMAGE_HEIGHT)
-          Dimensions(width, height)
-        }
-
-        case _ => None
-
-      }
-    }
-
   def getColorModelInformation(image: File, metadata: Metadata, mimeType: MimeType)(implicit logMarker: LogMarker): Future[Map[String, String]] = {
-
+    val stopWatch = Stopwatch.start
     val source = addImage(image)
 
     val formatter = format(source)("%r")
 
-    runIdentifyCmd(formatter, useImageMagick = false).map{ imageType => getColourInformation(metadata, imageType.headOption, mimeType) }
-      .recover { case _ => getColourInformation(metadata, None, mimeType) }
+    runIdentifyCmd(formatter, useImageMagick = false).map { imageType => getColourInformation(metadata, imageType.headOption, mimeType) }
+      .recover { case _ => getColourInformation(metadata, None, mimeType) }.map { result =>
+      logger.info(addLogMarkers(stopWatch.elapsed), "Finished getColorModelInformation")
+      result
+    }
   }
 
   // bits per sample might be a useful value, eg. "1", "8"; or it might be annoying like "1 bits/component/pixel", "8 8 8 bits/component/pixel"
@@ -287,14 +237,26 @@ object FileMetadataReader extends GridLogging {
           "photometricInterpretation" -> photometricInterpretation,
           "bitsPerSample" -> getFromExifDirectory(ExifDirectoryBase.TAG_BITS_PER_SAMPLE).flatMap(extractBitsPerSample)
         ).flattenOptions
+      case Heif =>
+        Map (
+          "hasAlpha" -> hasAlpha,
+          "colorType" -> maybeImageType,
+          "photometricInterpretation" -> photometricInterpretation,
+      ).flattenOptions
     }
   }
 
   private def nonEmptyTrimmed(nullableStr: String): Option[String] =
     Option(nullableStr) map (_.trim) filter (_.nonEmpty)
 
-  private def readMetadata(file: File): Future[Metadata] = Future {
-    ImageMetadataReader.readMetadata(file)
+  private def readMetadata(file: File)(implicit logMarker: LogMarker): Future[Metadata] = {
+    val stopwatch = Stopwatch.start
+    Future {
+      ImageMetadataReader.readMetadata(file)
+    }.map { result =>
+      logger.info(addLogMarkers(stopwatch.elapsed),"Finished readMetadata")
+      result
+    }
   }
 
   // Helper to flatten maps of options
diff --git a/image-loader/app/model/Projector.scala b/image-loader/app/model/Projector.scala
index 20b222382a1..833693621b8 100644
--- a/image-loader/app/model/Projector.scala
+++ b/image-loader/app/model/Projector.scala
@@ -1,44 +1,43 @@
 package model
 
-import java.io.{File, FileOutputStream}
-import com.amazonaws.services.s3.AmazonS3
-import com.gu.mediaservice.{GridClient, ImageDataMerger}
-import com.gu.mediaservice.lib.auth.Authentication
-import com.amazonaws.services.s3.model.{GetObjectRequest, ObjectMetadata, S3Object => AwsS3Object}
+import _root_.play.api.libs.ws.WSRequest
+import com.amazonaws.services.s3.model.{ObjectMetadata, S3Object => AwsS3Object}
 import com.gu.mediaservice.lib.ImageIngestOperations.{fileKeyFromId, optimisedPngKeyFromId}
-import com.gu.mediaservice.lib.{ImageIngestOperations, ImageStorageProps, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage}
-import com.gu.mediaservice.lib.aws.{Embedder, EmbedderMessage, S3Ops}
+import com.gu.mediaservice.lib._
+import com.gu.mediaservice.lib.auth.Authentication
+import com.gu.mediaservice.lib.aws.{Embedder, EmbedderMessage, S3, S3Bucket, S3Object}
 import com.gu.mediaservice.lib.cleanup.ImageProcessor
+import com.gu.mediaservice.lib.config.InstanceForRequest
 import com.gu.mediaservice.lib.imaging.ImageOperations
 import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch}
 import com.gu.mediaservice.lib.net.URI
-import com.gu.mediaservice.model.{Image, MimeType, UploadInfo}
+import com.gu.mediaservice.model.{Image, Instance, MimeType, Png, UploadInfo}
+import com.gu.mediaservice.{GridClient, ImageDataMerger}
 import lib.imaging.{MimeTypeDetection, NoSuchImageExistsInS3}
 import lib.{DigestedFile, ImageLoaderConfig}
-import model.upload.UploadRequest
+import model.upload.{OptimiseOps, UploadRequest}
 import org.apache.commons.io.IOUtils
 import org.joda.time.{DateTime, DateTimeZone}
-import play.api.libs.ws.WSRequest
-import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse
 
-import java.nio.file.Path
-import scala.jdk.CollectionConverters._
+import java.io.{File, FileOutputStream}
 import scala.concurrent.duration.Duration
 import scala.concurrent.{Await, ExecutionContext, Future}
+import scala.jdk.CollectionConverters._
 
 object Projector {
 
   import Uploader.toImageUploadOpsCfg
 
-  def apply(config: ImageLoaderConfig, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, maybeEmbedder: Option[Embedder])(implicit ec: ExecutionContext): Projector
-  = new Projector(toImageUploadOpsCfg(config), S3Ops.buildS3Client(config), imageOps, processor, auth, maybeEmbedder)
+  def apply(config: ImageLoaderConfig, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, maybeEmbedder: Option[Embedder], s3: S3, optimiseOps: OptimiseOps)(implicit ec: ExecutionContext): Projector
+  = new Projector(toImageUploadOpsCfg(config), s3, imageOps, processor, auth, maybeEmbedder, optimiseOps)
 }
 
 case class S3FileExtractedMetadata(
   uploadedBy: String,
   uploadTime: DateTime,
   uploadFileName: Option[String],
-  identifiers: Map[String, String]
+  identifiers: Map[String, String],
+  isFeedUpload: Option[Boolean],
 )
 
 object S3FileExtractedMetadata {
@@ -68,6 +67,7 @@ object S3FileExtractedMetadata {
     }.map{ case (key, value) =>
       key.stripPrefix(ImageStorageProps.identifierMetadataKeyPrefix) -> value
     }
+    val isFeedUpload = fileUserMetadata.get(ImageStorageProps.isFeedUploadMetadataKey).map(_.toBoolean)
 
     val uploadFileName = fileUserMetadata.get(ImageStorageProps.filenameMetadataKey)
 
@@ -76,27 +76,29 @@ object S3FileExtractedMetadata {
       uploadTime = uploadTime,
       uploadFileName = uploadFileName,
       identifiers = identifiers,
+      isFeedUpload = isFeedUpload,
     )
   }
 }
 
 class Projector(config: ImageUploadOpsCfg,
-                s3: AmazonS3,
+                s3: S3,
                 imageOps: ImageOperations,
                 processor: ImageProcessor,
                 auth: Authentication,
-                maybeEmbedder: Option[Embedder]) extends GridLogging {
+                maybeEmbedder: Option[Embedder],
+                optimiseOps: OptimiseOps) extends GridLogging with InstanceForRequest {
 
-  private val imageUploadProjectionOps = new ImageUploadProjectionOps(config, imageOps, processor, s3, maybeEmbedder)
+  private val imageUploadProjectionOps = new ImageUploadProjectionOps(config, imageOps, processor, s3, maybeEmbedder, optimiseOps)
 
   def projectS3ImageById(imageId: String, tempFile: File, gridClient: GridClient, onBehalfOfFn: WSRequest => WSRequest)
-                        (implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[Image]] = {
+                        (implicit ec: ExecutionContext, logMarker: LogMarker, instance: Instance): Future[Option[Image]] = {
     Future {
       import ImageIngestOperations.fileKeyFromId
       val s3Key = fileKeyFromId(imageId)
 
       if (!s3.doesObjectExist(config.originalFileBucket, s3Key))
-        throw new NoSuchImageExistsInS3(config.originalFileBucket, s3Key)
+        throw new NoSuchImageExistsInS3(config.originalFileBucket.bucket, s3Key)
 
       val s3Source = Stopwatch(s"object exists, getting s3 object at s3://${config.originalFileBucket}/$s3Key to perform Image projection"){
         s3.getObject(config.originalFileBucket, s3Key)
@@ -130,11 +132,11 @@ class Projector(config: ImageUploadOpsCfg,
                    extractedS3Meta: S3FileExtractedMetadata,
                    gridClient: GridClient,
                    onBehalfOfFn: WSRequest => WSRequest)
-                  (implicit ec: ExecutionContext, logMarker: LogMarker): Future[Image] = {
+                  (implicit ec: ExecutionContext, logMarker: LogMarker, instance: Instance): Future[Image] = {
     val DigestedFile(tempFile_, id_) = srcFileDigest
 
     val identifiers_ = extractedS3Meta.identifiers
-    val uploadInfo_ = UploadInfo(filename = extractedS3Meta.uploadFileName)
+    val uploadInfo_ = UploadInfo(filename = extractedS3Meta.uploadFileName, isFeedUpload = extractedS3Meta.isFeedUpload)
 
     MimeTypeDetection.guessMimeType(tempFile_) match {
       case util.Left(unsupported) => Future.failed(unsupported)
@@ -146,7 +148,8 @@ class Projector(config: ImageUploadOpsCfg,
           uploadTime = extractedS3Meta.uploadTime,
           uploadedBy = extractedS3Meta.uploadedBy,
           identifiers = identifiers_,
-          uploadInfo = uploadInfo_
+          uploadInfo = uploadInfo_,
+          instance = instance, // TODO careful with this one!
         )
 
         imageUploadProjectionOps.projectImageFromUploadRequest(uploadRequest) flatMap (
@@ -159,26 +162,29 @@ class Projector(config: ImageUploadOpsCfg,
 class ImageUploadProjectionOps(config: ImageUploadOpsCfg,
                                imageOps: ImageOperations,
                                processor: ImageProcessor,
-                               s3: AmazonS3,
+                               s3: S3,
                                maybeEmbedder: Option[Embedder],
+                               optimiseOps: OptimiseOps
 ) extends GridLogging {
 
-  import Uploader.{fromUploadRequestShared, toMetaMap}
+  import Uploader.fromUploadRequestShared
 
 
   def projectImageFromUploadRequest(uploadRequest: UploadRequest)
-                                   (implicit ec: ExecutionContext, logMarker: LogMarker): Future[Image] = {
-    val dependenciesWithProjectionsOnly = ImageUploadOpsDependencies(
+                                   (implicit ec: ExecutionContext, logMarker: LogMarker, instance: Instance): Future[Image] = {
+    val dependenciesWithProjectionsOnly: ImageUploadOpsDependencies = ImageUploadOpsDependencies(
       config,
       imageOps,
       projectOriginalFileAsS3Model,
       projectThumbnailFileAsS3Model,
       projectOptimisedPNGFileAsS3Model,
+      projectEmbeddingSourceAsS3Model,
       tryFetchThumbFile = fetchThumbFile,
-      tryFetchOptimisedFile = fetchOptimisedFile
+      tryFetchOptimisedFile = fetchOptimisedFile,
+      maybeEmbedder = maybeEmbedder
     )
 
-    fromUploadRequestShared(uploadRequest, dependenciesWithProjectionsOnly, processor)
+    fromUploadRequestShared(uploadRequest, dependenciesWithProjectionsOnly, processor, optimiseOps).map(_._1)
   }
 
   private def projectOriginalFileAsS3Model(storableOriginalImage: StorableOriginalImage) =
@@ -190,24 +196,30 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg,
   private def projectOptimisedPNGFileAsS3Model(storableOptimisedImage: StorableOptimisedImage) =
     Future.successful(storableOptimisedImage.toProjectedS3Object(config.originalFileBucket))
 
-  private def fetchThumbFile(
-    imageId: String, outFile: File
-  )(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[(File, MimeType)]] = {
-    val key = fileKeyFromId(imageId)
+  private def projectEmbeddingSourceAsS3Model(storableEmbeddingSourceImage: Option[StorableEmbeddingSourceImage]): Future[Option[S3Object]] = {
+    Future.successful {
+      storableEmbeddingSourceImage.map { storableEmbeddingSourceImage =>
+        storableEmbeddingSourceImage.toProjectedS3Object(config.embedSourceBucket)
+      }
+    }
+  }
 
+  private def fetchThumbFile(
+    imageId: String, outFile: File, instance: Instance)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[(File, MimeType)]] = {
+    val key = fileKeyFromId(imageId)(instance)
     fetchFile(config.thumbBucket, key, outFile)
   }
 
   private def fetchOptimisedFile(
-    imageId: String, outFile: File
+    imageId: String, outFile: File, instance: Instance
   )(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[(File, MimeType)]] = {
-    val key = optimisedPngKeyFromId(imageId)
+    val key = optimisedPngKeyFromId(imageId)(instance)
 
     fetchFile(config.originalFileBucket, key, outFile)
   }
 
   private def fetchFile(
-    bucket: String, key: String, outFile: File
+                         bucket: S3Bucket, key: String, outFile: File
   )(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[(File, MimeType)]] = {
     logger.info(logMarker, s"Trying fetch existing image from S3 bucket - $bucket at key $key")
     val doesFileExist = Future { s3.doesObjectExist(bucket, key) } recover { case _ => false }
@@ -216,7 +228,7 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg,
         logger.warn(logMarker, s"image did not exist in bucket $bucket at key $key")
         Future.successful(None) // falls back to creating from original file
       case true =>
-        val obj = s3.getObject(new GetObjectRequest(bucket, key))
+        val obj = s3.getObject(bucket, key)
         val fos = new FileOutputStream(outFile)
         try {
           IOUtils.copy(obj.getObjectContent, fos)
diff --git a/image-loader/app/model/QuarantineUploader.scala b/image-loader/app/model/QuarantineUploader.scala
index af75e44a73c..b18079981ee 100644
--- a/image-loader/app/model/QuarantineUploader.scala
+++ b/image-loader/app/model/QuarantineUploader.scala
@@ -18,6 +18,8 @@ import play.api.Logger
 import play.api.libs.json.{JsObject, Json}
 import com.gu.mediaservice.lib.formatting._
 import model.upload.UploadRequest
+import play.api.mvc.{AnyContent, Request}
+
 import java.net.URLEncoder
 import java.nio.charset.StandardCharsets
 import scala.concurrent.{ExecutionContext, Future}
@@ -28,6 +30,7 @@ class QuarantineUploader(val store: QuarantineStore,
 
   private def storeQuarantineFile(uploadRequest: UploadRequest)
                          (implicit logMarker: LogMarker) = {
+    implicit val instance: Instance = uploadRequest.instance
     val meta = Uploader.toMetaMap(uploadRequest)
     store.storeQuarantineImage(
       uploadRequest.imageId,
@@ -37,7 +40,7 @@ class QuarantineUploader(val store: QuarantineStore,
     )
   }
 
-  def quarantineFile(uploadRequest: UploadRequest)(
+  def quarantineFile(uploadRequest: UploadRequest)(instance: Instance) (
     implicit ec: ExecutionContext,
     logMarker: LogMarker): Future[JsObject] = {
 
@@ -45,7 +48,7 @@ class QuarantineUploader(val store: QuarantineStore,
 
     for {
       _ <- storeQuarantineFile(uploadRequest)
-      uri = s"${config.rootUri}/uploadStatus/${uploadRequest.imageId}"
+      uri = s"${config.rootUri(instance)}/uploadStatus/${uploadRequest.imageId}"
     } yield {
       Json.obj("uri" -> uri)
     }
diff --git a/image-loader/app/model/S3IngestObject.scala b/image-loader/app/model/S3IngestObject.scala
index 7621ac2067e..53f6047a216 100644
--- a/image-loader/app/model/S3IngestObject.scala
+++ b/image-loader/app/model/S3IngestObject.scala
@@ -14,7 +14,8 @@ case class S3IngestObject (
   uploadTime: java.util.Date,
   contentLength: Long,
   getInputStream: () => java.io.InputStream,
-  identifiers: Map[String, String] = Map.empty
+  identifiers: Map[String, String] = Map.empty,
+  isFeedUpload: Boolean,
 )
 
 object S3IngestObject {
@@ -26,19 +27,33 @@ object S3IngestObject {
     val s3Object = store.getS3Object(key)
     val metadata = s3Object.getObjectMetadata
 
+    val mediaIdFromUiUpload = metadata.getUserMetadata.asScala.get("media-id")
+    val isFeedUpload = mediaIdFromUiUpload.isEmpty  // TODO Not concise
+
     S3IngestObject(
       key,
-      uploadedBy = keyParts.head,
+      uploadedBy = uploadedFromPath(keyParts),
       filename = keyParts.last,
-      maybeMediaIdFromUiUpload = metadata.getUserMetadata.asScala.get("media-id"), // set by the client in upload in manager.js
+      maybeMediaIdFromUiUpload = mediaIdFromUiUpload, // set by the client in upload in manager.js
       uploadTime = metadata.getLastModified,
       contentLength = metadata.getContentLength,
       getInputStream = () => s3Object.getObjectContent,
       identifiers = metadata.getUserMetadata.asScala.collect{
         case (key, value) if key.startsWith(ImageStorageProps.identifierMetadataKeyPrefix) =>
           key.stripPrefix(ImageStorageProps.identifierMetadataKeyPrefix) -> value
-      }.toMap
+      }.toMap,
+      isFeedUpload = isFeedUpload
     )
   }
+
+  def uploadedFromPath(keyParts: Array[String]): String = {
+    val indexOfFeedProviderName = if (keyParts.contains("feeds")) {
+      keyParts.indexOf("feeds") + 1
+    } else {
+      0
+    }
+
+    keyParts(Seq(indexOfFeedProviderName, 0).max)
+  }
 }
 
diff --git a/image-loader/app/model/UploadStatus.scala b/image-loader/app/model/UploadStatus.scala
index 044acacf713..af28678e23f 100644
--- a/image-loader/app/model/UploadStatus.scala
+++ b/image-loader/app/model/UploadStatus.scala
@@ -10,7 +10,8 @@ case class UploadStatusRecord(
                                identifiers: Option[String],
                                status: StatusType,
                                errorMessage: Option[String],
-                               expires: Long
+                               expires: Long,
+                               instance: String
                              )
 
 object UploadStatusRecord {
diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala
index aaa811e52c6..31729e4fac1 100644
--- a/image-loader/app/model/Uploader.scala
+++ b/image-loader/app/model/Uploader.scala
@@ -1,38 +1,34 @@
 package model
 
-import com.gu.mediaservice.{GridClient, ImageDataMerger}
+import _root_.play.api.libs.json.Json
+import _root_.play.api.libs.ws.WSRequest
 import com.gu.mediaservice.lib.Files.createTempFile
-import com.gu.mediaservice.lib.ImageIngestOperations.fileKeyFromId
-
-import java.io.File
-import java.nio.file.{Files, Path}
+import com.gu.mediaservice.lib._
 import com.gu.mediaservice.lib.argo.ArgoHelpers
 import com.gu.mediaservice.lib.auth.Authentication
-import com.gu.mediaservice.lib.{BrowserViewableImage, ImageStorageProps, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage}
-import com.gu.mediaservice.lib.aws.{Embedder, EmbedderMessage, S3Object, S3Vectors, UpdateMessage}
+import com.gu.mediaservice.lib.aws._
 import com.gu.mediaservice.lib.cleanup.ImageProcessor
 import com.gu.mediaservice.lib.formatting._
 import com.gu.mediaservice.lib.imaging.ImageOperations
 import com.gu.mediaservice.lib.imaging.ImageOperations.{optimisedMimeType, thumbMimeType}
 import com.gu.mediaservice.lib.logging._
-import com.gu.mediaservice.lib.metadata.{FileMetadataHelper, ImageMetadataConverter}
+import com.gu.mediaservice.lib.metadata.ImageMetadataConverter
 import com.gu.mediaservice.lib.net.URI
 import com.gu.mediaservice.model._
 import com.gu.mediaservice.syntax.MessageSubjects
-import lib.{DigestedFile, ImageLoaderConfig, Notifications}
+import com.gu.mediaservice.{GridClient, ImageDataMerger}
 import lib.imaging.{FileMetadataReader, MimeTypeDetection}
 import lib.storage.ImageLoaderStore
+import lib.{DigestedFile, ImageLoaderConfig, Notifications}
 import model.Uploader.{fromUploadRequestShared, toImageUploadOpsCfg}
-import model.upload.{OptimiseOps, OptimiseWithPngQuant, UploadRequest}
+import model.upload.{OptimiseOps, UploadRequest}
 import org.joda.time.DateTime
-import play.api.libs.json.{JsObject, Json}
-import play.api.libs.ws.WSRequest
-import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse
 
-import scala.collection.compat._
+import java.io.File
+import java.nio.file.Files
 import scala.concurrent.{ExecutionContext, Future}
 
-case class ImageUpload(uploadRequest: UploadRequest, image: Image)
+ case class ImageUpload(uploadRequest: UploadRequest, image: Image, embeddingSource: Option[S3Object])
 
 case object ImageUpload {
 
@@ -68,9 +64,9 @@ case class ImageUploadOpsCfg(
   tempDir: File,
   thumbWidth: Int,
   thumbQuality: Double,
-  transcodedMimeTypes: List[MimeType],
-  originalFileBucket: String,
-  thumbBucket: String
+  originalFileBucket: S3Bucket,
+  thumbBucket: S3Bucket,
+  embedSourceBucket: S3Bucket
 )
 
 case class ImageUploadOpsDependencies(
@@ -79,8 +75,10 @@ case class ImageUploadOpsDependencies(
   storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object],
   storeOrProjectThumbFile: StorableThumbImage => Future[S3Object],
   storeOrProjectOptimisedImage: StorableOptimisedImage => Future[S3Object],
-  tryFetchThumbFile: (String, File) => Future[Option[(File, MimeType)]] = (_, _) => Future.successful(None),
-  tryFetchOptimisedFile: (String, File) => Future[Option[(File, MimeType)]] = (_, _) => Future.successful(None),
+  storeEmbeddingSource: Option[StorableEmbeddingSourceImage] => Future[Option[S3Object]],
+  tryFetchThumbFile: (String, File, Instance) => Future[Option[(File, MimeType)]] = (_, _, _) => Future.successful(None),
+  tryFetchOptimisedFile: (String, File, Instance) => Future[Option[(File, MimeType)]] = (_, _, _) => Future.successful(None),
+  maybeEmbedder: Option[Embedder]
 )
 
 
@@ -95,87 +93,94 @@ object Uploader extends GridLogging {
       config.tempDir,
       config.thumbWidth,
       config.thumbQuality,
-      config.transcodedMimeTypes,
       config.imageBucket,
-      config.thumbnailBucket
+      config.thumbnailBucket,
+      config.embeddingSourceBucket
     )
   }
 
-  def fromUploadRequestShared(uploadRequest: UploadRequest, deps: ImageUploadOpsDependencies, processor: ImageProcessor)
-                             (implicit ec: ExecutionContext, logMarker: LogMarker): Future[Image] = {
+  def fromUploadRequestShared(uploadRequest: UploadRequest, deps: ImageUploadOpsDependencies, processor: ImageProcessor, optimiseOps: OptimiseOps)
+                             (implicit ec: ExecutionContext, logMarker: LogMarker): Future[(Image, Option[S3Object])] = {
 
     import deps._
 
     logger.info(logMarker, "Starting image ops")
-
-    val fileMetadataFuture = toFileMetadata(uploadRequest.tempFile, uploadRequest.imageId, uploadRequest.mimeType)
-
     logger.info(logMarker, "Have read file headers")
 
-    fileMetadataFuture.flatMap(fileMetadata => {
-      uploadAndStoreImage(
-        storeOrProjectOriginalFile,
-        storeOrProjectThumbFile,
-        storeOrProjectOptimisedImage,
-        OptimiseWithPngQuant,
-        uploadRequest,
-        deps,
-        fileMetadata,
-        processor)(ec, addLogMarkers(fileMetadata.toLogMarker))
-    })
+    uploadAndStoreImage(
+      storeOrProjectOriginalFile,
+      storeOrProjectThumbFile,
+      storeOrProjectOptimisedImage,
+      storeEmbeddingSource,
+      uploadRequest,
+      deps,
+      processor,
+      optimiseOps)
   }
 
+
   private[model] def uploadAndStoreImage(storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object],
                                          storeOrProjectThumbFile: StorableThumbImage => Future[S3Object],
                                          storeOrProjectOptimisedFile: StorableOptimisedImage => Future[S3Object],
-                                         optimiseOps: OptimiseOps,
+                                         storeEmbeddingSource: Option[StorableEmbeddingSourceImage] => Future[Option[S3Object]],
                                          uploadRequest: UploadRequest,
                                          deps: ImageUploadOpsDependencies,
-                                         fileMetadata: FileMetadata,
-                                         processor: ImageProcessor)
-                  (implicit ec: ExecutionContext, logMarker: LogMarker) = {
+                                         processor: ImageProcessor,
+                                         optimiseOps: OptimiseOps)
+                  (implicit ec: ExecutionContext, logMarker: LogMarker): Future[(Image, Option[S3Object])] = {
     val originalMimeType = uploadRequest.mimeType
       .orElse(MimeTypeDetection.guessMimeType(uploadRequest.tempFile).toOption)
     match {
       case Some(a) => a
       case None => throw new Exception("File of unknown and undetectable mime type")
     }
+    logger.info("Original Mime type: " + originalMimeType)
 
     val tempDirForRequest: File = Files.createTempDirectory(deps.config.tempDir.toPath, "upload").toFile
 
-    val colourModelFuture = ImageOperations.identifyColourModel(uploadRequest.tempFile, originalMimeType)
-    val sourceDimensionsFuture = FileMetadataReader.dimensions(uploadRequest.tempFile, Some(originalMimeType))
-    val sourceOrientationMetadataFuture = FileMetadataReader.orientation(uploadRequest.tempFile)
-
     val storableOriginalImage = StorableOriginalImage(
       uploadRequest.imageId,
       uploadRequest.tempFile,
       originalMimeType,
       uploadRequest.uploadTime,
-      toMetaMap(uploadRequest)
+      toMetaMap(uploadRequest),
+      uploadRequest.instance
     )
     val sourceStoreFuture = storeOrProjectOriginalFile(storableOriginalImage)
-    val eventualBrowserViewableImage = createBrowserViewableFileFuture(uploadRequest, tempDirForRequest, deps)
+    val eventualBrowserViewableImage = createBrowserViewableFileFuture(uploadRequest)
 
     val eventualImage = for {
       browserViewableImage <- eventualBrowserViewableImage
       s3Source <- sourceStoreFuture
       mergedUploadRequest = patchUploadRequestWithS3Metadata(uploadRequest, s3Source)
-      optimisedFileMetadata <- FileMetadataReader.fromIPTCHeadersWithColorInfo(browserViewableImage)
-      sourceDimensions <- sourceDimensionsFuture
-      sourceOrientationMetadata <- sourceOrientationMetadataFuture
-      thumbViewableImage <- createThumbFuture(optimisedFileMetadata, colourModelFuture, browserViewableImage, deps, tempDirForRequest, orientationMetadata = sourceOrientationMetadata)
+      imageInformation <- ImageOperations.getImageInformation(uploadRequest.tempFile)
+      sourceDimensions = imageInformation._1
+      sourceOrientationMetadata = imageInformation._2
+      colourModel = imageInformation._3
+      colourModelInformation = imageInformation._4
+      fileMetadata <- toFileMetadata(uploadRequest.tempFile, uploadRequest.imageId, uploadRequest.mimeType)
+      thumbViewableImageAndDimensions <- createThumbFuture(browserViewableImage, deps, tempDirForRequest, uploadRequest.instance, orientationMetadata = sourceOrientationMetadata)
+      thumbViewableImage = thumbViewableImageAndDimensions._1
+      maybeThumbDimensions = thumbViewableImageAndDimensions._2
+      thumbDimensions <- {
+        maybeThumbDimensions.map { dimensions =>
+          Future.successful(Some(dimensions))
+        }.getOrElse {
+          ImageOperations.getImageInformation(thumbViewableImage.file).map(_._1)
+        }
+      }
       s3Thumb <- storeOrProjectThumbFile(thumbViewableImage)
       maybeStorableOptimisedImage <- getStorableOptimisedImage(
-        tempDirForRequest, optimiseOps, browserViewableImage, optimisedFileMetadata, deps.tryFetchOptimisedFile)
+      tempDirForRequest, browserViewableImage, deps.tryFetchOptimisedFile, optimiseOps, uploadRequest.instance)
       s3PngOption <- maybeStorableOptimisedImage match {
         case Some(storableOptimisedImage) => storeOrProjectOptimisedFile(storableOptimisedImage).map(a=>Some(a))
         case None => Future.successful(None)
       }
-      thumbDimensions <- FileMetadataReader.dimensions(thumbViewableImage.file, Some(thumbViewableImage.mimeType))
-      colourModel <- colourModelFuture
+      embeddingSource <- createEmbeddingsSource(browserViewableImage, sourceOrientationMetadata, deps)
+      storedEmbeddingSource <- storeEmbeddingSource(embeddingSource)
+
     } yield {
-      val fullFileMetadata = fileMetadata.copy(colourModel = colourModel)
+      val fullFileMetadata = fileMetadata.copy(colourModel = colourModel).copy(colourModelInformation = colourModelInformation)
       val metadata = ImageMetadataConverter.fromFileMetadata(fullFileMetadata, s3Source.metadata.objectMetadata.lastModified)
 
       val sourceAsset = Asset.fromS3Object(s3Source, sourceDimensions, sourceOrientationMetadata)
@@ -192,12 +197,14 @@ object Uploader extends GridLogging {
       )
       val processedImage = processor(baseImage)
 
-      logger.info(logMarker, s"Ending image ops")
+      logger.info(addLogMarkers(fileMetadata.toLogMarker), s"Ending image ops")
       // FIXME: dirty hack to sync the originalUsageRights and originalMetadata as well
-      processedImage.copy(
+      val image = processedImage.copy(
         originalMetadata = processedImage.metadata,
         originalUsageRights = processedImage.usageRights
       )
+
+      (image, storedEmbeddingSource)
     }
     eventualImage.onComplete{ _ =>
       tempDirForRequest.listFiles().map(f => f.delete())
@@ -208,15 +215,15 @@ object Uploader extends GridLogging {
 
   private def getStorableOptimisedImage(
                                          tempDir: File,
-                                         optimiseOps: OptimiseOps,
                                          browserViewableImage: BrowserViewableImage,
-                                         optimisedFileMetadata: FileMetadata,
-                                         tryFetchOptimisedFile: (String, File) => Future[Option[(File, MimeType)]]
+                                         tryFetchOptimisedFile: (String, File, Instance) => Future[Option[(File, MimeType)]],
+                                         optimiseOps: OptimiseOps,
+                                         instance: Instance
   )(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[StorableOptimisedImage]] = {
-    if (optimiseOps.shouldOptimise(Some(browserViewableImage.mimeType), optimisedFileMetadata)) {
+    if (optimiseOps.shouldOptimise(Some(browserViewableImage.mimeType))) {
       for {
         tempFile <- createTempFile("optimisedpng-", optimisedMimeType.fileExtension, tempDir)
-        maybeDownloadedOptimisedFile <- tryFetchOptimisedFile(browserViewableImage.id, tempFile)
+        maybeDownloadedOptimisedFile <- tryFetchOptimisedFile(browserViewableImage.id, tempFile, instance)
         (optimisedFile, optimisedMimeType) <- {
           maybeDownloadedOptimisedFile match {
             case Some(optData) => Future.successful(optData)
@@ -238,41 +245,41 @@ object Uploader extends GridLogging {
   def toMetaMap(uploadRequest: UploadRequest): Map[String, String] = {
     val baseMeta = Map(
       ImageStorageProps.uploadedByMetadataKey -> uploadRequest.uploadedBy,
-      ImageStorageProps.uploadTimeMetadataKey -> printDateTime(uploadRequest.uploadTime)
+      ImageStorageProps.uploadTimeMetadataKey -> printDateTime(uploadRequest.uploadTime),
     ) ++
       uploadRequest.identifiersMeta ++
-      uploadRequest.uploadInfo.filename.map(ImageStorageProps.filenameMetadataKey -> _)
+      uploadRequest.uploadInfo.filename.map(ImageStorageProps.filenameMetadataKey -> _) ++
+      uploadRequest.uploadInfo.isFeedUpload.map(ImageStorageProps.isFeedUploadMetadataKey -> _.toString)
 
     baseMeta.view.mapValues(URI.encode).toMap
   }
 
-  private def toFileMetadata(f: File, imageId: String, mimeType: Option[MimeType])(implicit logMarker: LogMarker): Future[FileMetadata] = {
-    mimeType match {
-      case Some(Png | Tiff | Jpeg) => FileMetadataReader.fromIPTCHeadersWithColorInfo(f, imageId, mimeType.get)
+  private def toFileMetadata(f: File, imageId: String, mimeType: Option[MimeType])(implicit ec: ExecutionContext, logMarker: LogMarker): Future[FileMetadata] = {
+    val stopwatch = Stopwatch.start
+    (mimeType match {
+      //case Some(Png | Tiff | Jpeg) => FileMetadataReader.fromIPTCHeadersWithColorInfo(f, imageId, mimeType.get)
       case _ => FileMetadataReader.fromIPTCHeaders(f, imageId)
+    }).map { result =>
+      logger.info(addLogMarkers(stopwatch.elapsed), "Finished toFileMetadata")
+      result
     }
   }
 
-  private def createThumbFuture(fileMetadata: FileMetadata,
-                                colourModelFuture: Future[Option[String]],
-                                browserViewableImage: BrowserViewableImage,
+  private def createThumbFuture(browserViewableImage: BrowserViewableImage,
                                 deps: ImageUploadOpsDependencies,
                                 tempDir: File,
-                                orientationMetadata: Option[OrientationMetadata],
-  )(implicit ec: ExecutionContext, logMarker: LogMarker) = {
+                                instance: Instance,
+                                orientationMetadata: Option[OrientationMetadata]
+  )(implicit ec: ExecutionContext, logMarker: LogMarker): Future[(StorableThumbImage, Option[Dimensions])] = {
     import deps._
 
     def generateThumbnail(tempFile: File) = {
       for {
-        colourModel <- colourModelFuture
-        iccColourSpace = FileMetadataHelper.normalisedIccColourSpace(fileMetadata)
-        thumbData <- imageOps.createThumbnail(
+        thumbData <- imageOps.createThumbnailVips(
           browserViewableImage,
           config.thumbWidth,
           config.thumbQuality,
           tempFile,
-          iccColourSpace,
-          colourModel,
           orientationMetadata,
         )
       } yield thumbData
@@ -280,40 +287,57 @@ object Uploader extends GridLogging {
 
     for {
       tempFile <- createTempFile(s"thumb-", thumbMimeType.fileExtension, tempDir)
-      maybeThumbFile <- deps.tryFetchThumbFile(browserViewableImage.id, tempFile)
-      (thumb, thumbMimeType) <- {
+      maybeThumbFile <- deps.tryFetchThumbFile(browserViewableImage.id, tempFile, instance)
+      (thumb, thumbMimeType, thumbDimensions) <- {
         maybeThumbFile match {
-          case Some(thumbData) => Future.successful(thumbData)
+          case Some(thumbData) => Future.successful((thumbData._1, thumbData._2, None))
           case None => generateThumbnail(tempFile)
         }
       }
-    } yield browserViewableImage
-      .copy(file = thumb, mimeType = thumbMimeType)
-      .asStorableThumbImage
+    } yield {
+      (browserViewableImage
+        .copy(file = thumb, mimeType = thumbMimeType)
+        .asStorableThumbImage, thumbDimensions)
+    }
+  }
+
+  private def createEmbeddingsSource(browserViewableImage: BrowserViewableImage,
+                                     orientationMetadata: Option[OrientationMetadata],
+                                     deps: ImageUploadOpsDependencies,
+                                    )(implicit ec: ExecutionContext): Future[Option[StorableEmbeddingSourceImage]] = {
+    import deps._
+    maybeEmbedder.map { embedder =>
+      val eventualMaybeSourceBytes = imageOps.createEmbeddingSource(browserViewableImage.file, orientationMetadata, embedder.embeddingSourceImageFormat())
+
+      eventualMaybeSourceBytes.map { maybeSourceBytes =>
+        maybeSourceBytes.map { sourceBytes =>
+          val tempPath = Files.createTempFile("embeddingsource-", ".tmp") // TODO push to image ops
+          tempPath.toFile.deleteOnExit()
+          Files.write(tempPath, sourceBytes)
+
+          browserViewableImage.copy(
+            file = tempPath.toFile,
+            mimeType = Png
+          ).asStorableEmbeddingSourceImage
+        }
+      }
+    }.getOrElse {
+      logger.info("Skipping createEmbeddingsSource because no embedder is configured")
+      Future.successful(None)
+    }
   }
 
   private def createBrowserViewableFileFuture(
-    uploadRequest: UploadRequest,
-    tempDir: File,
-    deps: ImageUploadOpsDependencies
+    uploadRequest: UploadRequest
   )(implicit ec: ExecutionContext, logMarker: LogMarker): Future[BrowserViewableImage] = {
-    import deps._
     uploadRequest.mimeType match {
-      case Some(mime) if config.transcodedMimeTypes.contains(mime) =>
-        for {
-          (file, mimeType) <- imageOps.transformImage(uploadRequest.tempFile, uploadRequest.mimeType, tempDir)
-        } yield BrowserViewableImage(
-          uploadRequest.imageId,
-          file = file,
-          mimeType = mimeType,
-          isTransformedFromSource = true
-        )
       case Some(mimeType) =>
         Future.successful(
           BrowserViewableImage(
             uploadRequest.imageId,
             file = uploadRequest.tempFile,
-            mimeType = mimeType)
+            mimeType = mimeType,
+            instance = uploadRequest.instance)
         )
       case None => Future.failed(new Exception("This file is not an image with an identifiable mime type"))
     }
@@ -338,7 +362,8 @@ class Uploader(
   val maybeEmbedder: Option[Embedder],
                imageProcessor: ImageProcessor,
   gridClient: GridClient,
-  auth: Authentication
+  auth: Authentication,
+  optimiseOps: OptimiseOps
 )(
   implicit val ec: ExecutionContext
 ) extends MessageSubjects with ArgoHelpers {
@@ -348,7 +373,7 @@ class Uploader(
     isReplacement: Boolean
   )(
     mediaIdToAddUsageTo: String
-  ) = {
+  )(implicit instance: Instance) = {
     gridClient.postUsage(
       usageType = "child",
       data = Json.obj(
@@ -365,11 +390,11 @@ class Uploader(
   }
 
   private def fromUploadRequest(uploadRequest: UploadRequest)
-                               (implicit logMarker: LogMarker): Future[ImageUpload] = {
+                               (implicit logMarker: LogMarker, instance: Instance): Future[ImageUpload] = {
     val sideEffectDependencies = ImageUploadOpsDependencies(toImageUploadOpsCfg(config), imageOps,
-      storeSource, storeThumbnail, storeOptimisedImage)
+      storeSource, storeThumbnail, storeOptimisedImage, storeEmbeddingSource, maybeEmbedder = maybeEmbedder)
     Stopwatch.async("finalImage") {
-      val finalImage = fromUploadRequestShared(uploadRequest, sideEffectDependencies, imageProcessor)
+      val finalImage = fromUploadRequestShared(uploadRequest, sideEffectDependencies, imageProcessor, optimiseOps)
       uploadRequest.identifiers.foreach{
         case (ImageStorageProps.derivativeOfMediaIdsIdentifierKey, commaSeparatedMediaIdsToAddUsagesTo) =>
           commaSeparatedMediaIdsToAddUsagesTo.split(",").map(_.trim).foreach(
@@ -378,7 +403,7 @@ class Uploader(
         case (ImageStorageProps.replacesMediaIdIdentifierKey, mediaIdToAddUsageTo) =>
           addChildUsageToParentImage(uploadRequest, isReplacement = true)(mediaIdToAddUsageTo)
       }
-      finalImage.map(img => ImageUpload(uploadRequest, img))
+      finalImage.map(img => ImageUpload(uploadRequest, img._1, img._2))
     }
   }
 
@@ -400,11 +425,20 @@ class Uploader(
   private def storeOptimisedImage(storableOptimisedImage: StorableOptimisedImage)
                                  (implicit logMarker: LogMarker) = store.store(storableOptimisedImage)
 
+  private def storeEmbeddingSource(storableEmbeddingSourceImage: Option[StorableEmbeddingSourceImage])(implicit logMarker: LogMarker): Future[Option[S3Object]] = {
+    storableEmbeddingSourceImage.map { storableEmbeddingSourceImage =>
+        store.store(storableEmbeddingSourceImage).map(Some(_))
+      }
+      .getOrElse(Future.successful(None))
+  }
+
   def loadFile(digestedFile: DigestedFile,
                uploadedBy: String,
                identifiers: Map[String, String],
                uploadTime: DateTime,
-               filename: Option[String])
+               filename: Option[String],
+               instance: Instance,
+               isFeedUpload: Boolean)
               (implicit ec:ExecutionContext,
                logMarker: LogMarker): Future[UploadRequest] = Future {
     val DigestedFile(tempFile, id) = digestedFile
@@ -427,44 +461,37 @@ class Uploader(
           uploadTime = uploadTime,
           uploadedBy = uploadedBy,
           identifiers = identifiersMap,
-          uploadInfo = UploadInfo(filename)
+          uploadInfo = UploadInfo(filename, Some(isFeedUpload)),
+          instance = instance
         )
     }
   }
 
   def storeFile(uploadRequest: UploadRequest)
                (implicit ec:ExecutionContext,
-                logMarker: LogMarker): Future[UploadStatusUri] = {
+                logMarker: LogMarker, instance: Instance): Future[UploadStatusUri] = {
 
     logger.info(logMarker, "Storing file")
 
     for {
       imageUpload <- fromUploadRequest(uploadRequest)
-      updateMessage = UpdateMessage(subject = Image, image = Some(imageUpload.image))
-      _ <- Future { notifications.publish(updateMessage) }
-      // Send the optimised PNG to the embedder if there is one (e.g. for TIFFs),
-      // otherwise send the original image.
-      assetForEmbedder = imageUpload.image.optimisedPng match {
-        case Some(optimisedPngAsset) =>
-          logger.info(logMarker, s"Queueing optimised PNG instead of original for embedding")
-          optimisedPngAsset
-        case _ =>
-          imageUpload.image.source
+      updateMessage = UpdateMessage(subject = Image, image = Some(imageUpload.image), instance = uploadRequest.instance)
+      _ <- Future {
+        notifications.publish(updateMessage)
       }
-      uriForEmbedder = assetForEmbedder.file
-      s3BucketForEmbedder = uriForEmbedder.getHost.split('.').head
-      s3KeyForEmbedder = uriForEmbedder.getPath.stripPrefix("/")
-      mimeTypeForEmbedder = assetForEmbedder.mimeType.getOrElse(
-        throw new Exception("Image for embedding has no mime type")
-      ).name
-      _ = queueImageToEmbed(EmbedderMessage(
-        uploadRequest.imageId,
-        mimeTypeForEmbedder,
-        s3BucketForEmbedder,
-        s3KeyForEmbedder,
-      ))
-      // TODO: centralise where all these URLs are constructed
+      // Send the embed source to the embedder
+      _ = imageUpload.embeddingSource.foreach { embeddingSource =>
+        queueImageToEmbed(EmbedderMessage(
+          uploadRequest.imageId,
+          Png.fileExtension,
+          config.embeddingSourceBucket.bucket,
+          config.embeddingSourceBucket.keyFromS3URL(embeddingSource.uri),
+          instance.id
+        ))
+      }
+
     } yield {
+      /*
       config.maybeLowerEnvironmentQueueBucketToSampleInto.foreach { lowerEnvironmentQueueBucket =>
         if (math.random() < config.lowerEnvironmentSamplingPercentageAsDecimal) {
           val mediaId = imageUpload.image.id
@@ -480,8 +507,8 @@ class Uploader(
           }
         }
       }
-
-      UploadStatusUri(s"${config.rootUri}/uploadStatus/${uploadRequest.imageId}")
+      */
+      UploadStatusUri(s"${config.rootUri(instance)}/uploadStatus/${uploadRequest.imageId}")
     }
 
   }
@@ -490,13 +517,14 @@ class Uploader(
                   gridClient: GridClient,
                   onBehalfOfFn: WSRequest => WSRequest)
                  (implicit ec: ExecutionContext,
-                  logMarker: LogMarker): Future[Unit] = for {
+                  logMarker: LogMarker,
+                  instance: Instance): Future[Unit] = for {
     imageUpload <- fromUploadRequest(uploadRequest)
     imageWithoutUserEdits = imageUpload.image
     imageWithUserEditsApplied <- ImageDataMerger.aggregate(imageWithoutUserEdits, gridClient, onBehalfOfFn)
     _ <- Future {
       notifications.publish(
-        UpdateMessage(subject = Image, image = Some(imageWithUserEditsApplied))
+        UpdateMessage(subject = Image, image = Some(imageWithUserEditsApplied), instance = uploadRequest.instance)
       )
     }
   } yield ()
diff --git a/image-loader/app/model/upload/OptimiseOps.scala b/image-loader/app/model/upload/OptimiseOps.scala
index 1b753dd38db..9705728131f 100644
--- a/image-loader/app/model/upload/OptimiseOps.scala
+++ b/image-loader/app/model/upload/OptimiseOps.scala
@@ -1,21 +1,24 @@
 package model.upload
 
+import app.photofox.vipsffm.enums.VipsIntent
+import app.photofox.vipsffm.{VImage, VipsHelper, VipsOption}
 import com.gu.mediaservice.lib.ImageWrapper
+import com.gu.mediaservice.lib.imaging.ImageOperations
 import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap, Stopwatch}
-import com.gu.mediaservice.model.{FileMetadata, MimeType, Png, Tiff}
+import com.gu.mediaservice.model.{MimeType, Png}
 
 import java.io.File
+import java.lang.foreign.Arena
 import scala.concurrent.{ExecutionContext, Future}
-import scala.sys.process._
 
 trait OptimiseOps {
   def toOptimisedFile(file: File, imageWrapper: ImageWrapper, tempDir: File)
                      (implicit ec: ExecutionContext, logMarker: LogMarker): Future[(File, MimeType)]
-  def shouldOptimise(mimeType: Option[MimeType], fileMetadata: FileMetadata): Boolean
+  def shouldOptimise(mimeType: Option[MimeType]): Boolean
   def optimiseMimeType: MimeType
 }
 
-object OptimiseWithPngQuant extends OptimiseOps {
+class OptimiseWithPngQuant(imageOperations: ImageOperations) extends OptimiseOps {
 
   override def optimiseMimeType: MimeType = Png
 
@@ -23,33 +26,35 @@ object OptimiseWithPngQuant extends OptimiseOps {
                      (implicit ec: ExecutionContext, logMarker: LogMarker): Future[(File, MimeType)] = Future {
 
     val marker = MarkerMap(
-      "fileName" -> file.getName()
+      "fileName" -> file.getName
     )
 
-    Stopwatch("pngquant") {
-      val result = Seq("pngquant", "-s10", "--quality", "1-85", file.getAbsolutePath,
-        "--force", "--output", optimisedFile.getAbsolutePath
-      ).!
-      if (result > 0)
-        throw new Exception(s"pngquant failed to convert to optimised png file (rc = $result)")
-    }(marker)
+    // Given a source file on any valid upload type, return a file of the optimised type
+    Stopwatch("toOptimisedFile") {
+      try {
+        val arena = Arena.ofConfined
+
+        val image = VImage.newFromFile(arena, file.getAbsolutePath)
+
+        // If we saw and ICC profile than we will need to transform
+        val needsICCTransform = VipsHelper.image_get_typeof(arena, image.getUnsafeStructAddress, "icc-profile-data") != 0
+        val correctedForICCProfile = if (needsICCTransform) {
+          image.iccTransform("srgb",
+            VipsOption.Enum("intent", VipsIntent.INTENT_PERCEPTUAL), // Helps with CMYK; see https://github.com/libvips/libvips/issues/1110
+          )
+        } else {
+          // LAB gets corrupted by a needless icc_transform
+          image
+        }
 
-    if (optimisedFile.exists()) {
-      (optimisedFile, optimiseMimeType)
-    } else {
-      throw new Exception(s"Attempted to optimise PNG file ${optimisedFile.getPath}")
-    }
+        imageOperations.saveImageToFile(correctedForICCProfile: VImage, optimiseMimeType, 85, optimisedFile, quantise = true)
+        (optimisedFile, optimiseMimeType)
+      } catch {
+        case _: Exception =>
+          throw new Exception(s"Failed to optimise PNG file ${file.getAbsolutePath}")
+      }
+    }(marker)
   }
 
-  def shouldOptimise(mimeType: Option[MimeType], fileMetadata: FileMetadata): Boolean =
-    mimeType match {
-      case Some(Png) =>
-        fileMetadata.colourModelInformation.get("colorType") match {
-          case Some("True Color") => true
-          case Some("True Color with Alpha") => true
-          case _ => false
-        }
-      case Some(Tiff) => true // TODO This should be done better, it could be better optimised into a jpeg if there is no transparency.
-      case _ => false
-    }
+  def shouldOptimise(mimeType: Option[MimeType]): Boolean = false
 }
diff --git a/image-loader/app/model/upload/UploadRequest.scala b/image-loader/app/model/upload/UploadRequest.scala
index cf5fb3848c6..57b83e72c0c 100644
--- a/image-loader/app/model/upload/UploadRequest.scala
+++ b/image-loader/app/model/upload/UploadRequest.scala
@@ -4,7 +4,7 @@ import com.gu.mediaservice.lib.ImageStorageProps
 
 import java.io.File
 import java.util.UUID
-import com.gu.mediaservice.model.{MimeType, UploadInfo}
+import com.gu.mediaservice.model.{Instance, MimeType, UploadInfo}
 import net.logstash.logback.marker.{LogstashMarker, Markers}
 import org.joda.time.format.ISODateTimeFormat
 import org.joda.time.{DateTime, DateTimeZone}
@@ -19,6 +19,7 @@ case class UploadRequest(
                           uploadedBy: String,
                           identifiers: Map[String, String],
                           uploadInfo: UploadInfo,
+                          instance: Instance
                         ) {
 
   val identifiersMeta: Map[String, String] = identifiers.map { case (k, v) =>
diff --git a/image-loader/conf/routes b/image-loader/conf/routes
index a76b80b5939..0eee3fe1442 100644
--- a/image-loader/conf/routes
+++ b/image-loader/conf/routes
@@ -17,7 +17,6 @@ GET     /uploadStatuses/:userId                       controllers.UploadStatusCo
 # Management
 GET     /management/healthcheck                       com.gu.mediaservice.lib.management.Management.healthCheck
 GET     /management/manifest                          com.gu.mediaservice.lib.management.Management.manifest
-GET     /management/whoAmI                            com.gu.mediaservice.lib.management.InnerServiceStatusCheckController.whoAmI(depth: Int)
 
 # Shoo robots away
 GET     /robots.txt                                   com.gu.mediaservice.lib.management.Management.disallowRobots
diff --git a/image-loader/test/scala/lib/imaging/FileMetadataReaderTest.scala b/image-loader/test/scala/lib/imaging/FileMetadataReaderTest.scala
index d9a570e2243..de0434db75f 100644
--- a/image-loader/test/scala/lib/imaging/FileMetadataReaderTest.scala
+++ b/image-loader/test/scala/lib/imaging/FileMetadataReaderTest.scala
@@ -23,63 +23,6 @@ class FileMetadataReaderTest extends AnyFunSpec with Matchers with ScalaFutures
   implicit override val patienceConfig: PatienceConfig = PatienceConfig(timeout = Span(1000, Millis), interval = Span(25, Millis))
   implicit val logMarker: LogMarker = MarkerMap()
 
-  it("should read the correct dimensions for a JPG image") {
-    val image = fileAt("getty.jpg")
-    val dimsFuture = FileMetadataReader.dimensions(image, Some(Jpeg))
-    whenReady(dimsFuture) { dimOpt =>
-      dimOpt should be(Symbol("defined"))
-      dimOpt.get.width should be(100)
-      dimOpt.get.height should be(60)
-    }
-  }
-
-  it("should capture exif orientation tag in JPG images") {
-    val image = fileAt("exif-orientated.jpg")
-    val orientationFuture = FileMetadataReader.orientation(image)
-    whenReady(orientationFuture) { orientationOpt =>
-      orientationOpt should be(Symbol("defined"))
-      orientationOpt.get.exifOrientation should be(Some(6))
-    }
-  }
-
-  it("should ignore 0 degree exif orientation tag as it has no material effect") {
-    val image = fileAt("exif-orientated-no-rotation.jpg")
-    val orientationFuture = FileMetadataReader.orientation(image)
-    whenReady(orientationFuture) { orientationOpt =>
-      orientationOpt should be(None)
-    }
-  }
-
-  it("should use uncorrected width and height as dimensions for exif 90 rotations") {
-    val image = fileAt("exif-orientated.jpg")
-    val dimsFuture = FileMetadataReader.dimensions(image, Some(Jpeg))
-    whenReady(dimsFuture) { dimOpt =>
-      dimOpt should be(Symbol("defined"))
-      dimOpt.get.width should be(3456)
-      dimOpt.get.height should be(2304)
-    }
-  }
-
-  it("should read the correct dimensions for a tiff image") {
-    val image = fileAt("flower.tif")
-    val dimsFuture = FileMetadataReader.dimensions(image, Some(Tiff))
-    whenReady(dimsFuture) { dimOpt =>
-      dimOpt should be(Symbol("defined"))
-      dimOpt.get.width should be(73)
-      dimOpt.get.height should be(43)
-    }
-  }
-
-  it("should read the correct dimensions for a png image") {
-    val image = fileAt("schaik.com_pngsuite/basn0g08.png")
-    val dimsFuture = FileMetadataReader.dimensions(image, Some(Png))
-    whenReady(dimsFuture) { dimOpt =>
-      dimOpt should be(Symbol("defined"))
-      dimOpt.get.width should be(32)
-      dimOpt.get.height should be(32)
-    }
-  }
-
   it("should read the correct metadata for Getty JPG images") {
     val image = fileAt("getty.jpg")
     val metadataFuture = FileMetadataReader.fromIPTCHeaders(image, "dummy")
diff --git a/image-loader/test/scala/model/ImageUploadTest.scala b/image-loader/test/scala/model/ImageUploadTest.scala
index 2430a911a2f..9d1eb7aaeb3 100644
--- a/image-loader/test/scala/model/ImageUploadTest.scala
+++ b/image-loader/test/scala/model/ImageUploadTest.scala
@@ -1,15 +1,13 @@
 package model
 
-import java.io.File
-import java.net.URI
-import java.util.UUID
+import com.amazonaws.services.s3.AmazonS3
 import com.drew.imaging.ImageProcessingException
-import com.gu.mediaservice.lib.{StorableImage, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage}
-import com.gu.mediaservice.lib.aws.{EmbedderMessage, S3Metadata, S3Object, S3ObjectMetadata, S3Ops}
+import com.gu.mediaservice.lib.aws.{S3, S3Bucket, S3Object}
 import com.gu.mediaservice.lib.cleanup.ImageProcessor
 import com.gu.mediaservice.lib.imaging.ImageOperations
 import com.gu.mediaservice.lib.logging.LogMarker
-import com.gu.mediaservice.model.{FileMetadata, Jpeg, MimeType, Png, Tiff, UploadInfo}
+import com.gu.mediaservice.lib.{StorableImage, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage}
+import com.gu.mediaservice.model._
 import lib.imaging.MimeTypeDetection
 import model.upload.{OptimiseWithPngQuant, UploadRequest}
 import org.joda.time.DateTime
@@ -17,10 +15,10 @@ import org.scalatest.Assertion
 import org.scalatest.funsuite.AsyncFunSuite
 import org.scalatest.matchers.should.Matchers
 import org.scalatestplus.mockito.MockitoSugar
-import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse
 import test.lib.ResourceHelpers
 
-import java.nio.file.Path
+import java.io.File
+import java.util.UUID
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.{Failure, Success}
 
@@ -31,10 +29,12 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar {
     override def markerContents: Map[String, Any] = Map()
   }
 
+  private val mockS3Client = mock[AmazonS3]
+
   private implicit val logMarker: MockLogMarker = new MockLogMarker()
     // For mime type info, see https://github.com/guardian/grid/pull/2568
     val tempDir = new File("/tmp")
-    val mockConfig: ImageUploadOpsCfg = ImageUploadOpsCfg(tempDir, 256, 85d, List(Tiff), "img-bucket", "thumb-bucket")
+    val mockConfig: ImageUploadOpsCfg = ImageUploadOpsCfg(tempDir, 256, 85d, S3Bucket("img-bucket", S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false, mockS3Client), S3Bucket("thumb-bucket", S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false, mockS3Client))
 
   /**
     * @todo: I flailed about until I found a path that worked, but
@@ -50,12 +50,9 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar {
 
     val randomId = UUID.randomUUID().toString + fileName
 
-    val mockS3Meta = S3Metadata(Map.empty, S3ObjectMetadata(None, None, None))
-    val mockS3Object = S3Object(new URI("innernets.com"), 12345, mockS3Meta)
-
     def mockStore = (a: StorableImage) =>
       Future.successful(
-        S3Object("madeupname", "madeupkey", a.file, Some(a.mimeType), None, a.meta, None)
+        S3Object(S3Bucket("madeupname", S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false, mockS3Client), "madeupkey", a.file, Some(a.mimeType), None, a.meta, None)
       )
 
     def storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object] = mockStore
@@ -68,10 +65,11 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar {
       storeOrProjectOriginalFile = storeOrProjectOriginalFile,
       storeOrProjectThumbFile = storeOrProjectThumbFile,
       storeOrProjectOptimisedImage = storeOrProjectOptimisedPNG,
+      maybeEmbedder = None
     )
 
     val tempFile = ResourceHelpers.fileAt(fileName)
-    val ul = UploadInfo(None)
+    val ul = UploadInfo(None, None)
 
     val uploadRequest = UploadRequest(
       imageId = randomId,
@@ -80,18 +78,18 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar {
       uploadTime = DateTime.now(),
       uploadedBy = "uploadedBy",
       identifiers = Map(),
-      uploadInfo = ul
+      uploadInfo = ul,
+      Instance("an-instance"),
     )
 
     val futureImage = Uploader.uploadAndStoreImage(
       storeOrProjectOriginalFile = mockDependencies.storeOrProjectOriginalFile,
       storeOrProjectThumbFile = mockDependencies.storeOrProjectThumbFile,
       storeOrProjectOptimisedFile = mockDependencies.storeOrProjectOptimisedImage,
-      optimiseOps = OptimiseWithPngQuant,
       uploadRequest = uploadRequest,
       deps = mockDependencies,
-      fileMetadata = FileMetadata(),
       processor = ImageProcessor.identity,
+      new OptimiseWithPngQuant(imageOps)
     )
 
     // Assertions; Failure will auto-fail
diff --git a/image-loader/test/scala/model/ProjectorTest.scala b/image-loader/test/scala/model/ProjectorTest.scala
index eccec5b65f0..ac07cbe7077 100644
--- a/image-loader/test/scala/model/ProjectorTest.scala
+++ b/image-loader/test/scala/model/ProjectorTest.scala
@@ -1,18 +1,22 @@
 package model
 
+import com.amazonaws.services.s3.AmazonS3
+
 import java.io.File
 import java.net.URI
-import java.util.{Date, UUID}
-import com.amazonaws.services.s3.AmazonS3
+import java.util.Date
 import com.amazonaws.services.s3.model.ObjectMetadata
 import com.gu.mediaservice.GridClient
 import com.gu.mediaservice.lib.auth.Authentication
+import com.gu.mediaservice.lib.aws.{Embedder, S3, S3Bucket, S3Vectors}
+import com.gu.mediaservice.lib.aws.S3Ops
 import com.gu.mediaservice.lib.cleanup.ImageProcessor
 import com.gu.mediaservice.lib.imaging.ImageOperations
 import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap}
-import com.gu.mediaservice.model._
+import com.gu.mediaservice.model.{Instance, _}
 import com.gu.mediaservice.model.leases.LeasesByMedia
 import lib.DigestedFile
+import model.upload.OptimiseWithPngQuant
 import org.joda.time.{DateTime, DateTimeZone}
 import org.mockito.ArgumentMatchers.any
 import org.mockito.Mockito.{times, verify, when}
@@ -40,13 +44,16 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc
 
   private val imageOperations = new ImageOperations(ctxPath)
 
-  private val config = ImageUploadOpsCfg(new File("/tmp"), 256, 85d, Nil, "img-bucket", "thumb-bucket")
+  private val mockS3Client = mock[AmazonS3]
+  private val config = ImageUploadOpsCfg(new File("/tmp"), 256, 85d, S3Bucket("img-bucket", S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false, mockS3Client), S3Bucket("thumb-bucket", S3.AmazonAwsS3Endpoint, usesPathStyleURLs = false, mockS3Client))
 
   private val maybeEmbedder = None
 
-  private val s3 = mock[AmazonS3]
+  private val s3 = mock[S3]
   private val auth = mock[Authentication]
-  private val projector = new Projector(config, s3, imageOperations, ImageProcessor.identity, auth, maybeEmbedder)
+  private val projector = new Projector(config, s3, imageOperations, ImageProcessor.identity, auth, maybeEmbedder, new OptimiseWithPngQuant(imageOperations))
+
+  private implicit val instance: Instance = Instance("an-instance")
 
   // FIXME temporary ignored as test is not executable in CI/CD machine
   // because graphic lib files like srgb.icc, cmyk.icc are in root directory instead of resources
@@ -135,7 +142,7 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc
       softDeletedMetadata = None,
       lastModified = Some(new DateTime("2020-01-24T17:36:08.456Z").withZone(DateTimeZone.UTC)),
       identifiers = Map(),
-      uploadInfo = UploadInfo(Some("getty.jpg")),
+      uploadInfo = UploadInfo(Some("getty.jpg"), Some(true)),
       source = Asset(new URI("http://img-bucket.s3.amazonaws.com/i/d/1/2/3/" + id),
         Some(12666),
         Some(Jpeg),
@@ -200,9 +207,11 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc
       uploadTime = uploadTime,
       uploadFileName = uploadFileName,
       identifiers = Map.empty,
+      isFeedUpload = Some(true),
     )
 
     implicit val logMarker: LogMarker = MarkerMap()
+    implicit val instance: Instance = Instance("an-instance")
 
     val gridClient = mock[GridClient]
     when(gridClient.getUsages(id, identity)).thenReturn(Future.successful(Nil))
diff --git a/image-loader/test/scala/model/S3IngestObjectTest.scala b/image-loader/test/scala/model/S3IngestObjectTest.scala
new file mode 100644
index 00000000000..ff0512ef379
--- /dev/null
+++ b/image-loader/test/scala/model/S3IngestObjectTest.scala
@@ -0,0 +1,14 @@
+package scala.model
+
+import model.S3IngestObject
+import org.scalatest.flatspec.AnyFlatSpec
+import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper
+
+class S3IngestObjectTest extends AnyFlatSpec {
+
+  it should "infer uploader from folder after feeds" in {
+    val keyParts = "fingerpost/feeds/20250226/PA/205350.SOCCER-Kilmarnock--20514432_PICTURES_PRI13.jpg".split("/")
+    S3IngestObject.uploadedFromPath(keyParts) shouldBe "20250226"
+  }
+
+}
diff --git a/kahuna/app/InstanceSpecificSecurityHeaderFilter.scala b/kahuna/app/InstanceSpecificSecurityHeaderFilter.scala
new file mode 100644
index 00000000000..3a28c8b1ec9
--- /dev/null
+++ b/kahuna/app/InstanceSpecificSecurityHeaderFilter.scala
@@ -0,0 +1,17 @@
+import com.gu.mediaservice.lib.config.InstanceForRequest
+import lib.KahunaConfig
+import play.api.Configuration
+import play.api.mvc.{EssentialAction, EssentialFilter, RequestHeader}
+import play.filters.headers._
+
+class InstanceSpecificSecurityHeaderFilter(config: KahunaConfig, playConfig: Configuration)
+  extends EssentialFilter with InstanceForRequest {
+
+  override def apply(next: EssentialAction): EssentialAction = (req: RequestHeader) => {
+    val instance = instanceOf(req)
+    val kahunaSecurityConfig = KahunaSecurityConfig.apply(config, playConfig, instance)
+    val instanceSpecificSecurityHeadersFilter = SecurityHeadersFilter(kahunaSecurityConfig: SecurityHeadersConfig).apply(next)
+    instanceSpecificSecurityHeadersFilter.apply(req)
+  }
+
+}
diff --git a/kahuna/app/KahunaComponents.scala b/kahuna/app/KahunaComponents.scala
index 8a10d6e6674..619db511599 100644
--- a/kahuna/app/KahunaComponents.scala
+++ b/kahuna/app/KahunaComponents.scala
@@ -1,51 +1,58 @@
-import com.gu.mediaservice.lib.management.InnerServiceStatusCheckController
 import com.gu.mediaservice.lib.net.URI
-import com.gu.mediaservice.lib.play.GridComponents
+import com.gu.mediaservice.lib.play.{ConnectionBrokenFilter, GridComponents, RequestLoggingFilter, RequestMetricFilter}
+import com.gu.mediaservice.model.Instance
 import controllers.{AssetsComponents, KahunaController}
 import lib.KahunaConfig
 import play.api.ApplicationLoader.Context
 import play.api.Configuration
+import play.api.mvc.EssentialFilter
 import play.filters.headers.SecurityHeadersConfig
 import router.Routes
 
 class KahunaComponents(context: Context) extends GridComponents(context, new KahunaConfig(_)) with AssetsComponents {
-  final override lazy val securityHeadersConfig: SecurityHeadersConfig = KahunaSecurityConfig(config, context.initialConfiguration)
-
   final override val buildInfo = utils.buildinfo.BuildInfo
 
+  override def httpFilters: Seq[EssentialFilter] = Seq(
+    instanceSpecificCorsFilter,
+    // csrfFilter,  TODO Ineffective as gateway is not setting correct hostname headers!
+    new InstanceSpecificSecurityHeaderFilter(config, context.initialConfiguration),
+    gzipFilter,
+    new RequestLoggingFilter(materializer),
+    new ConnectionBrokenFilter(materializer),
+    new RequestMetricFilter(config, materializer, actorSystem, applicationLifecycle)
+  )
+
   val controller = new KahunaController(auth, config, controllerComponents, authorisation)
-  val InnerServiceStatusCheckController = new InnerServiceStatusCheckController(auth, controllerComponents, config.services, wsClient)
 
-  final override val router = new Routes(httpErrorHandler, controller, assets, management, InnerServiceStatusCheckController)
+  final override val router = new Routes(httpErrorHandler, controller, assets, management)
 
 }
 
 object KahunaSecurityConfig {
-  def apply(config: KahunaConfig, playConfig: Configuration): SecurityHeadersConfig = {
+  def apply(config: KahunaConfig, playConfig: Configuration, instance: Instance): SecurityHeadersConfig = {
     val base = SecurityHeadersConfig.fromConfiguration(playConfig)
 
     val services = List(
-      config.services.apiBaseUri,
-      config.services.loaderBaseUri,
-      config.services.cropperBaseUri,
-      config.services.metadataBaseUri,
-      config.services.imgopsBaseUri,
-      config.services.usageBaseUri,
-      config.services.collectionsBaseUri,
-      config.services.leasesBaseUri,
-      config.services.authBaseUri,
+      config.services.apiBaseUri(instance),
+      config.services.loaderBaseUri(instance),
+      config.services.cropperBaseUri(instance),
+      config.services.metadataBaseUri(instance),
+      config.services.imgopsBaseUri(instance),
+      config.services.usageBaseUri(instance),
+      config.services.collectionsBaseUri(instance),
+      config.services.leasesBaseUri(instance),
+      config.services.authBaseUri(instance),
       config.services.guardianWitnessBaseUri
     )
 
-    val frameSources = s"frame-src ${config.services.authBaseUri} ${config.services.kahunaBaseUri} https://accounts.google.com https://www.youtube.com ${config.scriptsToLoad.map(_.host).mkString(" ")}"
+    val frameSources = s"frame-src https://accounts.google.com https://www.youtube.com ${config.scriptsToLoad.map(_.host).mkString(" ")}"
     val frameAncestors = s"frame-ancestors ${config.frameAncestors.mkString(" ")}"
-    val connectSources = s"connect-src 'self' ${(services :+ config.imageOrigin).mkString(" ")} ${config.connectSources.mkString(" ")}"
+    val connectSources = s"connect-src 'self' ${config.connectSources.mkString(" ")}"
 
-    val imageSources = s"img-src ${List(
+    val imageSources =  s"img-src ${List(
       "data:",
       "blob:",
-      URI.ensureSecure(config.services.imgopsBaseUri).toString,
-      URI.ensureSecure(config.fullOrigin).toString,
+      URI.ensureSecure(config.services.imgopsBaseUri(instance)).toString,
       URI.ensureSecure(config.thumbOrigin).toString,
       URI.ensureSecure(config.cropOrigin).toString,
       URI.ensureSecure("app.getsentry.com").toString,
diff --git a/kahuna/app/controllers/KahunaController.scala b/kahuna/app/controllers/KahunaController.scala
index 7674168c42a..cb2a8aa8118 100644
--- a/kahuna/app/controllers/KahunaController.scala
+++ b/kahuna/app/controllers/KahunaController.scala
@@ -3,13 +3,14 @@ package controllers
 import com.gu.mediaservice.lib.argo.ArgoHelpers
 import com.gu.mediaservice.lib.auth.Authentication.Principal
 import com.gu.mediaservice.lib.auth.{Authentication, Authorisation, BaseControllerWithLoginRedirects}
-import lib.{EnableAISearch, ExampleSwitch, FeatureSwitches, KahunaConfig, UseCqlChips}
+import lib.{EnableAISearch, ExampleSwitch, FeatureSwitches, KahunaConfig, KahunaClientServiceUrls, UseCqlChips}
 import play.api.mvc.ControllerComponents
 import play.api.libs.json._
 
 import scala.concurrent.ExecutionContext
 import com.gu.mediaservice.lib.config.FieldAlias._
-import com.gu.mediaservice.lib.config.Services
+import com.gu.mediaservice.lib.config.{InstanceForRequest, Services}
+import com.gu.mediaservice.model.Instance
 import play.api.mvc.Security.AuthenticatedRequest
 import play.twirl.api.Html
 
@@ -20,13 +21,14 @@ class KahunaController(
   authorisation: Authorisation
 )(
   implicit val ec: ExecutionContext
-) extends BaseControllerWithLoginRedirects with ArgoHelpers {
+) extends BaseControllerWithLoginRedirects with ArgoHelpers with InstanceForRequest {
 
   override def auth: Authentication = authentication
 
   override def services: Services = config.services
 
   def index(ignored: String) = withOptionalLoginRedirect { request =>
+    implicit val instance: Instance = instanceOf(request)
 
     val maybeUser: Option[Authentication.Principal] = request match {
       case authedRequest: AuthenticatedRequest[_, _] => authedRequest.user match {
@@ -55,19 +57,27 @@ class KahunaController(
     val metadataTemplates: String = Json.toJson(config.metadataTemplates).toString()
     val announcements: String = Json.toJson(config.announcements).toString()
     val interimFilterOptions: String = Json.toJson(config.interimFilterOptions).toString()
-    val returnUri = config.rootUri + okPath
+    val returnUri = config.rootUri(instance) + okPath
     val costFilterLabel = config.costFilterLabel.getOrElse("Free to use only")
     val costFilterChargeable = config.costFilterChargeable.getOrElse(false)
     val maybeOrgOwnedValue =
       if(config.shouldDisplayOrgOwnedCountAndFilterCheckbox)
-        Html(s""""${config.staffPhotographerOrganisation}-owned"""")
+        Html(""""owned"""")
       else
         Html("undefined")
     val imageTypes = Json.toJson(config.imageTypes).toString()
     val agencyPicksIngredients = Json.toJson(config.agencyPicksIngredients).toString()
 
+    val rootUri = config.rootUri(instance)
+
+    val kahunaClientServiceUrls = KahunaClientServiceUrls(
+      rootUri = rootUri,
+      mediaApiUri = config.mediaApiUri(instance),
+      authUri = config.authUri(instance)
+    )
+
     Ok(views.html.main(
-      s"${config.authUri}/login?redirectUri=$returnUri",
+      s"${config.authUri(instance)}/login?redirectUri=$returnUri",
       fieldAliases,
       scriptsToLoad,
       domainMetadataSpecs,
@@ -81,12 +91,13 @@ class KahunaController(
       config,
       featureSwitchesJson,
       imageTypes,
-      agencyPicksIngredients
+      agencyPicksIngredients,
+      kahunaClientServiceUrls
     ))
   }
 
   def quotas = authentication { req =>
-    Ok(views.html.quotas(config.mediaApiUri))
+    Ok(views.html.quotas(config.mediaApiUri(instanceOf(req))))
   }
 
   def notifications = authentication { req =>
diff --git a/kahuna/app/lib/KahunaClientServiceUrls.scala b/kahuna/app/lib/KahunaClientServiceUrls.scala
new file mode 100644
index 00000000000..d2a2ec0c223
--- /dev/null
+++ b/kahuna/app/lib/KahunaClientServiceUrls.scala
@@ -0,0 +1,3 @@
+package lib
+
+case class KahunaClientServiceUrls(rootUri: String, mediaApiUri: String, authUri: String)
diff --git a/kahuna/app/lib/KahunaConfig.scala b/kahuna/app/lib/KahunaConfig.scala
index 0422a5b1453..de367fddffa 100644
--- a/kahuna/app/lib/KahunaConfig.scala
+++ b/kahuna/app/lib/KahunaConfig.scala
@@ -2,7 +2,9 @@ package lib
 
 import com.gu.mediaservice.lib.auth.Permissions.Pinboard
 import com.gu.mediaservice.lib.auth.SimplePermission
+import com.gu.mediaservice.lib.aws.S3Bucket
 import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources}
+import com.gu.mediaservice.model.Instance
 import play.api.libs.json._
 
 case class ScriptToLoad(
@@ -14,16 +16,14 @@ case class ScriptToLoad(
 )
 
 class KahunaConfig(resources: GridConfigResources) extends CommonConfig(resources) {
-  val rootUri: String = services.kahunaBaseUri
-  val mediaApiUri: String = services.apiBaseUri
-  val authUri: String = services.authBaseUri
+  val rootUri: Instance => String = services.kahunaBaseUri
+  def mediaApiUri: Instance => String = services.apiBaseUri
+  val authUri: Instance => String = services.authBaseUri
 
   val sentryDsn: Option[String] = stringOpt("sentry.dsn").filterNot(_.isEmpty)
 
   val thumbOrigin: String = string("origin.thumb")
-  val fullOrigin: String = string("origin.full")
   val cropOrigin: String = string("origin.crops")
-  val imageOrigin: String = string("origin.images")
 
   val costFilterLabel: Option[String] = stringOpt("costFilter.label")
   val costFilterChargeable: Option[Boolean] = booleanOpt("costFilter.chargeable")
@@ -48,11 +48,11 @@ class KahunaConfig(resources: GridConfigResources) extends CommonConfig(resource
 
   val showDenySyndicationWarning: Option[Boolean] = booleanOpt("showDenySyndicationWarning")
   val showSendToPhotoSales: Option[Boolean] = booleanOpt("showSendToPhotoSales")
+  val aiSearchResultLimit: Int = intOpt("ai.search.resultLimit").getOrElse(200)
 
   val frameAncestors: Set[String] = getStringSet("security.frameAncestors")
-  val connectSources: Set[String] = getStringSet("security.connectSources") ++ maybeBucketForUIUploads.map { bucket =>
-    if (isDev) "https://localstack.media.local.dev-gutools.co.uk"
-    else s"https://$bucket.s3.$awsRegion.amazonaws.com"
+  val connectSources: Set[String] = getStringSet("security.connectSources") ++ maybeIngestBucket.map { ingestBucket =>
+    ingestBucket.bucketURL().toURL.toExternalForm
   } ++ telemetryUri
   val fontSources: Set[String] = getStringSet("security.fontSources")
   val imageSources: Set[String] = getStringSet("security.imageSources")
diff --git a/kahuna/app/views/main.scala.html b/kahuna/app/views/main.scala.html
index 859f47fabba..88de715ac90 100644
--- a/kahuna/app/views/main.scala.html
+++ b/kahuna/app/views/main.scala.html
@@ -1,6 +1,7 @@
 @import lib.ScriptToLoad
 
 @import lib.KahunaConfig
+@import lib.KahunaClientServiceUrls
 @(
   reauthUri: String,
   fieldAliases: String,
@@ -16,7 +17,8 @@
   kahunaConfig: KahunaConfig,
   featureSwitches: String,
   imageTypes: String,
-  agencyPicksIngredients: String
+  agencyPicksIngredients: String,
+  kahunaClientServiceUrls: KahunaClientServiceUrls
 )
 
 
@@ -29,15 +31,15 @@
 
 
     
-    
-    
+    
+    
 
     
     
     
-    
+    
     
-    
+    
 
     @kahunaConfig.sentryDsn.map { dsn =>  }
     
@@ -53,7 +55,7 @@