diff --git a/build.sbt b/build.sbt index 69e4f7c7394..59fd392e6b6 100644 --- a/build.sbt +++ b/build.sbt @@ -52,7 +52,7 @@ val commonSettings = Seq( "org.scalatestplus" %% "mockito-3-4" % "3.1.4.0" % Test, "org.mockito" % "mockito-core" % "2.18.0" % Test, "org.scalamock" %% "scalamock" % "5.1.0" % Test, - "org.testcontainers" % "localstack" % "1.21.4" % Test + "org.testcontainers" % "testcontainers-localstack" % "2.0.5" % Test ), dependencyOverrides ++= jacksonOverrides, @@ -124,10 +124,10 @@ lazy val commonLib = project("common-lib").settings( "software.amazon.awssdk" % "cloudwatch" % awsSdkV2Version, // declare explicit dependency on desired version of aws sdk v2 bedrock runtime "software.amazon.awssdk" % "bedrockruntime" % awsSdkV2Version, - "software.amazon.awssdk" % "s3vectors" % awsSdkV2Version, "com.adobe.xmp" % "xmpcore" % "6.1.11", ws, - "org.testcontainers" % "testcontainers-elasticsearch" % "2.0.2" % Test, + "com.google.genai" % "google-genai" % "1.53.0" exclude("com.fasterxml.jackson.module", "jackson-module-kotlin"), + "org.testcontainers" % "testcontainers-elasticsearch" % "2.0.5" % Test, ), dependencyOverrides += "ch.qos.logback" % "logback-classic" % "1.2.13" % Test ) @@ -185,10 +185,14 @@ lazy val thrall = playProject("thrall", 9002) "com.gu" %% "kcl-pekko-stream" % "0.1.2", "org.testcontainers" % "testcontainers-elasticsearch" % "2.0.2" % Test, "com.google.protobuf" % "protobuf-java" % "3.19.6", - "software.amazon.awssdk" % "sqs" % awsSdkV2Version + "software.amazon.awssdk" % "sqs" % awsSdkV2Version, + "org.apache.pekko" %% "pekko-connectors-sqs" % "1.0.2" ), dependencyOverrides ++= Seq( - "org.apache.pekko" %% "pekko-stream" % "1.0.3" + "org.apache.pekko" %% "pekko-stream" % "1.0.3", + "org.apache.pekko" %% "pekko-http" % "1.0.1", + "org.apache.pekko" %% "pekko-http-core" % "1.0.1", + "org.apache.pekko" %% "pekko-parsing" % "1.0.1" ) ) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index afff3526883..8d7059624b5 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,5 +1,6 @@ options: machineType: 'N1_HIGHCPU_8' +timeout: '1200s' steps: - name: 'node:24-alpine' entrypoint: 'npm' @@ -15,9 +16,11 @@ steps: args: [ 'run', 'dist' ] - name: 'gcr.io/$PROJECT_ID/scala-sbt:1.11.7-jdk-25' - args: ['docker:publishLocal'] + args: ['compile', 'collections/test', 'leases/test', 'media-api/test', 'docker:publishLocal'] env: - - 'DOCKER_API_VERSION=1.41' + - 'DOCKER_HOST=unix:///var/run/docker.sock' + - 'DOCKER_API_VERSION=1.41' + - name: 'gcr.io/cloud-builders/docker' args: ['tag', 'auth:0.1', 'eu.gcr.io/$PROJECT_ID/auth:$BRANCH_NAME'] - name: 'gcr.io/cloud-builders/docker' diff --git a/collections/test/store/CollectionsStoreTest.scala b/collections/test/store/CollectionsStoreTest.scala index 78c2f5a9e71..a5e40e8c53d 100644 --- a/collections/test/store/CollectionsStoreTest.scala +++ b/collections/test/store/CollectionsStoreTest.scala @@ -7,8 +7,7 @@ import org.scalatest.concurrent.ScalaFutures import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers import org.scalatest.time.{Millis, Seconds, Span} -import org.testcontainers.containers.localstack.LocalStackContainer -import org.testcontainers.containers.localstack.LocalStackContainer.Service.DYNAMODB +import org.testcontainers.localstack.LocalStackContainer import org.testcontainers.utility.DockerImageName import software.amazon.awssdk.auth.credentials.{AwsBasicCredentials, StaticCredentialsProvider} import software.amazon.awssdk.regions.Region @@ -25,11 +24,11 @@ class CollectionsStoreTest extends AnyFunSpec with Matchers with ScalaFutures wi implicit val defaultPatience: PatienceConfig = PatienceConfig(timeout = Span(2, Seconds), interval = Span(100, Millis)) - private val dynamoContainer = new LocalStackContainer(DockerImageName.parse("localstack/localstack:1.4.0")).withServices(DYNAMODB) + private val dynamoContainer = new LocalStackContainer(DockerImageName.parse("localstack/localstack:1.4.0")).withServices("dynamodb") dynamoContainer.start() private val dynamoClient = DynamoDbAsyncClient.builder(). - endpointOverride(dynamoContainer.getEndpointOverride(DYNAMODB)). + endpointOverride(dynamoContainer.getEndpoint). region(Region.of(dynamoContainer.getRegion)). credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(dynamoContainer.getAccessKey, dynamoContainer.getSecretKey))).build() diff --git a/collections/test/store/ImageCollectionsStoreTest.scala b/collections/test/store/ImageCollectionsStoreTest.scala index 77e366f42ba..9a9ce684717 100644 --- a/collections/test/store/ImageCollectionsStoreTest.scala +++ b/collections/test/store/ImageCollectionsStoreTest.scala @@ -7,8 +7,7 @@ import org.scalatest.concurrent.ScalaFutures import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers import org.scalatest.time.{Millis, Seconds, Span} -import org.testcontainers.containers.localstack.LocalStackContainer -import org.testcontainers.containers.localstack.LocalStackContainer.Service.DYNAMODB +import org.testcontainers.localstack.LocalStackContainer import org.testcontainers.utility.DockerImageName import software.amazon.awssdk.auth.credentials.{AwsBasicCredentials, StaticCredentialsProvider} import software.amazon.awssdk.regions.Region @@ -22,11 +21,11 @@ class ImageCollectionsStoreTest extends AnyFunSpec with Matchers with ScalaFutur implicit val defaultPatience: PatienceConfig = PatienceConfig(timeout = Span(5, Seconds), interval = Span(500, Millis)) - private val dynamoContainer = new LocalStackContainer(DockerImageName.parse("localstack/localstack:1.4.0")).withServices(DYNAMODB) + private val dynamoContainer = new LocalStackContainer(DockerImageName.parse("localstack/localstack:1.4.0")).withServices("dynamodb") dynamoContainer.start() private val dynamoClient = DynamoDbAsyncClient.builder(). - endpointOverride(dynamoContainer.getEndpointOverride(DYNAMODB)). + endpointOverride(dynamoContainer.getEndpoint). region(Region.of(dynamoContainer.getRegion)). credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(dynamoContainer.getAccessKey, dynamoContainer.getSecretKey))).build() diff --git a/common-lib/src/main/resources/application.conf b/common-lib/src/main/resources/application.conf index 4b2b868f210..7e2ec759ff7 100644 --- a/common-lib/src/main/resources/application.conf +++ b/common-lib/src/main/resources/application.conf @@ -33,7 +33,10 @@ authentication.providers { machine { className = "com.gu.mediaservice.lib.auth.provider.ApiKeyAuthenticationProvider" config { - # authKeyStoreBucket = + # authKeyStoreBucket.name = + # authKeyStoreBucket.endpoint = + # authKeyStoreBucket.pathStyleUrls = false + # authKeyStoreBucket.region = } } # TODO: short term we put panda here for backwards compatibility but the default provider should be something better diff --git a/common-lib/src/main/scala/com/gu/mediaservice/GridClient.scala b/common-lib/src/main/scala/com/gu/mediaservice/GridClient.scala index f3df2707dda..72dcdc956b7 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/GridClient.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/GridClient.scala @@ -150,8 +150,9 @@ class GridClient(services: Services, originDomain: Instance => String)(implicit def getImageLoaderProjection(mediaId: String, imageLoaderEndpoint: String, authFn: WSRequest => WSRequest) (implicit ec: ExecutionContext): Future[Option[Image]] = { - logger.info("attempt to get image projection from image-loader") - val url = new URL(s"$imageLoaderEndpoint/images/project/$mediaId") + val projectUrl = s"$imageLoaderEndpoint/images/project/$mediaId" + logger.info(s"attempt to get image projection from image-loader: $projectUrl") + val url = new URL(projectUrl) makeGetRequestAsync(url, authFn, requestTimeout = Some(300.seconds)) map { case Found(json, _) => Some(json.as[Image]) case NotFound(_, _) => None diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala index 3bff1d70d4a..6576cddce53 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala @@ -1,12 +1,14 @@ package com.gu.mediaservice.lib - +import _root_.play.api.libs.json._ import com.gu.mediaservice.lib.aws.{S3Bucket, S3Object} import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.LogMarker -import com.gu.mediaservice.model.{Instance, MimeType} +import com.gu.mediaservice.model.{Embedding, Instance, MimeType} import com.typesafe.scalalogging.StrictLogging import org.joda.time.DateTime +import software.amazon.awssdk.core.ResponseInputStream +import software.amazon.awssdk.services.s3.model.GetObjectResponse import java.io.File import scala.concurrent.Future @@ -16,19 +18,22 @@ object ImageIngestOperations { def optimisedPngKeyFromId(id: String)(implicit instance: Instance): String = instance.id + "/" + "optimised/" + snippetForId(id: String) + def embeddingKeyFromId(id: String)(implicit instance: Instance): String = instance.id + "/" + snippetForId(id) + private def snippetForId(id: String) = id.take(6).mkString("/") + "/" + id } -class ImageIngestOperations(imageBucket: S3Bucket, thumbnailBucket: S3Bucket, config: CommonConfig, isVersionedS3: Boolean = false) +class ImageIngestOperations(imageBucket: S3Bucket, thumbnailBucket: S3Bucket, embeddingSourceBucket: S3Bucket, embeddingsBucket: S3Bucket, config: CommonConfig, isVersionedS3: Boolean = false) extends S3ImageStorage(config) with StrictLogging { - import ImageIngestOperations.{fileKeyFromId, optimisedPngKeyFromId} + import ImageIngestOperations.{embeddingKeyFromId, fileKeyFromId, optimisedPngKeyFromId} def store(storableImage: StorableImage) (implicit logMarker: LogMarker): Future[S3Object] = storableImage match { case s:StorableOriginalImage => storeOriginalImage(s) case s:StorableThumbImage => storeThumbnailImage(s) case s:StorableOptimisedImage => storeOptimisedImage(s) + case s:StorableEmbeddingSourceImage => storeEmbeddingSourceImage(s) } private def storeOriginalImage(storableImage: StorableOriginalImage) @@ -55,6 +60,23 @@ class ImageIngestOperations(imageBucket: S3Bucket, thumbnailBucket: S3Bucket, co overwrite = true) } + private def storeEmbeddingSourceImage(storableImage: StorableEmbeddingSourceImage) + (implicit logMarker: LogMarker): Future[S3Object] = { + val instanceSpecificKey = fileKeyFromId(storableImage.id)(storableImage.instance) + logger.info(s"Storing embedding source to instance specific key: ${embeddingSourceBucket.name} / $instanceSpecificKey") + storeImage(embeddingSourceBucket, instanceSpecificKey, storableImage.file, Some(storableImage.mimeType), + overwrite = true) + } + + def getEmbeddingStoreImage(key: String): ResponseInputStream[GetObjectResponse] = { + getObject(embeddingSourceBucket, key) + } + + def storeEmbedding(key: String, embedding: Embedding): Unit = { + logger.info(s"Storing embedding source to key: ${embeddingsBucket.name} / $key") + putString(embeddingsBucket, key, Json.stringify(Json.toJson(embedding))) + } + private def bulkDelete(bucket: S3Bucket, keys: List[String]): Future[Map[String, Boolean]] = keys match { case Nil => Future.successful(Map.empty) case _ => Future { @@ -68,6 +90,7 @@ class ImageIngestOperations(imageBucket: S3Bucket, thumbnailBucket: S3Bucket, co def deleteThumbnails(ids: Set[String])(implicit instance: Instance) = bulkDelete(thumbnailBucket, ids.map(id => fileKeyFromId(id)).toList) def deletePNG(id: String)(implicit logMarker: LogMarker, instance: Instance): Future[Unit] = deleteImage(imageBucket, optimisedPngKeyFromId(id)) def deletePNGs(ids: Set[String])(implicit instance: Instance) = bulkDelete(imageBucket, ids.map(id => optimisedPngKeyFromId(id)).toList) + def deleteEmbeddings(ids: Set[String])(implicit instance: Instance) = bulkDelete(embeddingsBucket, ids.map(id => embeddingKeyFromId(id)).toList) def doesOriginalExist(id: String)(implicit instance: Instance): Boolean = this.doesObjectExist(imageBucket, fileKeyFromId(id)) @@ -121,7 +144,16 @@ case class StorableOptimisedImage(id: String, file: File, mimeType: MimeType, me meta = meta ) } - +case class StorableEmbeddingSourceImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty, instance: Instance) extends StorableImage { + override def toProjectedS3Object(embeddingSourcesBucket: S3Bucket): S3Object = S3Object( + embeddingSourcesBucket, + ImageIngestOperations.fileKeyFromId(id)(instance), + file, + Some(mimeType), + lastModified = None, + meta = meta + ) +} /** * @param id @@ -135,5 +167,6 @@ case class StorableOptimisedImage(id: String, file: File, mimeType: MimeType, me case class BrowserViewableImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty, isTransformedFromSource: Boolean = false, instance: Instance) extends ImageWrapper { def asStorableOptimisedImage = StorableOptimisedImage(id, file, mimeType, meta, instance) def asStorableThumbImage = StorableThumbImage(id, file, mimeType, meta, instance) + def asStorableEmbeddingSourceImage = StorableEmbeddingSourceImage(id, file, mimeType, meta, instance) } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala index 930ff508033..f1fbb4af5b2 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala @@ -1,16 +1,17 @@ package com.gu.mediaservice.lib.aws -import software.amazon.awssdk.services.bedrockruntime.model._ -import software.amazon.awssdk.services.bedrockruntime._ import com.gu.mediaservice.lib.config.CommonConfig -import play.api.libs.json.Json -import software.amazon.awssdk.core.SdkBytes - -import java.net.URI +import com.gu.mediaservice.lib.embeddings.{EmbeddingImplementation, EmbeddingSourceImageFormat} import com.gu.mediaservice.lib.logging.LogMarker +import com.gu.mediaservice.model.{CohereV4Embedding, Embedding, ImageMetadata, Jpeg, MimeType} +import org.apache.commons.codec.binary.Base64 import play.api.libs.json.OFormat.oFormatFromReadsAndOWrites import play.api.libs.json._ +import software.amazon.awssdk.core.SdkBytes +import software.amazon.awssdk.services.bedrockruntime._ +import software.amazon.awssdk.services.bedrockruntime.model._ +import java.net.URI import scala.concurrent.{ExecutionContext, Future} object Bedrock { @@ -22,10 +23,19 @@ object Bedrock { ) private implicit val bedrockTextRequestFormat: OFormat[BedrockTextRequest] = Json.format[BedrockTextRequest] + + case class BedrockImageRequest( + input_type: String, + embedding_types: List[String], + images: List[String], + output_dimension: Int + ) + + private implicit val bedrockImageRequestFormat: OFormat[BedrockImageRequest] = Json.format[BedrockImageRequest] } class Bedrock(config: CommonConfig) - extends AwsClientBuilderUtils { + extends EmbeddingImplementation with AwsClientBuilderUtils { // TODO: figure out what the more usual pattern for turning off localstack behaviour is override def awsLocalEndpointUri: Option[URI] = None @@ -37,7 +47,7 @@ class Bedrock(config: CommonConfig) .build() } - private def createRequestBody(inputData: String): InvokeModelRequest = { + private def createSearchQueryRequestBody(inputData: String): InvokeModelRequest = { val body = Bedrock.BedrockTextRequest( input_type = "search_query", embedding_types = List("float"), @@ -58,6 +68,30 @@ class Bedrock(config: CommonConfig) request } + private def createImageSearchDocumentRequestBody(base64Image: String, imageMimeType: MimeType): InvokeModelRequest = { + val body = Bedrock.BedrockImageRequest( + input_type = "search_document", + embedding_types = List("float"), + images = List( + s"`data:${imageMimeType.name};base64,$base64Image`" + ), + output_dimension = 1536 + ) + + val jsonBody = Json.toJson(body).toString() + + val request: InvokeModelRequest = { + InvokeModelRequest + .builder() + .accept("*/*") + .body(SdkBytes.fromUtf8String(jsonBody)) + .contentType("application/json") + .modelId("global.cohere.embed-v4:0") + .build() + } + request + } + private def sendBedrockEmbeddingRequest(requestBody: InvokeModelRequest)( implicit logMarker: LogMarker ): InvokeModelResponse = { @@ -77,7 +111,7 @@ class Bedrock(config: CommonConfig) } def createTextEmbedding(inputData: String)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[List[Double]] = { - val requestBody = createRequestBody(inputData) + val requestBody = createSearchQueryRequestBody(inputData) val bedrockFuture = Future { sendBedrockEmbeddingRequest(requestBody) } bedrockFuture.map { response => val responseBody = response.body().asUtf8String() @@ -91,4 +125,32 @@ class Bedrock(config: CommonConfig) embedding } } + + override def createImageEmbeddings(source: Array[Byte], mimeType: MimeType, maybeMetadata: Option[ImageMetadata])(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Embedding] = { + val base64ImageData = Base64.encodeBase64String(source) + val requestBody = createImageSearchDocumentRequestBody( + base64ImageData, embeddingSourceImageFormat().format + ) + val bedrockFuture = Future { + sendBedrockEmbeddingRequest(requestBody) + } + bedrockFuture.map { response => + val responseBody = response.body().asUtf8String() + val json = Json.parse(responseBody) + // Extract the embedding array (first element since it's an array of arrays) + val embeddings = (json \ "embeddings" \ "float")(0).as[List[Double]] + logger.info( + logMarker, + s"Successfully created image embedding. Vector size: ${embeddings.size}" + ) + embeddings + }.map { embeddings => + Embedding( + cohereEmbedV4 = Some(CohereV4Embedding(embeddings)) + ) + } + } + + override def embeddingSourceImageFormat(): EmbeddingSourceImageFormat = EmbeddingSourceImageFormat(longestAxis = 3000, format = Jpeg, letterBox = false) + } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/ContentDisposition.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/ContentDisposition.scala index da2a73ad96a..796a1fae85d 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/ContentDisposition.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/ContentDisposition.scala @@ -19,14 +19,16 @@ trait ContentDisposition extends GridLogging { getContentDisposition(filename, fallbackLatin1Filename(image, extension)) } - def getContentDisposition(image: Image, crop: Crop, asset: Asset, shortenDownloadFilename: Boolean): String = { - val cropId: String = crop.id.map(id => s"($id)").getOrElse("") + def getContentDisposition(image: Image, crop: Crop, asset: Asset): String = { val extension: String = getExtension(image, asset) - val dimensions: String = asset.dimensions.map(dims => s"(${dims.width} x ${dims.height})").getOrElse("") - val filenameSuffix: String = s"(${image.id})$cropId$dimensions$extension" - val filename = getBaseFilename(image, filenameSuffix, shortenDownloadFilename) + val filename = image.uploadInfo.filename match { + case Some(filename) => filename + case _ => image.id + } - getContentDisposition(filename, fallbackLatin1Filename(image, extension)) + // Drop original file's extension and replace with the crops actual extension + val withCropsExtension = removeExtension(filename) + extension + getContentDisposition(withCropsExtension, fallbackLatin1Filename(image, extension)) } private def getExtension(image: Image, asset: Asset): String = asset.mimeType match { diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala index efe5a287eca..198186df003 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala @@ -1,33 +1,38 @@ package com.gu.mediaservice.lib.aws +import com.gu.mediaservice.lib.embeddings.{EmbeddingImplementation, EmbeddingSourceImageFormat} import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} -import com.gu.mediaservice.model.{Jpeg, MimeType, Png, Tiff} +import com.gu.mediaservice.model.{Embedding, ImageMetadata, MimeType} import play.api.libs.json.{Json, OFormat} -import software.amazon.awssdk.services.s3vectors.model.QueryVectorsResponse -import software.amazon.awssdk.services.s3vectors.model.{QueryOutputVector, QueryVectorsResponse, VectorData} import software.amazon.awssdk.services.sqs.model.SendMessageResponse -import java.nio.file.{Files, Path} import scala.concurrent.{ExecutionContext, Future} -import scala.jdk.CollectionConverters.CollectionHasAsScala -case class EmbedderMessage(imageId: String, fileType: String, s3Bucket: String, s3Key: String, instance: String) +case class EmbedderMessage(imageId: String, s3Bucket: String, s3Key: String, instance: String, imageMetadata: Option[ImageMetadata]) object EmbedderMessage { implicit val format: OFormat[EmbedderMessage] = Json.format[EmbedderMessage] } -class Embedder(bedrock: Bedrock, sqs: SimpleSqsMessageConsumer)(implicit ec: ExecutionContext) extends GridLogging { +class Embedder(embedding: EmbeddingImplementation, sqs: SimpleSqsMessageConsumer)(implicit ec: ExecutionContext) extends GridLogging { def createQueryEmbedding(query: String)(implicit logMarker: LogMarker): Future[List[Double]] = { logger.info(logMarker, s"Creating text embedding for query: $query") for { - embedding <- bedrock.createTextEmbedding(query) + embedding <- embedding.createTextEmbedding(query) } yield embedding } - def queueImageToEmbed(message: EmbedderMessage)(implicit logMarker: LogMarker) = { + def createImageEmbedding(source: Array[Byte], mimeType: MimeType, maybeMetadata: Option[ImageMetadata])(implicit logMarker: LogMarker): Future[Embedding] = { + logger.info(logMarker, s"Creating image embedding") + embedding.createImageEmbeddings(source, mimeType, maybeMetadata) + } + + def queueImageToEmbed(message: EmbedderMessage)(implicit logMarker: LogMarker): Unit = { val messageBody = Json.stringify(Json.toJson(message)) val result: SendMessageResponse = sqs.sendMessage(messageBody) logger.info(logMarker, s"Queued image for embedding with message ID: ${result.messageId()}") } + + def embeddingSourceImageFormat(): EmbeddingSourceImageFormat = embedding.embeddingSourceImageFormat() + } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Kinesis.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Kinesis.scala index 3f86055d4e5..c82bd4cd96f 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Kinesis.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Kinesis.scala @@ -1,22 +1,22 @@ package com.gu.mediaservice.lib.aws -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider -import software.amazon.awssdk.core.SdkBytes -import software.amazon.awssdk.regions.Region - -import java.nio.ByteBuffer -import java.util.UUID -import software.amazon.awssdk.services.kinesis.model.PutRecordRequest -import software.amazon.awssdk.services.kinesis.KinesisClient import com.gu.mediaservice.lib.json.JsonByteArrayUtil -import com.gu.mediaservice.model.usage.UsageNotice -import net.logstash.logback.marker.{LogstashMarker, Markers} -import play.api.libs.json.{JodaWrites, Json, Writes} import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} import com.gu.mediaservice.model.Instance +import com.gu.mediaservice.model.usage.UsageNotice +import net.logstash.logback.marker.{LogstashMarker, Markers} import org.joda.time.DateTime +import play.api.libs.json.{JodaWrites, Json, Writes} +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider +import software.amazon.awssdk.core.SdkBytes +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.kinesis.KinesisClient +import software.amazon.awssdk.services.kinesis.model.{PutRecordRequest, PutRecordsRequest, PutRecordsRequestEntry} import java.net.URI +import java.nio.ByteBuffer +import java.util.UUID +import scala.jdk.CollectionConverters.SeqHasAsJava case class KinesisSenderConfig( override val awsRegion: Region, @@ -61,5 +61,38 @@ class Kinesis(config: KinesisSenderConfig) extends GridLogging{ throw e } } + + def publish[T <: LogMarker](messages: Seq[T])(implicit messageWrites: Writes[T]): Unit = { + implicit val yourJodaDateWrites: Writes[DateTime] = JodaWrites.JodaDateTimeWrites + implicit val iw: Writes[Instance] = Json.writes[Instance] + implicit val unw: Writes[UsageNotice] = Json.writes[UsageNotice] + + val records: Seq[PutRecordsRequestEntry] = messages.map { message => + val payload = JsonByteArrayUtil.toByteArray(message) + + PutRecordsRequestEntry.builder() + .partitionKey(UUID.randomUUID().toString) + .data(SdkBytes.fromByteArray(payload)) + .build() + } + + logger.info(s"Publishing ${messages.size} messages to kinesis: ${config.streamName}") + + val request = PutRecordsRequest.builder() + .streamName(config.streamName) + .records(records.asJava) + .build() + + try { + val result = kinesisClient.putRecords(request) + logger.info(s"Published kinesis message: $result") + } catch { + case e: Exception => + logger.error(s"kinesis putRecords failed", e) + // propagate error forward to the client + throw e + } + } + } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index a0ec134d539..9891030b1b1 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -283,8 +283,24 @@ object S3Ops extends GridLogging { val s3Endpoint = "s3.amazonaws.com" def buildS3Client(config: CommonConfig, endpointOverride: Option[String] = None, usesPathStyleURLs: Boolean = false, maybeRegionOverride: Option[Region] = None): S3Client = { + val credentials = endpointOverride.flatMap { endpoint => + // Use static credentials if the end point is overridden to a non AWS provider + config.googleS3AccessKey.flatMap { accessKey => + config.googleS3SecretKey.map { secretKey => + // create credentials provider + logger.info("Usign static credentials for custom endpoint: " + endpoint) + import software.amazon.awssdk.auth.credentials.{AwsBasicCredentials, StaticCredentialsProvider} + val credentials = AwsBasicCredentials.create(accessKey, secretKey) + StaticCredentialsProvider.create(credentials) + + } + } + }.getOrElse{ + config.awsCredentials + } + val builder = S3Client.builder() - .credentialsProvider(config.awsCredentials) + .credentialsProvider(credentials) .region(maybeRegionOverride.getOrElse(config.awsRegion)) .forcePathStyle(usesPathStyleURLs) @@ -299,8 +315,24 @@ object S3Ops extends GridLogging { } def buildPresignerClientV2(config: CommonConfig, endpointOverride: Option[String] = None, usesPathStyleURLs: Boolean = false, maybeRegionOverride: Option[Region] = None): S3Presigner = { + val credentials = endpointOverride.flatMap { endpoint => + // Use static credentials if the end point is overridden to a non AWS provider + config.googleS3AccessKey.flatMap { accessKey => + config.googleS3SecretKey.map { secretKey => + // create credentials provider + logger.info("Usign static credentials for custom endpoint: " + endpoint) + import software.amazon.awssdk.auth.credentials.{AwsBasicCredentials, StaticCredentialsProvider} + val credentials = AwsBasicCredentials.create(accessKey, secretKey) + StaticCredentialsProvider.create(credentials) + + } + } + }.getOrElse{ + config.awsCredentials + } + val builder = S3Presigner.builder() - .credentialsProvider(config.awsCredentials) + .credentialsProvider(credentials) .region(maybeRegionOverride.getOrElse(config.awsRegion)) .serviceConfiguration(S3Configuration.builder() .pathStyleAccessEnabled(usesPathStyleURLs) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala index e3d82b53d55..0bd8dfa6838 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Bucket.scala @@ -1,6 +1,8 @@ package com.gu.mediaservice.lib.aws import com.gu.mediaservice.lib.config.CommonConfig +import com.typesafe.config.ConfigException +import play.api.Configuration import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3.presigner.S3Presigner @@ -52,4 +54,54 @@ object S3Bucket { client = S3Ops.buildS3Client(config, endpointOverride, usesPathStyleURLs, maybeRegionOverride), presigner = S3Ops.buildPresignerClientV2(config, endpointOverride, usesPathStyleURLs, maybeRegionOverride) ) + + /** + * Build a bucket from a config block of the form: + * {{{ + * s3.image.bucket { + * name = "media-service-image-bucket" # required + * endpoint = "s3.eu-west-1.amazonaws.com" # optional + * pathStyleUrls = false # optional, defaults to `endpoint` being set + * region = "eu-west-1" # optional + * } + * }}} + * where `bucketConfigPath` is the path to the block, e.g. "s3.image.bucket". + * + * When `endpoint` is absent the bucket falls back to the localstack endpoint in DEV and the + * real AWS S3 endpoint otherwise (the same behaviour as `S3Bucket(name, config)`). + */ + def fromConfig(bucketConfigPath: String, config: CommonConfig): S3Bucket = + fromConfig(config.configuration, bucketConfigPath, config) + + def fromConfigOpt(bucketConfigPath: String, config: CommonConfig): Option[S3Bucket] = + fromConfigOpt(config.configuration, bucketConfigPath, config) + + /** As `fromConfig` but reading the block from an arbitrary (possibly scoped) `Configuration` + * - e.g. an auth provider's own config block. */ + def fromConfig(source: Configuration, bucketConfigPath: String, config: CommonConfig): S3Bucket = + fromConfigOpt(source, bucketConfigPath, config).getOrElse( + throw new IllegalStateException(s"Missing required S3 bucket config: '$bucketConfigPath.name'") + ) + + def fromConfigOpt(source: Configuration, bucketConfigPath: String, config: CommonConfig): Option[S3Bucket] = { + val maybeName = try { + source.getOptional[String](s"$bucketConfigPath.name") + } catch { + case e: ConfigException.WrongType => + throw new IllegalStateException( + s"S3 bucket config '$bucketConfigPath' must be an object with a 'name' key, " + + s"""e.g. `$bucketConfigPath { name = "my-bucket" }`. """ + + s"""The legacy string form `$bucketConfigPath = "my-bucket"` is no longer supported.""", + e + ) + } + maybeName.map { name => + val endpointOverride = source.getOptional[String](s"$bucketConfigPath.endpoint").filter(_.nonEmpty) + .orElse(config.awsLocalEndpoint) + val usesPathStyleURLs = source.getOptional[Boolean](s"$bucketConfigPath.pathStyleUrls") + .getOrElse(endpointOverride.isDefined) + val maybeRegionOverride = source.getOptional[String](s"$bucketConfigPath.region").filter(_.nonEmpty).map(Region.of) + apply(name, config, endpointOverride, usesPathStyleURLs, maybeRegionOverride) + } + } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala deleted file mode 100644 index 5c8a0922510..00000000000 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala +++ /dev/null @@ -1,143 +0,0 @@ -package com.gu.mediaservice.lib.aws -import com.gu.mediaservice.lib.config.CommonConfig -import com.gu.mediaservice.lib.logging.LogMarker -import software.amazon.awssdk.regions.Region -import software.amazon.awssdk.services.s3vectors._ -import software.amazon.awssdk.services.s3vectors.model.{ - DeleteVectorsRequest, DeleteVectorsResponse, GetOutputVector, GetVectorsRequest, PutInputVector, PutVectorsRequest, - PutVectorsResponse, VectorData, QueryVectorsRequest, QueryVectorsResponse -} - -import java.net.URI -import scala.concurrent.{ExecutionContext, Future} -import scala.jdk.CollectionConverters._ -import scala.util.Try - - -object S3Vectors { - object DeletionStatus extends Enumeration { - val deleted: Value = Value("deleted") - val notFound: Value = Value("not found") - val failed: Value = Value("failed to delete") - - def fromBeforeAndAfter(key: String, vectorsBefore: Set[String], vectorsAfter: Set[String]): DeletionStatus.Value = - if (!vectorsBefore.contains(key)) DeletionStatus.notFound - else if (vectorsAfter.contains(key)) DeletionStatus.failed - else DeletionStatus.deleted - } -} - -class S3Vectors(config: CommonConfig)(implicit ec: ExecutionContext) - extends AwsClientBuilderUtils { - - import S3Vectors.DeletionStatus - - // TODO: figure out what the more usual pattern for turning off localstack behaviour is - override def awsLocalEndpointUri: Option[URI] = None - - override def isDev: Boolean = config.isDev - - // The S3 Vector Store is not yet available in eu-west-1, so we are using eu-central-1 because it's closest to us. - override def awsRegion: Region = Region.EU_CENTRAL_1 - - val client: S3VectorsClient = { - withAWSCredentials(S3VectorsClient.builder()) - .build() - } - - private val vectorBucketName: String = s"image-embeddings-${config.stage.toLowerCase}" - private val indexName: String = "cohere-embed-english-v3" - - private def getVectors(keys: Set[String], returnData: Boolean, returnMetadata: Boolean): List[GetOutputVector] = - // GetVectors has a max of 100 - // https://docs.aws.amazon.com/AmazonS3/latest/API/API_S3VectorBuckets_GetVectors.html - keys.grouped(100).flatMap { batch => - val request = GetVectorsRequest.builder() - .indexName(indexName) - .returnData(returnData) - .returnMetadata(returnMetadata) - .vectorBucketName(vectorBucketName) - .keys(batch.asJavaCollection) - .build() - - val response = client.getVectors(request) - response.vectors().asScala.toList - }.toList - - private def deleteVectors(keys: Set[String]): DeleteVectorsResponse = { - val request = DeleteVectorsRequest.builder() - .indexName(indexName) - .vectorBucketName(vectorBucketName) - .keys(keys.asJavaCollection) - .build() - - client.deleteVectors(request) - } - - private def getExistingVectorKeys(keys: Set[String]): Set[String] = - getVectors(keys, returnData = false, returnMetadata = false).map(_.key).toSet - - private def deleteBatch(batch: Set[String], batchCount: String)(implicit logMarker: LogMarker): Map[String, DeletionStatus.Value] = { - // Currently AWS don't provide any information on which deletes succeeded or failed. - // It returns 200 even if none of the provided keys currently exist. - // So in order to tell what was actually deleted, we need to make GetVectors requests before & after. - val vectorsBefore = getExistingVectorKeys(batch) - - logger.info(logMarker, s"${vectorsBefore.size} vectors to delete from batch of ${batch.size} ($batchCount)") - - if (vectorsBefore.isEmpty) { - batch.map(_ -> DeletionStatus.notFound).toMap - } else { - try { - deleteVectors(batch) - } catch { - // Swallow this error. Because there is a low write throughput across Puts and Deletes - // (5 per second), we may get failures here (though hopefully mitigated by the SDK retry logic). - // By recovering at this granularity, we can still try and report exactly - // what did and didn't get deleted through the subsequent GetVectors call. - case e: Exception => - logger.error(logMarker, s"Exception during S3 Vector Store API call to delete batch of ${batch.size} vectors ($batchCount)", e) - } - val vectorsAfter = getExistingVectorKeys(batch) - - if (vectorsAfter.nonEmpty) { - logger.warn(logMarker, s"${vectorsAfter.size} of ${vectorsBefore.size} failed to delete ($batchCount)") - } - logger.info(logMarker, s"${vectorsBefore.size - vectorsAfter.size} vectors deleted from batch of ${batch.size} ($batchCount)") - - batch.map(key => key -> DeletionStatus.fromBeforeAndAfter(key, vectorsBefore, vectorsAfter)).toMap - } - } - - def deleteEmbeddings(imageIds: Set[String])(implicit logMarker: LogMarker): Future[Map[String, DeletionStatus.Value]] = Future { - try { - val startTime = System.currentTimeMillis() - // We can only delete 500 keys at once - // https://docs.aws.amazon.com/AmazonS3/latest/API/API_S3VectorBuckets_DeleteVectors.html - val batches = imageIds.grouped(500).toList - - val result = batches.zipWithIndex.flatMap { case (batch, i) => - val batchCount = s"batch ${i + 1} of ${batches.length}" - deleteBatch(batch, batchCount) - }.toMap - - val duration = System.currentTimeMillis() - startTime - val stats = DeletionStatus.values.map { deletionStatus => - val count = result.values.count(_ == deletionStatus) - s"${deletionStatus}: ${count}" - } - logger.info( - logMarker, - s"deleteEmbeddings took ${duration}ms for ${imageIds.size} images, ${stats.mkString(", ")}" - ) - result - } catch { - case e: Exception => - // Swallow this error and assume all images failed to delete, - // so we don't affect the rest of the reaping process and can still report the results. - logger.error("Unexpected exception when deleting embeddings", e) - imageIds.map(key => key -> DeletionStatus.failed).toMap - } - - } -} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/ThrallMessageSender.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/ThrallMessageSender.scala index e84bef4d164..fe9d2958cdc 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/ThrallMessageSender.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/ThrallMessageSender.scala @@ -16,7 +16,11 @@ class ThrallMessageSender(config: KinesisSenderConfig) { kinesis.publish(updateMessage)(UpdateMessage.writes) } - def publish(externalThrallMessage: ExternalThrallMessage) = { + def publish(updateMessages: Seq[UpdateMessage]): Unit = { + kinesis.publish(updateMessages)(UpdateMessage.writes) + } + + def publish(externalThrallMessage: ExternalThrallMessage): Unit = { kinesis.publish(externalThrallMessage) } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/collections/CollectionPaths.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/collections/CollectionPaths.scala new file mode 100644 index 00000000000..c9bb0584b5d --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/collections/CollectionPaths.scala @@ -0,0 +1,18 @@ +package com.gu.mediaservice.lib.collections + +trait CollectionPaths { + + private val delimiter = "/" + private val doublequotes = "\"" + + def stringToPath(s: String): List[String] = s.split(delimiter).toList + + def pathToString(path: Seq[String]): String = path.mkString(delimiter) + + def pathToPathId(path: Seq[String]): String = pathToString(path).toLowerCase + + + // We could use `ValidationNel`s here, but that's overkill + def isValidPathBit(s: String): Boolean = if (s.contains(delimiter) || s.contains(doublequotes)) false else true + +} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/collections/CollectionsManager.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/collections/CollectionsManager.scala index aad2db048ad..d8100351c0d 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/collections/CollectionsManager.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/collections/CollectionsManager.scala @@ -4,14 +4,10 @@ import com.gu.mediaservice.lib.net.URI.{encode, decode} import com.gu.mediaservice.model.Collection -object CollectionsManager { - val delimiter = "/" - val doublequotes = "\"" +object CollectionsManager extends CollectionPaths with CssColours { - def stringToPath(s: String) = s.split(delimiter).toList - def pathToString(path: List[String]) = path.mkString(delimiter) - def pathToPathId(path: List[String]) = pathToString(path).toLowerCase def pathToUri(path: List[String]) = pathToString(path.map(encode)) + def uriToPath(uri: String) = stringToPath(decode(uri)) def sortBy(c: Collection) = c.pathId @@ -41,22 +37,4 @@ object CollectionsManager { }} } - // We could use `ValidationNel`s here, but that's overkill - def isValidPathBit(s: String) = if (s.contains(delimiter) || s.contains(doublequotes)) false else true - - // These use Source swatches - private val collectionColours = Map( - "australia" -> "#185E36", - "culture" -> "#BB3B80", - "film & music" -> "#6B5840", - "g2" -> "#121212", - "guide" -> "#7D0068", - "observer" -> "#052962", - "sport" -> "#22874D", - "travel" -> "#041F4A" - ) - - private def getCollectionColour(s: String) = collectionColours.get(s) - - def getCssColour(path: List[String]): Option[String] = path.headOption.map(_.toLowerCase).flatMap(getCollectionColour) } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/collections/CssColours.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/collections/CssColours.scala new file mode 100644 index 00000000000..2c0216882b3 --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/collections/CssColours.scala @@ -0,0 +1,35 @@ +package com.gu.mediaservice.lib.collections + +trait CssColours extends CollectionPaths { + + // These use Source swatches + private val collectionColours = Map( + "home/biz & cash" -> "#c98a07", + "home/home news" -> "#022164", + "home/international" -> "#022164", + "home/ofm" -> "#f2327d", + "home/magazine" -> "#5b1e4a", + "home/sensemakers" -> "#3c9bf9", + "home/sport" -> "#00663b", + "home/supplements" -> "#008083", + "home" -> "#052962" + ) + + def getCssColour(path: List[String]): Option[String] = { + def forPath(depth: Int, default: Option[String]): Option[String] = { + if (depth > path.size) { + default + } else { + val pathId = pathToPathId(path.take(depth)) + collectionColours.get(pathId).map { colour => + forPath(depth + 1, Some(colour)) + }.getOrElse { + default + } + } + } + // recurse drop the path return the furthest leaf node + forPath(depth = 1, default = None) + } + +} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala index bf26b0b039a..0468e6e6ad5 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala @@ -60,15 +60,20 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientBui lazy val softDeletedMetadataTable: String = string("dynamo.table.softDelete.metadata") - val imageBucket: S3Bucket = S3Bucket(string("s3.image.bucket"), this) - val thumbnailBucket: S3Bucket = S3Bucket(string("s3.thumb.bucket"), this) - val imgPublishingBucket: S3Bucket = S3Bucket(string("publishing.image.bucket"), this) + val googleS3AccessKey: Option[String] = stringOpt("s3.accessKey") + val googleS3SecretKey: Option[String] = stringOpt("s3.secretKey") + + val imageBucket: S3Bucket = S3Bucket.fromConfig("s3.image.bucket", this) + val thumbnailBucket: S3Bucket = S3Bucket.fromConfig("s3.thumb.bucket", this) + val imgPublishingBucket: S3Bucket = S3Bucket.fromConfig("publishing.image.bucket", this) + val embeddingSourcesBucket: S3Bucket = S3Bucket.fromConfig("s3.embeddingSources.bucket", this) + val embeddingsBucket: S3Bucket = S3Bucket.fromConfig("s3.embeddings.bucket", this) val maybeIngestSqsQueueUrl: Option[String] = stringOpt("sqs.ingest.queue.url") - val maybeIngestBucket: Option[S3Bucket] = stringOpt("s3.ingest.bucket").map(S3Bucket(_, this)) - val maybeFailBucket: Option[S3Bucket] = stringOpt("s3.fail.bucket").map(S3Bucket(_, this)) + val maybeIngestBucket: Option[S3Bucket] = S3Bucket.fromConfigOpt("s3.ingest.bucket", this) + val maybeFailBucket: Option[S3Bucket] = S3Bucket.fromConfigOpt("s3.fail.bucket", this) - val maybeQuarantineBucket: Option[S3Bucket] = stringOpt("s3.quarantine.bucket").map(S3Bucket(_, this)) + val maybeQuarantineBucket: Option[S3Bucket] = S3Bucket.fromConfigOpt("s3.quarantine.bucket", this) val maybeBucketForUIUploads: Option[S3Bucket] = maybeQuarantineBucket orElse maybeIngestBucket diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/Services.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/Services.scala index 8a912f28181..6e4edf3cb65 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/Services.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/Services.scala @@ -47,7 +47,7 @@ protected class SingleHostServices(val domain: String) extends Services { override def loaderBaseUri(instance: Instance): String = vhostServiceName("image-loader", instance) - override def projectionBaseUri(instance: Instance): String = vhostServiceName("projection", instance) + override def projectionBaseUri(instance: Instance): String = vhostServiceName("image-loader", instance) override def cropperBaseUri(instance: Instance): String = vhostServiceName("cropper", instance) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchClient.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchClient.scala index afa97cfc8db..d8839eff11d 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchClient.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchClient.scala @@ -7,11 +7,14 @@ import com.sksamuel.elastic4s.http.JavaClient import com.sksamuel.elastic4s.requests.common.HealthStatus import com.sksamuel.elastic4s.requests.indexes.CreateIndexResponse import com.sksamuel.elastic4s.requests.indexes.admin.IndexExistsResponse +import com.sksamuel.elastic4s.requests.searches.SearchHit import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration._ import scala.concurrent.{Await, Future} +case class ScrolledSearchResults(hits: List[SearchHit], scrollId: Option[String]) + case class ElasticSearchImageCounts( catCount: Long, searchResponseCount: Long, @@ -23,6 +26,7 @@ trait ElasticSearchClient extends ElasticSearchExecutions with GridLogging { private val tenSeconds = Duration(10, SECONDS) private val thirtySeconds = Duration(30, SECONDS) + protected val scrollKeepAlive = 5.minutes def url: String @@ -192,6 +196,20 @@ trait ElasticSearchClient extends ElasticSearchExecutions with GridLogging { logger.info("Got alias action response: " + aliasActionResponse) } + def continueScrolling(scrollId: String)(implicit logMarker: LogMarker = MarkerMap()) = { + val query = searchScroll(scrollId).keepAlive(scrollKeepAlive) + executeAndLog(query, "retrieving next batch of image ids to migrate, continuation of scroll").map { response => + ScrolledSearchResults(response.result.hits.hits.toList, response.result.scrollId) + } + } + + def closeScroll(scrollId: String)(implicit logMarker: LogMarker = MarkerMap()) = { + val close = clearScroll(scrollId) + executeAndLog(close, s"Closing unwanted scroll").failed.foreach { e => + logger.error(logMarker, "ES closeScroll request failed", e) + } + } + def removeAliasFrom(index: String, alias: String) = { logger.info(s"Removing alias $alias from $index") val removeAliasResponse = Await.result(client.execute { diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MappingTest.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MappingTest.scala index 95b063f4b5a..c1569539b2f 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MappingTest.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MappingTest.scala @@ -257,7 +257,8 @@ object MappingTest { embedding = Some( Embedding( cohereEmbedEnglishV3 = Some(CohereV3Embedding(image = (0 until 1024).map(_ * 0.001).toList)), - cohereEmbedV4 = Some(CohereV4Embedding(image = (0 until 256).map(_ * 0.001).toList)) + cohereEmbedV4 = Some(CohereV4Embedding(image = (0 until 256).map(_ * 0.001).toList)), + geminiEmbedding2 = Some(GeminiEmbedding2(image = (0 until 768).map(_ * 0.001).toList)) ) ) ) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/Mappings.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/Mappings.scala index 4472412a0a0..3e5f35d3458 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/Mappings.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/Mappings.scala @@ -105,6 +105,19 @@ object Mappings { efConstruction = Some(100) )) ) + )), + nonDynamicObjectField("geminiEmbedding2").copy(properties = Seq( + new DenseVectorField( + name = "image", + dims = Some(768), + index = Some(true), + similarity = Some(Cosine), + indexOptions = Some(DenseVectorIndexOptions( + `type` = DenseVectorField.Int8Hnsw, + m = Some(16), + efConstruction = Some(100) + )) + ) )) )) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MigrationStatusProvider.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MigrationStatusProvider.scala index f0bfcb54c74..3080df636f4 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MigrationStatusProvider.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MigrationStatusProvider.scala @@ -78,7 +78,7 @@ trait MigrationStatusProvider { private val migrationStatusRefresher = scheduler.scheduleAtFixedRate( initialDelay = 0.seconds, - interval = 5.seconds + interval = 1.minutes ) { () => { val instances = Await.result(instancesClient.getInstances(), Duration(10, SECONDS)) instances.foreach(refreshMigrationStatus) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/embeddings/EmbeddingImplementation.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/embeddings/EmbeddingImplementation.scala new file mode 100644 index 00000000000..6b23f11534f --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/embeddings/EmbeddingImplementation.scala @@ -0,0 +1,14 @@ +package com.gu.mediaservice.lib.embeddings + +import com.gu.mediaservice.lib.logging.LogMarker +import com.gu.mediaservice.model.{Embedding, ImageMetadata, Jpeg, MimeType} + +import scala.concurrent.{ExecutionContext, Future} + +trait EmbeddingImplementation { + def createImageEmbeddings(source: Array[Byte], mimeType: MimeType, maybeMetadata: Option[ImageMetadata])(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Embedding] + def createTextEmbedding(query: String)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[List[Double]] + def embeddingSourceImageFormat(): EmbeddingSourceImageFormat +} + +case class EmbeddingSourceImageFormat(longestAxis: Int, format: MimeType = Jpeg, letterBox: Boolean) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/embeddings/GoogleCloudEmbedding.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/embeddings/GoogleCloudEmbedding.scala new file mode 100644 index 00000000000..23ce934ae57 --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/embeddings/GoogleCloudEmbedding.scala @@ -0,0 +1,62 @@ +package com.gu.mediaservice.lib.embeddings + +import com.google.genai.Client +import com.google.genai.types._ +import com.gu.mediaservice.lib.logging.LogMarker +import com.gu.mediaservice.model.{Embedding, GeminiEmbedding2, ImageMetadata, Jpeg, MimeType} + +import scala.compat.java8.OptionConverters.RichOptionalGeneric +import scala.concurrent.{ExecutionContext, Future} +import scala.jdk.CollectionConverters._ + +class GoogleCloudEmbedding(projectId: String, location: String) extends EmbeddingImplementation { + private val client = Client.builder().vertexAI(true).project(projectId).location(location).build() + + private val modelId = "gemini-embedding-2" + + private val indexingEmbedContentConfig = EmbedContentConfig.builder() + .outputDimensionality(768) + .taskType("RETRIEVAL_DOCUMENT") + .build() + + private val queryEmbedContentConfig = EmbedContentConfig.builder() + .outputDimensionality(768) + .taskType("RETRIEVAL_QUERY") + .build() + + def createImageEmbeddings(source: Array[Byte], mimeType: MimeType, maybeMetadata: Option[ImageMetadata])(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Embedding] = { + Future { + val imagePart = Some(Part.fromBytes(source, mimeType.name)) + val titlePart = maybeMetadata.flatMap(_.title.map(Part.fromText)) + val descriptionPart = maybeMetadata.flatMap(_.description.map(Part.fromText)) + + val parts = List(imagePart, titlePart, descriptionPart).flatten.asJava + + val content = Content.builder(). + parts(parts). + build() + + val response = client.models.embedContent(modelId, content, indexingEmbedContentConfig) + + val embeddings = firstEmbeddingFromResponse(response) + Embedding(geminiEmbedding2 = Some(GeminiEmbedding2(embeddings.map(_.toDouble)))) + } + } + + def createTextEmbedding(query: String)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[List[Double]] = { + Future { + val q = query + val response = client.models.embedContent(modelId, q, queryEmbedContentConfig) + firstEmbeddingFromResponse(response) + } + } + + def embeddingSourceImageFormat(): EmbeddingSourceImageFormat = EmbeddingSourceImageFormat(longestAxis = 1200, format = Jpeg, letterBox = false) + + private def firstEmbeddingFromResponse(response: EmbedContentResponse): List[Double] = { + val a: Seq[ContentEmbedding] = response.embeddings().asScala.map(_.asScala.toSeq).getOrElse(Seq.empty) + val v = a.head.values().asScala.map(_.asScala).getOrElse(Seq.empty).toSeq + v.map(_.doubleValue()).toList + } + +} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/guardian/GuardianUsageRightsConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/guardian/GuardianUsageRightsConfig.scala index e5fe3318a3e..2a3aa1b1a07 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/guardian/GuardianUsageRightsConfig.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/guardian/GuardianUsageRightsConfig.scala @@ -129,7 +129,6 @@ object GuardianUsageRightsConfig extends UsageRightsConfigProvider { PublicationPhotographer("Steve Bell"), )), PublicationPhotographers(ObserverPublication, List( - PublicationPhotographer("Chris Riddell"), PublicationPhotographer("David Foldvari"), PublicationPhotographer("David Simonds"), )) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/imaging/ImageOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/imaging/ImageOperations.scala index e1c5d5b6e3d..8f58de48e05 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/imaging/ImageOperations.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/imaging/ImageOperations.scala @@ -1,11 +1,12 @@ package com.gu.mediaservice.lib.imaging -import app.photofox.vipsffm.enums.{VipsIntent, VipsInterpretation} +import app.photofox.vipsffm.enums.{VipsCompassDirection, VipsIntent, VipsInterpretation} import app.photofox.vipsffm.jextract.VipsRaw import app.photofox.vipsffm.{VBlob, VImage, VipsHelper, VipsOption} import com.adobe.internal.xmp.options.SerializeOptions import com.adobe.internal.xmp.{XMPConst, XMPMetaFactory} import com.gu.mediaservice.lib.BrowserViewableImage +import com.gu.mediaservice.lib.embeddings.EmbeddingSourceImageFormat import com.gu.mediaservice.lib.imaging.ImageOperations.thumbMimeType import com.gu.mediaservice.lib.imaging.im4jwrapper.ImageMagick import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch, addLogMarkers} @@ -213,6 +214,70 @@ class ImageOperations(playPath: String) extends GridLogging { } } + // Given an original image return a rendering of it which + // can be ingested by an embedding prediction end point. + def createEmbeddingSource(originalImageFile: File, + orientationMetadata: Option[OrientationMetadata], + embeddingSourceImageFormat: EmbeddingSourceImageFormat, + outputFile: File + ): Future[File] = { + Future { + val arena = Arena.ofConfined + + val embeddingLongestAxis = embeddingSourceImageFormat.longestAxis + val embeddingFormat = embeddingSourceImageFormat.format + + try { + val thumbnail = VImage.thumbnail(arena, originalImageFile.getAbsolutePath, embeddingLongestAxis, + VipsOption.Boolean("auto-rotate", false), + VipsOption.Enum("intent", VipsIntent.INTENT_PERCEPTUAL), + VipsOption.String("export-profile", "srgb") + ) + + val inMemoryCopy = VImage.newFromMemory(arena, thumbnail.writeToMemory(), + thumbnail.getWidth, thumbnail.getHeight, + VipsHelper.image_get_bands(thumbnail.getUnsafeStructAddress), + VipsHelper.image_get_format(thumbnail.getUnsafeStructAddress) + ) + + val rotated = orientationMetadata.map(_.orientationCorrection()).map { angle => + logger.info("Rotating thumbnail: " + angle) + inMemoryCopy.rotate(angle) + }.getOrElse { + inMemoryCopy + } + logger.info("Created embedding source: " + rotated.getWidth + "x" + rotated.getHeight) + + // Letter box to preserve aspect ratio of subjects + val letterBoxed = if (embeddingSourceImageFormat.letterBox) { + rotated.gravity( + VipsCompassDirection.COMPASS_DIRECTION_CENTRE, + embeddingLongestAxis, + embeddingLongestAxis, + ) + } else { + rotated + } + + saveImageToFile(letterBoxed, embeddingFormat, 90, outputFile) + arena.close() + + logger.info("Created embedding source with length: " + outputFile.length()) + outputFile + + } catch { + case e: Throwable => + arena.close() + throw e + } + + }.recoverWith { + case e: Throwable => + logger.error("Error creating embedding source", e) + Future.failed(e) + } + } + def saveImageToFile(image: VImage, mimeType: MimeType, quality: Int, outputFile: File, quantise: Boolean = false, keep: Option[Int] = None): File = { val k = keep.getOrElse(VipsRaw.VIPS_FOREIGN_KEEP_NONE) mimeType match { diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/tortoise/TortoiseUsageRightsConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/tortoise/TortoiseUsageRightsConfig.scala new file mode 100644 index 00000000000..e3d17043ba0 --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/tortoise/TortoiseUsageRightsConfig.scala @@ -0,0 +1,555 @@ +package com.gu.mediaservice.lib.tortoise + +import com.gu.mediaservice.lib.config.{PublicationPhotographer, PublicationPhotographers, UsageRightsConfigProvider} +import org.joda.time.LocalDate + +object TortoiseUsageRightsConfig extends UsageRightsConfigProvider { + private val ObserverPublication = "The Observer" + + val externalStaffPhotographers: List[PublicationPhotographers] = List( + PublicationPhotographers(ObserverPublication, List( + )) + ) + + // these are people who aren't photographers by trade, but have taken photographs for us. + // This is mainly used so when we ingest photos from Picdar, we make sure we categorise + // them correctly. + // TODO: Think about removin these once Picdar is dead. + val internalStaffPhotographers = List( + PublicationPhotographers(ObserverPublication, List( + )) + ) + + val contractedPhotographers = List( + PublicationPhotographers(ObserverPublication, List( + PublicationPhotographer("Andy Hall", from = Some(LocalDate.parse("2025-04-22"))), + PublicationPhotographer("Gary Calton", from = Some(LocalDate.parse("2025-04-22"))), + PublicationPhotographer("Suki Dhanda", from = Some(LocalDate.parse("2025-04-22"))), + PublicationPhotographer("Richard Saker", from = Some(LocalDate.parse("2025-04-22"))), + PublicationPhotographer("Karen Robinson", from = Some(LocalDate.parse("2025-04-22"))), + PublicationPhotographer("Sophia Evans", from = Some(LocalDate.parse("2025-04-22"))), + PublicationPhotographer("Katherine Anne Rose", from = Some(LocalDate.parse("2025-04-22"))), + PublicationPhotographer("Antonio Olmos", from = Some(LocalDate.parse("2025-04-22"))), + PublicationPhotographer("Jonathan Lovekin", from = Some(LocalDate.parse("2025-04-22"))) + )) + ) + + val staffIllustrators = List( + ) + + val contractIllustrators = List( + PublicationPhotographers(ObserverPublication, List( + PublicationPhotographer("Chris Riddell", from = Some(LocalDate.parse("2025-04-22"))) + )) + ) + + val creativeCommonsLicense = List( + "CC BY-4.0", "CC BY-SA-4.0", "CC BY-ND-4.0" + ) + + /* These are currently hardcoded */ + val payGettySourceList = List( + "ABC News", + "AFPTV", + "Alinari", + "Arnold Newman Collection", + "Baim Collection", + "Barrett-Jackson", + "Bob Thomas Sports Photography", + "Catwalking", + "CBS Television Stations Group RR", + "Contour", + "Contour RA", + "Corbis Premium Historical", + "Editorial Specials", + "Ercole Colombo", + "Extreme E", + "First Freedom", + "Formula E", + "Gamma-Legends", + "Getty Images Sport Classic", + "Icon Sport", + "J.LEAGUE", + "KBC - Japan", + "Klaud9", + "Kyodo News", + "Kyodo News Stills", + "LAT Image", + "Lichfield Studios Limited", + "Lonely Planet RF", + "Maggi & Maggi", + "Major League Baseball Platinum", + "Manchester City FC", + "Mondadori Portfolio Premium", + "NBA Classic", + "NBC News Archives Clips", + "Neil Leifer Collection", + "Newspix", + "NHK Video Bank Creative", + "NHK Video Bank Editorial", + "NHK Video Bank Premium", + "PA Images", + "Papixs", + "Paris Match Archive", + "Pele 10", + "Popperfoto", + "Premium Archive", + "Premium Archive Films Editorial", + "Rainer Schlegelmilch", + "Reportage Archive", + "SAMURAI JAPAN", + "SNS Group", + "Sports Illustrated", + "Sports Illustrated Classic", + "Storyful", + "Sutton Images", + "Sygma Premium", + "The Asahi Shimbun Premium", + "The Asahi Shimbun Video", + "Tottenham Hotspur FC", + "UEFA Exclusive", + "ullstein bild Premium", + "Ulrich Baumgarten", + "Vision Media", + // Here goes the list of inactive collections too + "#girlgaze", + "2VISTA", + "360cities.net Editorial", + "360cities.net RM", + "3D4Medical.com", + "3DClinic", + "40260 RF", + "ABC News", + "ABSODELS RM", + "ACP", + "Action Plus", + "Addictive Stock", + "Aflo Foto Agency RM", + "AFP Creative", + "age fotostock RM", + "Alaska Stock Images RF", + "Alaska Stock Images RR", + "Alaskan Express RF", + "All Canada Photos RM", + "Allsport Concepts", + "Altrendo", + "Altrendo RR", + "amana images RM", + "America 24-7", + "arabianEye RM", + "Arcaid Images", + "Arcaid RR", + "Arcangel Images RR", + "Archive Photos Creative", + "Arena Football League", + "Aridi", + "Art Images", + "Artville", + "ASAblanca", + "Asia Images RF", + "Asia Images RM", + "Astrakan", + "Aurora", + "Aurora Plus", + "Author's Image RF", + "AWL Images RM", + "Axiom Photographic Agency", + "Barcroft", + "Barcroft Media", + "Bettmann Creative", + "bilderlounge RR", + "Biosphoto RM", + "Black Box", + "Blend Images RM", + "Blend Images RR", + "Bloomberg Creative Photos RM", + "Boost", + "Botanica", + "Broadway.com RM", + "Built.Images RF", + "BuzzFoto", + "Caiaimage", + "Canopy RM", + "Car Culture", + "Cavan Images RM", + "CBS Watch Magazine", + "CCN Images RR", + "CGIBackgrounds", + "Check Six", + "Chic Sketch Editorial", + "Chic Sketch RF", + "China Span RM", + "Christian Science Monitor", + "CI BuzzFoto", + "CI Europa Press", + "CI FilmMagic", + "CI FilmMagic, Inc", + "CI FM Europa Press", + "CI Getty Images Entertainment", + "CI Getty Images Sport", + "CI News Feature", + "CI WI Europa Press", + "CI WireImage", + "Citizen Stock RM", + "Clerkenwell", + "clipart.com", + "Code Red", + "Codex", + "Collection Mix Subjects RM", + "Collection Vogue Paris", + "Collegiate Images", + "Colorsport", + "Comstock Images", + "Conde Nast Collection Editorial", + "Conde Nast Collection RM", + "Conde Nast Lifestyle Collection", + "Construction Photography RF", + "Contour Style", + "Contour Style Creative", + "Corbis Documentary", + "Corbis Historical Creative", + "Corbis NX", + "Corbis RF", + "Corbis RM Stills", + "Cote", + "Country Music Hall of Fame and Museum", + "CSA Images RM", + "Cuboimages RM", + "Cultura Exclusive", + "Cultura RF", + "Cultura RM", + "Cusp RM", + "Custom Medical Stock Photo RF", + "Custom Medical Stock Photo RM", + "Cut and Deal RF", + "Da Vinci Codex Atlanticus", + "Daily Express", + "DAJ", + "DAJ RM", + "De Agostini RM", + "Denkou RF", + "DigitalGlobe", + "Discovery Channel Images RM", + "DK Stock", + "Dorling Kindersley", + "Eastphoto RF", + "Eastphoto RM", + "Ecoscene RR", + "El Universal", + "Emotive Images RF", + "ESTADÃO CONTEÚDO", + "Everyday Projects", + "Eye Ubiquitous RR", + "EyeEm", + "EyeEm Premium", + "EyeEm RM", + "Eyewire", + "EyeWire Other", + "F1online RM", + "Fame Flynet Stills", + "Fancy RF", + "Federugby", + "Fever Images RF", + "Feyenoord", + "Finanzen Verlag", + "First Light", + "Flickr Flash", + "Flickr Prime", + "Flickr State", + "Flirt RF", + "FM Europa Press", + "FogStock", + "Folio Images RF", + "Folio Images RM", + "FoodPix", + "FoodShapes RF", + "Fototrove", + "Fox Entertainment Group", + "Gallo Images", + "Gamma-Features", + "GAP Photos RM", + "Garden Picture Library RM", + "Genuine Japan Creative Stills", + "Genuine Japan Editorial Stills", + "George Steinmetz", + "Getty Images - NASCAR Partners", + "Getty Images Multimedia Footage", + "Getty Images Special Access", + "Global Cricket Ventures - BCCI", + "Globe Photos", + "Globo", + "Glow RM", + "Glowimages RM", + "GoGo Images RF", + "Golden Boy Promotions", + "GoodSalt RR", + "GraphEast RF", + "GraphEast RM", + "Gulf Images RM", + "Hemera", + "Hemis.fr RM", + "Her Og Nu", + "Hero Images", + "Hero Images Corbis", + "HillCreek Pictures RF", + "Historic Map Works", + "Hoberman Collection UK RR", + "Hola Images RM", + "Hoxton", + "I Love Images RF", + "Iconic Images", + "Iconica", + "Iconotec RF", + "Ikon Images", + "Illustration Works", + "Image Farm RF", + "Image Ideas RF", + "Image Partner Media", + "Image100", + "imageBROKER RM", + "ImageDJ RF", + "Imagemore", + "ImageRite RF", + "Images Bazaar", + "Images.com RF", + "imageshop RF", + "imagesouk", + "ImageState RF", + "ImageState RM", + "Imagezoo RM", + "ImaZinS RM", + "Index Stock Images RR", + "Indian Premier League", + "Ingram Publishing RF", + "Inmagineasia", + "InsideOutPix RF", + "Inspirestock RF", + "Interact Images", + "International Speedway Corp.", + "Iromaya RF", + "IS Stock RF", + "iStock Exclusive RF", + "iStock Main", + "iStock Signature", + "iStock Signature Plus", + "iStock Vectors Plus", + "ItaliaStock", + "JLPGA", + "John Warburton-Lee RR", + "Johner Images", + "Jon Arnold Images RF", + "JTB Photo RM", + "Juice Images RF", + "Juniors Bildarchiv RM", + "Kablonk RF", + "Kallista Images", + "Keith Levit Photography RF", + "Keystone RF", + "Kobal Collection", + "Las Vegas Stock RR", + "LAT", + "LatinContent RM", + "Laughing Stock RM", + "Lifesize", + "Link Image RM", + "London Stills RR", + "Lonely Planet Images", + "LOOK", + "LuckyPix RR", + "Luxy", + "Map Resources", + "Mary Evans Picture Library RM", + "Masterfile", + "Masters", + "mauritius images RM", + "Mayo Clinic Collection", + "MedioImages", + "Melba Photo Agency RF", + "Mike King", + "Minden Pictures II", + "Minden Pictures RM", + "Mint Images RM", + "Mise En Beaute RR", + "MLBPA - The Players Choice", + "Moment RM", + "Moment Select", + "Moment Unreleased", + "Mondadori Portfolio", + "National Geographic", + "National Geographic Magazines", + "National Geographic RF", + "Nativestock", + "Nature Picture Library", + "Neovision RM", + "Nettavisen", + "New York Cosmos", + "newstockimages RF", + "NFL", + "Nordic Life", + "Nordic Photos", + "NucleusMedicalArt.com RM", + "NYonAir", + "Oceans-Image RR", + "Offside Live", + "OJO Images RM", + "OJO Plus RF", + "Old Visuals RF", + "Olive Images RF", + "Open Door Images RF", + "Open Mike Productions", + "Oxford Scientific RM", + "Pacific Stock RM", + "PANAPRESS", + "PanoramaStock RF", + "Panoramic Images RM", + "Panoramic Images RR", + "Panther Media RF", + "Paris Match Collection", + "Passage RM", + "Perspectives", + "Peter Arnold", + "Photo Exchange Bank Germany", + "PhotoAlto Agency RM", + "Photodisc", + "Photographer's Choice", + "Photographer's Choice RR", + "Photolibrary RF", + "Photolibrary RM", + "Photonica", + "Photonica World", + "Photononstop RM", + "Phototake RM", + "Phovoir RF", + "Picture Press RM", + "Pixmann RF", + "Pixta", + "Popperfoto Creative", + "Popstar Pictures", + "Popular Science", + "Portsmouth FC", + "Premium Ent", + "Private Label", + "Publisher Mix RM", + "Queerstock", + "QuickImage RF", + "QuickImage RR", + "Radius Images RF", + "Rainer Schlegelmilch", + "Real Latino RF", + "Realistic Reflections", + "Red Cover RM", + "Redlink RM", + "Refinery29 RM", + "relaximages", + "Reportage by Getty Images", + "Retrofile", + "Reunion Images", + "Riser", + "Robert Harding World Imagery", + "RooM RM", + "SAKIstyle RM", + "SambaPhoto", + "Science Faction", + "Science Faction Jewels", + "Science Photo Library RM", + "Science Source", + "ScienceFoto RM", + "Scoopt", + "Sebun", + "simple stock shots RF", + "Sites & Photos", + "Smart.MAGNA RF", + "Snapwi.re", + "SodaStyle", + "SoFood Collection RF", + "Solus", + "Somos RF", + "Sony BMG Music Entertainment", + "SPL Creative RM", + "Sport Plus", + "Starface Image Collection", + "Stock Illustration RF", + "Stock Illustration Source", + "Stock4B", + "stockbrokerXtra RF", + "Stockbyte", + "Stockbyte Global", + "stockbyway RF", + "Stockdisc", + "StockFood Creative RM", + "StockFood Creative RR", + "StockImage", + "Stone", + "Studio Harcourt", + "SuperStock RF", + "SuperStock RM", + "Swimwear by Popstar", + "Tango Stock RM", + "TAO Images RM", + "TASS", + "Taxi", + "Taxi Japan RM", + "Televisa", + "Terry O'Neill", + "TF-Images", + "the Agency Collection", + "The Axel Springer Collection", + "The England Collection", + "The Gruner & Jahr Collection", + "The Image Bank", + "The LIFE Images Collection", + "The LIFE Picture Collection", + "The LIFE Premium Collection", + "The New York Post", + "The Stock Connection RR", + "The StockPile Collection RF", + "Thinkstock", + "Tim de Waele", + "Tohoku Colour Agency RM", + "TongRo Images RF", + "Topic Images", + "Triangle", + "Trond Tandberg", + "Twenty20 RF", + "Universal Images Group", + "Untitled X-Ray", + "UpperCut RF", + "Uppercut RM", + "Urban CGI RF", + "View Stock RM", + "VII", + "VII Premium", + "VisitBritain RF", + "VisitBritain RM", + "Visual China Group Video", + "Visual Language RF", + "Visuals Unlimited", + "Warner Bros. Entertainment", + "WaterFrame RM", + "Wembley National Stadium Ltd", + "West Ham United FC", + "Westend61 RM", + "WI Europa Press", + "WIN-Initiative RM", + "WNET Collection RF", + "Workbook Stock", + "World Kabbadi League", + "World Sport Group", + "Yann Arthus-Bertrand", + "Zefa RF", + "Zen Shui RF" + ) + + val freeSuppliers = List( + "Alamy", + "AP", + "Getty Images", + "GNM", + "PA" + ) + + val suppliersCollectionExcl = Map( + "Getty Images" -> payGettySourceList + ) + +} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/Embedding.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/Embedding.scala index 562ca0dc871..0e9bc300d2e 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/Embedding.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/Embedding.scala @@ -7,7 +7,8 @@ import play.api.libs.json.OFormat // We currently only write V4 to ES, see the Embedding type in image-embedder-lambda case class Embedding( cohereEmbedEnglishV3: Option[CohereV3Embedding] = None, - cohereEmbedV4: Option[CohereV4Embedding] = None + cohereEmbedV4: Option[CohereV4Embedding] = None, + geminiEmbedding2: Option[GeminiEmbedding2] = None, ) case class CohereV3Embedding( @@ -26,6 +27,14 @@ object CohereV4Embedding { implicit val format: OFormat[CohereV4Embedding] = Json.format[CohereV4Embedding] } +case class GeminiEmbedding2( + image: List[Double] +) + +object GeminiEmbedding2 { + implicit val format: OFormat[GeminiEmbedding2] = Json.format[GeminiEmbedding2] +} + object Embedding { implicit val format: OFormat[Embedding] = Json.format[Embedding] } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/Image.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/Image.scala index a55929cc979..79de137fc47 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/Image.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/Image.scala @@ -44,9 +44,13 @@ case class Image( def hasNonInferredRights: Boolean = !hasInferredSyndicationRightsOrNoRights def syndicationStatus: SyndicationStatus = { - val isRightsAcquired: Boolean = syndicationRights.exists(_.isRightsAcquired) + // TODO Deduplicate this with syndicationFilter + val isOwned: Boolean = usageRights match { + case _: Photographer => true + case _ => false + } - if (!isRightsAcquired) { + if (!isOwned) { UnsuitableForSyndication } else { val hasSyndicationUsage = usages.exists(_.platform == SyndicationUsage) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/MimeType.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/MimeType.scala index 13e527625f8..14df82982d7 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/MimeType.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/MimeType.scala @@ -10,6 +10,7 @@ sealed trait MimeType { case Jpeg => "image/jpeg" case Png => "image/png" case Tiff => "image/tiff" + case Heif => "image/heif" } def fileExtension: String = s".${name.split('/').reverse.head}" @@ -22,6 +23,8 @@ object MimeType extends GridLogging { case "image/jpeg" => Jpeg case "image/png" => Png case "image/tiff" => Tiff + case "image/heif" => Heif + case "image/heic" => Heif // Support crops created in the early years of Grid (~2016) which state mime type w/out an 'image/' prefix // TODO correct these values in a reindex @@ -51,3 +54,4 @@ object Jpeg extends MimeType { object Png extends MimeType object Tiff extends MimeType +object Heif extends MimeType diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/ThrallMessage.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/ThrallMessage.scala index 5a4344ef13f..434f51ef0fb 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/ThrallMessage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/ThrallMessage.scala @@ -90,6 +90,7 @@ object ExternalThrallMessage{ implicit val upsertFromProjectionMessage: OFormat[UpsertFromProjectionMessage] = Json.format[UpsertFromProjectionMessage] implicit val createInstanceMessage: OFormat[CreateInstanceMessage] = Json.format[CreateInstanceMessage] + implicit val reindexImageMessage: OFormat[ReindexImageMessage] = Json.format[ReindexImageMessage] implicit val writes: OWrites[ExternalThrallMessage] = Json.writes[ExternalThrallMessage] implicit val reads: Reads[ExternalThrallMessage] = Json.reads[ExternalThrallMessage] @@ -171,3 +172,5 @@ case class CompleteMigrationMessage(lastModified: DateTime, instance: Instance) } case class CreateInstanceMessage(id: String, lastModified: DateTime, instance: Instance) extends ExternalThrallMessage + +case class ReindexImageMessage(id: String, lastModified: DateTime, instance: Instance) extends ExternalThrallMessage diff --git a/common-lib/src/main/scala/com/gu/mediaservice/syntax/MessageSubjects.scala b/common-lib/src/main/scala/com/gu/mediaservice/syntax/MessageSubjects.scala index c40ff2de17b..b0a2f32c6ee 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/syntax/MessageSubjects.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/syntax/MessageSubjects.scala @@ -22,7 +22,7 @@ trait MessageSubjects { val UpdateImageSyndicationMetadata = "update-image-syndication-metadata" val UpdateImagePhotoshootMetadata = "update-image-photoshoot-metadata" val CreateInstance = "create-instance" - + val ReindexImage = "reindex-image" } object MessageSubjects extends MessageSubjects diff --git a/common-lib/src/test/resources/IMG_0128.HEIC b/common-lib/src/test/resources/IMG_0128.HEIC new file mode 100644 index 00000000000..a5fa30b48a5 Binary files /dev/null and b/common-lib/src/test/resources/IMG_0128.HEIC differ diff --git a/common-lib/src/test/resources/application.conf b/common-lib/src/test/resources/application.conf index 92e1d78ac76..414ce5a8a5a 100644 --- a/common-lib/src/test/resources/application.conf +++ b/common-lib/src/test/resources/application.conf @@ -3,9 +3,11 @@ grid.appName: "test" thrall.kinesis.stream.name: "not-used" thrall.kinesis.lowPriorityStream.name: "not-used" domain.root: "notused.example.com" -s3.image.bucket: "not-used" -s3.thumb.bucket: "not-used" -publishing.image.bucket: "not-used" +s3.image.bucket.name: "not-used" +s3.thumb.bucket.name: "not-used" +publishing.image.bucket.name: "not-used" +s3.embeddingSources.bucket.name: "not-used" +s3.embeddings.bucket.name: "non-used" image.processors = [ "com.gu.mediaservice.lib.cleanup.GuardianMetadataCleaners", diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/ContentDispositionTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/ContentDispositionTest.scala index 90b5142e8af..efccf8d7b59 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/ContentDispositionTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/ContentDispositionTest.scala @@ -25,17 +25,17 @@ class ContentDispositionTest extends AnyFunSuiteLike with ContentDisposition { header shouldBe """attachment; filename="abcdef1234567890.jpg"; filename*=UTF-8''%C2%A9House%20of%20Commons_240508_MU_PMQs-09_42668%20%28abcdef1234567890%29.jpg""" } - test("include crop id and dimensions in main crop asset filename") { + test("crop asset filename is the image uploaded filename with the correct file extension for the crop mime type") { val image = withFilename(MappingTest.testImage, "imagefilename.jpg") val crop = image.exports.head val cropAsset = crop.assets.head - val header = getContentDisposition(image, crop, cropAsset, shortenDownloadFilename = false) + val header = getContentDisposition(image, crop, cropAsset) // Latin1 fallback wants to remain simple header.contains("""filename="abcdef1234567890.jpg";""") shouldBe true val decoded = URLDecoder.decode(header, "UTF-8") - decoded shouldBe """attachment; filename="abcdef1234567890.jpg"; filename*=UTF-8''imagefilename (abcdef1234567890)(1234567890987654321)(1000 x 2000).jpg""" + decoded shouldBe """attachment; filename="abcdef1234567890.jpg"; filename*=UTF-8''imagefilename.jpg""" } test("use just the generated filename suffix as filename if short filenames are requested") { diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/collections/CssColoursTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/collections/CssColoursTest.scala new file mode 100644 index 00000000000..314fd4dd191 --- /dev/null +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/collections/CssColoursTest.scala @@ -0,0 +1,25 @@ +package com.gu.mediaservice.lib.collections + +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers + +class CssColoursTest extends AnyFunSpec with Matchers with CssColours { + + describe("CssColours") { + describe("getCssColour") { + it("should return exact match for specific collections") { + getCssColour(List("Home", "Supplements")) shouldBe Some("#008083") + } + it("should default of none for collections with no specific colour preferences or parents with a colour") { + getCssColour(List("Unknown")) shouldBe None + } + it("should return colour of parent for collection with no specific colour") { + getCssColour(List("Home", "Something")) shouldBe Some("#052962") + } + it("should return colour of closet parent for collection with no specific colour") { + getCssColour(List("Home", "Supplements", "Something", "Something else")) shouldBe Some("#008083") + } + } + } + +} diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/imaging/ImageOperationsTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/imaging/ImageOperationsTest.scala index af39f652082..174d3d5c18a 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/imaging/ImageOperationsTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/imaging/ImageOperationsTest.scala @@ -3,8 +3,10 @@ package com.gu.mediaservice.lib.imaging import app.photofox.vipsffm.jextract.VipsRaw import app.photofox.vipsffm.{VImage, Vips} import com.gu.mediaservice.lib.BrowserViewableImage +import com.gu.mediaservice.lib.embeddings.EmbeddingSourceImageFormat import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} import com.gu.mediaservice.model._ +import org.apache.commons.io.FileUtils import org.scalatest.concurrent.ScalaFutures import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers @@ -13,7 +15,6 @@ import org.scalatest.time.{Millis, Span} import java.io.File import java.lang.foreign.Arena import scala.concurrent.ExecutionContext.Implicits.global -import scala.concurrent.Future // This test is disabled for now as it doesn't run on our CI environment, because GraphicsMagick is not present... class ImageOperationsTest extends AnyFunSpec with Matchers with ScalaFutures { @@ -102,6 +103,39 @@ class ImageOperationsTest extends AnyFunSpec with Matchers with ScalaFutures { r._1.isFile should be(true) } } + + it("render Heif correctly") { + val image = fileAt("IMG_0128.HEIC") + + val outputFile = new File("/Users/tony/Desktop/thumbnail-heic.jpg") + val browserViewableImageImage = BrowserViewableImage("TODO", image, Jpeg, Map.empty, false, Instance("TODO")) + + val eventualThumbnail = new ImageOperations("").createThumbnailVips(browserViewableImageImage, 1000, 95, outputFile, None) + whenReady(eventualThumbnail) { r => + r._1.isFile should be(true) + } + } + + } + + describe("embeddings") { + it("should produce embedding sources from original images") { + implicit val arena: Arena = Arena.ofShared() + val fullSizedImage = fileAt("exif-orientated.jpg") + val imageOperations = new ImageOperations("") + + val format = EmbeddingSourceImageFormat( + longestAxis = 2800, format = Jpeg, letterBox = true + ) + val outputFile = new File("/Users/tony/Desktop/embedding-source.jpg") + + val eventualEmbeddingSource = imageOperations.createEmbeddingSource(fullSizedImage, orientationMetadata = Some(OrientationMetadata(exifOrientation = Some(6))), embeddingSourceImageFormat = format, outputFile) + + whenReady(eventualEmbeddingSource) { embeddingSource => + arena.close() + outputFile.length > 100 should be(true) + } + } } describe("resize") { diff --git a/common-lib/src/test/scala/com/gu/mediaservice/model/ImageTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/model/ImageTest.scala index 31fc0b492dc..54ba392fff4 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/model/ImageTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/model/ImageTest.scala @@ -14,7 +14,9 @@ class ImageTest extends AnyFunSpec with Matchers { describe("Image syndication status") { it("should be UnsuitableForSyndication by default") { - val image = createImage() + val image = createImage( + usageRights = NoRights + ) image.usages.length shouldBe 0 image.syndicationRights shouldBe None @@ -23,23 +25,21 @@ class ImageTest extends AnyFunSpec with Matchers { image.syndicationStatus shouldBe UnsuitableForSyndication } - it("should be AwaitingReviewForSyndication if syndication rights are acquired") { - val image = createImage( - syndicationRights = Some(rightsAcquired) - ) + it("should be AwaitingReviewForSyndication if owned") { + val image = createImage() image.syndicationStatus shouldBe AwaitingReviewForSyndication } - it("should be UnsuitableForSyndication if syndication rights are not acquired") { + it("should be UnsuitableForSyndication if not owned") { val image = createImage( - syndicationRights = Some(noRightsAcquired) + usageRights = Agency("An agency") ) image.syndicationStatus shouldBe UnsuitableForSyndication } - it("should be UnsuitableForSyndication if there is no syndication rights") { + it("should be UnsuitableForSyndication if not owned even if has syndication usages") { val imageId = UUID.randomUUID().toString val usages = List( @@ -60,7 +60,8 @@ class ImageTest extends AnyFunSpec with Matchers { val image = createImage( id = imageId, usages = usages, - leases = Some(leaseByMedia) + leases = Some(leaseByMedia), + usageRights = NoRights ) image.syndicationStatus shouldBe UnsuitableForSyndication @@ -135,7 +136,7 @@ class ImageTest extends AnyFunSpec with Matchers { } object ImageTest { - def createImage(id: String = UUID.randomUUID().toString, usages: List[Usage] = List(), leases: Option[LeasesByMedia] = None, syndicationRights: Option[SyndicationRights] = None): Image = { + def createImage(id: String = UUID.randomUUID().toString, usages: List[Usage] = List(), leases: Option[LeasesByMedia] = None, syndicationRights: Option[SyndicationRights] = None, usageRights: UsageRights = StaffPhotographer("T. Hanks", "The Guardian")): Image = { Image( id = id, uploadTime = DateTime.now(), @@ -157,8 +158,8 @@ object ImageTest { userMetadata = None, metadata = ImageMetadata(dateTaken = None, title = Some(s"Test image $id"), keywords = None), originalMetadata = ImageMetadata(), - usageRights = StaffPhotographer("T. Hanks", "The Guardian"), - originalUsageRights = StaffPhotographer("T. Hanks", "The Guardian"), + usageRights = usageRights, + originalUsageRights = usageRights, exports = Nil, syndicationRights = syndicationRights, diff --git a/common-lib/src/test/scala/com/gu/mediaservice/testlib/ElasticSearchDockerBase.scala b/common-lib/src/test/scala/com/gu/mediaservice/testlib/ElasticSearchDockerBase.scala index 6717b14c222..4b380807f38 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/testlib/ElasticSearchDockerBase.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/testlib/ElasticSearchDockerBase.scala @@ -35,8 +35,9 @@ trait ElasticSearchDockerBase extends BeforeAndAfterAll { } } else None + val esHost = esContainer.map(_.getHost).getOrElse("localhost") val esPort = esContainer.map(_.getMappedPort(9200)).getOrElse(9200) - val esTestUrl = Properties.envOrElse("ES6_TEST_URL", s"http://localhost:$esPort") + val esTestUrl = Properties.envOrElse("ES6_TEST_URL", s"http://$esHost:$esPort") override protected def afterAll(): Unit = { super.afterAll() diff --git a/cropper/app/controllers/CropperController.scala b/cropper/app/controllers/CropperController.scala index 7c784f62320..4268a240506 100644 --- a/cropper/app/controllers/CropperController.scala +++ b/cropper/app/controllers/CropperController.scala @@ -103,6 +103,8 @@ class CropperController(auth: Authentication, crops: Crops, store: CropStore, no private val canDeleteCrops: PrincipalFilter = authorisation.hasPermissionTo(DeleteCropsOrUsages) + private def downloadExportMasterLink(imageId: String, exportId: String)(implicit instance: Instance) = Link(s"crop-download-$exportId-master", s"${config.apiUri(instance)}/images/$imageId/export/$exportId/master/download") + private def downloadExportLink(imageId: String, exportId: String, width: Int)(implicit instance: Instance) = Link(s"crop-download-$exportId-$width", s"${config.apiUri(instance)}/images/$imageId/export/$exportId/asset/$width/download") def getCrops(id: String) = auth.async { httpRequest => @@ -119,20 +121,27 @@ class CropperController(auth: Authentication, crops: Crops, store: CropStore, no val deleteCropsAction = ArgoAction("delete-crops", URI.create(s"${config.rootUri(instance)}/crops/$id"), "DELETE") - lazy val cropDownloadLinks = for { - crop <- crops - asset <- crop.assets - dimensions <- asset.dimensions - width = dimensions.width - cropId <- crop.id - } yield downloadExportLink(id, cropId, width) + lazy val cropDownloadAssetsLinks = + for { + crop <- crops + asset <- crop.assets + dimensions <- asset.dimensions + width = dimensions.width + cropId <- crop.id + } yield downloadExportLink(id, cropId, width) + + lazy val cropDownloadMasterLinks = + for { + crop <- crops + cropId <- crop.id + } yield downloadExportMasterLink(id, cropId) val links = (for { crop <- crops.headOption link = Link("image", crop.specification.uri) } yield { if (config.canDownloadCrop) { - link :: cropDownloadLinks + link :: cropDownloadAssetsLinks ++ cropDownloadMasterLinks } else List(link) }) getOrElse List() diff --git a/cropper/app/lib/AspectRatio.scala b/cropper/app/lib/AspectRatio.scala index b093f1b5875..28161afe301 100644 --- a/cropper/app/lib/AspectRatio.scala +++ b/cropper/app/lib/AspectRatio.scala @@ -9,7 +9,8 @@ object AspectRatio { Ratio("5:3", 5, 3), Ratio("2:3", 2, 3), Ratio("16:9", 16, 9), - Ratio("1:1", 1, 1) + Ratio("1:1", 1, 1), + Ratio("3:2", 3, 2) ) def clean(aspect: String): Option[Float] = knownRatios diff --git a/dev/script/generate-config/service-config.js b/dev/script/generate-config/service-config.js index 6e34288838c..667a4ca6131 100644 --- a/dev/script/generate-config/service-config.js +++ b/dev/script/generate-config/service-config.js @@ -32,7 +32,7 @@ function getCommonConfig(config) { const useLocalPermissions = !isNoAuthorisation && Boolean(config.coreStackProps.PermissionsBucket); return `domain.root="${config.DOMAIN}" - |authentication.providers.machine.config.authKeyStoreBucket="${config.coreStackProps.KeyBucket}" + |authentication.providers.machine.config.authKeyStoreBucket.name="${config.coreStackProps.KeyBucket}" |aws.local.endpoint="https://localstack.media.${config.DOMAIN}" |thrall.kinesis.stream.name="${config.coreStackProps.ThrallMessageStream}" |thrall.kinesis.lowPriorityStream.name="${config.coreStackProps.ThrallLowPriorityMessageStream}" @@ -48,8 +48,8 @@ function getCommonConfig(config) { ${useLocalPermissions ? '|auth.useLocal=true' : ''} ${useLocalPermissions ? `|permissions.bucket="${config.coreStackProps.PermissionsBucket}"` : ''} |sqs.ingest.queue.url="${config.coreStackProps.IngestSqsQueue.replace("http://localhost:4576", `https://localstack.media.${config.DOMAIN}`)}" - |s3.ingest.bucket="${config.coreStackProps.IngestQueueBucket}" - |s3.fail.bucket="${config.coreStackProps.IngestQueueFailBucket}" + |s3.ingest.bucket.name="${config.coreStackProps.IngestQueueBucket}" + |s3.fail.bucket.name="${config.coreStackProps.IngestQueueFailBucket}" |usageRightsConfigProvider.config.freeSuppliers=[ | "AAP", | "Alamy", @@ -104,7 +104,7 @@ function getCommonConfig(config) { function getAuthConfig(config) { return stripMargin`${getCommonConfig(config)} - |s3.config.bucket="${config.coreStackProps.ConfigBucket}" + |s3.config.bucket.name="${config.coreStackProps.ConfigBucket}" |aws.region="${config.AWS_DEFAULT_REGION}" |security.cors.allowedOrigins="${getCorsAllowedOriginString(config)}" |metrics.request.enabled=false @@ -114,7 +114,7 @@ function getAuthConfig(config) { function getCollectionsConfig(config) { return stripMargin`${getCommonConfig(config)} |aws.region="${config.AWS_DEFAULT_REGION}" - |s3.collections.bucket="${config.coreStackProps.CollectionsBucket}" + |s3.collections.bucket.name="${config.coreStackProps.CollectionsBucket}" |dynamo.table.collections="CollectionsTable" |dynamo.table.imageCollections="ImageCollectionsTable" |security.cors.allowedOrigins="${getCorsAllowedOriginString(config)}" @@ -125,9 +125,9 @@ function getCollectionsConfig(config) { function getCropperConfig(config) { return stripMargin`${getCommonConfig(config)} |aws.region="${config.AWS_DEFAULT_REGION}" - |publishing.image.bucket="${config.coreStackProps.ImageOriginBucket}" + |publishing.image.bucket.name="${config.coreStackProps.ImageOriginBucket}" |publishing.image.host="public.media.${config.DOMAIN}" - |s3.config.bucket="${config.coreStackProps.ConfigBucket}" + |s3.config.bucket.name="${config.coreStackProps.ConfigBucket}" |security.cors.allowedOrigins="${getCorsAllowedOriginString(config)}" |metrics.request.enabled=false |`; @@ -136,11 +136,11 @@ function getCropperConfig(config) { function getImageLoaderConfig(config) { return stripMargin`${getCommonConfig(config)} |aws.region="${config.AWS_DEFAULT_REGION}" - |s3.image.bucket="${config.coreStackProps.ImageBucket}" - |s3.thumb.bucket="${config.coreStackProps.ThumbBucket}" + |s3.image.bucket.name="${config.coreStackProps.ImageBucket}" + |s3.thumb.bucket.name="${config.coreStackProps.ThumbBucket}" |#uncomment to enable the quarantine bucket - |#s3.quarantine.bucket="${config.coreStackProps.QuarantineBucket}" - |s3.config.bucket="${config.coreStackProps.ConfigBucket}" + |#s3.quarantine.bucket.name="${config.coreStackProps.QuarantineBucket}" + |s3.config.bucket.name="${config.coreStackProps.ConfigBucket}" |dynamo.table.upload.status="UploadStatusTable" |aws.local.endpoint="https://localstack.media.${config.DOMAIN}" |security.cors.allowedOrigins="${getCorsAllowedOriginString(config)}" @@ -204,10 +204,10 @@ function getLeasesConfig(config) { function getMediaApiConfig(config) { return stripMargin`${getCommonConfig(config)} |aws.region="${config.AWS_DEFAULT_REGION}" - |s3.image.bucket="${config.coreStackProps.ImageBucket}" - |s3.thumb.bucket="${config.coreStackProps.ThumbBucket}" - |s3.config.bucket="${config.coreStackProps.ConfigBucket}" - |s3.usagemail.bucket="${config.coreStackProps.UsageMailBucket}" + |s3.image.bucket.name="${config.coreStackProps.ImageBucket}" + |s3.thumb.bucket.name="${config.coreStackProps.ThumbBucket}" + |s3.config.bucket.name="${config.coreStackProps.ConfigBucket}" + |s3.usagemail.bucket.name="${config.coreStackProps.UsageMailBucket}" |es6.url="${config.es6.url}" |es6.shards=${config.es6.shards} |es6.replicas=${config.es6.replicas} @@ -225,7 +225,7 @@ function getMediaApiConfig(config) { function getMetadataEditorConfig(config) { return stripMargin`${getCommonConfig(config)} |aws.region="${config.AWS_DEFAULT_REGION}" - |s3.collections.bucket="${config.coreStackProps.CollectionsBucket}" + |s3.collections.bucket.name="${config.coreStackProps.CollectionsBucket}" |dynamo.table.edits="EditsTable" |dynamo.globalsecondaryindex.edits.photoshoots="Photoshoots" |dynamo.table.syndication="SyndicationTable" @@ -238,9 +238,9 @@ function getMetadataEditorConfig(config) { function getThrallConfig(config) { return stripMargin`${getCommonConfig(config)} |aws.region="${config.AWS_DEFAULT_REGION}" - |s3.image.bucket="${config.coreStackProps.ImageBucket}" - |s3.thumb.bucket="${config.coreStackProps.ThumbBucket}" - |s3.reaper.bucket="${config.coreStackProps.ReaperBucket}" + |s3.image.bucket.name="${config.coreStackProps.ImageBucket}" + |s3.thumb.bucket.name="${config.coreStackProps.ThumbBucket}" + |s3.reaper.bucket.name="${config.coreStackProps.ReaperBucket}" |indexed.image.sns.topic.arn="${config.coreStackProps.IndexedImageTopic}" |es6.url="${config.es6.url}" |es6.shards=${config.es6.shards} diff --git a/docs/06-objects-of-interest/02-config.md b/docs/06-objects-of-interest/02-config.md index 7b8d78d4524..cf61118089d 100644 --- a/docs/06-objects-of-interest/02-config.md +++ b/docs/06-objects-of-interest/02-config.md @@ -23,6 +23,26 @@ Anything that's specific to an organisation and to a stage, but is common across #### `//.conf` Service-specific configs. These will override all other config files. +### S3 bucket configuration + +Each S3 bucket is configured as a config object rather than a bare string. The bucket name +lives under a `name` key, alongside optional keys for talking to a non-default endpoint: + +```hocon +s3.image.bucket { + name = "media-service-image-bucket" # required + endpoint = "s3.eu-west-1.amazonaws.com" # optional; defaults to the localstack endpoint in DEV, real S3 otherwise + pathStyleUrls = false # optional; defaults to `endpoint` being set + region = "eu-west-1" # optional; defaults to `aws.region` +} +``` + +This applies to every `*.bucket` key below (`s3.image.bucket`, `s3.thumb.bucket`, +`publishing.image.bucket`, `s3.config.bucket`, `s3.usagemail.bucket`, `s3.ingest.bucket`, +`s3.fail.bucket`, `s3.quarantine.bucket`, `s3.reaper.bucket`, `s3.embeddings.bucket`, +`s3.embeddingSources.bucket`, `s3.image.replicaBucket`) and to +`authentication.providers.machine.config.authKeyStoreBucket`. + ## Config documentation ### Common configuration @@ -121,7 +141,7 @@ Service-specific configs. These will override all other config files. - authentication.providers.machine.config.authKeyStoreBucket + authentication.providers.machine.config.authKeyStoreBucket.name string @@ -178,7 +198,7 @@ Service-specific configs. These will override all other config files. - authentication.providers.machine.config.authKeyStoreBucket + authentication.providers.machine.config.authKeyStoreBucket.name @@ -214,7 +234,7 @@ Service-specific configs. These will override all other config files. - s3.config.bucket + s3.config.bucket.name @@ -259,7 +279,7 @@ Service-specific configs. These will override all other config files. - authentication.providers.machine.config.authKeyStoreBucket + authentication.providers.machine.config.authKeyStoreBucket.name @@ -301,7 +321,7 @@ Service-specific configs. These will override all other config files. - s3.collections.bucket + s3.collections.bucket.name @@ -352,7 +372,7 @@ Service-specific configs. These will override all other config files. - authentication.providers.machine.config.authKeyStoreBucket + authentication.providers.machine.config.authKeyStoreBucket.name @@ -394,7 +414,7 @@ Service-specific configs. These will override all other config files. - publishing.image.bucket + publishing.image.bucket.name @@ -406,7 +426,7 @@ Service-specific configs. These will override all other config files. - s3.config.bucket + s3.config.bucket.name @@ -445,7 +465,7 @@ Service-specific configs. These will override all other config files. - authentication.providers.machine.config.authKeyStoreBucket + authentication.providers.machine.config.authKeyStoreBucket.name @@ -487,25 +507,25 @@ Service-specific configs. These will override all other config files. - s3.image.bucket + s3.image.bucket.name - s3.thumb.bucket + s3.thumb.bucket.name - s3.quarantine.bucket + s3.quarantine.bucket.name - s3.config.bucket + s3.config.bucket.name @@ -562,7 +582,7 @@ Service-specific configs. These will override all other config files. - authentication.providers.machine.config.authKeyStoreBucket + authentication.providers.machine.config.authKeyStoreBucket.name @@ -604,25 +624,25 @@ Service-specific configs. These will override all other config files. - s3.image.bucket + s3.image.bucket.name - s3.thumb.bucket + s3.thumb.bucket.name - s3.quarantine.bucket + s3.quarantine.bucket.name - s3.config.bucket + s3.config.bucket.name @@ -731,7 +751,7 @@ Service-specific configs. These will override all other config files. - authentication.providers.machine.config.authKeyStoreBucket + authentication.providers.machine.config.authKeyStoreBucket.name @@ -812,7 +832,7 @@ Service-specific configs. These will override all other config files. - authentication.providers.machine.config.authKeyStoreBucket + authentication.providers.machine.config.authKeyStoreBucket.name @@ -854,25 +874,25 @@ Service-specific configs. These will override all other config files. - s3.image.bucket + s3.image.bucket.name - s3.thumb.bucket + s3.thumb.bucket.name - s3.config.bucket + s3.config.bucket.name - s3.usagemail.bucket + s3.usagemail.bucket.name @@ -935,7 +955,7 @@ Service-specific configs. These will override all other config files. - authentication.providers.machine.config.authKeyStoreBucket + authentication.providers.machine.config.authKeyStoreBucket.name @@ -977,7 +997,7 @@ Service-specific configs. These will override all other config files. - s3.collections.bucket + s3.collections.bucket.name @@ -1040,7 +1060,7 @@ Service-specific configs. These will override all other config files. - authentication.providers.machine.config.authKeyStoreBucket + authentication.providers.machine.config.authKeyStoreBucket.name @@ -1088,13 +1108,13 @@ Service-specific configs. These will override all other config files. - s3.ingest.bucket + s3.ingest.bucket.name - s3.fail.bucket + s3.fail.bucket.name @@ -1121,7 +1141,7 @@ Service-specific configs. These will override all other config files. - authentication.providers.machine.config.authKeyStoreBucket + authentication.providers.machine.config.authKeyStoreBucket.name @@ -1163,13 +1183,13 @@ Service-specific configs. These will override all other config files. - s3.image.bucket + s3.image.bucket.name - s3.thumb.bucket + s3.thumb.bucket.name @@ -1237,7 +1257,7 @@ Service-specific configs. These will override all other config files. - authentication.providers.machine.config.authKeyStoreBucket + authentication.providers.machine.config.authKeyStoreBucket.name diff --git a/image-loader/app/ImageLoaderComponents.scala b/image-loader/app/ImageLoaderComponents.scala index 4cd7d5b61a6..503dbb70c5a 100644 --- a/image-loader/app/ImageLoaderComponents.scala +++ b/image-loader/app/ImageLoaderComponents.scala @@ -1,6 +1,7 @@ import app.photofox.vipsffm.{Vips, VipsHelper} import com.gu.mediaservice.GridClient -import com.gu.mediaservice.lib.aws.{Bedrock, Embedder, S3, SimpleSqsMessageConsumer} +import com.gu.mediaservice.lib.aws.{Embedder, S3, SimpleSqsMessageConsumer} +import com.gu.mediaservice.lib.embeddings.GoogleCloudEmbedding import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.GridLogging import com.gu.mediaservice.lib.play.GridComponents @@ -34,11 +35,24 @@ class ImageLoaderComponents(context: Context) extends GridComponents(context, ne val notifications = new Notifications(config) val downloader = new Downloader()(ec,wsClient) - val maybeEmbedder: Option[Embedder] = config.maybeImageEmbedderQueueUrl - .filter(_ => config.shouldEmbed) - .map {queueUrl => - new Embedder(new Bedrock(config), new SimpleSqsMessageConsumer(queueUrl, config)) - } + private val maybeGcpProjectId = config.gcpProjectId + private val vertexApiLocation = "eu" + private val maybeGoogleCloudEmbedding = for { + gcpProjectId <- maybeGcpProjectId + } yield { + new GoogleCloudEmbedding(projectId = gcpProjectId, location = vertexApiLocation) + } + + private val maybeEmbedding = maybeGoogleCloudEmbedding + + val maybeEmbedder: Option[Embedder] = for { + embedding <- maybeEmbedding + queueUrl <- config.maybeImageEmbedderQueueUrl.filter(_ => config.shouldEmbed) + } yield { + + logger.info("Image loader is configured to queue embedding requests to: " + queueUrl) + new Embedder(embedding, new SimpleSqsMessageConsumer(queueUrl, config)) + } private val s3 = new S3(config) diff --git a/image-loader/app/lib/ImageLoaderConfig.scala b/image-loader/app/lib/ImageLoaderConfig.scala index 26146326991..0afe18b96b1 100644 --- a/image-loader/app/lib/ImageLoaderConfig.scala +++ b/image-loader/app/lib/ImageLoaderConfig.scala @@ -1,6 +1,6 @@ package lib -import com.gu.mediaservice.lib.aws.{S3Bucket, S3Ops} +import com.gu.mediaservice.lib.aws.S3Bucket import java.io.File import com.gu.mediaservice.lib.cleanup.{ComposedImageProcessor, ImageProcessor, ImageProcessorResources} @@ -13,8 +13,14 @@ import software.amazon.awssdk.regions.Region import scala.concurrent.duration.FiniteDuration class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(resources) with StrictLogging { - val maybeImageReplicaBucket: Option[S3Bucket] = stringOpt("s3.image.replicaBucket").map{ replicaBucketName => - S3Bucket.apply(replicaBucketName, this, None, usesPathStyleURLs = false, maybeRegionOverride = Some(Region.US_WEST_1)) + val maybeImageReplicaBucket: Option[S3Bucket] = stringOpt("s3.image.replicaBucket.name").map { replicaBucketName => + S3Bucket.apply( + replicaBucketName, + this, + endpointOverride = stringOpt("s3.image.replicaBucket.endpoint").filter(_.nonEmpty), + usesPathStyleURLs = booleanOpt("s3.image.replicaBucket.pathStyleUrls").getOrElse(false), + maybeRegionOverride = stringOpt("s3.image.replicaBucket.region").filter(_.nonEmpty).map(Region.of).orElse(Some(Region.US_WEST_1)) + ) } val lowerEnvironmentSamplingPercentageAsDecimal = intOpt("s3.sampling.percentage").getOrElse(1) / 100.0 @@ -37,6 +43,8 @@ class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(res val shouldEmbed: Boolean = boolean("s3.vectors.shouldEmbed") val maybeImageEmbedderQueueUrl: Option[String] = stringOpt("sqs.image.embedder.queue.url") + val gcpProjectId: Option[String] = stringOpt("gcp.project.id") + /** * Load in the chain of image processors from config. This can be a list of * companion objects, class names, both with and without config. diff --git a/image-loader/app/lib/ImageLoaderStore.scala b/image-loader/app/lib/ImageLoaderStore.scala index 39fd800e9ce..2125400698d 100644 --- a/image-loader/app/lib/ImageLoaderStore.scala +++ b/image-loader/app/lib/ImageLoaderStore.scala @@ -16,7 +16,7 @@ import scala.jdk.CollectionConverters.MapHasAsJava class S3FileDoesNotExistException extends Exception() -class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperations(config.imageBucket, config.thumbnailBucket, config) with GridLogging { +class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperations(config.imageBucket, config.thumbnailBucket, config.embeddingSourcesBucket, config.embeddingsBucket, config) with GridLogging { private def handleNotFound[T](key: String)(doWork: => T)(loggingIfNotFound: => Unit): T = { try { @@ -77,5 +77,6 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati } { logger.warn(logMarker, s"Attempted to delete $key from ingest bucket, but it does not exist.") } + } diff --git a/image-loader/app/lib/imaging/FileMetadataReader.scala b/image-loader/app/lib/imaging/FileMetadataReader.scala index fbee2b77772..cf05fd14296 100644 --- a/image-loader/app/lib/imaging/FileMetadataReader.scala +++ b/image-loader/app/lib/imaging/FileMetadataReader.scala @@ -214,6 +214,12 @@ object FileMetadataReader extends GridLogging { "photometricInterpretation" -> photometricInterpretation, "bitsPerSample" -> getFromExifDirectory(ExifDirectoryBase.TAG_BITS_PER_SAMPLE).flatMap(extractBitsPerSample) ).flattenOptions + case Heif => + Map ( + "hasAlpha" -> hasAlpha, + "colorType" -> maybeImageType, + "photometricInterpretation" -> photometricInterpretation, + ).flattenOptions } } diff --git a/image-loader/app/model/Projector.scala b/image-loader/app/model/Projector.scala index 6dcc311ab10..cf9d6c6255a 100644 --- a/image-loader/app/model/Projector.scala +++ b/image-loader/app/model/Projector.scala @@ -1,16 +1,17 @@ package model +import _root_.play.api.libs.json._ import _root_.play.api.libs.ws.WSRequest -import com.gu.mediaservice.lib.ImageIngestOperations.{fileKeyFromId, optimisedPngKeyFromId} +import com.gu.mediaservice.lib.ImageIngestOperations.{embeddingKeyFromId, fileKeyFromId, optimisedPngKeyFromId} import com.gu.mediaservice.lib._ import com.gu.mediaservice.lib.auth.Authentication -import com.gu.mediaservice.lib.aws.{Embedder, S3, S3Bucket} +import com.gu.mediaservice.lib.aws.{Embedder, S3, S3Bucket, S3Object} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.config.InstanceForRequest import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch} import com.gu.mediaservice.lib.net.URI -import com.gu.mediaservice.model.{Image, Instance, MimeType, UploadInfo} +import com.gu.mediaservice.model.{Embedding, Image, Instance, MimeType, Png, UploadInfo} import com.gu.mediaservice.{GridClient, ImageDataMerger} import lib.imaging.{MimeTypeDetection, NoSuchImageExistsInS3} import lib.{DigestedFile, ImageLoaderConfig} @@ -135,8 +136,8 @@ class Projector(config: ImageUploadOpsCfg, val uploadInfo_ = UploadInfo(filename = extractedS3Meta.uploadFileName, isFeedUpload = extractedS3Meta.isFeedUpload) MimeTypeDetection.guessMimeType(tempFile_) match { - case util.Left(unsupported) => Future.failed(unsupported) - case util.Right(mimeType) => + case scala.util.Left(unsupported) => Future.failed(unsupported) + case scala.util.Right(mimeType) => val uploadRequest = UploadRequest( imageId = id_, tempFile = tempFile_, @@ -167,18 +168,24 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg, def projectImageFromUploadRequest(uploadRequest: UploadRequest) - (implicit ec: ExecutionContext, logMarker: LogMarker): Future[Image] = { + (implicit ec: ExecutionContext, logMarker: LogMarker, instance: Instance): Future[Image] = { val dependenciesWithProjectionsOnly: ImageUploadOpsDependencies = ImageUploadOpsDependencies( config, imageOps, projectOriginalFileAsS3Model, projectThumbnailFileAsS3Model, projectOptimisedPNGFileAsS3Model, + createEmbeddingsSource = (_, _, _) => Future.successful(None), + projectEmbeddingSourceAsS3Model, tryFetchThumbFile = fetchThumbFile, - tryFetchOptimisedFile = fetchOptimisedFile + tryFetchOptimisedFile = fetchOptimisedFile, + tryFetchEmbedding = fetchEmbeddingResult, + maybeEmbedder = maybeEmbedder, + // Projection reuses a previously-computed embedding via fetchEmbedding/tryFetchEmbedding above, + // and projectEmbeddingSourceAsS3Model never uploads the source image, so recreating it here would be wasted work. ) - fromUploadRequestShared(uploadRequest, dependenciesWithProjectionsOnly, processor, optimiseOps) + fromUploadRequestShared(uploadRequest, dependenciesWithProjectionsOnly, processor, optimiseOps).map(_._1) } private def projectOriginalFileAsS3Model(storableOriginalImage: StorableOriginalImage) = @@ -190,6 +197,30 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg, private def projectOptimisedPNGFileAsS3Model(storableOptimisedImage: StorableOptimisedImage) = Future.successful(storableOptimisedImage.toProjectedS3Object(config.originalFileBucket)) + private def projectEmbeddingSourceAsS3Model(storableEmbeddingSourceImage: Option[StorableEmbeddingSourceImage]): Future[Option[S3Object]] = { + Future.successful { + storableEmbeddingSourceImage.map { storableEmbeddingSourceImage => + storableEmbeddingSourceImage.toProjectedS3Object(config.embedSourceBucket) + } + } + } + + private def fetchEmbeddingResult(imageId: String, instance: Instance)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[Embedding]] = { + val key = embeddingKeyFromId(imageId)(instance) + val doesResultExist = Future { s3.doesObjectExist(config.embeddingsBucket, key) } recover { case _ => false } + doesResultExist.flatMap { + case false => + logger.warn(logMarker, s"embedding did not exist in bucket ${config.embeddingsBucket} at key $key") + Future.successful(None) // falls back to no previously computed embedding + case true => + Future { + s3.getObjectAsString(config.embeddingsBucket, key).flatMap { json => + Json.parse(json).validate[Embedding].asOpt + } + } + } + } + private def fetchThumbFile( imageId: String, outFile: File, instance: Instance)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[(File, MimeType)]] = { val key = fileKeyFromId(imageId)(instance) diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala index c9f97eea1c1..9bbac3b4fbc 100644 --- a/image-loader/app/model/Uploader.scala +++ b/image-loader/app/model/Uploader.scala @@ -28,12 +28,12 @@ import java.io.File import java.nio.file.Files import scala.concurrent.{ExecutionContext, Future} -case class ImageUpload(uploadRequest: UploadRequest, image: Image) + case class ImageUpload(uploadRequest: UploadRequest, image: Image, embeddingSource: Option[S3Object]) case object ImageUpload { def createImage(uploadRequest: UploadRequest, source: Asset, thumbnail: Asset, png: Option[Asset], - fileMetadata: FileMetadata, metadata: ImageMetadata): Image = { + fileMetadata: FileMetadata, metadata: ImageMetadata, embedding: Option[Embedding] = None): Image = { val usageRights = NoRights Image( uploadRequest.imageId, @@ -54,8 +54,9 @@ case object ImageUpload { usageRights, List(), List(), - // ImageEmbedding will be written by lambda later - embedding = None + // For a fresh upload there is no embedding yet - it is written later once computed. + // For a projected (re-indexed) image, an embedding fetched from a prior run may be supplied. + embedding = embedding ) } } @@ -65,7 +66,9 @@ case class ImageUploadOpsCfg( thumbWidth: Int, thumbQuality: Double, originalFileBucket: S3Bucket, - thumbBucket: S3Bucket + thumbBucket: S3Bucket, + embedSourceBucket: S3Bucket, + embeddingsBucket: S3Bucket ) case class ImageUploadOpsDependencies( @@ -74,8 +77,12 @@ case class ImageUploadOpsDependencies( storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object], storeOrProjectThumbFile: StorableThumbImage => Future[S3Object], storeOrProjectOptimisedImage: StorableOptimisedImage => Future[S3Object], + createEmbeddingsSource: (BrowserViewableImage, Option[OrientationMetadata], File) => Future[Option[StorableEmbeddingSourceImage]], + storeEmbeddingSource: Option[StorableEmbeddingSourceImage] => Future[Option[S3Object]], tryFetchThumbFile: (String, File, Instance) => Future[Option[(File, MimeType)]] = (_, _, _) => Future.successful(None), tryFetchOptimisedFile: (String, File, Instance) => Future[Option[(File, MimeType)]] = (_, _, _) => Future.successful(None), + tryFetchEmbedding: (String, Instance) => Future[Option[Embedding]] = (_, _) => Future.successful(None), + maybeEmbedder: Option[Embedder], ) @@ -91,41 +98,43 @@ object Uploader extends GridLogging { config.thumbWidth, config.thumbQuality, config.imageBucket, - config.thumbnailBucket + config.thumbnailBucket, + config.embeddingSourcesBucket, + config.embeddingsBucket ) } def fromUploadRequestShared(uploadRequest: UploadRequest, deps: ImageUploadOpsDependencies, processor: ImageProcessor, optimiseOps: OptimiseOps) - (implicit ec: ExecutionContext, logMarker: LogMarker): Future[Image] = { + (implicit ec: ExecutionContext, logMarker: LogMarker): Future[(Image, Option[S3Object])] = { import deps._ logger.info(logMarker, "Starting image ops") - val fileMetadataFuture = toFileMetadata(uploadRequest.tempFile, uploadRequest.imageId, uploadRequest.mimeType) logger.info(logMarker, "Have read file headers") - fileMetadataFuture.flatMap(fileMetadata => { - uploadAndStoreImage( - storeOrProjectOriginalFile, - storeOrProjectThumbFile, - storeOrProjectOptimisedImage, - uploadRequest, - deps, - processor, - optimiseOps)(ec, addLogMarkers(fileMetadata.toLogMarker)) - }) + uploadAndStoreImage( + storeOrProjectOriginalFile, + storeOrProjectThumbFile, + storeOrProjectOptimisedImage, + storeEmbeddingSource, + uploadRequest, + deps, + processor, + optimiseOps) } + private[model] def uploadAndStoreImage(storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object], storeOrProjectThumbFile: StorableThumbImage => Future[S3Object], storeOrProjectOptimisedFile: StorableOptimisedImage => Future[S3Object], + storeEmbeddingSource: Option[StorableEmbeddingSourceImage] => Future[Option[S3Object]], uploadRequest: UploadRequest, deps: ImageUploadOpsDependencies, processor: ImageProcessor, optimiseOps: OptimiseOps) - (implicit ec: ExecutionContext, logMarker: LogMarker) = { + (implicit ec: ExecutionContext, logMarker: LogMarker): Future[(Image, Option[S3Object])] = { val originalMimeType = uploadRequest.mimeType .orElse(MimeTypeDetection.guessMimeType(uploadRequest.tempFile).toOption) match { @@ -174,6 +183,10 @@ object Uploader extends GridLogging { case Some(storableOptimisedImage) => storeOrProjectOptimisedFile(storableOptimisedImage).map(a=>Some(a)) case None => Future.successful(None) } + embeddingSource <- deps.createEmbeddingsSource(browserViewableImage, sourceOrientationMetadata, tempDirForRequest) + storedEmbeddingSource <- storeEmbeddingSource(embeddingSource) + previouslyComputedEmbedding <- deps.tryFetchEmbedding(uploadRequest.imageId, uploadRequest.instance) + } yield { val fullFileMetadata = fileMetadata.copy(colourModel = colourModel).copy(colourModelInformation = colourModelInformation) val metadata = ImageMetadataConverter.fromFileMetadata(fullFileMetadata, s3Source.metadata.objectMetadata.lastModified) @@ -188,16 +201,19 @@ object Uploader extends GridLogging { thumbAsset, pngAsset, fullFileMetadata, - metadata + metadata, + previouslyComputedEmbedding ) val processedImage = processor(baseImage) logger.info(addLogMarkers(fileMetadata.toLogMarker), s"Ending image ops") // FIXME: dirty hack to sync the originalUsageRights and originalMetadata as well - processedImage.copy( + val image = processedImage.copy( originalMetadata = processedImage.metadata, originalUsageRights = processedImage.usageRights ) + + (image, storedEmbeddingSource) } eventualImage.onComplete{ _ => tempDirForRequest.listFiles().map(f => f.delete()) @@ -359,8 +375,7 @@ class Uploader( private def fromUploadRequest(uploadRequest: UploadRequest) (implicit logMarker: LogMarker, instance: Instance): Future[ImageUpload] = { - val sideEffectDependencies = ImageUploadOpsDependencies(toImageUploadOpsCfg(config), imageOps, - storeSource, storeThumbnail, storeOptimisedImage) + val sideEffectDependencies = ImageUploadOpsDependencies(toImageUploadOpsCfg(config), imageOps, storeSource, storeThumbnail, storeOptimisedImage, createEmbeddingsSource = createEmbeddingsSource, storeEmbeddingSource, maybeEmbedder = maybeEmbedder) Stopwatch.async("finalImage") { val finalImage = fromUploadRequestShared(uploadRequest, sideEffectDependencies, imageProcessor, optimiseOps) uploadRequest.identifiers.foreach{ @@ -371,7 +386,7 @@ class Uploader( case (ImageStorageProps.replacesMediaIdIdentifierKey, mediaIdToAddUsageTo) => addChildUsageToParentImage(uploadRequest, isReplacement = true)(mediaIdToAddUsageTo) } - finalImage.map(img => ImageUpload(uploadRequest, img)) + finalImage.map(img => ImageUpload(uploadRequest, img._1, img._2)) } } @@ -393,6 +408,32 @@ class Uploader( private def storeOptimisedImage(storableOptimisedImage: StorableOptimisedImage) (implicit logMarker: LogMarker) = store.store(storableOptimisedImage) + private def createEmbeddingsSource(browserViewableImage: BrowserViewableImage, + orientationMetadata: Option[OrientationMetadata], + tempDir: File): Future[Option[StorableEmbeddingSourceImage]] = { + maybeEmbedder.map { embedder => + createTempFile("embeddingsource-", embedder.embeddingSourceImageFormat().format.fileExtension, tempDir).flatMap { tempFile => + val eventualEmbeddingSource = imageOps.createEmbeddingSource(browserViewableImage.file, orientationMetadata, embedder.embeddingSourceImageFormat(), tempFile) + eventualEmbeddingSource.map { embeddingSource => + Some(browserViewableImage.copy( + file = embeddingSource, + mimeType = embedder.embeddingSourceImageFormat().format + ).asStorableEmbeddingSourceImage) + } + } + }.getOrElse { + logger.info("Skipping createEmbeddingsSource because no embedder is configured") + Future.successful(None) + } + } + + private def storeEmbeddingSource(storableEmbeddingSourceImage: Option[StorableEmbeddingSourceImage])(implicit logMarker: LogMarker): Future[Option[S3Object]] = { + storableEmbeddingSourceImage.map { storableEmbeddingSourceImage => + store.store(storableEmbeddingSourceImage).map(Some(_)) + } + .getOrElse(Future.successful(None)) + } + def loadFile(digestedFile: DigestedFile, uploadedBy: String, identifiers: Map[String, String], @@ -437,30 +478,25 @@ class Uploader( for { imageUpload <- fromUploadRequest(uploadRequest) updateMessage = UpdateMessage(subject = Image, image = Some(imageUpload.image), instance = uploadRequest.instance) - _ <- Future { notifications.publish(updateMessage) } - // Send the optimised PNG to the embedder if there is one (e.g. for TIFFs), - // otherwise send the original image. - assetForEmbedder = imageUpload.image.optimisedPng match { - case Some(optimisedPngAsset) => - logger.info(logMarker, s"Queueing optimised PNG instead of original for embedding") - optimisedPngAsset - case _ => - imageUpload.image.source + _ <- Future { + notifications.publish(updateMessage) + } + // Send the embed source to the embedder + _ = imageUpload.embeddingSource.foreach { embeddingSource => + + val imageMetadata = imageUpload.image.metadata + val imageMetadataJson = Json.prettyPrint(Json.toJson(imageMetadata)) + logger.info("Putting imageMetadata onto EmbedderMessage: " + imageMetadataJson.length) + + queueImageToEmbed(EmbedderMessage( + uploadRequest.imageId, + config.embeddingSourcesBucket.name, + config.embeddingSourcesBucket.keyFromURL(embeddingSource.uri), + instance.id, + Some(imageMetadata) // TODO SQS size limit and billing optimization + )) } - uriForEmbedder = assetForEmbedder.file - s3BucketForEmbedder = uriForEmbedder.getHost.split('.').head - s3KeyForEmbedder = uriForEmbedder.getPath.stripPrefix("/") - mimeTypeForEmbedder = assetForEmbedder.mimeType.getOrElse( - throw new Exception("Image for embedding has no mime type") - ).name - _ = queueImageToEmbed(EmbedderMessage( - uploadRequest.imageId, - mimeTypeForEmbedder, - s3BucketForEmbedder, - s3KeyForEmbedder, - instance.id - )) - // TODO: centralise where all these URLs are constructed + } yield { /* config.maybeLowerEnvironmentQueueBucketToSampleInto.foreach { lowerEnvironmentQueueBucket => diff --git a/image-loader/test/scala/model/ImageUploadTest.scala b/image-loader/test/scala/model/ImageUploadTest.scala index 54607cf4aed..4dd494c5a22 100644 --- a/image-loader/test/scala/model/ImageUploadTest.scala +++ b/image-loader/test/scala/model/ImageUploadTest.scala @@ -1,11 +1,11 @@ package model import com.drew.imaging.ImageProcessingException -import com.gu.mediaservice.lib.aws.{S3Bucket, S3Metadata, S3Object, S3ObjectMetadata} +import com.gu.mediaservice.lib._ +import com.gu.mediaservice.lib.aws.{S3Metadata, S3Object, S3ObjectMetadata} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.LogMarker -import com.gu.mediaservice.lib.{StorableImage, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage} import com.gu.mediaservice.model._ import lib.imaging.MimeTypeDetection import model.upload.{OptimiseWithPngQuant, UploadRequest} @@ -32,7 +32,7 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar { private implicit val logMarker: MockLogMarker = new MockLogMarker() // For mime type info, see https://github.com/guardian/grid/pull/2568 val tempDir = new File("/tmp") - val mockConfig: ImageUploadOpsCfg = ImageUploadOpsCfg(tempDir, 256, 85d, ResourceHelpers.dummyBucket("img-bucket"), ResourceHelpers.dummyBucket("thumb-bucket")) + val mockConfig: ImageUploadOpsCfg = ImageUploadOpsCfg(tempDir, 256, 85d, ResourceHelpers.dummyBucket("img-bucket"), ResourceHelpers.dummyBucket("thumb-bucket"), ResourceHelpers.dummyBucket("embeddingSources-bucket"), ResourceHelpers.dummyBucket("embeddings-bucket")) /** * @todo: I flailed about until I found a path that worked, but @@ -56,9 +56,15 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar { mockS3Object ) + def mockOptionalStore = (a: Option[StorableImage]) => + Future.successful( + a.map(a => mockS3Object) + ) + def storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object] = mockStore def storeOrProjectThumbFile: StorableThumbImage => Future[S3Object] = mockStore def storeOrProjectOptimisedPNG: StorableOptimisedImage => Future[S3Object] = mockStore + def storeOrProjectEmbeddingSource: Option[StorableEmbeddingSourceImage] => Future[Option[S3Object]] = mockOptionalStore val mockDependencies = ImageUploadOpsDependencies( config = mockConfig, @@ -66,6 +72,9 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar { storeOrProjectOriginalFile = storeOrProjectOriginalFile, storeOrProjectThumbFile = storeOrProjectThumbFile, storeOrProjectOptimisedImage = storeOrProjectOptimisedPNG, + createEmbeddingsSource = (_, _, _) => Future.successful(None), + storeEmbeddingSource = storeOrProjectEmbeddingSource, + maybeEmbedder = None, ) val tempFile = ResourceHelpers.fileAt(fileName) @@ -86,14 +95,17 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar { storeOrProjectOriginalFile = mockDependencies.storeOrProjectOriginalFile, storeOrProjectThumbFile = mockDependencies.storeOrProjectThumbFile, storeOrProjectOptimisedFile = mockDependencies.storeOrProjectOptimisedImage, + storeEmbeddingSource = mockDependencies.storeEmbeddingSource, uploadRequest = uploadRequest, deps = mockDependencies, processor = ImageProcessor.identity, - new OptimiseWithPngQuant(imageOps) + new OptimiseWithPngQuant(imageOps), + ) // Assertions; Failure will auto-fail - futureImage.map(i => { + futureImage.map( ie => { + val i = ie._1 // Assertions on original request assert(i.id == randomId, "Correct id comes back") assert(i.source.mimeType.contains(expectedOriginalMimeType), "Should have the correct mime type") diff --git a/image-loader/test/scala/model/ProjectorTest.scala b/image-loader/test/scala/model/ProjectorTest.scala index 13c711df249..70fd3504d65 100644 --- a/image-loader/test/scala/model/ProjectorTest.scala +++ b/image-loader/test/scala/model/ProjectorTest.scala @@ -1,20 +1,16 @@ package model -import java.io.File -import java.net.URI -import java.util.{Date, UUID} import com.gu.mediaservice.GridClient import com.gu.mediaservice.lib.auth.Authentication import com.gu.mediaservice.lib.aws.S3 import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} -import com.gu.mediaservice.model.{Instance, _} +import com.gu.mediaservice.model._ import com.gu.mediaservice.model.leases.LeasesByMedia import lib.DigestedFile import model.upload.OptimiseWithPngQuant import org.joda.time.{DateTime, DateTimeZone} -import org.mockito.ArgumentMatchers.any import org.mockito.Mockito.{times, verify, when} import org.scalatest.concurrent.ScalaFutures import org.scalatest.freespec.AnyFreeSpec @@ -22,15 +18,12 @@ import org.scalatest.matchers.should.Matchers import org.scalatest.time.{Millis, Span} import org.scalatestplus.mockito.MockitoSugar import play.api.libs.json.{JsArray, JsString} -import software.amazon.awssdk.services.s3.S3Client -import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse -import play.api.mvc.RequestHeader import test.lib.ResourceHelpers -import java.nio.file.Path +import java.io.File +import java.net.URI import scala.concurrent.ExecutionContext.Implicits.global -import scala.jdk.CollectionConverters._ -import scala.concurrent.{ExecutionContext, Future} +import scala.concurrent.Future class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with MockitoSugar { @@ -42,7 +35,7 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc private val imageOperations = new ImageOperations(ctxPath) - private val config = ImageUploadOpsCfg(new File("/tmp"), 256, 85d, dummyBucket("img-bucket"), dummyBucket("thumb-bucket")) + private val config = ImageUploadOpsCfg(new File("/tmp"), 256, 85d, dummyBucket("img-bucket"), dummyBucket("thumb-bucket"), dummyBucket("embeddingSource-bucket"), dummyBucket("embeddings-bucket")) private val maybeEmbedder = None diff --git a/kahuna/public/js/components/gr-downloader/gr-downloader.html b/kahuna/public/js/components/gr-downloader/gr-downloader.html index 2974341eaf6..4021617a0f1 100644 --- a/kahuna/public/js/components/gr-downloader/gr-downloader.html +++ b/kahuna/public/js/components/gr-downloader/gr-downloader.html @@ -34,14 +34,14 @@
  • - local_library - phonelink + local_library + phonelink {{usage.title}} diff --git a/kahuna/public/js/components/gr-image-usage/gr-image-usage.js b/kahuna/public/js/components/gr-image-usage/gr-image-usage.js index ab5e0c8226e..f672fd316e1 100644 --- a/kahuna/public/js/components/gr-image-usage/gr-image-usage.js +++ b/kahuna/public/js/components/gr-image-usage/gr-image-usage.js @@ -138,10 +138,12 @@ module.controller('grImageUsageListCtrl', [ return moment(timestamp).fromNow(); }; - ctrl.isRecent = (timestamp) => { + ctrl.isRecent = (usage) => { const nowtime = new Date(); - return moment(timestamp) + let recent = moment(usage.dateAdded) .isAfter(moment(nowtime).subtract(imageUsagesService.recentTime, 'days')); + let ignored = usage.platform === "print" && usage.status !== "published"; + return recent && !ignored; }; }]); diff --git a/kahuna/public/js/components/gr-more-like-this/gr-more-like-this.html b/kahuna/public/js/components/gr-more-like-this/gr-more-like-this.html index 54776e3156a..a00e54d2851 100644 --- a/kahuna/public/js/components/gr-more-like-this/gr-more-like-this.html +++ b/kahuna/public/js/components/gr-more-like-this/gr-more-like-this.html @@ -1,6 +1,6 @@ diff --git a/kahuna/public/js/components/gr-more-like-this/gr-more-like-this.js b/kahuna/public/js/components/gr-more-like-this/gr-more-like-this.js index b6f24de0550..fc30dba3fc4 100644 --- a/kahuna/public/js/components/gr-more-like-this/gr-more-like-this.js +++ b/kahuna/public/js/components/gr-more-like-this/gr-more-like-this.js @@ -12,7 +12,8 @@ moreLikeThis.controller('MoreLikeThisCtrl', [ let ctrl = this; ctrl.$onInit = () => { - ctrl.showMoreLikeThis = window._clientConfig.aiSearchEnabled; + const imageHasEmbedding = !!(ctrl.image.data.embedding && ctrl.image.data.embedding.geminiEmbedding2 && ctrl.image.data.embedding.geminiEmbedding2.image); + ctrl.showMoreLikeThis = imageHasEmbedding; ctrl.getMoreLikeThisQuery = function() { return `similar:${ctrl.image.data.id}`; }; diff --git a/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx index 95e3115d283..9435c2499b7 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx @@ -63,7 +63,6 @@ const ExtendedSortControl: React.FC = ({ props }) => { const handleLogoClick = (e: any) => { setSortOption(DefaultSortOption); setUserTakenSelect(false); - props.onSortSelect(DefaultSortOption, 'with', false); }; const handleQueryChange = (e: any) => { diff --git a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx index 4bef8d6132d..b5c70d84d10 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx @@ -48,7 +48,6 @@ const SortControl: React.FC = ({ props }) => { useEffect(() => { const handleLogoClick = (e: any) => { setSortOption(DefaultSortOption); - props.onSortSelect(DefaultSortOption); }; const handleQueryChange = (e: any) => { diff --git a/kahuna/public/js/components/gu-lazy-preview-shortcuts/gu-lazy-preview-shortcuts.js b/kahuna/public/js/components/gu-lazy-preview-shortcuts/gu-lazy-preview-shortcuts.js new file mode 100644 index 00000000000..d54820d6bbd --- /dev/null +++ b/kahuna/public/js/components/gu-lazy-preview-shortcuts/gu-lazy-preview-shortcuts.js @@ -0,0 +1,39 @@ +import angular from 'angular'; + +import '../gr-keyboard-shortcut/gr-keyboard-shortcut'; + +export var lazyPreviewShortcuts = angular.module('gu.lazyPreviewShortcuts', [ + 'gr.keyboardShortcut' +]); + +lazyPreviewShortcuts.directive('guLazyPreviewShortcuts', + ['keyboardShortcut', + function(keyboardShortcut) { + return { + restrict: 'EA', + require: '^guLazyPreview', + link: function (scope, element, attrs, lazyGalleryCtrl) { + function invoke(fnName) { + return (event) => { + // Must cancel any scrolling caused by the key + event.preventDefault(); + lazyGalleryCtrl[fnName](); + }; + } + + keyboardShortcut.bindTo(scope) + .add({ + combo: 'left', + description: 'Go to the previous image', + allowIn: ['INPUT'], + callback: invoke('prevItem') + }) + .add({ + combo: 'right', + description: 'Go to the next image', + allowIn: ['INPUT'], + callback: invoke('nextItem') + }); + } + }; +}]); diff --git a/kahuna/public/js/components/gu-lazy-preview/gu-lazy-preview.css b/kahuna/public/js/components/gu-lazy-preview/gu-lazy-preview.css new file mode 100644 index 00000000000..52cf28d1d9d --- /dev/null +++ b/kahuna/public/js/components/gu-lazy-preview/gu-lazy-preview.css @@ -0,0 +1,171 @@ +/* ========================================================================== + preview-view + ========================================================================== */ + +.preview-view { + position: relative; + height: calc(100vh - 100px); /* screen - top bar */ + top: 35px; /* results bar height */ + background-color: #333333; +} + +.preview-view-item { + width: 100%; + height: 100%; + position: relative; + box-sizing: border-box; + display: flex; + align-items: baseline; + align-content: center; + padding: 10px; +} + +.preview-view-item__preview { + width: 100%; + box-sizing: border-box; + border: 5px solid transparent; +} + +.preview-view-item__preview--selected { + border-color: #00adee; +} + +.preview-view__controls { + position: absolute; + bottom: 20px; + left: 50%; + transform: translateX(-50%); + opacity: .5; + transition: opacity .1s; +} + +.preview-view__controls:hover { + opacity: 1; +} + +.preview-view__control { + padding: 10px; + border: 1px solid #565656; + background: #333333; + transition: opacity .1s; +} + +/* TODO: abstract repeated results styles */ +.preview-view-item__select input[type=checkbox] { + visibility: hidden; +} + +.preview-view-item__select { + position: absolute; + display: none; + + /* above thumbnail */ + z-index: 1; +} + +.preview-view-item__select--no-pointer-events { + pointer-events: none; +} + +.preview-view-item__select__checkbox__label { + position: absolute; + top: 30px; + left: 30px; + cursor: pointer; +} + +.preview-view-item__select .preview-view-item__select__checkbox__label gr-icon { + font-size: 25px; + color: white; +} + +.preview-view-item__select--selected .preview-view-item__select__checkbox__label gr-icon { + color: #00adee; +} + +.preview-view-item--selected, +.preview-view-item--selected:hover { + border-color: #00adee; +} + +/* Hacky: Hide gr-archiver-status "Add to Library" (unarchived state) unless hovering */ +.preview-view-item .gr-archiver-status--unarchived { + display: none; +} + +.preview-view-item:hover .preview-view-item__select, +.preview-view-item:hover .preview__fade, +.preview-view-item:hover .image-actions-container, +.preview-view-item:hover .gr-archiver-status--unarchived { + display: block; +} + +.preview-view-item:hover .gr-add-label--inactive { + display: inline-block; +} + +.preview-view-item--selected:hover .preview__fade { + display: none; +} + +.preview-view-item .preview__image { + max-height: calc(100vh - 190px); +} + +.preview-view-item .loading .preview__image { + display: none; +} + +.preview-view-item .preview__loading { + height: calc(100vh - 190px); + position:relative; + display: none; +} + +.preview-view-item .loading .preview__loading { + display: block; +} + +.preview-view-item .preview__loading__icon { + height: 30px; + width: 30px; + display: block; + position: absolute; + left: 50%; + top: 50%; + transform: translate(-50%, -50%); + max-width: 100%; + margin: 0 auto; + animation: spin 1000ms infinite linear; + font-size: 30px; +} + +@keyframes spin { + from {transform:rotate(0deg);} + to {transform:rotate(360deg);} +} + +.preview-view-item__select--selected { + display: initial; +} + +.preview-view-item__preview .preview__info-container { + position: absolute; + bottom: 0; + left: 50%; + transform: translateX(-50%); + width: 100%; + background: rgba(0, 0, 0, 0.5); + padding: 20px; + box-sizing: border-box; + overflow: auto; +} + +.preview-view-item__preview .preview__info-container, +.preview-view-item__preview .loading .preview__info-container { + display: none; +} + +.preview-view-item__preview:hover .preview__info-container { + display: block; +} diff --git a/kahuna/public/js/components/gu-lazy-preview/gu-lazy-preview.html b/kahuna/public/js/components/gu-lazy-preview/gu-lazy-preview.html new file mode 100644 index 00000000000..d327fd896a2 --- /dev/null +++ b/kahuna/public/js/components/gu-lazy-preview/gu-lazy-preview.html @@ -0,0 +1,58 @@ +
    + +
    + +
    + + +
    +
    + + +
    + + + + + +
    + +
    + + + + +
    + +
    diff --git a/kahuna/public/js/components/gu-lazy-preview/gu-lazy-preview.js b/kahuna/public/js/components/gu-lazy-preview/gu-lazy-preview.js new file mode 100644 index 00000000000..215ba495760 --- /dev/null +++ b/kahuna/public/js/components/gu-lazy-preview/gu-lazy-preview.js @@ -0,0 +1,142 @@ +import angular from 'angular'; +import Rx from 'rx'; +import 'rx-dom'; + +import template from './gu-lazy-preview.html'; + +import './gu-lazy-preview.css'; + +import '../../util/rx'; + +export var lazyPreview = angular.module('gu.lazyPreview', ['util.rx']); + +function asInt(string) { + return parseInt(string, 10); +} + +lazyPreview.controller('GuLazyPreviewCtrl', [function() { + let ctrl = this; + + ctrl.init = function({items$, totalItems$, preloadedItems$, currentIndex$}) { + const itemsCount$ = items$.map(items => items.length).distinctUntilChanged(); + const totalItemsCount$ = totalItems$.map(totalItems => { + return totalItems.length; + }).distinctUntilChanged(); + + const buttonCommands$ = Rx.Observable.create(observer => { + ctrl.prevItem = () => observer.onNext('prevItem'); + ctrl.nextItem = () => observer.onNext('nextItem'); + + // Make sure we start at the beginning + observer.onNext('previewStart'); + }); + + + const itemsOffset$ = buttonCommands$.combineLatest( + itemsCount$, + (command, itemsCount) => { + return {command, itemsCount}; + }).withLatestFrom( + currentIndex$, + ({command}, currentIndex) => { + return { + prevItem: -1, + nextItem: +1, + previewStart: currentIndex * -1 + }[command] || 0; + } + ); + + + + + const updatedIndex$ = itemsOffset$.withLatestFrom( + currentIndex$, itemsCount$, totalItemsCount$, + (itemsOffset, currentIndex, itemsCount, totalItemsCount) => { + const updatedIndex = currentIndex + itemsOffset; + // Update the index if it's in the range of items + if (updatedIndex >= 0 && updatedIndex < totalItemsCount) { + return updatedIndex; + } else { + return currentIndex; + } + }); + + const item$ = updatedIndex$.withLatestFrom( + totalItemsCount$, items$, + (updatedIndex, totalItemsCount, items) => { + currentIndex$.onNext(updatedIndex); + return items[updatedIndex]; + }); + + const currentPage$ = currentIndex$.withLatestFrom( + preloadedItems$, + (currentIndex, preloadedItems) => { + return Math.floor(currentIndex / preloadedItems); + }); + + const rangeToLoad$ = currentPage$.withLatestFrom( + preloadedItems$, + (currentPage, preloadedItems) => { + const start = currentPage * preloadedItems; + const end = ((currentPage + 1) * preloadedItems) - 1; + return {start, end}; + }). + // Debounce range loading, which also helps discard + // erroneous large ranges while combining + // loadedRangeStart$ and loadedRangeEnd$ changes (one after the other) + debounce(10). + // Ignore if either end isn't set (whole range already loaded) + filter(({start, end}) => start !== -1 && end !== -1). + // Ignore if $start after $end (incomplete combine$ state) + filter(({start, end}) => start <= end). + distinctUntilChanged(({start, end}) => `${start}-${end}`); + + return { + item$, + rangeToLoad$ + }; + }; + +}]); + +lazyPreview.directive('guLazyPreview', ['observe$', 'observeCollection$', 'subscribe$', 'inject$', + function( + observe$, observeCollection$, subscribe$, inject$) { + return { + restrict: 'E', + controller: 'GuLazyPreviewCtrl', + controllerAs: 'previewCtrl', + transclude: true, + template: template, + link: function(scope, element, attrs, ctrl) { + // Map attributes as Observable streams + const { + guLazyPreviewItems: itemsAttr, + guLazyPreviewItemsTotal: totalItemsAttr, + guLazyPreviewSelectionMode: selectionMode, + guLazyPreviewLoadRange: loadRangeFn, + guLazyPreviewPreloadedItems: preloadedItemsAttr + } = attrs; + + const items$ = observeCollection$(scope, itemsAttr); + const selectionMode$ = observe$(scope, selectionMode); + const totalItems$ = observe$(scope, totalItemsAttr); + const preloadedItems$ = observe$(scope, preloadedItemsAttr).map(asInt); + + const currentIndex$ = new Rx.BehaviorSubject(0); + + const {item$, rangeToLoad$} = ctrl.init( + {items$, totalItems$, preloadedItems$, currentIndex$} + ); + + subscribe$(scope, rangeToLoad$, range => { + scope.$eval(loadRangeFn, range); + }); + + inject$(scope, currentIndex$, ctrl, 'currentIndex'); + inject$(scope, selectionMode$, ctrl, 'selectionMode'); + inject$(scope, item$, ctrl, 'item'); + } + }; +}]); diff --git a/kahuna/public/js/crop/controller.js b/kahuna/public/js/crop/controller.js index 8f43516385b..fdb5d02f2ef 100644 --- a/kahuna/public/js/crop/controller.js +++ b/kahuna/public/js/crop/controller.js @@ -52,7 +52,6 @@ crop.controller('ImageCropCtrl', [ const imageId = $stateParams.imageId; const circularMaskKey = 'crop.shouldUseCircularMask'; - ctrl.shouldUseCircularMask = true; try { const stored = storage.getJs(circularMaskKey); if (typeof stored === 'boolean') { @@ -74,10 +73,14 @@ crop.controller('ImageCropCtrl', [ const storageCropType = cropSettings.getCropType(); const storageDefaultCropType = cropSettings.getDefaultCropType(); + const cropOptionDisplayValue = cropOption => cropOption.ratioString + ? `${cropOption.displayName} (${cropOption.ratioString})` + : cropOption.displayName; + ctrl.cropOptions = allCropOptions .filter(option => (!storageCropType || storageCropType === option.key) && !option.isHidden ) .map(option => Object.assign(option, { - value: option.ratioString ? `${option.key} (${option.ratioString})` : option.key, + value: cropOptionDisplayValue(option), minimalValue: option.ratioString || option.key, tooltip: `${option.key} [${option.key.charAt(0)}]`, disabled: storageCropType && storageCropType !== option.key diff --git a/kahuna/public/js/image/controller.js b/kahuna/public/js/image/controller.js index 35d522d7c2c..1e473abebcb 100644 --- a/kahuna/public/js/image/controller.js +++ b/kahuna/public/js/image/controller.js @@ -27,7 +27,8 @@ import '../components/gr-display-crops/gr-display-crops'; import '../components/gu-date/gu-date'; import {radioList} from '../components/gr-radio-list/gr-radio-list'; import {cropUtil} from '../util/crop'; -import { List } from 'immutable'; +import {List} from 'immutable'; + const image = angular.module('kahuna.image.controller', [ 'util.rx', 'util.storage', @@ -292,7 +293,7 @@ image.controller('ImageCtrl', [ ' does not have the same dimensions as the master. Using the next largest cropped asset with width ' + largestWidth + 'Please correct this inconsistency.'); } - crop.downloadLink = largestAsset.downloadLink; + crop.downloadLink = cropsResource.links.find(link => link.rel.includes(`crop-download-${crop.id}-master`))?.href; } }); } diff --git a/kahuna/public/js/preview/image-large.html b/kahuna/public/js/preview/image-large.html new file mode 100644 index 00000000000..ca7e19a11d1 --- /dev/null +++ b/kahuna/public/js/preview/image-large.html @@ -0,0 +1,165 @@ +
    + + + + +
    +
    + autorenew +
    + {{ctrl.image.data.metadata.description}} +     + + +
    +
    + autorenew +
    + {{ctrl.image.data.metadata.description}} +     + +
    + +
    +
    + + + + + + + +
    + +

    + + {{ctrl.image.data.metadata.description || ctrl.image.data.metadata.title || ' '}} +

    +
    + +
    +
    + + Uploaded: {{ctrl.image.data.uploadTime | date:'dd/MM/yy'}} + {{ctrl.image.data.uploadTime | date:'HH:mm'}} + + + + + crop + + + + + local_library + + + + + phonelink + +
    +
    + + + +
    +
    + + £ +
    + +
    + + + warning +
    + +
    + trending_up +
    + +
    + + flag +
    +
    +
    +
    +
    +
    diff --git a/kahuna/public/js/preview/image.js b/kahuna/public/js/preview/image.js index b673150b149..3bb05bf0742 100644 --- a/kahuna/public/js/preview/image.js +++ b/kahuna/public/js/preview/image.js @@ -1,10 +1,12 @@ import angular from 'angular'; +import Rx from 'rx'; import '../util/rx'; import '../util/storage'; import {restrictionsText} from '../util/rights-categories'; import template from './image.html'; +import templateLarge from './image-large.html'; import '../image/service'; import '../imgops/service'; @@ -181,6 +183,41 @@ image.directive('uiPreviewImage', function() { }; }); +image.directive('uiPreviewImageLarge', ['observe$', 'inject$', 'imgops', + function(observe$, inject$, imgops) { + return { + restrict: 'E', + scope: { + image: '=', + hideInfo: '=', + selectionMode: '=' + }, + // extra actions can be transcluded in + transclude: true, + template: templateLarge, + controller: 'uiPreviewImageCtrl', + controllerAs: 'ctrl', + bindToController: true, + link: function(scope, element, attrs, ctrl) { + ctrl.loading = false; + const image$ = new Rx.Subject(); + + const optimisedImage$ = image$.flatMap((image) => { + return Rx.Observable.fromPromise(imgops.getFullScreenUri(image)); + }); + + scope.$watch(() => ctrl.image.data.id, () => { + ctrl.loading = true; + image$.onNext(ctrl.image); + }); + + inject$(scope, optimisedImage$, ctrl, 'optimisedImage'); + + scope.$watch(() => ctrl.optimisedImage, () => ctrl.loading = false); + } + }; +}]); + image.directive('grStopPropagation', function() { return { restrict: 'A', diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index c0666a0fe25..089f00b9f00 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -262,10 +262,7 @@ query.controller('SearchQueryCtrl', [ // eslint-disable-next-line complexity function watchSearchChange(newFilter, sender) { - let showPaid = newFilter.nonFree ? newFilter.nonFree : false; - if (sender && sender == "filterChange" && !newFilter.nonFree) { - showPaid = ctrl.user.permissions.showPaid; - } + const showPaid = newFilter.nonFree ? newFilter.nonFree : false; storage.setJs("isNonFree", showPaid, true); // check for taken date sort contradiction diff --git a/kahuna/public/js/search/results.html b/kahuna/public/js/search/results.html index 1d4e707d3b8..9d3a3d980ff 100644 --- a/kahuna/public/js/search/results.html +++ b/kahuna/public/js/search/results.html @@ -141,6 +141,12 @@ ng-if="ctrl.selectionCount > 0 && !ctrl.isDeleted"> + + + + + +
    Too many results to display
    diff --git a/kahuna/public/js/search/results.js b/kahuna/public/js/search/results.js index 5e322d46807..8ff85d85e03 100644 --- a/kahuna/public/js/search/results.js +++ b/kahuna/public/js/search/results.js @@ -9,7 +9,9 @@ import '../util/seq'; import '../util/storage'; import '../util/constants/sendToCapture-config'; import '../components/gu-lazy-table/gu-lazy-table'; +import '../components/gu-lazy-preview/gu-lazy-preview'; import '../components/gu-lazy-table-shortcuts/gu-lazy-table-shortcuts'; +import '../components/gu-lazy-preview-shortcuts/gu-lazy-preview-shortcuts'; import '../components/gr-archiver/gr-archiver'; import '../components/gr-delete-image/gr-delete-image'; import '../components/gr-undelete-image/gr-un-delete-image'; @@ -48,6 +50,8 @@ export var results = angular.module('kahuna.search.results', [ 'util.seq', 'gu.lazyTable', 'gu.lazyTableShortcuts', + 'gu.lazyPreview', + 'gu.lazyPreviewShortcuts', 'gr.archiver', 'gr.downloader', 'gr.moreLikeThis', @@ -203,6 +207,9 @@ results.controller('SearchResultsCtrl', [ ctrl.newImagesCount = 0; ctrl.newImagesLastCheckedMoment = moment(); + // Preview control + ctrl.previewView = false; + ctrl.needsQuery = $stateParams.useAISearch && (!$stateParams.query || !$stateParams.query.trim()); // Map to track image->position and help remove duplicates diff --git a/kahuna/public/js/search/structured-query/query-suggestions.ts b/kahuna/public/js/search/structured-query/query-suggestions.ts index 6f8c4ab2e1c..b3a6f723d1a 100644 --- a/kahuna/public/js/search/structured-query/query-suggestions.ts +++ b/kahuna/public/js/search/structured-query/query-suggestions.ts @@ -52,13 +52,10 @@ const isSearch = [ `owned-illustration`, `owned`, "under-quota", - "deleted" + "deleted", + "reapable" ]; -if (window._clientConfig.useReaper === true) { - isSearch.push("reapable"); -} - if (window._clientConfig.agencyPicksIngredients) { isSearch.push("agency-pick"); } @@ -159,6 +156,10 @@ querySuggestions.factory("querySuggestions", [ { fieldName: "is", resolver: isSearch }, { fieldName: "dateTaken" } as const, { fieldName: "date" }, + { + fieldName: "syndicationStatus", + resolver: ['review', 'queued', 'sent', 'unsuitable'] + }, ...Object.keys(fieldAliases).map((fieldName) => { const staticAliasOptions = suggestFieldAliasOptions(fieldName); diff --git a/kahuna/public/js/services/image/usages.js b/kahuna/public/js/services/image/usages.js index 07519cb329e..5648853753f 100644 --- a/kahuna/public/js/services/image/usages.js +++ b/kahuna/public/js/services/image/usages.js @@ -76,7 +76,7 @@ imageUsagesService.factory('imageUsagesService', [function() { return usagesList.filter(item=> { const timestamp = item.get('dateAdded'); const recentIfAfter = moment().subtract(recentDays, 'days'); - return moment(timestamp).isAfter(recentIfAfter); + return item.get('status') === 'published' && moment(timestamp).isAfter(recentIfAfter); }); }); const recentDigitalUsages$ = filterByPlatform('digital').map((usagesList) => { diff --git a/kahuna/public/js/util/constants/cropOptions.js b/kahuna/public/js/util/constants/cropOptions.js index dc1bbea2b16..1945b3ac2ae 100644 --- a/kahuna/public/js/util/constants/cropOptions.js +++ b/kahuna/public/js/util/constants/cropOptions.js @@ -1,9 +1,10 @@ // `ratioString` is sent to the server, being `undefined` for `freeform` is expected 🙈 -export const landscapeOld = {key: 'landscape OLD', ratio: 5 / 3, ratioString: '5:3', isHidden: true}; -export const landscape = {key: 'landscape', ratio: 5 / 4, ratioString: '5:4'}; -export const portrait = {key: 'portrait', ratio: 4 / 5, ratioString: '4:5'}; -export const video = {key: 'video', ratio: 16 / 9, ratioString: '16:9'}; -export const square = {key: 'square', ratio: 1, ratioString: '1:1'}; -export const freeform = {key: 'freeform', ratio: null, isDefault: true}; +export const landscapeOld = {key: 'landscape OLD', ratio: 5 / 3, ratioString: '5:3', displayName: "Landscape 5:3"}; +export const landscape32 = {key: 'landscape32', ratio: 3 / 2, ratioString: '3:2', displayName: "landscape 3:2"}; +export const landscape = {key: 'landscape', ratio: 5 / 4, ratioString: '5:4', isHidden: true, displayName: 'Landscape'}; +export const portrait = {key: 'portrait', ratio: 4 / 5, ratioString: '4:5', displayName: 'Portrait'}; +export const video = {key: 'video', ratio: 16 / 9, ratioString: '16:9', displayName: 'Video'}; +export const square = {key: 'square', ratio: 1, ratioString: '1:1', displayName: 'Square'}; +export const freeform = {key: 'freeform', ratio: null, isDefault: true, displayName: 'Freeform'}; -export const cropOptions = [landscapeOld, landscape, portrait, video, square, freeform]; +export const cropOptions = [landscapeOld, landscape32, landscape, portrait, video, square, freeform]; diff --git a/leases/test/lib/LeaseStoreSpec.scala b/leases/test/lib/LeaseStoreSpec.scala index d9ae3e3e85f..050da7e6d27 100644 --- a/leases/test/lib/LeaseStoreSpec.scala +++ b/leases/test/lib/LeaseStoreSpec.scala @@ -9,8 +9,7 @@ import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers import org.scalatest.time.{Millis, Seconds, Span} import org.scalatestplus.mockito.MockitoSugar -import org.testcontainers.containers.localstack.LocalStackContainer -import org.testcontainers.containers.localstack.LocalStackContainer.Service.DYNAMODB +import org.testcontainers.localstack.LocalStackContainer import org.testcontainers.utility.DockerImageName import software.amazon.awssdk.auth.credentials.{AwsBasicCredentials, StaticCredentialsProvider} import software.amazon.awssdk.regions.Region @@ -25,11 +24,11 @@ class LeaseStoreSpec extends AnyFunSpec with Matchers with ScalaFutures with Bef implicit val defaultPatience: PatienceConfig = PatienceConfig(timeout = Span(2, Seconds), interval = Span(100, Millis)) - private val dynamoContainer = new LocalStackContainer(DockerImageName.parse("localstack/localstack:1.4.0")).withServices(DYNAMODB) + private val dynamoContainer = new LocalStackContainer(DockerImageName.parse("localstack/localstack:1.4.0")).withServices("dynamodb") dynamoContainer.start() private val dynamoClient = DynamoDbAsyncClient.builder(). - endpointOverride(dynamoContainer.getEndpointOverride(DYNAMODB)). + endpointOverride(dynamoContainer.getEndpoint). region(Region.of(dynamoContainer.getRegion)). credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(dynamoContainer.getAccessKey, dynamoContainer.getSecretKey))).build() diff --git a/media-api/app/MediaApiComponents.scala b/media-api/app/MediaApiComponents.scala index b73420d5ffd..aeea235efcf 100644 --- a/media-api/app/MediaApiComponents.scala +++ b/media-api/app/MediaApiComponents.scala @@ -1,6 +1,6 @@ -import com.gu.mediaservice.lib.aws.{Bedrock, Embedder, S3Vectors, ThrallMessageSender} -import com.gu.mediaservice.lib.instances.InstancesClient import com.gu.mediaservice.lib.aws._ +import com.gu.mediaservice.lib.embeddings.GoogleCloudEmbedding +import com.gu.mediaservice.lib.instances.InstancesClient import com.gu.mediaservice.lib.management.{ElasticSearchHealthCheck, Management} import com.gu.mediaservice.lib.metadata.SoftDeletedMetadataTable import com.gu.mediaservice.lib.play.GridComponents @@ -31,9 +31,25 @@ class MediaApiComponents(context: Context) extends GridComponents(context, new M val imageResponse = new ImageResponse(config, s3, usageQuota) val softDeletedMetadataTable = new SoftDeletedMetadataTable(config) - val embedder = new Embedder(new Bedrock(config), new SimpleSqsMessageConsumer(config.queueUrl, config)) - val mediaApi = new MediaApi(auth, messageSender, softDeletedMetadataTable, elasticSearch, imageResponse, config, controllerComponents, s3, mediaApiMetrics, wsClient, authorisation, embedder, usageEvents) + private val maybeGcpProjectId = config.gcpProjectId + private val vertexApiLocation = "eu" + private val maybeGoogleCloudEmbedding = for { + gcpProjectId <- maybeGcpProjectId + } yield { + new GoogleCloudEmbedding(projectId = gcpProjectId, location = vertexApiLocation) + } + + private val maybeEmbedding = maybeGoogleCloudEmbedding + + val maybeEmbedder: Option[Embedder] = for { + embedding <- maybeEmbedding + queueUrl <- config.embedderQueueUrl + } yield { + new Embedder(embedding, new SimpleSqsMessageConsumer(queueUrl, config)) + } + + val mediaApi = new MediaApi(auth, messageSender, softDeletedMetadataTable, elasticSearch, imageResponse, config, controllerComponents, s3, mediaApiMetrics, wsClient, authorisation, maybeEmbedder, usageEvents) val suggestionController = new SuggestionController(auth, elasticSearch, controllerComponents) val aggController = new AggregationController(auth, elasticSearch, controllerComponents) val usageController = new UsageController(auth, config, elasticSearch, usageQuota, controllerComponents) diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index e9e8ebd612d..c699f50adc7 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -22,6 +22,7 @@ import com.sksamuel.elastic4s.requests.searches.queries.Query import lib._ import lib.elasticsearch._ import lib.querysyntax.Condition +import lib.querysyntax.{Match, SimilarField, SimilarValue} import org.apache.http.entity.ContentType import org.apache.pekko.stream.scaladsl.StreamConverters import org.http4s.UriTemplate @@ -48,7 +49,7 @@ class MediaApi( mediaApiMetrics: MediaApiMetrics, ws: WSClient, authorisation: Authorisation, - embedder: Embedder, + maybeEmbedder: Option[Embedder], events: UsageEvents, )(implicit val ec: ExecutionContext) extends BaseController with MessageSubjects with ArgoHelpers with ContentDisposition with InstanceForRequest { @@ -57,11 +58,15 @@ class MediaApi( // Process-local cache keyed on normalised query text. Stores the Bedrock Future so that // concurrent requests for the same query share a single in-flight Bedrock call, and // subsequent requests within the TTL window skip Bedrock entirely. - private val embeddingCache: AsyncLoadingCache[String, List[Double]] = Scaffeine() - .maximumSize(config.aiSearchEmbeddingCacheMaxSize) - .buildAsyncFuture((normQuery: String) => - embedder.createQueryEmbedding(normQuery)(MarkerMap()) - ) + private val maybeEmbeddingCache: Option[AsyncLoadingCache[String, List[Double]]] = { + maybeEmbedder.map { embedder => + Scaffeine() + .maximumSize(config.aiSearchEmbeddingCacheMaxSize) + .buildAsyncFuture((normQuery: String) => + embedder.createQueryEmbedding(normQuery)(MarkerMap()) + ) + } + } private val searchParamList = List( "q", @@ -310,6 +315,36 @@ class MediaApi( .recover{ case error => respondError(InternalServerError, "cannot-get", s"Cannot get soft-deleted metadata ${error}") } } + def downloadImageExportMaster(imageId: String, exportId: String) = auth.async { implicit request => + implicit val instance: Instance = instanceOf(request) + implicit val logMarker: LogMarker = MarkerMap( + "requestType" -> "download-image-export-master", + "requestId" -> RequestLoggingFilter.getRequestId(request), + "imageId" -> imageId, + "exportId" -> exportId, + ) ++ RequestLoggingFilter.loggablePrincipal(request.user) + + elasticSearch.getImageById(imageId) map { + case Some(source) if isVisibleToAccessor(request.user, source) => + val maybeResult = for { + export <- source.exports.find(_.id.contains(exportId)) + asset <- export.master + key = config.imgPublishingBucket.keyFromURL(asset.file) + s3Object <- Try(s3.getObject(config.imgPublishingBucket, key)).toOption + file = StreamConverters.fromInputStream(() => s3Object) + entity = HttpEntity.Streamed(file, asset.size, asset.mimeType.map(_.name)) + result = Result(ResponseHeader(OK), entity).withHeaders("Content-Disposition" -> getContentDisposition(source, export, asset)) + } yield { + if(config.recordDownloadAsUsage) { + postToUsages(config.usageUri(instance) + "/usages/download", auth.getOnBehalfOfPrincipal(request.user), source.id, Authentication.getIdentity(request.user)) + } + result + } + maybeResult.getOrElse(ExportNotFound) + case _ => ImageNotFound(imageId) + } + } + def downloadImageExport(imageId: String, exportId: String, width: Int) = auth.async { implicit request => implicit val instance: Instance = instanceOf(request) implicit val logMarker: LogMarker = MarkerMap( @@ -332,7 +367,7 @@ class MediaApi( s3Object <- s3Res.toOption file = StreamConverters.fromInputStream(() => s3Object) entity = HttpEntity.Streamed(file, asset.size, asset.mimeType.map(_.name)) - result = Result(ResponseHeader(OK), entity).withHeaders("Content-Disposition" -> getContentDisposition(source, export, asset, config.shortenDownloadFilename)) + result = Result(ResponseHeader(OK), entity).withHeaders("Content-Disposition" -> getContentDisposition(source, export, asset)) } yield { if(config.recordDownloadAsUsage) { postToUsages(config.usageUri(instance) + "/usages/download", auth.getOnBehalfOfPrincipal(request.user), source.id, Authentication.getIdentity(request.user)) @@ -603,11 +638,12 @@ class MediaApi( EmbeddedEntity(uri = imageUri, data = Some(imageData), imageLinks, imageActions) } - def performSearchAndRespond(searchParams: SearchParams)(implicit instance: Instance) = for { + def performSearchAndRespond(searchParams: SearchParams, maybeSimilarToVector: Option[Seq[Float]])(implicit instance: Instance) = for { SearchResults(hits, totalCount, extraCounts) <- elasticSearch.search( searchParams.copy( shouldFlagGraphicImages = shouldFlagGraphicImages, - ) + ), + maybeSimilarToVector ) imageEntities = hits map (hitToImageEntity _).tupled prevLink = getPrevLink(searchParams) @@ -684,6 +720,7 @@ class MediaApi( ) val filterOpt = buildAiFilter(parts.filterConditions, params) + val eventualMaybeEmbedding = embeddingForImageId(imageId) // Compute the filtered pool total and ticker count badges in parallel with the // KNN search, so similar-image results show the same "Best k of N matches" total @@ -691,16 +728,11 @@ class MediaApi( val filterTotalAndCounts = elasticSearch.countMatchingFilterWithExtraCounts(filterOpt) for { - maybeImage <- elasticSearch.getImageById(imageId) - maybeEmbedding = maybeImage - .filter(image => isVisibleToAccessor(request.user, image)) - .flatMap(_.embedding) - .flatMap(_.cohereEmbedV4) - .map(_.image) + maybeEmbedding <- eventualMaybeEmbedding searchResults <- maybeEmbedding match { // If we have an embedding, perform the KNN search. If not, return an empty result set. case Some(embedding) => - elasticSearch.semanticSearch(embedding, k = k, numCandidates = Math.max(k * 2, 100), filterOpt = filterOpt) + elasticSearch.knnSearch(embedding, k = k, numCandidates = Math.max(k * 2, 100), filterOpt = filterOpt) case None => Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) } @@ -712,11 +744,14 @@ class MediaApi( def semanticSearchByText(k: Int, parts: AiQueryParts, params: SearchParams): Future[SearchResults] = { // Separate the chips from the main query text // So that we can embed just the query text - parts.semanticQuery match { - case None => + (parts.semanticQuery, maybeEmbeddingCache) match { + case (_, None) => logger.info(logMarker, s"No semantic query found in structured query; returning no AI results") Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) - case Some(semanticQuery) => + case (None, _) => + logger.info(logMarker, s"No semantic query found in structured query; returning no AI results") + Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) + case (Some(semanticQuery), Some(embeddingCache)) => val vecWeight = params.vecWeight.getOrElse(0.85) val outerMarker = logMarker @@ -747,7 +782,7 @@ class MediaApi( query = semanticQuery, queryEmbedding = embedding, k = k, - numCandidates = Math.max(k * 2, 100), + numCandidates = k * 2, vecWeight = vecWeight, filterOpt = filterOpt ) @@ -796,13 +831,47 @@ class MediaApi( } else { _searchParams } - SearchParams.validate(searchParams).fold( + + val x: Future[Result] = SearchParams.validate(searchParams).fold( // TODO: respondErrorCollection? errors => Future.successful(respondError(UnprocessableEntity, InvalidUriParams.errorKey, errors.map(_.message).mkString(", ")) - ), - params => performSearchAndRespond(params) + ), { params: SearchParams => + // Extract the similar parameter from the structured query + val maybeSimilarImageId = params.structuredQuery.flatMap { + case Match(SimilarField, SimilarValue(imageId)) => + logger.info("Saw similar parameter: " + imageId) + Some(imageId) + case _ => None + }.headOption + + val eventualMaybeSimilarToVector = maybeSimilarImageId.map { imageId => + embeddingForImageId(imageId) + }.getOrElse { + Future { + None + } + } + + eventualMaybeSimilarToVector.flatMap { maybeSimilarToVector => + performSearchAndRespond(params, maybeSimilarToVector) + } + } ) + x + } + } + + private def embeddingForImageId(imageId: String)(implicit logMarker: LogMarker, instance: Instance, request: Authentication.Request[AnyContent]): Future[Option[List[Float]]] = { + for { + maybeImage <- elasticSearch.getImageById(imageId) + maybeEmbedding: Option[List[Float]] = maybeImage + .filter(image => isVisibleToAccessor(request.user, image)) + .flatMap(_.embedding) + .flatMap(_.geminiEmbedding2) + .map(_.image.map(_.toFloat)) + } yield { + maybeEmbedding } } diff --git a/media-api/app/lib/MediaApiConfig.scala b/media-api/app/lib/MediaApiConfig.scala index 13ea92abaae..83e027d6b01 100644 --- a/media-api/app/lib/MediaApiConfig.scala +++ b/media-api/app/lib/MediaApiConfig.scala @@ -3,15 +3,12 @@ package lib import com.gu.mediaservice.lib.aws.S3Bucket import com.gu.mediaservice.lib.config.{CommonConfigWithElastic, GridConfigResources} import com.gu.mediaservice.lib.elasticsearch.filters -import com.sksamuel.elastic4s.ElasticApi.{matchPhraseQuery, should} -import com.sksamuel.elastic4s.ElasticDsl.matchQuery -import com.sksamuel.elastic4s.requests.searches.queries.Query -import com.sksamuel.elastic4s.requests.searches.queries.matches.MatchQuery import com.gu.mediaservice.model.Instance +import com.sksamuel.elastic4s.ElasticApi.matchPhraseQuery +import com.sksamuel.elastic4s.requests.searches.queries.Query import org.joda.time.DateTime import scalaz.NonEmptyList -import scala.collection.immutable import scala.util.Try case class StoreConfig( @@ -20,8 +17,8 @@ case class StoreConfig( ) class MediaApiConfig(resources: GridConfigResources) extends CommonConfigWithElastic(resources) { - val configBucket: S3Bucket = S3Bucket(string("s3.config.bucket"), this) - val usageMailBucket: S3Bucket = S3Bucket(string("s3.usagemail.bucket"), this) + val configBucket: S3Bucket = S3Bucket.fromConfig("s3.config.bucket", this) + val usageMailBucket: S3Bucket = S3Bucket.fromConfig("s3.usagemail.bucket", this) val quotaStoreKey: String = string("quota.store.key") val quotaStoreConfig: StoreConfig = StoreConfig(configBucket, quotaStoreKey) @@ -70,10 +67,14 @@ class MediaApiConfig(resources: GridConfigResources) extends CommonConfigWithEla val restrictDownload: Boolean = boolean("restrictDownload") - val queueUrl: String = stringOpt("sqs.embedder.queue.url").getOrElse("") + val embedderQueueUrl: Option[String] = stringOpt("sqs.embedder.queue.url") + + val gcpProjectId: Option[String] = stringOpt("gcp.project.id") val aiSearchResultLimit: Int = intOpt("ai.search.resultLimit").getOrElse(200) val aiSearchEmbeddingCacheMaxSize: Int = intOpt("ai.search.embeddingCache.maxSize").getOrElse(500) + val aiSearchMinimumSimilarity: Float = stringOpt("ai.search.minimumSimilarity").map(_.toFloat).getOrElse(0.40f) + val aiSimilarImagesMinimumSimilarity: Float = stringOpt("ai.similarImages.minimumSimilarity").map(_.toFloat).getOrElse(0.80f) val maybeAgencyPickQuery: Option[Query] = agencyPicksIngredients.map { ingredients => filters.or( diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 2e37ef34b0e..01f8decc1d6 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -45,6 +45,8 @@ class ElasticSearch( val instancesClient: InstancesClient, ) extends ElasticSearchClient with ImageFields with MatchFields with FutureSyntax with GridLogging with MigrationStatusProvider { + private val knnSearchFieldName = "embedding.geminiEmbedding2.image" + private val maybeOrgOwnedExtraCount: Option[(String, ExtraCountConfig)] = if (config.shouldDisplayOrgOwnedCountAndFilterCheckbox) Some(s"owned" -> ExtraCountConfig( @@ -164,7 +166,7 @@ class ElasticSearch( def lookupIds(ids: List[String], offset: Int, length: Int)(implicit ex: ExecutionContext, logMarker: LogMarker, instance: Instance): Future[SearchResults] = { val query = filters.pinnedIds(ids) - val searchRequest = prepareSearch(query) + val searchRequest = prepareSearch(query, None) .trackTotalHits(true) .storedFields("_source") .from(offset) @@ -183,10 +185,7 @@ class ElasticSearch( logger.warn(logMarker, "knnSearch called but includeDenseVectorMappings=false, returning empty results") Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) } else { - val knn = Knn("embedding.cohereEmbedV4.image", filter = filterOpt) - .queryVector(queryEmbedding.map(_.toDouble)) - .k(k) - .numCandidates(numCandidates) + val knn = knnSimilarClause(queryEmbedding, k, numCandidates) val searchRequest = ElasticDsl.search(imagesCurrentAlias(instance)) .knn(knn) @@ -247,10 +246,11 @@ class ElasticSearch( )(implicit instance: Instance): SearchRequest = ElasticDsl .search(imagesCurrentAlias(instance)) - .knn(Knn("embedding.cohereEmbedV4.image", filter = filterOpt) + .knn(Knn(knnSearchFieldName, filter = filterOpt) .queryVector(queryEmbedding) .k(k) .numCandidates(numCandidates) + .similarity(config.aiSearchMinimumSimilarity) ) .size(k) @@ -394,11 +394,28 @@ class ElasticSearch( } } - def search(params: SearchParams)(implicit ex: ExecutionContext, instance: Instance, logMarker:MarkerMap = MarkerMap()): Future[SearchResults] = { + private def knnSimilarClause(queryEmbedding: List[Float], k: Int, numCandidates: Int) = { + Knn(knnSearchFieldName) + .queryVector(queryEmbedding.map(_.toDouble)) + .k(k) + .numCandidates(numCandidates) + .similarity(config.aiSimilarImagesMinimumSimilarity) + } + + def search(params: SearchParams, maybeSimilarToVector: Option[Seq[Float]] = None)(implicit ex: ExecutionContext, instance: Instance, logMarker:MarkerMap = MarkerMap()): Future[SearchResults] = { val query: Query = queryBuilder.makeQuery(params.structuredQuery) + + val similarTo: Option[Knn] = maybeSimilarToVector.map { s => + knnSimilarClause(s.toList, 1000, 2000) + } + val filterOpt: Option[Query] = queryBuilder.buildFilterOpt(params, searchFilters, syndicationFilter) + val withFilter = filterOpt.map { f => + boolQuery() must query filter f + }.getOrElse(query) + val sort = params.orderBy match { case Some("dateAddedToCollection") => sorts.dateAddedToCollectionDescending case _ => sorts.createSort(params.orderBy) @@ -429,7 +446,7 @@ class ElasticSearch( Seq.empty } - val searchRequest = prepareSearch(maybeWithFilter(query, filterOpt)) + val searchRequest: SearchRequest = prepareSearch(withFilter, similarTo) .trackTotalHits(trackTotalHits) .runtimeMappings(runtimeMappings) .storedFields("_source") // this needs to be explicit when using script fields @@ -437,11 +454,16 @@ class ElasticSearch( .aggregations(extraCountAggregations) .from(params.offset) .size(params.length) - .sortBy(sort) - executeAndLog(searchRequest, "image search"). + val withKnn: SearchRequest = similarTo.map { _ => + searchRequest + }.getOrElse { + searchRequest.sortBy(sort) + } + + executeAndLog(withKnn, "image search"). toMetric(Some(mediaApiMetrics.searchQueries), List(mediaApiMetrics.searchTypeDimension("results")))(_.result.took).map { r => - logSearchQueryIfTimedOut(searchRequest, r.result) + logSearchQueryIfTimedOut(withKnn, r.result) val imageHits = r.result.hits.hits.map(resolveHit).toSeq.flatten.map(i => (i.instance.id, i)) // setting trackTotalHits to false means we don't get any hit count at all. // Requester has explicitly opted into not caring about the total hits, so give them what they want (nothing). @@ -494,7 +516,7 @@ class ElasticSearch( val query = boolQuery().must(matchAllQuery()).filter(boolQuery().must(beSupplier, haveNestedUsage)) - val search = prepareSearch(query) size 0 + val search = prepareSearch(query, None) size 0 executeAndLog(search, s"$id usage search").map { r => import r.result @@ -568,7 +590,7 @@ class ElasticSearch( ) ) - val search = prepareSearch(query).size(0).aggs(composerQuotaAgg, frontsQuotaAgg, printQuotaAgg) + val search = prepareSearch(query, None).size(0).aggs(composerQuotaAgg, frontsQuotaAgg, printQuotaAgg) executeAndLog(search, s"$id quota count by supplier search").map { r => import r.result @@ -622,7 +644,7 @@ class ElasticSearch( val query = boolQuery().must(matchAllQuery()).filter(boolQuery().must(beSupplier, haveQualifyingUsage)) - val search = prepareSearch(query).trackTotalHits(true).from(offset).size(length) + val search = prepareSearch(query, None).trackTotalHits(true).from(offset).size(length) def isQualifyingUsage(usage: Usage): Boolean = UsageStore.countQualifyingStatuses.contains(usage.status) && @@ -676,7 +698,7 @@ class ElasticSearch( )(implicit ex: ExecutionContext, logMarker: LogMarker, instance: Instance): Future[AggregateSearchResults] = { logger.info(logMarker, "aggregate search: " + name + " / " + params + " / " + aggregation) val query = queryBuilder.makeQuery(params.structuredQuery) - val search = prepareSearch(query) aggregations aggregation size 0 + val search = prepareSearch(query, None) aggregations aggregation size 0 executeAndLog(search, s"$name aggregate search") .toMetric(Some(mediaApiMetrics.searchQueries), List(mediaApiMetrics.searchTypeDimension("aggregate")))(_.result.took).map { r => @@ -712,7 +734,7 @@ class ElasticSearch( def withSearchQueryTimeout(sr: SearchRequest): SearchRequest = sr timeout SearchQueryTimeout - private def prepareSearch(query: Query)(implicit instance: Instance): SearchRequest = { + private def prepareSearch(query: Query, maybeSimilarTo: Option[Knn])(implicit instance: Instance): SearchRequest = { val indexes = migrationStatus match { case completionPreview: CompletionPreview => List(completionPreview.migrationIndexName) case running: Running => List(imagesCurrentAlias(instance), running.migrationIndexName) @@ -722,7 +744,13 @@ class ElasticSearch( case running: Running => filters.and(query, filters.mustNot(filters.term("esInfo.migration.migratedTo", running.migrationIndexName))) case _ => query } - val searchRequest = ElasticDsl.search(indexes) query migrationAwareQuery + + val searchRequest = maybeSimilarTo.map { knn => + ElasticDsl.search(indexes).knn(knn.filter(migrationAwareQuery)) + }.getOrElse { + ElasticDsl.search(indexes) query migrationAwareQuery + } + withSearchQueryTimeout(searchRequest) } diff --git a/media-api/app/lib/elasticsearch/HybridResult.scala b/media-api/app/lib/elasticsearch/HybridResult.scala index 61f3ff0d6f7..4bb34b92107 100644 --- a/media-api/app/lib/elasticsearch/HybridResult.scala +++ b/media-api/app/lib/elasticsearch/HybridResult.scala @@ -53,7 +53,7 @@ object HybridResult extends GridLogging { )(implicit logMarker: LogMarker): Option[HybridResult] = resolveHit(hit).map { image => val semanticScore = image.instance.embedding - .flatMap(_.cohereEmbedV4) + .flatMap(_.geminiEmbedding2) // TODO deduplicate // We can't use the dot product shortcut because image vectors // are truncated 256-dim versions of a normalised 1536-dim vector, // meaning they will not have magnitude 1. diff --git a/media-api/app/lib/elasticsearch/QueryBuilder.scala b/media-api/app/lib/elasticsearch/QueryBuilder.scala index ed8989282b1..526cff4fa8e 100644 --- a/media-api/app/lib/elasticsearch/QueryBuilder.scala +++ b/media-api/app/lib/elasticsearch/QueryBuilder.scala @@ -4,14 +4,14 @@ import com.gu.mediaservice.lib.ImageFields import com.gu.mediaservice.lib.elasticsearch.filters import com.gu.mediaservice.lib.formatting.printDateTime import com.gu.mediaservice.lib.logging.GridLogging -import com.gu.mediaservice.model.Agency +import com.gu.mediaservice.model.{Agency, SyndicationStatus} import com.sksamuel.elastic4s.ElasticDsl import com.sksamuel.elastic4s.ElasticDsl._ import com.sksamuel.elastic4s.requests.common.Operator import com.sksamuel.elastic4s.requests.searches.queries.Query import com.sksamuel.elastic4s.requests.searches.queries.matches.{MultiMatchQuery, MultiMatchQueryBuilderType} -import lib.querysyntax._ import lib.MediaApiConfig +import lib.querysyntax.{SimilarField, _} import scalaz.NonEmptyList import scalaz.syntax.std.list._ @@ -61,6 +61,7 @@ class QueryBuilder(matchFields: Seq[String], overQuotaAgencies: () => List[Agenc case _ => matchPhraseQuery(resolveFieldPath(field), value) } case DateRange(start, end) => rangeQuery(resolveFieldPath(field)).gte(printDateTime(start)).lte(printDateTime(end)) + case SyndicationStatusValue(status) => new SyndicationFilter(config).statusFilter(SyndicationStatus.apply(status)) case e => throw InvalidQuery(s"Cannot do single field query on $e") } case HierarchyField => condition.value match { @@ -116,7 +117,16 @@ class QueryBuilder(matchFields: Seq[String], overQuotaAgencies: () => List[Agenc }.toList - val queryWithNormal = normal.foldLeft(boolQuery()) { + val withoutSimilar = normal.filter { + case Match(field, _) => + field match { + case SimilarField => false + case _ => true + } + case _ => true + } + + val queryWithNormal = withoutSimilar.foldLeft(boolQuery()) { case (query, Negation(cond)) => query.withNot(makeQueryBit(cond)) case (query, cond@Match(_, _)) => query.withMust(makeQueryBit(cond)) case (query, _) => query diff --git a/media-api/app/lib/elasticsearch/SyndicationFilter.scala b/media-api/app/lib/elasticsearch/SyndicationFilter.scala index 7d194f76bb6..f0de27f58de 100644 --- a/media-api/app/lib/elasticsearch/SyndicationFilter.scala +++ b/media-api/app/lib/elasticsearch/SyndicationFilter.scala @@ -5,6 +5,7 @@ import com.gu.mediaservice.lib.elasticsearch.filters import com.gu.mediaservice.model._ import com.gu.mediaservice.model.leases.{AllowSyndicationLease, DenySyndicationLease} import com.gu.mediaservice.model.usage.SyndicationUsage +import com.sksamuel.elastic4s.ElasticApi.not import com.sksamuel.elastic4s.requests.searches.RuntimeMapping import com.sksamuel.elastic4s.requests.searches.queries.Query import lib.MediaApiConfig @@ -14,18 +15,6 @@ class SyndicationFilter(config: MediaApiConfig) extends ImageFields { val isSyndicationDateFilterActive = config.isProd - private def syndicationRightsAcquired(acquired: Boolean): Query = filters.boolTerm( - field = "syndicationRights.rights.acquired", - value = acquired - ) - - private val noRightsAcquired: Query = filters.or( - filters.existsOrMissing("syndicationRights.rights.acquired", exists = false), - syndicationRightsAcquired(false) - ) - - private val hasRightsAcquired: Query = syndicationRightsAcquired(true) - private val hasAllowLease: Query = filters.term( "leases.leases.access", AllowSyndicationLease.name @@ -72,30 +61,22 @@ class SyndicationFilter(config: MediaApiConfig) extends ImageFields { filters.date("leases.leases.endDate", Some(DateTime.now), None).get ) - private def syndicationRightsPublished: Query = filters.or( - filters.existsOrMissing("syndicationRights.published", exists = false), - filters.date("syndicationRights.published", None, Some(DateTime.now)).get - ) - private val syndicatableCategory: Query = IsOwnedPhotograph().query def statusFilter(status: SyndicationStatus): Query = status match { case SentForSyndication => filters.and( - hasRightsAcquired, hasAllowLease, hasSyndicationUsage ) case QueuedForSyndication => filters.and( - hasRightsAcquired, filters.mustNot(hasSyndicationUsage), filters.and( hasAllowLease, leaseHasStarted, - syndicationRightsPublished + syndicatableCategory, ) ) case BlockedForSyndication => filters.and( - hasRightsAcquired, hasDenyLease ) case AwaitingReviewForSyndication => { @@ -114,7 +95,6 @@ class SyndicationFilter(config: MediaApiConfig) extends ImageFields { ) val rightsAcquiredNoLeaseFilter = filters.and( - hasRightsAcquired, syndicatableCategory, filters.mustNot(mustNotClauses:_*), ) @@ -127,7 +107,7 @@ class SyndicationFilter(config: MediaApiConfig) extends ImageFields { case _ => rightsAcquiredNoLeaseFilter } } - case UnsuitableForSyndication => noRightsAcquired + case UnsuitableForSyndication => not(IsOwnedPhotograph().query) } } diff --git a/media-api/app/lib/querysyntax/QuerySyntax.scala b/media-api/app/lib/querysyntax/QuerySyntax.scala index 73dfc89080d..edc8243081b 100644 --- a/media-api/app/lib/querysyntax/QuerySyntax.scala +++ b/media-api/app/lib/querysyntax/QuerySyntax.scala @@ -38,6 +38,7 @@ class QuerySyntax(val input: ParserInput) extends Parser with ImageFields { DateRangeMatch ~> Match | AtMatch | FileTypeMatch ~> Match | CollectionRule | + SyndicationStatusMatch ~> Match | ScopedMatch ~> Match | HashMatch | AnyMatch } @@ -180,6 +181,11 @@ class QuerySyntax(val input: ParserInput) extends Parser with ImageFields { MatchMimeTypeField ~ ':' ~ MatchMimeTypeValue } + def SyndicationStatusMatch = rule { + MatchSyndicationStatusField ~ ':' ~ MatchSyndicationStatusValue + } + + def AtMatch = rule { '@' ~ MatchDateRangeValue ~> (range => Match(SingleField(getFieldPath("uploadTime")), range)) } def MatchDateField = rule { capture(AllowedDateFieldName) ~> resolveDateField _ } @@ -188,10 +194,18 @@ class QuerySyntax(val input: ParserInput) extends Parser with ImageFields { capture("fileType") ~> resolveMimeTypeField _ } + def MatchSyndicationStatusField = rule { + capture("syndicationStatus") ~> resolveSyndicationStatusField _ + } + def resolveMimeTypeField(name: String): Field = name match { case "fileType" => SingleField(getFieldPath("mimeType")) } + def resolveSyndicationStatusField(name: String): Field = name match { + case "syndicationStatus" => SingleField("syndicationStatus") + } + def resolveDateField(name: String): Field = name match { case "date" | "uploaded" => SingleField("uploadTime") case "taken" => SingleField("dateTaken") @@ -218,6 +232,12 @@ class QuerySyntax(val input: ParserInput) extends Parser with ImageFields { def AllowedFileTypesValues = rule { "tiff" | "tif" | "jpg" | "jpeg" | "png" } + def MatchSyndicationStatusValue = rule { + capture(AllowedSyndicationStatusValues) ~> parseSyndicationStatus _ + } + + def AllowedSyndicationStatusValues = rule { "review" | "queued" | "unsuitable" | "sent" } + def translateMimeType(expr: String): MimeType = expr match { case s if s.equals("tif") || s.equals("tiff") => Tiff case s if s.equals("jpg") || s.equals("jpeg") => Jpeg @@ -226,6 +246,8 @@ class QuerySyntax(val input: ParserInput) extends Parser with ImageFields { def parseMimeType(expr: String): Value = Words(translateMimeType(expr).toString) + def parseSyndicationStatus(expr: String): Value = SyndicationStatusValue(expr) + def normaliseDateExpr(expr: String): String = expr.replaceAll("\\.", " ") val todayParser = DateAliasParser("today", today, tomorrow) diff --git a/media-api/app/lib/querysyntax/model.scala b/media-api/app/lib/querysyntax/model.scala index 871cb79e5ed..b663b7e6284 100644 --- a/media-api/app/lib/querysyntax/model.scala +++ b/media-api/app/lib/querysyntax/model.scala @@ -25,3 +25,4 @@ final case class DateRange(startDate: DateTime, endDate: DateTime) extends Value final case class HasValue(string: String) extends Value final case class IsValue(string: String) extends Value final case class SimilarValue(string: String) extends Value +final case class SyndicationStatusValue(string: String) extends Value diff --git a/media-api/conf/routes b/media-api/conf/routes index 9e84a5d8422..b4b5b5e683d 100644 --- a/media-api/conf/routes +++ b/media-api/conf/routes @@ -19,6 +19,7 @@ GET /images/:id/projection/diff controllers. GET /images/:id/fileMetadata controllers.MediaApi.getImageFileMetadata(id: String) GET /images/:imageId/uploadedBy controllers.MediaApi.uploadedBy(imageId: String) GET /images/:imageId/export/:exportId controllers.MediaApi.getImageExport(imageId: String, exportId: String) +GET /images/:imageId/export/:exportId/master/download controllers.MediaApi.downloadImageExportMaster(imageId: String, exportId: String) GET /images/:imageId/export/:exportId/asset/:width/download controllers.MediaApi.downloadImageExport(imageId: String, exportId: String, width: Int) GET /images/:imageId/export controllers.MediaApi.getImageExports(imageId: String) GET /images/:imageId/download controllers.MediaApi.downloadOriginalImage(imageId: String) diff --git a/media-api/test/lib/elasticsearch/ConditionFixtures.scala b/media-api/test/lib/elasticsearch/ConditionFixtures.scala index d702831ab21..f61c5c2e5c7 100644 --- a/media-api/test/lib/elasticsearch/ConditionFixtures.scala +++ b/media-api/test/lib/elasticsearch/ConditionFixtures.scala @@ -29,4 +29,6 @@ trait ConditionFixtures { val nestedCondition: Condition = Nested(SingleField("usages"), SingleField("usages.status"), Words("pending")) val anotherNestedCondition: Condition = Nested(SingleField("something"), SingleField("something.field"), Phrase("dogs")) + val syndicationStatusCondition: Condition = Match(SingleField("syndicationStatus"), SyndicationStatusValue("review")) + } diff --git a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala index 7cd2df6e6cf..938c663634a 100644 --- a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala +++ b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala @@ -546,12 +546,12 @@ class ElasticSearchTest extends ElasticSearchTestBase with Eventually with Elast } } - it("should return 3 images if an Internal tier queries for AwaitingReviewForSyndication images") { + it("should return 6 owned images if an Internal tier queries for AwaitingReviewForSyndication images") { // Elastic1 implementation is returning the images with reviewed and blocked syndicationStatus val search = SearchParams(tier = Internal, syndicationStatus = Some(AwaitingReviewForSyndication)) val searchResult = ES.search(search) whenReady(searchResult, timeout, interval) { result => - result.total shouldBe 3 + result.total shouldBe 6 } } } diff --git a/media-api/test/lib/elasticsearch/Fixtures.scala b/media-api/test/lib/elasticsearch/Fixtures.scala index 70375788be0..cbc05f30f6a 100644 --- a/media-api/test/lib/elasticsearch/Fixtures.scala +++ b/media-api/test/lib/elasticsearch/Fixtures.scala @@ -29,15 +29,17 @@ trait Fixtures { "thrall.kinesis.stream.name", "thrall.kinesis.lowPriorityStream.name", "domain.root", - "s3.config.bucket", - "s3.usagemail.bucket", + "s3.config.bucket.name", + "s3.usagemail.bucket.name", "quota.store.key", "es.index.aliases.current", "es.index.aliases.migration", "es6.url", - "s3.image.bucket", - "s3.thumb.bucket", - "publishing.image.bucket", + "s3.image.bucket.name", + "s3.thumb.bucket.name", + "publishing.image.bucket.name", + "s3.embeddingSources.bucket.name", + "s3.embeddings.bucket.name", "grid.stage", "grid.appName", "instance.service.my", @@ -92,7 +94,7 @@ trait Fixtures { syndicationRights = syndicationRights, leases = leases.getOrElse(LeasesByMedia.build(Nil)), usages = usages, - embedding = vector.map(v => Embedding(cohereEmbedV4 = Some(CohereV4Embedding(image = v)))), + embedding = vector.map(v => Embedding(geminiEmbedding2 = Some(GeminiEmbedding2(image = v)))), ) } diff --git a/media-api/test/lib/elasticsearch/HybridResultTest.scala b/media-api/test/lib/elasticsearch/HybridResultTest.scala index 715e5d41db8..576a1853c10 100644 --- a/media-api/test/lib/elasticsearch/HybridResultTest.scala +++ b/media-api/test/lib/elasticsearch/HybridResultTest.scala @@ -1,7 +1,7 @@ package lib.elasticsearch import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} -import com.gu.mediaservice.model.{CohereV4Embedding, Embedding, Handout, Image} +import com.gu.mediaservice.model.{Embedding, GeminiEmbedding2, Handout, Image} import com.sksamuel.elastic4s.requests.searches.SearchHit import org.scalactic.Tolerance import org.scalatest.OptionValues @@ -39,7 +39,7 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T private def imageWithEmbedding(id: String, embedding: Option[List[Double]]): Image = createImage(id, Handout()).copy( - embedding = embedding.map(vec => Embedding(cohereEmbedV4 = Some(CohereV4Embedding(image = vec)))) + embedding = embedding.map(vec => Embedding(geminiEmbedding2 = Some(GeminiEmbedding2(image = vec)))) ) private def sourceWrapperFor(image: Image): SourceWrapper[Image] = diff --git a/media-api/test/lib/elasticsearch/HybridSearchTest.scala b/media-api/test/lib/elasticsearch/HybridSearchTest.scala index 86a8a079251..2353c0060a2 100644 --- a/media-api/test/lib/elasticsearch/HybridSearchTest.scala +++ b/media-api/test/lib/elasticsearch/HybridSearchTest.scala @@ -65,17 +65,17 @@ class HybridSearchTest extends AnyFunSpec private val fiveSeconds = Duration(5, SECONDS) // The vector we'll "search" with - represents the user's query embedding. - private val queryEmbedding: List[Double] = firstBasisVector(256) + private val queryEmbedding: List[Double] = firstBasisVector(768) // TODO push to the in use embedder implementation private def aiImage(id: String, title: String, vector: List[Double]): Image = { val base = createImage(id = id, usageRights = Handout(), vector = Some(vector)) base.copy(metadata = base.metadata.copy(title = Some(title))) } - // 256-dim vectors to match the `embedding.cohereEmbedV4.image` dense_vector + // 768-dim vectors to match the `embedding.geminiEmbedding2.image` dense_vector // mapping, each constructed to have a known cosine similarity to the queryEmbedding. private def vectorWithScore(score: Double): List[Double] = { - vectorWithCosineSimilarity(256, score) + vectorWithCosineSimilarity(768, score) } // The full set of images shared by every spec below; indexed once in beforeAll. diff --git a/media-api/test/lib/elasticsearch/QueryBuilderTest.scala b/media-api/test/lib/elasticsearch/QueryBuilderTest.scala index ace13ce3005..5340a7d6226 100644 --- a/media-api/test/lib/elasticsearch/QueryBuilderTest.scala +++ b/media-api/test/lib/elasticsearch/QueryBuilderTest.scala @@ -59,6 +59,15 @@ class QueryBuilderTest extends AnyFunSpec with Matchers with ConditionFixtures w query.must.head.asInstanceOf[MatchPhraseQuery].value shouldBe "avalue" } + it("syndication status condition should give a syndicationStatusFilter query") { + val conditions = List(syndicationStatusCondition) + + val query = queryBuilder.makeQuery(conditions).asInstanceOf[BoolQuery] + + println(query) + query.must.size shouldBe 1 + } + it("multiple conditions should give multiple must conditions") { val query = queryBuilder.makeQuery(List(fieldPhraseMatchCondition, anotherFieldPhraseMatchCondition)).asInstanceOf[BoolQuery] diff --git a/media-api/test/lib/querysyntax/ParserTest.scala b/media-api/test/lib/querysyntax/ParserTest.scala index 4cfea97da10..e496d83b30c 100644 --- a/media-api/test/lib/querysyntax/ParserTest.scala +++ b/media-api/test/lib/querysyntax/ParserTest.scala @@ -504,6 +504,15 @@ class ParserTest extends AnyFunSpec with Matchers with BeforeAndAfter with Image } } + describe("syndicationStatus filter") { + it("should find review status images") { + Parser.run("syndicationStatus:review") should be (List( + Match(SingleField(getFieldPath("syndicationStatus")), SyndicationStatusValue("review")), + Negation(Match(IsField,IsValue("deleted"))), + NegationNested(Nested(SingleField("usages"), SingleField("usages.status"), Phrase("replaced"))))) + } + } + describe("fileType filter") { it("should find jpegs images") { Parser.run("fileType:jpeg") should be (List( diff --git a/metadata-editor/test/lib/EditsStoreTest.scala b/metadata-editor/test/lib/EditsStoreTest.scala index 4bbe56a5379..d692daaec29 100644 --- a/metadata-editor/test/lib/EditsStoreTest.scala +++ b/metadata-editor/test/lib/EditsStoreTest.scala @@ -6,8 +6,7 @@ import org.scalatest.concurrent.ScalaFutures import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers import org.scalatest.time.{Millis, Seconds, Span} -import org.testcontainers.containers.localstack.LocalStackContainer -import org.testcontainers.containers.localstack.LocalStackContainer.Service.DYNAMODB +import org.testcontainers.localstack.LocalStackContainer import org.testcontainers.utility.DockerImageName import software.amazon.awssdk.auth.credentials.{AwsBasicCredentials, StaticCredentialsProvider} import software.amazon.awssdk.regions.Region @@ -23,13 +22,13 @@ class EditsStoreTest extends AnyFunSpec with Matchers with ScalaFutures with Bef implicit val defaultPatience: PatienceConfig = PatienceConfig(timeout = Span(5, Seconds), interval = Span(100, Millis)) implicit val instance: Instance = Instance(id = "an-instance") - private val dynamoContainer = new LocalStackContainer(DockerImageName.parse("localstack/localstack:1.4.0")).withServices(DYNAMODB) + private val dynamoContainer = new LocalStackContainer(DockerImageName.parse("localstack/localstack:1.4.0")).withServices("dynamodb") dynamoContainer.start() val testTableName: String = "test-edits-table-" + UUID.randomUUID().toString - private val dynamoClient = DynamoDbClient.builder(). - endpointOverride(dynamoContainer.getEndpointOverride(DYNAMODB)). + private val dynamoClient: DynamoDbClient = DynamoDbClient.builder(). + endpointOverride(dynamoContainer.getEndpoint). region(Region.of(dynamoContainer.getRegion)). credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(dynamoContainer.getAccessKey, dynamoContainer.getSecretKey))).build() diff --git a/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala b/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala index d315662b740..f23bc806881 100644 --- a/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala +++ b/rest-lib/src/main/scala/com/gu/mediaservice/lib/auth/provider/ApiKeyAuthenticationProvider.scala @@ -26,7 +26,7 @@ class ApiKeyAuthenticationProvider(configuration: Configuration, resources: Auth var keyStorePlaceholder: Option[KeyStore] = _ override def initialise(): Unit = { - val store = new KeyStore(S3Bucket(configuration.get[String]("authKeyStoreBucket"), resources.commonConfig), resources.commonConfig, new S3(resources.commonConfig)) + val store = new KeyStore(S3Bucket.fromConfig(configuration, "authKeyStoreBucket", resources.commonConfig), resources.commonConfig, new S3(resources.commonConfig)) store.scheduleUpdates(resources.actorSystem.scheduler) keyStorePlaceholder = Some(store) } diff --git a/rest-lib/src/test/resources/application.conf b/rest-lib/src/test/resources/application.conf index 1bff942991f..0c02ca8dac3 100644 --- a/rest-lib/src/test/resources/application.conf +++ b/rest-lib/src/test/resources/application.conf @@ -3,6 +3,8 @@ grid.appName: "test" thrall.kinesis.stream.name: "not-used" thrall.kinesis.lowPriorityStream.name: "not-used" domain.root: "notused.example.com" -s3.image.bucket: "not-used" -s3.thumb.bucket: "not-used" -publishing.image.bucket: "not-used" +s3.image.bucket.name: "not-used" +s3.thumb.bucket.name: "not-used" +publishing.image.bucket.name: "not-used" +s3.embeddingSources.bucket.name: "not-used" +s3.embeddings.bucket.name: "not-used" diff --git a/thrall/app/ThrallComponents.scala b/thrall/app/ThrallComponents.scala index 0f908c99a41..590e47dee67 100644 --- a/thrall/app/ThrallComponents.scala +++ b/thrall/app/ThrallComponents.scala @@ -1,6 +1,7 @@ import com.gu.kinesis.{KinesisRecord, KinesisSource, ConsumerConfig => KclPekkoStreamConfig} import com.gu.mediaservice.GridClient -import com.gu.mediaservice.lib.aws.{S3Ops, S3Vectors, ThrallMessageSender} +import com.gu.mediaservice.lib.aws._ +import com.gu.mediaservice.lib.embeddings.GoogleCloudEmbedding import com.gu.mediaservice.lib.instances.{Instances, InstancesClient} import com.gu.mediaservice.lib.logging.MarkerMap import com.gu.mediaservice.lib.metadata.SoftDeletedMetadataTable @@ -11,16 +12,17 @@ import instances.{InstanceMessageSender, InstanceUsageMessage} import lib._ import lib.elasticsearch._ import lib.kinesis.{KinesisConfig, ThrallEventConsumer} +import lib.sqs.EmbeddingSqsConsumer import org.apache.pekko.Done import org.apache.pekko.stream.scaladsl.Source import play.api.ApplicationLoader.Context import router.Routes import software.amazon.awssdk.regions.Region -import software.amazon.awssdk.services.sqs.SqsClient import software.amazon.awssdk.services.sqs.model.GetQueueUrlRequest +import software.amazon.awssdk.services.sqs.{SqsAsyncClient, SqsClient} -import scala.concurrent.{Await, Future} import scala.concurrent.duration._ +import scala.concurrent.{Await, Future} import scala.language.postfixOps class ThrallComponents(context: Context) extends GridComponents(context, new ThrallConfig(_)) with StrictLogging with AssetsComponents @@ -78,7 +80,8 @@ class ThrallComponents(context: Context) extends GridComponents(context, new Thr gridClient, auth, instanceMessageSender, - usageEvents + usageEvents, + messageSender ) val thrallStreamProcessor = new ThrallStreamProcessor( @@ -91,7 +94,7 @@ class ThrallComponents(context: Context) extends GridComponents(context, new Thr val streamRunning: Future[Done] = thrallStreamProcessor.run() - val s3Vectors = new S3Vectors(config) + private val s3 = new S3(config) Source.repeat(()).throttle(1, per = 5.minute).map(_ => { implicit val logMarker: MarkerMap = MarkerMap() @@ -117,13 +120,43 @@ class ThrallComponents(context: Context) extends GridComponents(context, new Thr }).run() + private val lowPriorityMessageSender = new ThrallMessageSender(config.thrallKinesisLowPriorityStreamConfig) + + private val sqsAsyncClient: SqsAsyncClient = SqsAsyncClient.builder() + .region(Region.EU_WEST_1) + .build() + + private val maybeGcpProjectId = config.gcpProjectId + private val vertexApiLocation = "eu" + private val maybeGoogleCloudEmbedding = for { + gcpProjectId <- maybeGcpProjectId + } yield { + new GoogleCloudEmbedding(projectId = gcpProjectId, location = vertexApiLocation) + } + + private val maybeEmbedding = maybeGoogleCloudEmbedding + + private val maybeEmbedder = for { + embedding <- maybeEmbedding + queueUrl <- config.embeddingsQueueUrl // TODO name inconsistant with imageloader + } yield { + new Embedder(embedding, new SimpleSqsMessageConsumer(queueUrl, config)) + } + + for { + queueUrl <- config.embeddingsQueueUrl + embedder <- maybeEmbedder // This could be the embedding direct? + } yield { + logger.info("Listening for embedding requests on queue: " + queueUrl) + new EmbeddingSqsConsumer(queueUrl, sqsAsyncClient, embedder, store, lowPriorityMessageSender)(actorSystem, materializer, executionContext).start() + } val softDeletedMetadataTable = new SoftDeletedMetadataTable(config) val maybeCustomReapableEligibility = config.maybeReapableEligibilityClass(applicationLifecycle) - val thrallController = new ThrallController(es, store, migrationSourceWithSender.send, messageSender, actorSystem, auth, config.services, controllerComponents, gridClient) - val reaperController = new ReaperController(es, store, s3Vectors, authorisation, config, actorSystem.scheduler, maybeCustomReapableEligibility, softDeletedMetadataTable, thrallMetrics, auth, config.services, controllerComponents, wsClient, usageEvents) + val thrallController = new ThrallController(es, store, migrationSourceWithSender.send, messageSender, actorSystem, auth, config.services, controllerComponents, gridClient, s3, config.imageBucket, lowPriorityMessageSender) + val reaperController = new ReaperController(es, store, authorisation, config, actorSystem.scheduler, maybeCustomReapableEligibility, softDeletedMetadataTable, thrallMetrics, auth, config.services, controllerComponents, wsClient, usageEvents) val healthCheckController = new HealthCheck(es, streamRunning.isCompleted, config, controllerComponents) - override lazy val router = new Routes(httpErrorHandler, thrallController, reaperController, healthCheckController, management, assets) + override lazy val router = new Routes(httpErrorHandler, thrallController, reaperController, healthCheckController, management, assets).withPrefix("/thrall/") } diff --git a/thrall/app/controllers/ReaperController.scala b/thrall/app/controllers/ReaperController.scala index 8e32cb5e78d..f9e27b359da 100644 --- a/thrall/app/controllers/ReaperController.scala +++ b/thrall/app/controllers/ReaperController.scala @@ -2,7 +2,6 @@ package controllers import com.gu.mediaservice.lib.auth.Permissions.DeleteImage import com.gu.mediaservice.lib.auth.{Authentication, Authorisation, BaseControllerWithLoginRedirects} -import com.gu.mediaservice.lib.aws.S3Vectors import com.gu.mediaservice.lib.config.{InstanceForRequest, Services} import com.gu.mediaservice.lib.elasticsearch.ReapableEligibility import com.gu.mediaservice.lib.events.UsageEvents @@ -30,7 +29,6 @@ import scala.util.{Failure, Success} class ReaperController( es: ElasticSearch, store: ThrallStore, - s3Vectors: S3Vectors, authorisation: Authorisation, val config: ThrallConfig, scheduler: Scheduler, @@ -147,14 +145,12 @@ class ReaperController( instance = instance.id ) )) - s3VectorsDeletions <- s3Vectors.deleteEmbeddings(esIdsActuallySoftDeleted) } yield { metrics.softReaped.increment(n = esIdsActuallySoftDeleted.size) esIds.map { id => val wasSoftDeletedInES = esIdsActuallySoftDeleted.contains(id) val detail = Json.obj( - "ES" -> wasSoftDeletedInES, - "s3Vectors" -> s3VectorsDeletions.get(id).map(_.toString) + "ES" -> wasSoftDeletedInES ) logger.info(s"Soft deleted image $id : ${Json.stringify(detail)}") id -> detail @@ -177,6 +173,7 @@ class ReaperController( mainImagesS3Deletions <- store.deleteOriginals(esIdsActuallyDeleted) thumbsS3Deletions <- store.deleteThumbnails(esIdsActuallyDeleted) pngsS3Deletions <- store.deletePNGs(esIdsActuallyDeleted) + embeddingsS3Deletions <- store.deleteEmbeddings(esIdsActuallyDeleted) _ <- softDeletedMetadataTable.clearStatuses(esIdsActuallyDeleted) // TODO No one has issued an image-deleted notification to metadata-editor? Metadata will persist forever? } yield { @@ -188,7 +185,8 @@ class ReaperController( "ES" -> wasHardDeletedFromES, "mainImage" -> mainImagesS3Deletions.get(ImageIngestOperations.fileKeyFromId(id)), "thumb" -> thumbsS3Deletions.get(ImageIngestOperations.fileKeyFromId(id)), - "optimisedPng" -> pngsS3Deletions.get(ImageIngestOperations.optimisedPngKeyFromId(id)) + "optimisedPng" -> pngsS3Deletions.get(ImageIngestOperations.optimisedPngKeyFromId(id)), + "embeddings" -> embeddingsS3Deletions.get(ImageIngestOperations.embeddingKeyFromId(id)) ) if (wasHardDeletedFromES) { usageEvents.hardDeleteImage(instance = i, image = id) diff --git a/thrall/app/controllers/ThrallController.scala b/thrall/app/controllers/ThrallController.scala index 9ca3b5ca258..34e28fc176a 100644 --- a/thrall/app/controllers/ThrallController.scala +++ b/thrall/app/controllers/ThrallController.scala @@ -1,24 +1,27 @@ package controllers -import org.apache.pekko.actor.ActorSystem -import org.apache.pekko.stream.Materializer -import org.apache.pekko.stream.scaladsl.{Sink, Source} import com.gu.mediaservice.GridClient import com.gu.mediaservice.lib.auth.{Authentication, BaseControllerWithLoginRedirects} -import com.gu.mediaservice.lib.aws.ThrallMessageSender +import com.gu.mediaservice.lib.aws.{S3, S3Bucket, ThrallMessageSender, UpdateMessage} import com.gu.mediaservice.lib.config.{InstanceForRequest, Services} -import com.gu.mediaservice.lib.elasticsearch.{NotRunning, Running} +import com.gu.mediaservice.lib.elasticsearch.{NotRunning, Running, ScrolledSearchResults} import com.gu.mediaservice.lib.logging.GridLogging -import com.gu.mediaservice.model.{CompleteMigrationMessage, CreateMigrationIndexMessage, Instance, UpsertFromProjectionMessage} +import com.gu.mediaservice.model._ +import com.gu.mediaservice.syntax.MessageSubjects.ReindexImage import lib.elasticsearch.ElasticSearch import lib.{MigrationRequest, OptionalFutureRunner, Paging, ThrallStore} +import org.apache.pekko.actor.ActorSystem +import org.apache.pekko.stream.Materializer +import org.apache.pekko.stream.scaladsl.{Sink, Source} import org.joda.time.{DateTime, DateTimeZone} import play.api.data.Form import play.api.data.Forms._ +import play.api.libs.Files import play.api.libs.json.Json -import play.api.mvc.{Action, AnyContent, ControllerComponents} +import play.api.mvc.{Action, AnyContent, ControllerComponents, MultipartFormData} -import scala.concurrent.duration.DurationInt +import java.util.concurrent.TimeUnit +import scala.concurrent.duration.{Duration, DurationInt} import scala.concurrent.{Await, ExecutionContext, Future} import scala.language.postfixOps @@ -33,7 +36,10 @@ class ThrallController( override val auth: Authentication, override val services: Services, override val controllerComponents: ControllerComponents, - gridClient: GridClient + gridClient: GridClient, + s3: S3, + imageBucket: S3Bucket, + lowPriorityMessageSender: ThrallMessageSender )(implicit val ec: ExecutionContext) extends BaseControllerWithLoginRedirects with GridLogging with InstanceForRequest { private val numberFormatter: Long => String = java.text.NumberFormat.getIntegerInstance().format @@ -142,7 +148,7 @@ class ThrallController( def startMigration = withLoginRedirectAsync { implicit request => val instance = instanceOf(request) - if(Form(single("start-confirmation" -> text)).bindFromRequest().get != "start"){ + if (Form(single("start-confirmation" -> text)).bindFromRequest().get != "start") { Future.successful(BadRequest("you did not enter 'start' in the text box")) } else { val msgFailedToFetchIndex = s"Could not fetch ES index details for alias '${es.imagesMigrationAlias(instance)}'" @@ -184,7 +190,7 @@ class ThrallController( def completeMigration(): Action[AnyContent] = withLoginRedirectAsync { implicit request => val instance = instanceOf(request) - if(Form(single("complete-confirmation" -> text)).bindFromRequest().get != "complete"){ + if (Form(single("complete-confirmation" -> text)).bindFromRequest().get != "complete") { Future.successful(BadRequest("you did not enter 'complete' in the text box")) } else { es.refreshAndRetrieveMigrationStatus(instance) match { @@ -220,11 +226,15 @@ class ThrallController( es.refreshAndRetrieveMigrationStatus(instance) Redirect(routes.ThrallController.index) } + def pauseMigration = { adjustMigration(es.pauseMigration) } + def resumeMigration = adjustMigration(es.resumeMigration) + def previewMigrationCompletion = adjustMigration(es.previewMigrationCompletion) + def unPreviewMigrationCompletion = adjustMigration(es.unPreviewMigrationCompletion) def migrateSingleImage: Action[AnyContent] = withLoginRedirectAsync { implicit request => @@ -291,4 +301,117 @@ class ThrallController( "id" -> text )(MigrateSingleImageForm.apply)(MigrateSingleImageForm.unapply) ) + + def reindex(): Action[AnyContent] = withLoginRedirect { implicit request => + Ok(views.html.reindexImage()) + } + + case class ReindexImageForm(id: String) + + val reindexImageFormReader: Form[ReindexImageForm] = Form( + mapping( + "id" -> text + )(ReindexImageForm.apply)(ReindexImageForm.unapply) + ) + + def reindexImage: Action[AnyContent] = withLoginRedirectAsync { implicit request => + implicit val instance: Instance = instanceOf(request) + val imageId = reindexImageFormReader.bindFromRequest().get.id + val reindexImageMessage = ReindexImageMessage(id = imageId, lastModified = DateTime.now(DateTimeZone.UTC), instance = instance) + messageSender.publish(reindexImageMessage) + Future.successful(Ok(s"reindex request for $imageId submitted")) + } + + def reindexAll(): Action[AnyContent] = withLoginRedirect { implicit request => + implicit val instance: Instance = instanceOf(request) + + logger.info(s"Reindex requested for instance ${instance.id}") + val s3Objects = Await.result(s3.listPaginating(imageBucket, Some(instance.id)), Duration(5, TimeUnit.MINUTES)) + + val mediaIds = s3Objects.flatMap { s3Object => + val key = imageBucket.keyFromURL(s3Object.uri) + logger.info("Reindexing s3 key: " + key) + key.split("/").lastOption.map(key => (key, s3Object.metadata.objectMetadata.lastModified.map(_.getMillis))) + }.sortBy(_._2).reverse.map(_._1) + + logger.info(s"Queuing reindex requests for ${mediaIds.size} images for instance ${instance.id}") + + val batches = mediaIds.grouped(100) + batches.foreach { batch => + val messages = batch.map { mediaId => + UpdateMessage( + subject = ReindexImage, + id = Some(mediaId), + instance = instance + ) + } + lowPriorityMessageSender.publish(messages) + } + Ok("ok") + } + + def reindexFromCsvPage: Action[AnyContent] = withLoginRedirect { implicit request => + Ok(views.html.reindexFromCsv()) + } + + def getAllDocumentIds(instanceId: String): Action[AnyContent] = withLoginRedirectAsync { implicit request => + implicit val instance: Instance = Instance(id = instanceId) + + def collectIds(results: ScrolledSearchResults, accumulated: List[String]): Future[List[String]] = { + val idsInTheScroll = results.hits.map(_.id) + val ids = accumulated ++ idsInTheScroll + logger.info("Scrolled up to: " + idsInTheScroll.lastOption) + + results.scrollId match { + case Some(scrollId) if results.hits.nonEmpty => + es.continueScrolling(scrollId).flatMap(next => collectIds(next, ids)) + case Some(scrollId) => + es.closeScroll(scrollId) + Future.successful(ids) + case None => + Future.successful(ids) + } + } + + es.startScrollingAllImageIds().flatMap(initial => collectIds(initial, List.empty)).map { ids => + val tmpFile = java.io.File.createTempFile("grid-document-ids-", ".json") + java.nio.file.Files.writeString(tmpFile.toPath, ids.mkString("\n")) + logger.info(s"getAllDocumentIds wrote ${ids.size} IDs to ${tmpFile.getAbsolutePath}") + Ok(Json.toJson(ids.size)) + } + } + + def reindexFromCsv: Action[MultipartFormData[Files.TemporaryFile]] = + Action(parse.multipartFormData).async { implicit request => + request.body + .file("csv") + .map { csv => + implicit val instance: Instance = instanceOf(request) + val mediaIds: Seq[String] = scala.io.Source.fromFile(csv.ref.toFile).getLines().toList + + val batches = mediaIds.grouped(100) + batches.foreach { batch => + val messages = batch.map { mediaId => + UpdateMessage( + subject = ReindexImage, + id = Some(mediaId), + instance = instance + ) + } + lowPriorityMessageSender.publish(messages) + } + + Future.successful { + Ok(s"reindex request for ${mediaIds.size} images submitted") + } + } + .getOrElse { + Future.successful( + Redirect(routes.ThrallController.reindexFromCsvPage()) + .flashing("error" -> "Missing file") + ) + } + } + + } diff --git a/thrall/app/lib/MigrationSourceWithSender.scala b/thrall/app/lib/MigrationSourceWithSender.scala index 48cbb29bfd1..02739a5424a 100644 --- a/thrall/app/lib/MigrationSourceWithSender.scala +++ b/thrall/app/lib/MigrationSourceWithSender.scala @@ -1,15 +1,15 @@ package lib -import org.apache.pekko.stream.scaladsl.Source -import org.apache.pekko.stream.{Materializer, OverflowStrategy, QueueOfferResult} -import org.apache.pekko.{Done, NotUsed} import com.gu.mediaservice.GridClient -import com.gu.mediaservice.lib.elasticsearch.{InProgress, Paused} -import com.gu.mediaservice.lib.instances.{InstancesClient, Instances} +import com.gu.mediaservice.lib.elasticsearch.{InProgress, Paused, ScrolledSearchResults} +import com.gu.mediaservice.lib.instances.InstancesClient import com.gu.mediaservice.lib.logging.GridLogging import com.gu.mediaservice.model.{Instance, MigrateImageMessage, MigrationMessage} import com.sksamuel.elastic4s.requests.searches.SearchHit -import lib.elasticsearch.{ElasticSearch, ScrolledSearchResults} +import lib.elasticsearch.ElasticSearch +import org.apache.pekko.stream.scaladsl.Source +import org.apache.pekko.stream.{Materializer, QueueOfferResult} +import org.apache.pekko.{Done, NotUsed} import play.api.libs.ws.WSRequest import java.time.Instant @@ -74,7 +74,7 @@ class MigrationSourceWithSenderFactory( case (InProgress(migrationIndexName), None) => es.startScrollingImageIdsToMigrate(migrationIndexName).map(handleScrollResponse) case (InProgress(_), Some(scrollId)) => - es.continueScrollingImageIdsToMigrate(scrollId).map(handleScrollResponse) + es.continueScrolling(scrollId).map(handleScrollResponse) case _ => Future.successful(List.empty) }).recover { case _ => // close existing scroll if it exists diff --git a/thrall/app/lib/ThrallConfig.scala b/thrall/app/lib/ThrallConfig.scala index 8518a442051..7bd85f9688f 100644 --- a/thrall/app/lib/ThrallConfig.scala +++ b/thrall/app/lib/ThrallConfig.scala @@ -11,9 +11,9 @@ import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider import software.amazon.awssdk.http.Protocol import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient import software.amazon.awssdk.regions.Region -import software.amazon.awssdk.services.cloudwatch.{CloudWatchAsyncClient, CloudWatchAsyncClientBuilder} -import software.amazon.awssdk.services.dynamodb.{DynamoDbAsyncClient, DynamoDbAsyncClientBuilder} -import software.amazon.awssdk.services.kinesis.{KinesisAsyncClient, KinesisAsyncClientBuilder} +import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient +import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient +import software.amazon.awssdk.services.kinesis.KinesisAsyncClient import software.amazon.kinesis.metrics.MetricsLevel import java.net.URI @@ -56,7 +56,7 @@ object KinesisReceiverConfig { } class ThrallConfig(resources: GridConfigResources) extends CommonConfigWithElastic(resources) { - val maybeReaperBucket: Option[S3Bucket] = stringOpt("s3.reaper.bucket").map(S3Bucket(_, this)) + val maybeReaperBucket: Option[S3Bucket] = S3Bucket.fromConfigOpt("s3.reaper.bucket", this) val maybeReaperCountPerRun: Option[Int] = intOpt("reaper.countPerRun") val metadataTopicArn: String = string("indexed.image.sns.topic.arn") @@ -82,4 +82,7 @@ class ThrallConfig(resources: GridConfigResources) extends CommonConfigWithElast val instanceUsageQueueName: String = string("instance.usage.queue.name") + val embeddingsQueueUrl: Option[String] = stringOpt("sqs.image.embedder.queue.url") + + val gcpProjectId: Option[String] = stringOpt("gcp.project.id") } diff --git a/thrall/app/lib/ThrallStore.scala b/thrall/app/lib/ThrallStore.scala index a221c0471c0..53a1407e05b 100644 --- a/thrall/app/lib/ThrallStore.scala +++ b/thrall/app/lib/ThrallStore.scala @@ -2,4 +2,4 @@ package lib import com.gu.mediaservice.lib -class ThrallStore(config: ThrallConfig) extends lib.ImageIngestOperations(config.imageBucket, config.thumbnailBucket, config, config.isVersionedS3) +class ThrallStore(config: ThrallConfig) extends lib.ImageIngestOperations(config.imageBucket, config.thumbnailBucket, config.embeddingSourcesBucket, config.embeddingsBucket, config, config.isVersionedS3) diff --git a/thrall/app/lib/ThrallStreamProcessor.scala b/thrall/app/lib/ThrallStreamProcessor.scala index 274d834ed87..84140a77158 100644 --- a/thrall/app/lib/ThrallStreamProcessor.scala +++ b/thrall/app/lib/ThrallStreamProcessor.scala @@ -1,17 +1,16 @@ package lib -import java.time.Instant -import org.apache.pekko.actor.ActorSystem -import org.apache.pekko.stream.scaladsl.{GraphDSL, MergePreferred, MergePrioritized, Source} -import org.apache.pekko.stream.{Materializer, SourceShape} -import org.apache.pekko.{Done, NotUsed} import com.gu.kinesis.KinesisRecord import com.gu.mediaservice.lib.DateTimeUtils -import com.gu.mediaservice.lib.aws.UpdateMessage import com.gu.mediaservice.lib.logging._ -import com.gu.mediaservice.model.{ExternalThrallMessage, InternalThrallMessage, ThrallMessage} -import lib.kinesis.{MessageTranslator, ThrallEventConsumer} +import com.gu.mediaservice.model.{ExternalThrallMessage, MigrationMessage, ThrallMessage} +import lib.kinesis.ThrallEventConsumer +import org.apache.pekko.actor.ActorSystem +import org.apache.pekko.stream.scaladsl.{GraphDSL, MergePreferred, Source} +import org.apache.pekko.stream.{Materializer, SourceShape} +import org.apache.pekko.{Done, NotUsed} +import java.time.Instant import scala.concurrent.{ExecutionContextExecutor, Future} import scala.util.{Failure, Success} @@ -61,25 +60,15 @@ class ThrallStreamProcessor( val mergedKinesisSource: Source[TaggedRecord[ThrallMessage], NotUsed] = Source.fromGraph(GraphDSL.create() { implicit graphBuilder => import GraphDSL.Implicits._ - val uiRecordSource = uiSource.map(kinesisRecord => + val uiRecordSource: Source[TaggedRecord[Array[Byte]], Future[Done]] = uiSource.map(kinesisRecord => TaggedRecord(kinesisRecord.data.toArray, kinesisRecord.approximateArrivalTimestamp, UiPriority, kinesisRecord.markProcessed)) - val automationRecordSource = automationSource.map(kinesisRecord => - TaggedRecord(kinesisRecord.data.toArray, kinesisRecord.approximateArrivalTimestamp, AutomationPriority, kinesisRecord.markProcessed)) - - val migrationMessagesSource = migrationSource.map { case MigrationRecord(internalThrallMessage, time) => + val migrationMessagesSource: Source[TaggedRecord[MigrationMessage], Future[Done]] = migrationSource.map { case MigrationRecord(internalThrallMessage, time) => TaggedRecord(internalThrallMessage, time, MigrationPriority, () => {}) } - // merge together ui and automation kinesis records - val uiAndAutomationRecordsMerge = graphBuilder.add(MergePreferred[TaggedRecord[Array[Byte]]](1)) - uiRecordSource ~> uiAndAutomationRecordsMerge.preferred - automationRecordSource ~> uiAndAutomationRecordsMerge.in(0) - // parse the kinesis records into thrall update messages (dropping those that fail) - val uiAndAutomationMessagesSource: PortOps[TaggedRecord[ExternalThrallMessage]] = - uiAndAutomationRecordsMerge.out - .map { taggedRecord => + val uiMessagesSource: Source[TaggedRecord[ExternalThrallMessage], Future[Done]] = uiRecordSource.map { taggedRecord => val parsedRecord = ThrallEventConsumer .parseRecord(taggedRecord.payload, taggedRecord.arrivalTimestamp) .map( @@ -98,35 +87,75 @@ class ThrallStreamProcessor( // merge in the re-ingestion source (preferring ui/automation) val mergePreferred = graphBuilder.add(MergePreferred[TaggedRecord[ThrallMessage]](1)) - uiAndAutomationMessagesSource ~> mergePreferred.preferred + uiMessagesSource ~> mergePreferred.preferred migrationMessagesSource ~> mergePreferred.in(0) - SourceShape(mergePreferred.out) + val out: SourceShape[TaggedRecord[ThrallMessage]] = SourceShape(mergePreferred.out) + out }) - def createStream(): Source[(TaggedRecord[ThrallMessage], Stopwatch, ThrallMessage), NotUsed] = { + val automationKinesisSource: Source[TaggedRecord[ThrallMessage], Future[Done]] = { + automationSource.map(kinesisRecord => + TaggedRecord(kinesisRecord.data.toArray, kinesisRecord.approximateArrivalTimestamp, AutomationPriority, kinesisRecord.markProcessed)) + .map { taggedRecord => + val parsedRecord = ThrallEventConsumer + .parseRecord(taggedRecord.payload, taggedRecord.arrivalTimestamp) + .map( + message => taggedRecord.copy(payload = message) + ) + // If we failed to parse the record (Left), we'll drop it below because we can't process it. + // However we still need to mark the record as processed, otherwise the kinesis stream can't progress + // and checkpoint will be stuck at this message forevermore. + parsedRecord.left.foreach(_ => taggedRecord.markProcessed()) + parsedRecord + } + // drop unparseable records + .collect { + case Right(taggedRecord) => taggedRecord + } + } + + def createUIStream(): Source[(TaggedRecord[ThrallMessage], Stopwatch, ThrallMessage), NotUsed] = { mergedKinesisSource.mapAsync(1) { result => val stopwatch = Stopwatch.start consumer.processMessage(result.payload) .recover { case _ => () } .map(_ => (result, stopwatch, result.payload)) - } - + } + } + def createAutomationStream(): Source[(TaggedRecord[ThrallMessage], Stopwatch, ThrallMessage), Future[Done]] = { + automationKinesisSource.mapAsync(5) { result => + val stopwatch = Stopwatch.start + consumer.processMessage(result.payload) + .recover { case _ => () } + .map(_ => (result, stopwatch, result.payload)) + } } + def run(): Future[Done] = { - val stream = this.createStream().runForeach { + val stream = this.createUIStream().runForeach { case (taggedRecord, stopwatch, _) => val markers = combineMarkers(taggedRecord, stopwatch.elapsed) logger.info(markers, "Record processed") taggedRecord.markProcessed() } - stream.onComplete { case Failure(exception) => logger.error("Thrall stream completed with failure", exception) case Success(_) => logger.info("Thrall stream completed with done, probably shutting down") } + val automationStream = this.createAutomationStream().runForeach { + case (taggedRecord, stopwatch, _) => + val markers = combineMarkers(taggedRecord, stopwatch.elapsed) + logger.info(markers, "Record processed") + taggedRecord.markProcessed() + } + automationStream.onComplete { + case Failure(exception) => logger.error("Thrall stream completed with failure", exception) + case Success(_) => logger.info("Thrall stream completed with done, probably shutting down") + } + stream } } diff --git a/thrall/app/lib/elasticsearch/ThrallMigrationClient.scala b/thrall/app/lib/elasticsearch/ThrallMigrationClient.scala index 75b012d4b62..7cdb79e928b 100644 --- a/thrall/app/lib/elasticsearch/ThrallMigrationClient.scala +++ b/thrall/app/lib/elasticsearch/ThrallMigrationClient.scala @@ -1,28 +1,29 @@ package lib.elasticsearch -import com.gu.mediaservice.lib.elasticsearch.{CompletionPreview, ElasticSearchClient, InProgress, MigrationAlreadyRunningError, MigrationNotRunningError, MigrationStatus, MigrationStatusProvider, NotRunning, Paused, Running} +import com.gu.mediaservice.lib.elasticsearch.{CompletionPreview, ElasticSearchClient, InProgress, MigrationAlreadyRunningError, MigrationNotRunningError, MigrationStatus, MigrationStatusProvider, NotRunning, Paused, Running, ScrolledSearchResults} import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} import com.gu.mediaservice.model.{Image, Instance} import com.sksamuel.elastic4s.ElasticApi.{existsQuery, matchQuery, not} import com.sksamuel.elastic4s.ElasticDsl import com.sksamuel.elastic4s.ElasticDsl.{addAlias, aliases, removeAlias, _} -import com.sksamuel.elastic4s.requests.searches.SearchHit import com.sksamuel.elastic4s.requests.searches.aggs.responses.bucket.Terms import com.sksamuel.elastic4s.requests.searches.aggs.responses.metrics.TopHits import lib.{FailedMigrationDetails, FailedMigrationSummary, FailedMigrationsGrouping, FailedMigrationsOverview} import play.api.libs.json.{JsObject, Json} import scala.concurrent.{ExecutionContext, Future} -import scala.concurrent.duration.DurationInt import scala.util.{Failure, Success} -final case class ScrolledSearchResults(hits: List[SearchHit], scrollId: Option[String]) - trait ThrallMigrationClient extends MigrationStatusProvider { self: ElasticSearchClient => - private val scrollKeepAlive = 5.minutes + def startScrollingAllImageIds()(implicit ex: ExecutionContext, logMarker: LogMarker = MarkerMap(), instance: Instance): Future[ScrolledSearchResults] = { + val query = search(imagesCurrentAlias(instance)).scroll(scrollKeepAlive).size(1000).fetchSource(false) + executeAndLog(query, "retrieving first batch of all image ids").map { response => + ScrolledSearchResults(response.result.hits.hits.toList, response.result.scrollId) + } + } def startScrollingImageIdsToMigrate(migrationIndexName: String) (implicit ex: ExecutionContext, logMarker: LogMarker = MarkerMap(), instance: Instance) = { @@ -35,19 +36,6 @@ trait ThrallMigrationClient extends MigrationStatusProvider { ScrolledSearchResults(response.result.hits.hits.toList, response.result.scrollId) } } - def continueScrollingImageIdsToMigrate(scrollId: String)(implicit ex: ExecutionContext, logMarker: LogMarker = MarkerMap()) = { - val query = searchScroll(scrollId).keepAlive(scrollKeepAlive) - executeAndLog(query, "retrieving next batch of image ids to migrate, continuation of scroll").map { response => - ScrolledSearchResults(response.result.hits.hits.toList, response.result.scrollId) - } - } - def closeScroll(scrollId: String)(implicit ex: ExecutionContext, logMarker: LogMarker = MarkerMap()) = { - val close = clearScroll(scrollId) - executeAndLog(close, s"Closing unwanted scroll").failed.foreach { e => - logger.error(logMarker, "ES closeScroll request failed", e) - } - } - private def adjustMigrationAlias(action: String)(handleIfApplicable: PartialFunction[MigrationStatus, Unit]) (implicit instance: Instance): Unit = { handleIfApplicable.applyOrElse( diff --git a/thrall/app/lib/kinesis/MessageProcessor.scala b/thrall/app/lib/kinesis/MessageProcessor.scala index cae60ac7575..8854aa83f7e 100644 --- a/thrall/app/lib/kinesis/MessageProcessor.scala +++ b/thrall/app/lib/kinesis/MessageProcessor.scala @@ -2,13 +2,15 @@ package lib.kinesis import com.gu.mediaservice.GridClient import com.gu.mediaservice.lib.auth.Authentication -import com.gu.mediaservice.lib.aws.EsResponse +import com.gu.mediaservice.lib.aws.{EsResponse, ThrallMessageSender, UpdateMessage} import com.gu.mediaservice.lib.elasticsearch.{ElasticNotFoundException, Running} import com.gu.mediaservice.lib.events.UsageEvents import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, combineMarkers} import com.gu.mediaservice.model.{AddImageLeaseMessage, CreateMigrationIndexMessage, DeleteImageExportsMessage, DeleteImageMessage, DeleteUsagesMessage, ImageMessage, MigrateImageMessage, RemoveImageLeaseMessage, ReplaceImageLeasesMessage, SetImageCollectionsMessage, SoftDeleteImageMessage, ThrallMessage, UnSoftDeleteImageMessage, UpdateImageExportsMessage, UpdateImagePhotoshootMetadataMessage, UpdateImageSyndicationMetadataMessage, UpdateImageUsagesMessage, UpdateImageUserMetadataMessage} import com.gu.mediaservice.model.usage.{Usage, UsageNotice} +import com.gu.mediaservice.syntax.MessageSubjects.Image import instances.{InstanceMessageSender, InstanceStatusMessage} +import org.joda.time.DateTime // import all except `Right`, which otherwise shadows the type used in `Either`s import com.gu.mediaservice.model.{Right => _, _} import com.gu.mediaservice.syntax.MessageSubjects @@ -33,7 +35,8 @@ class MessageProcessor( gridClient: GridClient, auth: Authentication, instanceMessageSender: InstanceMessageSender, - usageEvents: UsageEvents + usageEvents: UsageEvents, + messageSender: ThrallMessageSender ) extends GridLogging with MessageSubjects { def process(updateMessage: ThrallMessage, logMarker: LogMarker)(implicit ec: ExecutionContext): Future[Any] = { @@ -61,6 +64,7 @@ class MessageProcessor( case message: CompleteMigrationMessage => completeMigration(message, logMarker) case message: UpdateEmbeddingMessage => updateEmbedding(message, logMarker) case message: CreateInstanceMessage => setupNewInstance(message, logMarker) + case message: ReindexImageMessage => reindexImage(message, logMarker) case _ => logger.info(s"Unmatched ThrallMessage type: ${updateMessage.subject}; ignoring") Future.successful(()) @@ -273,4 +277,24 @@ class MessageProcessor( } } + private def reindexImage(message: ReindexImageMessage, logMarker: LogMarker)(implicit ec: ExecutionContext): Future[Boolean] = { + val mediaId = message.id + implicit val instance: Instance = message.instance + logger.info(s"Reindexing from s3 ${instance.id} / $mediaId") + + gridClient.getImageLoaderProjection(mediaId, auth.innerServiceCall).map { maybeImage => + logger.info(s"Projected ${instance.id} / $mediaId to ${maybeImage.map(_.id)}") + maybeImage.exists { image => + val updateMessage = UpsertFromProjectionMessage(image.id, image, DateTime.now, instance) + logger.info(s"Publishing projected image as a thrall image message: ${updateMessage.id}") + messageSender.publish(updateMessage) + true + } + }.recover { + case t: Throwable => + logger.warn(s"Error while reindexing ${instance.id} / $mediaId - Image has not been reindexed!") + throw t + } + } + } diff --git a/thrall/app/lib/kinesis/MessageTranslator.scala b/thrall/app/lib/kinesis/MessageTranslator.scala index 0081621c3ec..ab3a88d47c8 100644 --- a/thrall/app/lib/kinesis/MessageTranslator.scala +++ b/thrall/app/lib/kinesis/MessageTranslator.scala @@ -81,6 +81,10 @@ object MessageTranslator extends GridLogging { case Some(id) => Right(CreateInstanceMessage(id, updateMessage.lastModified, updateMessage.instance)) case _ => Left(MissingFieldsException(updateMessage.subject)) } + case ReindexImage => (updateMessage.id) match { + case Some(id) => Right(ReindexImageMessage(id, updateMessage.lastModified, updateMessage.instance)) + case _ => Left(MissingFieldsException(updateMessage.subject)) + } case _ => Left(ProcessorNotFoundException(updateMessage.subject)) } } diff --git a/thrall/app/lib/kinesis/ThrallEventConsumer.scala b/thrall/app/lib/kinesis/ThrallEventConsumer.scala index fc4f0ddb9ed..f9cd3985c65 100644 --- a/thrall/app/lib/kinesis/ThrallEventConsumer.scala +++ b/thrall/app/lib/kinesis/ThrallEventConsumer.scala @@ -3,11 +3,11 @@ package lib.kinesis import org.apache.pekko.actor.ActorSystem import com.gu.mediaservice.GridClient import com.gu.mediaservice.lib.auth.Authentication -import com.gu.mediaservice.lib.aws.UpdateMessage +import com.gu.mediaservice.lib.aws.{ThrallMessageSender, UpdateMessage} import com.gu.mediaservice.lib.events.UsageEvents import com.gu.mediaservice.lib.json.{JsonByteArrayUtil, PlayJsonHelpers} import com.gu.mediaservice.lib.logging._ -import com.gu.mediaservice.model.{ExternalThrallMessage, ThrallMessage} +import com.gu.mediaservice.model.{ExternalThrallMessage, ReindexImageMessage, ThrallMessage} import instances.InstanceMessageSender import lib._ import lib.elasticsearch._ @@ -26,15 +26,16 @@ class ThrallEventConsumer(es: ElasticSearch, gridClient: GridClient, auth: Authentication, instanceMessageSender: InstanceMessageSender, - usageEvents: UsageEvents + usageEvents: UsageEvents, + messageSender: ThrallMessageSender ) extends PlayJsonHelpers with GridLogging { private val attemptTimeout = FiniteDuration(20, SECONDS) private val delay = FiniteDuration(1, MILLISECONDS) - private val attempts = 2 - private val timeout = attemptTimeout * attempts + delay * (attempts - 1) + private val retries = 2 + private val timeout = attemptTimeout * retries + delay * (retries - 1) - private val messageProcessor = new MessageProcessor(es, store, metadataEditorNotifications, gridClient, auth, instanceMessageSender, usageEvents) + private val messageProcessor = new MessageProcessor(es, store, metadataEditorNotifications, gridClient, auth, instanceMessageSender, usageEvents, messageSender) private implicit val implicitActorSystem: ActorSystem = actorSystem @@ -55,7 +56,7 @@ class ThrallEventConsumer(es: ElasticSearch, */ (marker) => { messageProcessor.process(message, marker) - }, attempts, attemptTimeout, delay, marker + }, retries, attemptTimeout, delay, marker ).transform { case Success(_) => { logger.info( @@ -83,6 +84,11 @@ class ThrallEventConsumer(es: ElasticSearch, combineMarkers(marker, stopwatch.elapsed), s"Failed to process ${message.subject} message; message will be ignored:", e ) + message match { + case r: ReindexImageMessage => + logger.error("Failed to process reindex of: " + r.id) + case _ => () + } Failure(e) } } diff --git a/thrall/app/lib/sqs/EmbeddingSqsConsumer.scala b/thrall/app/lib/sqs/EmbeddingSqsConsumer.scala new file mode 100644 index 00000000000..00e4c70adc4 --- /dev/null +++ b/thrall/app/lib/sqs/EmbeddingSqsConsumer.scala @@ -0,0 +1,82 @@ +package lib.sqs + +import com.gu.mediaservice.lib.ImageIngestOperations +import com.gu.mediaservice.lib.aws.{Embedder, EmbedderMessage, ThrallMessageSender} +import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} +import com.gu.mediaservice.model.{Instance, MimeType, UpdateEmbeddingMessage} +import com.typesafe.scalalogging.StrictLogging +import lib.ThrallStore +import org.apache.pekko.actor.ActorSystem +import org.apache.pekko.stream.Materializer +import org.apache.pekko.stream.connectors.sqs.scaladsl.{SqsAckFlow, SqsSource} +import org.apache.pekko.stream.connectors.sqs.{MessageAction, SqsSourceSettings} +import org.apache.pekko.stream.scaladsl.{Keep, Sink} +import org.joda.time.DateTime +import play.api.libs.json.Json +import software.amazon.awssdk.services.sqs.SqsAsyncClient + +import scala.concurrent.{ExecutionContext, Future} + +class EmbeddingSqsConsumer(queueUrl: String, sqsClient: SqsAsyncClient, embedder: Embedder, thrallStore: ThrallStore, lowPriorityMessageSender: ThrallMessageSender) + (implicit system: ActorSystem, mat: Materializer, ec: ExecutionContext) + extends StrictLogging { + + private val sourceSettings = SqsSourceSettings.Defaults + private implicit val logMarker: LogMarker = MarkerMap() + + def start(): Future[_] = { + logger.info(s"Starting Pekko Connectors SQS consumer on $queueUrl") + SqsSource(queueUrl, sourceSettings)(sqsClient) + .map { message => + logger.info(s"Received SQS message id=${message.messageId()} body=${message.body()}") + + val maybeParsed = Json.parse(message.body()).validate[EmbedderMessage].asOpt + logger.info("Parsed: " + maybeParsed) + + maybeParsed.map { parsed => + // TODO check file exists + val s3Object = thrallStore.getEmbeddingStoreImage(parsed.s3Key) // TODO imageid to keep knowledge of path in the store + val response = s3Object.response() + val bytes = + try s3Object.readAllBytes() + finally s3Object.close() + + // Take the source image mimeType from S3 metadata for embedders who want it + val maybeMimeTypeHeader = Option(response.contentType()) + .filterNot(_.equalsIgnoreCase("application/octet-stream")) + val maybeMimeType = maybeMimeTypeHeader.map(MimeType(_)) // TODO recover to None + logger.info(s"Got embedding source with mineType $maybeMimeTypeHeader / $maybeMimeType and image metadata title: ${parsed.imageMetadata.flatMap(_.title)}") + + maybeMimeType.map { mimeType => + val eventualEmbedding = embedder.createImageEmbedding(bytes, mimeType, parsed.imageMetadata) + eventualEmbedding.map { embedding => + logger.info("Got embedding: " + embedding) + + // Store the embedding for reindexing + thrallStore.storeEmbedding(ImageIngestOperations.embeddingKeyFromId(parsed.imageId)(Instance(parsed.instance)), embedding) + + // Issue an UpdateEmbedding message + val updateEmbeddingMessage = UpdateEmbeddingMessage( + id = parsed.imageId, + lastModified = DateTime.now, // TODO check this against the lambda + embedding = embedding, + instance = Instance(id = parsed.instance) + ) + lowPriorityMessageSender.publish(updateEmbeddingMessage) + } + + }.getOrElse { + logger.warn("Skipping embedding source with missing mimeType: " + parsed.s3Key) + Future.successful(()) + } + } + + MessageAction.delete(message) + } + .via(SqsAckFlow(queueUrl)(sqsClient)) + .toMat(Sink.foreach { result => + logger.debug(s"Acked SQS message: $result") + })(Keep.right) + .run() + } +} diff --git a/thrall/app/views/index.scala.html b/thrall/app/views/index.scala.html index 45c935b0137..fb96e28a8fd 100644 --- a/thrall/app/views/index.scala.html +++ b/thrall/app/views/index.scala.html @@ -48,6 +48,10 @@