diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 2b47bee63e..6ea0cc9835 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -57,9 +57,9 @@ jobs: - 'infra/security/**' infra1: - 'infra/orchestrator-data/**' - - 'infra/enrolment-records:repository/**' - - 'infra/enrolment-records:realm-store/**' - - 'infra/enrolment-records:room-store/**' + - 'infra/enrolment-records/repository/**' + - 'infra/enrolment-records/realm-store/**' + - 'infra/enrolment-records/room-store/**' - 'infra/recent-user-activity/**' - 'infra/config-store/**' - 'infra/credential-store/**' diff --git a/infra/enrolment-records/realm-store/src/main/java/com/simprints/infra/enrolment/records/realm/store/config/RealmConfig.kt b/infra/enrolment-records/realm-store/src/main/java/com/simprints/infra/enrolment/records/realm/store/config/RealmConfig.kt index ff6e0b7881..081d148319 100644 --- a/infra/enrolment-records/realm-store/src/main/java/com/simprints/infra/enrolment/records/realm/store/config/RealmConfig.kt +++ b/infra/enrolment-records/realm-store/src/main/java/com/simprints/infra/enrolment/records/realm/store/config/RealmConfig.kt @@ -38,6 +38,6 @@ class RealmConfig @Inject constructor() { .build() companion object { - private const val REALM_SCHEMA_VERSION: Long = 18 + private const val REALM_SCHEMA_VERSION: Long = 19 } } diff --git a/infra/enrolment-records/realm-store/src/main/java/com/simprints/infra/enrolment/records/realm/store/migration/RealmMigrations.kt b/infra/enrolment-records/realm-store/src/main/java/com/simprints/infra/enrolment/records/realm/store/migration/RealmMigrations.kt index add23befc5..57f57831e9 100644 --- a/infra/enrolment-records/realm-store/src/main/java/com/simprints/infra/enrolment/records/realm/store/migration/RealmMigrations.kt +++ b/infra/enrolment-records/realm-store/src/main/java/com/simprints/infra/enrolment/records/realm/store/migration/RealmMigrations.kt @@ -10,11 +10,18 @@ import com.simprints.infra.enrolment.records.realm.store.models.toRealmInstant import io.realm.kotlin.dynamic.DynamicMutableRealmObject import io.realm.kotlin.dynamic.getValue import io.realm.kotlin.migration.AutomaticSchemaMigration +import io.realm.kotlin.types.RealmList import io.realm.kotlin.types.RealmUUID import java.util.Date import java.util.UUID internal class RealmMigrations : AutomaticSchemaMigration { + companion object { + private const val SUBJECT_FINGERPRINT_SAMPLES_FIELD = "fingerprintSamples" + private const val SUBJECT_FACE_SAMPLES_FIELD = "faceSamples" + private const val REFERENCE_ID_FIELD = "referenceId" + } + override fun migrate(migrationContext: AutomaticSchemaMigration.MigrationContext) { val oldVersion = migrationContext.oldRealm.schemaVersion() val newVersion = migrationContext.newRealm.schemaVersion() @@ -25,6 +32,7 @@ internal class RealmMigrations : AutomaticSchemaMigration { 10 -> migrateTo11(migrationContext) 11 -> migrateTo12(migrationContext) 12 -> migrateTo13(migrationContext) + 18 -> migrateTo19(migrationContext) } } } @@ -106,4 +114,30 @@ internal class RealmMigrations : AutomaticSchemaMigration { newObject?.set(SubjectsSchemaV13.SUBJECT_ID_FIELD, RealmUUID.from(subjectId)) } } + + /** + * Because of a missing Realm migration when `referenceId` was introduced, samples created + * before that schema change were left with an empty referenceId. + * + * Backfill a stable, shared UUID for all samples of the same subject and modality that still + * have an empty referenceId, mirroring the grouping that historically applied before + * referenceId existed. + */ + + private fun migrateTo19(migrationContext: AutomaticSchemaMigration.MigrationContext) { + migrationContext.enumerate(SubjectsSchemaV13.SUBJECT_TABLE) { _, newObject -> + newObject?.let { + backfillEmptyReferenceIds(it.getObjectList(SUBJECT_FINGERPRINT_SAMPLES_FIELD)) + backfillEmptyReferenceIds(it.getObjectList(SUBJECT_FACE_SAMPLES_FIELD)) + } + } + } + + private fun backfillEmptyReferenceIds(samples: RealmList) { + val samplesWithEmptyReferenceId = samples.filter { it.getValue(REFERENCE_ID_FIELD).isEmpty() } + if (samplesWithEmptyReferenceId.isEmpty()) return + + val newReferenceId = UUID.randomUUID().toString() + samplesWithEmptyReferenceId.forEach { it.set(REFERENCE_ID_FIELD, newReferenceId) } + } } diff --git a/infra/enrolment-records/realm-store/src/test/java/com/simprints/infra/enrolment/records/realm/store/migration/RealmMigrationsTest.kt b/infra/enrolment-records/realm-store/src/test/java/com/simprints/infra/enrolment/records/realm/store/migration/RealmMigrationsTest.kt index 4a803e16d8..f53f011b94 100644 --- a/infra/enrolment-records/realm-store/src/test/java/com/simprints/infra/enrolment/records/realm/store/migration/RealmMigrationsTest.kt +++ b/infra/enrolment-records/realm-store/src/test/java/com/simprints/infra/enrolment/records/realm/store/migration/RealmMigrationsTest.kt @@ -147,6 +147,42 @@ class RealmMigrationsTest { verify { newObject.set(eq(SubjectsSchemaV13.SUBJECT_ID_FIELD), eq(RealmUUID.from(GUID))) } } + @Test + fun `when migrating 18 to 19 backfills empty referenceId shared per modality`() { + setVersions(18L, 19L) + + val fingerprintSample1 = createSampleMock(referenceId = "") + val fingerprintSample2 = createSampleMock(referenceId = "") + val faceSample = createSampleMock(referenceId = "") + val untouchedFingerprintSample = createSampleMock(referenceId = "existing-ref") + + val newObject = mockk() + every { newObject.getObjectList(REFERENCE_ID_FINGERPRINT_SAMPLES_FIELD) } returns + realmListOf(fingerprintSample1, fingerprintSample2, untouchedFingerprintSample) + every { newObject.getObjectList(REFERENCE_ID_FACE_SAMPLES_FIELD) } returns realmListOf(faceSample) + + val enumeratorSlot = slot<(DynamicRealmObject, DynamicMutableRealmObject?) -> Unit>() + every { migrationContext.enumerate(any(), capture(enumeratorSlot)) } answers { + enumeratorSlot.captured.invoke(mockk(), newObject) + } + + realmMigrations.migrate(migrationContext) + + val fingerprintReferenceIdSlot = slot() + verify(exactly = 1) { fingerprintSample1.set(REFERENCE_ID_FIELD, capture(fingerprintReferenceIdSlot)) } + verify(exactly = 1) { fingerprintSample2.set(REFERENCE_ID_FIELD, eq(fingerprintReferenceIdSlot.captured)) } + verify(exactly = 0) { untouchedFingerprintSample.set(REFERENCE_ID_FIELD, any()) } + verify(exactly = 1) { faceSample.set(REFERENCE_ID_FIELD, any()) } + assertThat(fingerprintReferenceIdSlot.captured).isNotEmpty() + } + + private fun createSampleMock(referenceId: String): DynamicMutableRealmObject { + val sample = mockk() + every { sample.getValue(REFERENCE_ID_FIELD) } returns referenceId + every { sample.set(REFERENCE_ID_FIELD, any()) } returns sample + return sample + } + private fun setVersions( old: Long, new: Long, @@ -198,5 +234,9 @@ class RealmMigrationsTest { private const val UNIQUE_ID = "uniqueId" private const val NOT_UNIQUE_ID = "nonUniqueId" + + private const val REFERENCE_ID_FIELD = "referenceId" + private const val REFERENCE_ID_FINGERPRINT_SAMPLES_FIELD = "fingerprintSamples" + private const val REFERENCE_ID_FACE_SAMPLES_FIELD = "faceSamples" } } diff --git a/infra/enrolment-records/room-store/schemas/com.simprints.infra.enrolment.records.room.store.SubjectsDatabase/3.json b/infra/enrolment-records/room-store/schemas/com.simprints.infra.enrolment.records.room.store.SubjectsDatabase/3.json new file mode 100644 index 0000000000..c994bc8d83 --- /dev/null +++ b/infra/enrolment-records/room-store/schemas/com.simprints.infra.enrolment.records.room.store.SubjectsDatabase/3.json @@ -0,0 +1,241 @@ +{ + "formatVersion": 1, + "database": { + "version": 3, + "identityHash": "8dad14b775cb4eea6cb7293fbcf84b30", + "entities": [ + { + "tableName": "DbSubject", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`subjectId` TEXT NOT NULL, `projectId` TEXT NOT NULL, `attendantId` TEXT NOT NULL, `moduleId` TEXT NOT NULL, `createdAt` INTEGER, `updatedAt` INTEGER, PRIMARY KEY(`subjectId`))", + "fields": [ + { + "fieldPath": "subjectId", + "columnName": "subjectId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "projectId", + "columnName": "projectId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "attendantId", + "columnName": "attendantId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "moduleId", + "columnName": "moduleId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "createdAt", + "columnName": "createdAt", + "affinity": "INTEGER" + }, + { + "fieldPath": "updatedAt", + "columnName": "updatedAt", + "affinity": "INTEGER" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "subjectId" + ] + }, + "indices": [ + { + "name": "index_DbSubject_projectId_subjectId", + "unique": false, + "columnNames": [ + "projectId", + "subjectId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_DbSubject_projectId_subjectId` ON `${TABLE_NAME}` (`projectId`, `subjectId`)" + }, + { + "name": "index_DbSubject_projectId_moduleId_subjectId", + "unique": false, + "columnNames": [ + "projectId", + "moduleId", + "subjectId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_DbSubject_projectId_moduleId_subjectId` ON `${TABLE_NAME}` (`projectId`, `moduleId`, `subjectId`)" + }, + { + "name": "index_DbSubject_projectId_attendantId_subjectId", + "unique": false, + "columnNames": [ + "projectId", + "attendantId", + "subjectId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_DbSubject_projectId_attendantId_subjectId` ON `${TABLE_NAME}` (`projectId`, `attendantId`, `subjectId`)" + } + ] + }, + { + "tableName": "DbBiometricTemplate", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`uuid` TEXT NOT NULL, `subjectId` TEXT NOT NULL, `identifier` INTEGER, `templateData` BLOB NOT NULL, `format` TEXT NOT NULL, `referenceId` TEXT NOT NULL, `modality` INTEGER NOT NULL, PRIMARY KEY(`uuid`), FOREIGN KEY(`subjectId`) REFERENCES `DbSubject`(`subjectId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "uuid", + "columnName": "uuid", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "subjectId", + "columnName": "subjectId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "identifier", + "columnName": "identifier", + "affinity": "INTEGER" + }, + { + "fieldPath": "templateData", + "columnName": "templateData", + "affinity": "BLOB", + "notNull": true + }, + { + "fieldPath": "format", + "columnName": "format", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "referenceId", + "columnName": "referenceId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "modality", + "columnName": "modality", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "uuid" + ] + }, + "indices": [ + { + "name": "index_DbBiometricTemplate_format_subjectId", + "unique": false, + "columnNames": [ + "format", + "subjectId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_DbBiometricTemplate_format_subjectId` ON `${TABLE_NAME}` (`format`, `subjectId`)" + }, + { + "name": "index_DbBiometricTemplate_subjectId", + "unique": false, + "columnNames": [ + "subjectId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_DbBiometricTemplate_subjectId` ON `${TABLE_NAME}` (`subjectId`)" + } + ], + "foreignKeys": [ + { + "table": "DbSubject", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "subjectId" + ], + "referencedColumns": [ + "subjectId" + ] + } + ] + }, + { + "tableName": "DbExternalCredential", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `value` TEXT NOT NULL, `subjectId` TEXT NOT NULL, `type` TEXT NOT NULL, PRIMARY KEY(`value`, `subjectId`), FOREIGN KEY(`subjectId`) REFERENCES `DbSubject`(`subjectId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "value", + "columnName": "value", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "subjectId", + "columnName": "subjectId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "type", + "columnName": "type", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "value", + "subjectId" + ] + }, + "indices": [ + { + "name": "index_DbExternalCredential_subjectId", + "unique": false, + "columnNames": [ + "subjectId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_DbExternalCredential_subjectId` ON `${TABLE_NAME}` (`subjectId`)" + } + ], + "foreignKeys": [ + { + "table": "DbSubject", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "subjectId" + ], + "referencedColumns": [ + "subjectId" + ] + } + ] + } + ], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '8dad14b775cb4eea6cb7293fbcf84b30')" + ] + } +} \ No newline at end of file diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabase.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabase.kt index 157af2c0ab..2139e9c694 100644 --- a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabase.kt +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabase.kt @@ -7,6 +7,7 @@ import androidx.room.Room import androidx.room.RoomDatabase import com.simprints.infra.enrolment.records.room.store.SubjectsDatabase.Companion.SUBJECT_DB_VERSION import com.simprints.infra.enrolment.records.room.store.migration.SubjectMigration1to2 +import com.simprints.infra.enrolment.records.room.store.migration.SubjectMigration2to3 import com.simprints.infra.enrolment.records.room.store.models.DbBiometricTemplate import com.simprints.infra.enrolment.records.room.store.models.DbExternalCredential import com.simprints.infra.enrolment.records.room.store.models.DbSubject @@ -35,12 +36,12 @@ abstract class SubjectsDatabase : RoomDatabase() { ): SubjectsDatabase { val builder = Room .databaseBuilder(context, SubjectsDatabase::class.java, dbName) - .addMigrations(SubjectMigration1to2()) + .addMigrations(SubjectMigration1to2(), SubjectMigration2to3()) if (BuildConfig.DB_ENCRYPTION) { builder.openHelperFactory(factory) } return builder.build() } - const val SUBJECT_DB_VERSION = 2 + const val SUBJECT_DB_VERSION = 3 } } diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/migration/SubjectMigration2to3.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/migration/SubjectMigration2to3.kt new file mode 100644 index 0000000000..23637f7df2 --- /dev/null +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/migration/SubjectMigration2to3.kt @@ -0,0 +1,32 @@ +package com.simprints.infra.enrolment.records.room.store.migration + +import androidx.room.migration.Migration +import androidx.sqlite.db.SupportSQLiteDatabase +import java.util.UUID + +/** + * Schema version 2 -> 3 + * + * Changes: + * - Backfills empty [DbBiometricTemplate.referenceId] values caused by a missing Realm migration. + * Each distinct (subjectId, modality) group with an empty referenceId is assigned a new UUID so + * that all templates from the same original capture session share the same reference ID. + */ +class SubjectMigration2to3 : Migration(2, 3) { + override fun migrate(db: SupportSQLiteDatabase) { + val cursor = db.query( + "SELECT DISTINCT subjectId, modality FROM DbBiometricTemplate WHERE referenceId = ''", + ) + cursor.use { + while (cursor.moveToNext()) { + val subjectId = cursor.getString(0) + val modality = cursor.getInt(1) + val newReferenceId = UUID.randomUUID().toString() + db.execSQL( + "UPDATE DbBiometricTemplate SET referenceId = ? WHERE subjectId = ? AND modality = ? AND referenceId = ''", + arrayOf(newReferenceId, subjectId, modality), + ) + } + } + } +} diff --git a/infra/enrolment-records/room-store/src/test/java/com/simprints/infra/enrolment/records/room/store/migration/Migration2to3Test.kt b/infra/enrolment-records/room-store/src/test/java/com/simprints/infra/enrolment/records/room/store/migration/Migration2to3Test.kt new file mode 100644 index 0000000000..dcd60d2e77 --- /dev/null +++ b/infra/enrolment-records/room-store/src/test/java/com/simprints/infra/enrolment/records/room/store/migration/Migration2to3Test.kt @@ -0,0 +1,92 @@ +package com.simprints.infra.enrolment.records.room.store.migration + +import androidx.room.testing.MigrationTestHelper +import androidx.sqlite.db.SupportSQLiteDatabase +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import com.google.common.truth.Truth.assertThat +import com.simprints.infra.enrolment.records.room.store.SubjectsDatabase +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class Migration2to3Test { + @get:Rule + val helper = MigrationTestHelper( + InstrumentationRegistry.getInstrumentation(), + SubjectsDatabase::class.java, + ) + + @Test + fun `when migrating from 2 to 3 then empty referenceIds are backfilled per subject and modality`() { + val db2 = helper.createDatabase(name = TEST_DB, version = 2) + insertSubject(db2, SUBJECT_1) + insertSubject(db2, SUBJECT_2) + + // Subject 1 has two fingerprint templates with an empty referenceId (should share the same new id) + insertTemplate(db2, uuid = "t1", subjectId = SUBJECT_1, referenceId = "", modality = FINGERPRINT_MODALITY) + insertTemplate(db2, uuid = "t2", subjectId = SUBJECT_1, referenceId = "", modality = FINGERPRINT_MODALITY) + // Subject 1 also has a face template with an empty referenceId (different modality -> different new id) + insertTemplate(db2, uuid = "t3", subjectId = SUBJECT_1, referenceId = "", modality = FACE_MODALITY) + // Subject 2 already has a valid referenceId which must not be touched + insertTemplate(db2, uuid = "t4", subjectId = SUBJECT_2, referenceId = "existing-ref", modality = FINGERPRINT_MODALITY) + + val db3 = helper.runMigrationsAndValidate( + name = TEST_DB, + version = 3, + validateDroppedTables = true, + SubjectMigration2to3(), + ) + + val referenceIds = mutableMapOf() + db3.query("SELECT uuid, referenceId FROM DbBiometricTemplate").use { cursor -> + while (cursor.moveToNext()) { + referenceIds[cursor.getString(0)] = cursor.getString(1) + } + } + + // No empty referenceIds should remain + assertThat(referenceIds.values).doesNotContain("") + // Templates from the same subject+modality group share the same backfilled referenceId + assertThat(referenceIds["t1"]).isEqualTo(referenceIds["t2"]) + // Templates from a different modality get a different backfilled referenceId + assertThat(referenceIds["t3"]).isNotEqualTo(referenceIds["t1"]) + // Existing valid referenceIds are preserved + assertThat(referenceIds["t4"]).isEqualTo("existing-ref") + + db2.close() + db3.close() + } + + private fun insertSubject( + db: SupportSQLiteDatabase, + subjectId: String, + ) { + db.execSQL( + "INSERT INTO DbSubject (subjectId, projectId, attendantId, moduleId) VALUES (?, ?, ?, ?)", + arrayOf(subjectId, "project", "attendant", "module"), + ) + } + + private fun insertTemplate( + db: SupportSQLiteDatabase, + uuid: String, + subjectId: String, + referenceId: String, + modality: Int, + ) { + db.execSQL( + "INSERT INTO DbBiometricTemplate (uuid, subjectId, templateData, format, referenceId, modality) VALUES (?, ?, ?, ?, ?, ?)", + arrayOf(uuid, subjectId, ByteArray(0), "format", referenceId, modality), + ) + } + + companion object { + private const val TEST_DB = "migration-test" + private const val SUBJECT_1 = "subject-1" + private const val SUBJECT_2 = "subject-2" + private const val FINGERPRINT_MODALITY = 0 + private const val FACE_MODALITY = 1 + } +}