diff --git a/OpenASO.xcodeproj/project.pbxproj b/OpenASO.xcodeproj/project.pbxproj index b90f3a0..37f57e6 100644 --- a/OpenASO.xcodeproj/project.pbxproj +++ b/OpenASO.xcodeproj/project.pbxproj @@ -138,6 +138,23 @@ D32A00000000000000000001 /* OpenASOSchemaV5.swift in Sources */ = {isa = PBXBuildFile; fileRef = D32A00000000000000000011 /* OpenASOSchemaV5.swift */; }; D32A00000000000000000002 /* OpenASOSchemaV5.swift in Sources */ = {isa = PBXBuildFile; fileRef = D32A00000000000000000011 /* OpenASOSchemaV5.swift */; }; D32A00000000000000000003 /* KeywordResearchProject.swift in Sources */ = {isa = PBXBuildFile; fileRef = D32A00000000000000000012 /* KeywordResearchProject.swift */; }; + D60000000000000000000001 /* OpenASOSchemaV6.swift in Sources */ = {isa = PBXBuildFile; fileRef = D60000000000000000000011 /* OpenASOSchemaV6.swift */; }; + D60000000000000000000002 /* RankingAppRevision.swift in Sources */ = {isa = PBXBuildFile; fileRef = D60000000000000000000012 /* RankingAppRevision.swift */; }; + D60000000000000000000003 /* RankingCrawlRecord.swift in Sources */ = {isa = PBXBuildFile; fileRef = D60000000000000000000013 /* RankingCrawlRecord.swift */; }; + D60000000000000000000004 /* RankingFact.swift in Sources */ = {isa = PBXBuildFile; fileRef = D60000000000000000000014 /* RankingFact.swift */; }; + D60000000000000000000005 /* TrackedRankingCrawlLink.swift in Sources */ = {isa = PBXBuildFile; fileRef = D60000000000000000000015 /* TrackedRankingCrawlLink.swift */; }; + D60000000000000000000006 /* RankingMigrationState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D60000000000000000000016 /* RankingMigrationState.swift */; }; + D60000000000000000000021 /* OpenASOSchemaV6.swift in Sources */ = {isa = PBXBuildFile; fileRef = D60000000000000000000011 /* OpenASOSchemaV6.swift */; }; + D60000000000000000000022 /* RankingAppRevision.swift in Sources */ = {isa = PBXBuildFile; fileRef = D60000000000000000000012 /* RankingAppRevision.swift */; }; + D60000000000000000000023 /* RankingCrawlRecord.swift in Sources */ = {isa = PBXBuildFile; fileRef = D60000000000000000000013 /* RankingCrawlRecord.swift */; }; + D60000000000000000000024 /* RankingFact.swift in Sources */ = {isa = PBXBuildFile; fileRef = D60000000000000000000014 /* RankingFact.swift */; }; + D60000000000000000000025 /* TrackedRankingCrawlLink.swift in Sources */ = {isa = PBXBuildFile; fileRef = D60000000000000000000015 /* TrackedRankingCrawlLink.swift */; }; + D60000000000000000000026 /* RankingMigrationState.swift in Sources */ = {isa = PBXBuildFile; fileRef = D60000000000000000000016 /* RankingMigrationState.swift */; }; + D60000000000000000000007 /* SearchRanking/RankingAppRevisionStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = D60000000000000000000017 /* SearchRanking/RankingAppRevisionStore.swift */; }; + D60000000000000000000027 /* SearchRanking/RankingAppRevisionStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = D60000000000000000000017 /* SearchRanking/RankingAppRevisionStore.swift */; }; + D60000000000000000000008 /* Persistence/RankingSchemaV6Migrator.swift in Sources */ = {isa = PBXBuildFile; fileRef = D60000000000000000000018 /* Persistence/RankingSchemaV6Migrator.swift */; }; + D60000000000000000000028 /* Persistence/RankingSchemaV6Migrator.swift in Sources */ = {isa = PBXBuildFile; fileRef = D60000000000000000000018 /* Persistence/RankingSchemaV6Migrator.swift */; }; + D60000000000000000000009 /* RankingSchemaV6MigrationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D60000000000000000000019 /* RankingSchemaV6MigrationTests.swift */; }; D32A00000000000000000004 /* KeywordResearchProject.swift in Sources */ = {isa = PBXBuildFile; fileRef = D32A00000000000000000012 /* KeywordResearchProject.swift */; }; D32A00000000000000000005 /* KeywordResearchKeyword.swift in Sources */ = {isa = PBXBuildFile; fileRef = D32A00000000000000000013 /* KeywordResearchKeyword.swift */; }; D32A00000000000000000006 /* KeywordResearchKeyword.swift in Sources */ = {isa = PBXBuildFile; fileRef = D32A00000000000000000013 /* KeywordResearchKeyword.swift */; }; @@ -440,6 +457,15 @@ D32A00000000000000000012 /* KeywordResearchProject.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeywordResearchProject.swift; sourceTree = ""; }; D32A00000000000000000013 /* KeywordResearchKeyword.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeywordResearchKeyword.swift; sourceTree = ""; }; D32A00000000000000000014 /* KeywordResearchPersistenceTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeywordResearchPersistenceTests.swift; sourceTree = ""; }; + D60000000000000000000011 /* OpenASOSchemaV6.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OpenASOSchemaV6.swift; sourceTree = ""; }; + D60000000000000000000012 /* RankingAppRevision.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RankingAppRevision.swift; sourceTree = ""; }; + D60000000000000000000013 /* RankingCrawlRecord.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RankingCrawlRecord.swift; sourceTree = ""; }; + D60000000000000000000014 /* RankingFact.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RankingFact.swift; sourceTree = ""; }; + D60000000000000000000015 /* TrackedRankingCrawlLink.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TrackedRankingCrawlLink.swift; sourceTree = ""; }; + D60000000000000000000016 /* RankingMigrationState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RankingMigrationState.swift; sourceTree = ""; }; + D60000000000000000000017 /* SearchRanking/RankingAppRevisionStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SearchRanking/RankingAppRevisionStore.swift; sourceTree = ""; }; + D60000000000000000000018 /* Persistence/RankingSchemaV6Migrator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Persistence/RankingSchemaV6Migrator.swift; sourceTree = ""; }; + D60000000000000000000019 /* RankingSchemaV6MigrationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RankingSchemaV6MigrationTests.swift; sourceTree = ""; }; D32C00000000000000000011 /* Persistence/KeywordResearchProjectStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Persistence/KeywordResearchProjectStore.swift; sourceTree = ""; }; D32C00000000000000000012 /* KeywordResearchProjectStoreTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeywordResearchProjectStoreTests.swift; sourceTree = ""; }; D33A00000000000000000011 /* SearchRanking/KeywordResearchRankingWorkflow.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SearchRanking/KeywordResearchRankingWorkflow.swift; sourceTree = ""; }; @@ -623,6 +649,7 @@ D02000000000000000000003 /* PersistenceMigrationTests.swift */, D02000000000000000000007 /* ReleasedV1StoreFixture.swift */, D77BC4C40000000000000011 /* ExactV4StoreFixtureTests.swift */, + D60000000000000000000019 /* RankingSchemaV6MigrationTests.swift */, D32A00000000000000000014 /* KeywordResearchPersistenceTests.swift */, D32C00000000000000000012 /* KeywordResearchProjectStoreTests.swift */, D33A00000000000000000012 /* KeywordResearchRankingWorkflowTests.swift */, @@ -790,6 +817,8 @@ 9838D42ED1A3EA1E6DDDD68C /* OpenASOError.swift */, 035877A6E7443B32A3980E39 /* SearchRanking/RankingMatcher.swift */, 26D4B2C87FDED9CFCFA50D7D /* SearchRanking/RankingRefreshCoordinator.swift */, + D60000000000000000000017 /* SearchRanking/RankingAppRevisionStore.swift */, + D60000000000000000000018 /* Persistence/RankingSchemaV6Migrator.swift */, D33A00000000000000000011 /* SearchRanking/KeywordResearchRankingWorkflow.swift */, D34A00000000000000000011 /* SearchRanking/KeywordResearchHistoryReader.swift */, A70000000000000000000002 /* RatingsReviews/ReviewLanguageDetectionService.swift */, @@ -980,6 +1009,12 @@ D09D00000000000000000001 /* OpenASOSchemaV3.swift */, D20B00000000000000000011 /* OpenASOSchemaV4.swift */, D32A00000000000000000011 /* OpenASOSchemaV5.swift */, + D60000000000000000000011 /* OpenASOSchemaV6.swift */, + D60000000000000000000012 /* RankingAppRevision.swift */, + D60000000000000000000013 /* RankingCrawlRecord.swift */, + D60000000000000000000014 /* RankingFact.swift */, + D60000000000000000000015 /* TrackedRankingCrawlLink.swift */, + D60000000000000000000016 /* RankingMigrationState.swift */, D32A00000000000000000012 /* KeywordResearchProject.swift */, D32A00000000000000000013 /* KeywordResearchKeyword.swift */, D20B00000000000000000012 /* EstimatedKeywordDifficultyMetric.swift */, @@ -1256,6 +1291,14 @@ D09D00000000000000000003 /* OpenASOSchemaV3.swift in Sources */, D20B00000000000000000001 /* OpenASOSchemaV4.swift in Sources */, D32A00000000000000000001 /* OpenASOSchemaV5.swift in Sources */, + D60000000000000000000001 /* OpenASOSchemaV6.swift in Sources */, + D60000000000000000000002 /* RankingAppRevision.swift in Sources */, + D60000000000000000000003 /* RankingCrawlRecord.swift in Sources */, + D60000000000000000000004 /* RankingFact.swift in Sources */, + D60000000000000000000005 /* TrackedRankingCrawlLink.swift in Sources */, + D60000000000000000000006 /* RankingMigrationState.swift in Sources */, + D60000000000000000000007 /* SearchRanking/RankingAppRevisionStore.swift in Sources */, + D60000000000000000000008 /* Persistence/RankingSchemaV6Migrator.swift in Sources */, D32A00000000000000000003 /* KeywordResearchProject.swift in Sources */, D32A00000000000000000005 /* KeywordResearchKeyword.swift in Sources */, D20B00000000000000000003 /* EstimatedKeywordDifficultyMetric.swift in Sources */, @@ -1388,6 +1431,7 @@ D13000000000000000000002 /* AppMetadataRefreshServiceTests.swift in Sources */, D35A00000000000000000003 /* HeadlessRefreshServiceTests.swift in Sources */, D35D00000000000000000003 /* HeadlessRefreshObservationTests.swift in Sources */, + D60000000000000000000009 /* RankingSchemaV6MigrationTests.swift in Sources */, 0A5C00000000000000000005 /* AppStoreConnectReviewServiceTests.swift in Sources */, 0A5C00000000000000000007 /* AppStoreWebMetadataProviderTests.swift in Sources */, A19000000000000000000003 /* AppStoreWebRankingProviderTests.swift in Sources */, @@ -1436,6 +1480,14 @@ D09D00000000000000000005 /* OpenASOSchemaV3.swift in Sources */, D20B00000000000000000002 /* OpenASOSchemaV4.swift in Sources */, D32A00000000000000000002 /* OpenASOSchemaV5.swift in Sources */, + D60000000000000000000021 /* OpenASOSchemaV6.swift in Sources */, + D60000000000000000000022 /* RankingAppRevision.swift in Sources */, + D60000000000000000000023 /* RankingCrawlRecord.swift in Sources */, + D60000000000000000000024 /* RankingFact.swift in Sources */, + D60000000000000000000025 /* TrackedRankingCrawlLink.swift in Sources */, + D60000000000000000000026 /* RankingMigrationState.swift in Sources */, + D60000000000000000000027 /* SearchRanking/RankingAppRevisionStore.swift in Sources */, + D60000000000000000000028 /* Persistence/RankingSchemaV6Migrator.swift in Sources */, D32A00000000000000000004 /* KeywordResearchProject.swift in Sources */, D32A00000000000000000006 /* KeywordResearchKeyword.swift in Sources */, D20B00000000000000000004 /* EstimatedKeywordDifficultyMetric.swift in Sources */, diff --git a/OpenASO/Features/AppDetail/AppDetailPreviewFixtures.swift b/OpenASO/Features/AppDetail/AppDetailPreviewFixtures.swift index 540760e..a7538bc 100644 --- a/OpenASO/Features/AppDetail/AppDetailPreviewFixtures.swift +++ b/OpenASO/Features/AppDetail/AppDetailPreviewFixtures.swift @@ -225,7 +225,7 @@ private struct AppDetailPreviewHarness: View { platform: .iphone, in: modelContext ) - let observation = KeywordRankingCrawl( + let observation = RankingCrawlRecord( keyword: keyword, storefront: storefront, platform: .iphone, @@ -238,16 +238,21 @@ private struct AppDetailPreviewHarness: View { let rank = max(1, min(9, keyword.count % 10)) for (index, app) in rankedApps(rank: rank, competitors: competitors).enumerated() { - let item = KeywordAppRanking( - position: index + 1, - appStoreID: app.id == 6448311069 ? trackedAppID : app.id, + let rankingAppStoreID = app.id == 6448311069 ? trackedAppID : app.id + let revision = RankingAppRevision( + appStoreID: rankingAppStoreID, bundleID: app.bundleID, name: app.name, subtitle: nil, - sellerName: app.seller, + sellerName: app.seller + ) + let item = RankingFact( + position: index + 1, + appStoreID: rankingAppStoreID, + revision: revision, observation: observation ) - observation.items.append(item) + modelContext.insert(revision) modelContext.insert(item) } } diff --git a/OpenASO/Features/AppDetail/Keywords/Table/KeywordRankingHistoryLoader.swift b/OpenASO/Features/AppDetail/Keywords/Table/KeywordRankingHistoryLoader.swift index d19650c..15a145f 100644 --- a/OpenASO/Features/AppDetail/Keywords/Table/KeywordRankingHistoryLoader.swift +++ b/OpenASO/Features/AppDetail/Keywords/Table/KeywordRankingHistoryLoader.swift @@ -9,24 +9,26 @@ enum KeywordRankingHistoryLoader { ) throws -> [KeywordRankingCrawlSummary] { let targetQueryKey = queryKey let targetAppStoreID = appStoreID - let crawlDescriptor = FetchDescriptor( + let crawlDescriptor = FetchDescriptor( predicate: #Predicate { crawl in crawl.queryKey == targetQueryKey }, sortBy: [ - SortDescriptor(\KeywordRankingCrawl.observedAt, order: .forward), - SortDescriptor(\KeywordRankingCrawl.observationKey, order: .forward) + SortDescriptor(\RankingCrawlRecord.observedAt, order: .forward), + SortDescriptor(\RankingCrawlRecord.observationKey, order: .forward) ] ) let crawls = try modelContext.fetch(crawlDescriptor) + .filter { !$0.isTrackedRecovery } guard !crawls.isEmpty else { return [] } - let rankingDescriptor = FetchDescriptor( + let rankingDescriptor = FetchDescriptor( predicate: #Predicate { ranking in - ranking.queryKey == targetQueryKey && ranking.appStoreID == targetAppStoreID + ranking.observation.queryKey == targetQueryKey + && ranking.appStoreID == targetAppStoreID } ) let rankings = try modelContext.fetch(rankingDescriptor) diff --git a/OpenASO/Features/AppDetail/Keywords/Table/KeywordRankingListLoader.swift b/OpenASO/Features/AppDetail/Keywords/Table/KeywordRankingListLoader.swift index 3796b26..1317a71 100644 --- a/OpenASO/Features/AppDetail/Keywords/Table/KeywordRankingListLoader.swift +++ b/OpenASO/Features/AppDetail/Keywords/Table/KeywordRankingListLoader.swift @@ -128,11 +128,11 @@ enum KeywordRankingListLoader { } let targetCrawlKey = crawlKey - let descriptor = FetchDescriptor( + let descriptor = FetchDescriptor( predicate: #Predicate { ranking in - ranking.crawlKey == targetCrawlKey + ranking.observation.observationKey == targetCrawlKey }, - sortBy: [SortDescriptor(\KeywordAppRanking.position, order: .forward)] + sortBy: [SortDescriptor(\RankingFact.position, order: .forward)] ) let rankings = try modelContext.fetch(descriptor) @@ -238,25 +238,31 @@ enum KeywordRankingListLoader { let uniqueAppStoreIDs = Array(Set(appStoreIDs)).sorted() snapshotsByID.reserveCapacity(uniqueAppStoreIDs.count) - for appStoreID in uniqueAppStoreIDs { - try checkCancellation() - let targetAppStoreID = appStoreID - var descriptor = FetchDescriptor( - predicate: #Predicate { snapshot in - snapshot.storefront == targetStorefront - && snapshot.appStoreID == targetAppStoreID - && snapshot.ratingCount != nil - }, - sortBy: [ - SortDescriptor(\.ratingDate, order: .reverse), - SortDescriptor(\.observedAt, order: .reverse) - ] - ) - descriptor.fetchLimit = ratingSnapshotLimit - let snapshots = try modelContext.fetch(descriptor) - try checkCancellation() - guard !snapshots.isEmpty else { continue } - snapshotsByID[appStoreID] = snapshots.reversed().map(RatingSnapshotDisplayValue.init) + guard !uniqueAppStoreIDs.isEmpty else { return [:] } + let descriptor = FetchDescriptor( + predicate: #Predicate { snapshot in + snapshot.storefront == targetStorefront + && uniqueAppStoreIDs.contains(snapshot.appStoreID) + && snapshot.ratingCount != nil + }, + sortBy: [ + SortDescriptor(\.appStoreID, order: .forward), + SortDescriptor(\.ratingDate, order: .reverse), + SortDescriptor(\.observedAt, order: .reverse) + ] + ) + for (index, snapshot) in try modelContext.fetch(descriptor).enumerated() { + if snapshotsByID[snapshot.appStoreID, default: []].count < ratingSnapshotLimit { + snapshotsByID[snapshot.appStoreID, default: []].append( + RatingSnapshotDisplayValue(snapshot) + ) + } + if index.isMultiple(of: 256) { + try checkCancellation() + } + } + for appStoreID in snapshotsByID.keys { + snapshotsByID[appStoreID]?.reverse() } return snapshotsByID diff --git a/OpenASO/Features/AppDetail/Keywords/Table/KeywordTableSupport.swift b/OpenASO/Features/AppDetail/Keywords/Table/KeywordTableSupport.swift index 723693b..2242e96 100644 --- a/OpenASO/Features/AppDetail/Keywords/Table/KeywordTableSupport.swift +++ b/OpenASO/Features/AppDetail/Keywords/Table/KeywordTableSupport.swift @@ -675,7 +675,7 @@ struct KeywordRankingCrawlSummary: Identifiable, Equatable, Sendable { ) } - init(crawl: KeywordRankingCrawl, rank: Int?, errorMessage: String? = nil) { + init(crawl: RankingCrawlRecord, rank: Int?, errorMessage: String? = nil) { self.init( id: crawl.observationKey, rank: rank, @@ -724,7 +724,7 @@ struct KeywordRankingAppSummary: Identifiable, Equatable, Sendable { ) } - init(_ ranking: KeywordAppRanking) { + init(_ ranking: RankingFact) { self.init( position: ranking.position, appStoreID: ranking.appStoreID, diff --git a/OpenASO/Models/KeywordAppRanking.swift b/OpenASO/Models/KeywordAppRanking.swift index b427983..dbdde47 100644 --- a/OpenASO/Models/KeywordAppRanking.swift +++ b/OpenASO/Models/KeywordAppRanking.swift @@ -64,3 +64,4 @@ final class KeywordAppRanking { } typealias KeywordAppRanking = OpenASOSchemaV1.KeywordAppRanking +typealias LegacyKeywordAppRanking = OpenASOSchemaV1.KeywordAppRanking diff --git a/OpenASO/Models/KeywordRankingCrawl.swift b/OpenASO/Models/KeywordRankingCrawl.swift index e27a11c..dd7b2c8 100644 --- a/OpenASO/Models/KeywordRankingCrawl.swift +++ b/OpenASO/Models/KeywordRankingCrawl.swift @@ -97,3 +97,4 @@ final class KeywordRankingCrawl { } typealias KeywordRankingCrawl = OpenASOSchemaV1.KeywordRankingCrawl +typealias LegacyKeywordRankingCrawl = OpenASOSchemaV1.KeywordRankingCrawl diff --git a/OpenASO/Models/OpenASOSchemaV1.swift b/OpenASO/Models/OpenASOSchemaV1.swift index 3e7660e..927322a 100644 --- a/OpenASO/Models/OpenASOSchemaV1.swift +++ b/OpenASO/Models/OpenASOSchemaV1.swift @@ -41,7 +41,8 @@ enum OpenASOMigrationPlan: SchemaMigrationPlan { OpenASOSchemaV2.self, OpenASOSchemaV3.self, OpenASOSchemaV4.self, - OpenASOSchemaV5.self + OpenASOSchemaV5.self, + OpenASOSchemaV6.self ] } @@ -68,6 +69,10 @@ enum OpenASOMigrationPlan: SchemaMigrationPlan { .lightweight( fromVersion: OpenASOSchemaV4.self, toVersion: OpenASOSchemaV5.self + ), + .lightweight( + fromVersion: OpenASOSchemaV5.self, + toVersion: OpenASOSchemaV6.self ) ] } diff --git a/OpenASO/Models/OpenASOSchemaV6.swift b/OpenASO/Models/OpenASOSchemaV6.swift new file mode 100644 index 0000000..aa8420c --- /dev/null +++ b/OpenASO/Models/OpenASOSchemaV6.swift @@ -0,0 +1,20 @@ +import SwiftData + +/// Adds a normalized ranking store alongside the frozen V1 ranking entities. +/// A resumable post-open migration copies and validates legacy ranking data +/// before the redundant V1 rows are removed. +enum OpenASOSchemaV6: VersionedSchema { + static var versionIdentifier: Schema.Version { + Schema.Version(6, 0, 0) + } + + static var models: [any PersistentModel.Type] { + OpenASOSchemaV5.models + [ + RankingAppRevision.self, + RankingCrawlRecord.self, + RankingFact.self, + TrackedRankingCrawlLink.self, + RankingMigrationState.self + ] + } +} diff --git a/OpenASO/Models/RankingAppRevision.swift b/OpenASO/Models/RankingAppRevision.swift new file mode 100644 index 0000000..7ab76b6 --- /dev/null +++ b/OpenASO/Models/RankingAppRevision.swift @@ -0,0 +1,77 @@ +import CryptoKit +import Foundation +import SwiftData + +extension OpenASOSchemaV6 { +@Model +final class RankingAppRevision { + #Index([\.appStoreID]) + + @Attribute(.unique) var revisionKey: String + var appStoreID: Int64 + var bundleID: String? + var name: String + var subtitle: String? + var sellerName: String? + + init( + appStoreID: Int64, + bundleID: String?, + name: String, + subtitle: String?, + sellerName: String? + ) { + self.revisionKey = Self.makeRevisionKey( + appStoreID: appStoreID, + bundleID: bundleID, + name: name, + subtitle: subtitle, + sellerName: sellerName + ) + self.appStoreID = appStoreID + self.bundleID = bundleID + self.name = name + self.subtitle = subtitle + self.sellerName = sellerName + } + + static func makeRevisionKey( + appStoreID: Int64, + bundleID: String?, + name: String, + subtitle: String?, + sellerName: String? + ) -> String { + let payload = [ + String(appStoreID), + encoded(bundleID), + encoded(name), + encoded(subtitle), + encoded(sellerName) + ].joined(separator: "|") + let digest = SHA256.hash(data: Data(payload.utf8)) + return digest.map { String(format: "%02x", $0) }.joined() + } + + func matches( + appStoreID: Int64, + bundleID: String?, + name: String, + subtitle: String?, + sellerName: String? + ) -> Bool { + self.appStoreID == appStoreID + && self.bundleID == bundleID + && self.name == name + && self.subtitle == subtitle + && self.sellerName == sellerName + } + + private static func encoded(_ value: String?) -> String { + guard let value else { return "nil" } + return "some:\(value.utf8.count):\(value)" + } +} +} + +typealias RankingAppRevision = OpenASOSchemaV6.RankingAppRevision diff --git a/OpenASO/Models/RankingCrawlRecord.swift b/OpenASO/Models/RankingCrawlRecord.swift new file mode 100644 index 0000000..ed22cb8 --- /dev/null +++ b/OpenASO/Models/RankingCrawlRecord.swift @@ -0,0 +1,100 @@ +import Foundation +import SwiftData + +extension OpenASOSchemaV6 { +@Model +final class RankingCrawlRecord { + static let trackedRecoveryObservationKeyPrefix = "tracked-recovery::" + + #Index( + [\.queryKey], + [\.queryKey, \.observedAt], + [\.observedAt] + ) + + @Attribute(.unique) var observationKey: String + var queryKey: String + var keyword: String + var storefront: String + var platformRaw: String + var observedAt: Date + var observedHour: Int + var sourceRaw: String + var resultCount: Int + var submissionCount: Int + var winningCount: Int + var confidenceRaw: String? + + init( + keyword: String, + storefront: String, + platform: AppPlatform, + observedAt: Date, + source: RankingSource, + resultCount: Int, + query: KeywordQuery? = nil, + observedHour: Int? = nil, + submissionCount: Int = 1, + winningCount: Int = 1, + confidence: String? = nil + ) { + let normalizedKeyword = keyword.trimmingCharacters(in: .whitespacesAndNewlines) + let normalizedStorefront = storefront.lowercased() + let queryKey = TrackedAppKeyword.makeQueryKey( + term: normalizedKeyword, + storefront: normalizedStorefront, + platform: platform + ) + self.queryKey = queryKey + self.observationKey = Self.makeObservationKey( + queryKey: queryKey, + observedAt: observedAt, + source: source + ) + self.keyword = normalizedKeyword + self.storefront = normalizedStorefront + self.platformRaw = platform.rawValue + self.observedAt = observedAt + self.observedHour = observedHour ?? Self.utcHourBucket(for: observedAt) + self.sourceRaw = source.rawValue + self.resultCount = resultCount + self.submissionCount = submissionCount + self.winningCount = winningCount + self.confidenceRaw = confidence + } + + static func makeObservationKey(queryKey: String, observedAt: Date, source: RankingSource) -> String { + [queryKey, String(utcDayBucket(for: observedAt)), source.rawValue] + .joined(separator: "::") + } + + static func makeTrackedRecoveryObservationKey(snapshotKey: String) -> String { + trackedRecoveryObservationKeyPrefix + snapshotKey + } + + var isTrackedRecovery: Bool { + observationKey.hasPrefix(Self.trackedRecoveryObservationKeyPrefix) + } + + static func utcHourBucket(for date: Date) -> Int { + Int(floor(date.timeIntervalSince1970 / 3_600)) + } + + static func utcDayBucket(for date: Date) -> Int { + Int(floor(date.timeIntervalSince1970 / 86_400)) + } + + var platform: AppPlatform { + get { AppPlatform(rawValue: platformRaw) ?? .iphone } + set { platformRaw = newValue.rawValue } + } + + var source: RankingSource { + get { RankingSource(rawValue: sourceRaw) ?? .iTunesFallback } + set { sourceRaw = newValue.rawValue } + } + +} +} + +typealias RankingCrawlRecord = OpenASOSchemaV6.RankingCrawlRecord diff --git a/OpenASO/Models/RankingFact.swift b/OpenASO/Models/RankingFact.swift new file mode 100644 index 0000000..4c4347e --- /dev/null +++ b/OpenASO/Models/RankingFact.swift @@ -0,0 +1,51 @@ +import Foundation +import SwiftData + +extension OpenASOSchemaV6 { +@Model +final class RankingFact { + #Index( + [\.observation], + [\.observation, \.position], + [\.appStoreID, \.observation] + ) + + var position: Int + var appStoreID: Int64 + var observation: RankingCrawlRecord + var revision: RankingAppRevision + + init( + position: Int, + appStoreID: Int64, + revision: RankingAppRevision, + observation: RankingCrawlRecord + ) { + self.position = position + self.appStoreID = appStoreID + self.revision = revision + self.observation = observation + } + + static func makeItemKey(observationKey: String, appStoreID: Int64) -> String { + [observationKey, String(appStoreID)].joined(separator: "::") + } + + var itemKey: String { + Self.makeItemKey(observationKey: observation.observationKey, appStoreID: appStoreID) + } + + var bundleID: String? { revision.bundleID } + var name: String { revision.name } + var subtitle: String? { revision.subtitle } + var sellerName: String? { revision.sellerName } + var crawlKey: String { observation.observationKey } + var queryKey: String { observation.queryKey } + var storefront: String { observation.storefront } + var platformRaw: String { observation.platformRaw } + var observedAt: Date { observation.observedAt } + var platform: AppPlatform { observation.platform } +} +} + +typealias RankingFact = OpenASOSchemaV6.RankingFact diff --git a/OpenASO/Models/RankingMigrationState.swift b/OpenASO/Models/RankingMigrationState.swift new file mode 100644 index 0000000..0a6cb05 --- /dev/null +++ b/OpenASO/Models/RankingMigrationState.swift @@ -0,0 +1,56 @@ +import Foundation +import SwiftData + +extension OpenASOSchemaV6 { +@Model +final class RankingMigrationState { + static let singletonKey = "normalized-ranking-v6" + + @Attribute(.unique) var migrationKey: String + var phaseRaw: String + var lastObservationKey: String? + var lastSnapshotKey: String? + var legacyCrawlCount: Int + var legacyFactCount: Int + var legacyTrackedFactCount: Int + var migratedCrawlCount: Int + var migratedFactCount: Int + var processedSnapshotCount: Int + var migratedTrackedLinkCount: Int + var recoveredCrawlCount: Int + var startedAt: Date + var completedAt: Date? + + init(startedAt: Date = .now) { + self.migrationKey = Self.singletonKey + self.phaseRaw = RankingMigrationPhase.copyingCrawls.rawValue + self.lastObservationKey = nil + self.lastSnapshotKey = nil + self.legacyCrawlCount = 0 + self.legacyFactCount = 0 + self.legacyTrackedFactCount = 0 + self.migratedCrawlCount = 0 + self.migratedFactCount = 0 + self.processedSnapshotCount = 0 + self.migratedTrackedLinkCount = 0 + self.recoveredCrawlCount = 0 + self.startedAt = startedAt + self.completedAt = nil + } + + var phase: RankingMigrationPhase { + get { RankingMigrationPhase(rawValue: phaseRaw) ?? .copyingCrawls } + set { phaseRaw = newValue.rawValue } + } +} +} + +enum RankingMigrationPhase: String, Sendable { + case copyingCrawls + case linkingSnapshots + case validating + case cleaningLegacyRows + case completed +} + +typealias RankingMigrationState = OpenASOSchemaV6.RankingMigrationState diff --git a/OpenASO/Models/TrackedRankingCrawlLink.swift b/OpenASO/Models/TrackedRankingCrawlLink.swift new file mode 100644 index 0000000..1d10187 --- /dev/null +++ b/OpenASO/Models/TrackedRankingCrawlLink.swift @@ -0,0 +1,19 @@ +import Foundation +import SwiftData + +extension OpenASOSchemaV6 { +@Model +final class TrackedRankingCrawlLink { + #Index([\.crawl]) + + @Attribute(.unique) var snapshotKey: String + var crawl: RankingCrawlRecord + + init(snapshotKey: String, crawl: RankingCrawlRecord) { + self.snapshotKey = snapshotKey + self.crawl = crawl + } +} +} + +typealias TrackedRankingCrawlLink = OpenASOSchemaV6.TrackedRankingCrawlLink diff --git a/OpenASO/Services/MCP/OpenASOMCPService.swift b/OpenASO/Services/MCP/OpenASOMCPService.swift index ddec35b..5de5968 100644 --- a/OpenASO/Services/MCP/OpenASOMCPService.swift +++ b/OpenASO/Services/MCP/OpenASOMCPService.swift @@ -587,84 +587,28 @@ final class OpenASOMCPService: Sendable { in modelContext: ModelContext ) throws -> [String: BoundedHistoryRankedApps] { guard !snapshots.isEmpty else { return [:] } - - let crawlKeyBySnapshotKey = Dictionary(uniqueKeysWithValues: snapshots.map { snapshot in - let track = tracksByIdentityKey[snapshot.trackIdentityKey] ?? snapshot.keywordTrack - return ( - snapshot.snapshotKey, - KeywordRankingCrawl.makeObservationKey( - queryKey: track.queryKey, - observedAt: snapshot.searchedAt, - source: snapshot.source - ) + _ = tracksByIdentityKey + let snapshotKeys = snapshots.map(\.snapshotKey) + let links = try modelContext.fetch( + FetchDescriptor( + predicate: #Predicate { link in + snapshotKeys.contains(link.snapshotKey) + } ) - }) - let candidateCrawlKeys = Array(Set(crawlKeyBySnapshotKey.values)) - let crawlDescriptor = FetchDescriptor( - predicate: #Predicate { crawl in - candidateCrawlKeys.contains(crawl.observationKey) - } ) - let crawlsByKey = Dictionary(uniqueKeysWithValues: try modelContext.fetch(crawlDescriptor).map { - ($0.observationKey, $0) - }) - let matchingCrawlKeyBySnapshotKey = Dictionary(uniqueKeysWithValues: snapshots.compactMap { - snapshot -> (String, String)? in - guard let crawlKey = crawlKeyBySnapshotKey[snapshot.snapshotKey], - crawlsByKey[crawlKey]?.observedAt == snapshot.searchedAt else { - return nil - } - return (snapshot.snapshotKey, crawlKey) - }) + let matchingCrawlKeyBySnapshotKey = Dictionary( + uniqueKeysWithValues: links.map { ($0.snapshotKey, $0.crawl.observationKey) } + ) let crawlAppsByKey = try crawlRankedApps( crawlKeys: Array(Set(matchingCrawlKeyBySnapshotKey.values)), limit: limit, in: modelContext ) - var rankedAppsBySnapshotKey = matchingCrawlKeyBySnapshotKey.reduce( + return matchingCrawlKeyBySnapshotKey.reduce( into: [String: BoundedHistoryRankedApps]() ) { result, entry in result[entry.key] = crawlAppsByKey[entry.value] } - - // Stores created before shared crawl history was introduced only have the - // tracked snapshot relationship. Fall back for those rows while preferring - // the canonical crawl so the MCP response still exposes the full page. - let missingSnapshotKeys = snapshots.map(\.snapshotKey).filter { - rankedAppsBySnapshotKey[$0] == nil - } - guard !missingSnapshotKeys.isEmpty else { return rankedAppsBySnapshotKey } - - var descriptor = FetchDescriptor( - predicate: #Predicate { result in - missingSnapshotKeys.contains(result.snapshotKey) - }, - sortBy: [ - SortDescriptor(\.snapshotKey, comparator: .lexical, order: .forward), - SortDescriptor(\.position, order: .forward), - SortDescriptor(\.appStoreID, order: .forward), - SortDescriptor(\.name, comparator: .lexical, order: .forward), - ] - ) - descriptor.fetchLimit = ResponseLimits.maximumHistoryStoredRankedAppScanRows + 1 - let results = try modelContext.fetch(descriptor) - guard results.count <= ResponseLimits.maximumHistoryStoredRankedAppScanRows else { - throw OpenASOError.providerUnavailable( - "Stored ranking history is too large for one response. Narrow the filters or page size." - ) - } - let legacyAppsBySnapshotKey: [String: BoundedHistoryRankedApps] = results.reduce( - into: [:] - ) { grouped, result in - var rankedApps = grouped[result.snapshotKey] ?? BoundedHistoryRankedApps() - rankedApps.total += 1 - if rankedApps.items.count < limit { - rankedApps.items.append(Self.storedRankedApp(result)) - } - grouped[result.snapshotKey] = rankedApps - } - rankedAppsBySnapshotKey.merge(legacyAppsBySnapshotKey) { current, _ in current } - return rankedAppsBySnapshotKey } private static func crawlRankedApps( @@ -673,15 +617,14 @@ final class OpenASOMCPService: Sendable { in modelContext: ModelContext ) throws -> [String: BoundedHistoryRankedApps] { guard !crawlKeys.isEmpty else { return [:] } - var descriptor = FetchDescriptor( + var descriptor = FetchDescriptor( predicate: #Predicate { result in - crawlKeys.contains(result.crawlKey) + crawlKeys.contains(result.observation.observationKey) }, sortBy: [ - SortDescriptor(\.crawlKey, comparator: .lexical, order: .forward), + SortDescriptor(\RankingFact.observation.observationKey, comparator: .lexical, order: .forward), SortDescriptor(\.position, order: .forward), SortDescriptor(\.appStoreID, order: .forward), - SortDescriptor(\.itemKey, comparator: .lexical, order: .forward), ] ) descriptor.fetchLimit = ResponseLimits.maximumHistoryStoredRankedAppScanRows + 1 @@ -1222,22 +1165,25 @@ final class OpenASOMCPService: Sendable { let minimumDate = dateFrom ?? .distantPast let maximumDate = dateTo ?? .distantFuture - let countPredicate = #Predicate { crawl in + let recoveryPrefix = RankingCrawlRecord.trackedRecoveryObservationKeyPrefix + let countPredicate = #Predicate { crawl in queryKeys.contains(crawl.queryKey) && crawl.observedAt >= minimumDate && crawl.observedAt <= maximumDate + && !crawl.observationKey.starts(with: recoveryPrefix) } let total = try modelContext.fetchCount( - FetchDescriptor(predicate: countPredicate) + FetchDescriptor(predicate: countPredicate) ) let hasCursor = historyCursor != nil let cursorTimestamp = historyCursor?.timestamp ?? .distantFuture let cursorTieKey = historyCursor?.tieKey ?? "" - var crawlDescriptor = FetchDescriptor( + var crawlDescriptor = FetchDescriptor( predicate: #Predicate { crawl in queryKeys.contains(crawl.queryKey) && crawl.observedAt >= minimumDate && crawl.observedAt <= maximumDate + && !crawl.observationKey.starts(with: recoveryPrefix) && (!hasCursor || crawl.observedAt < cursorTimestamp || (crawl.observedAt == cursorTimestamp && crawl.observationKey > cursorTieKey)) @@ -4257,7 +4203,7 @@ extension OpenASOMCPService { } fileprivate static func storedRankedApp( - _ result: KeywordAppRanking + _ result: RankingFact ) -> OpenASOMCPStoredRankedApp { OpenASOMCPStoredRankedApp( id: result.itemKey, @@ -4297,7 +4243,7 @@ extension OpenASOMCPService { } fileprivate static func rankingCrawlSnapshot( - _ crawl: KeywordRankingCrawl, + _ crawl: RankingCrawlRecord, rankedApps: [OpenASOMCPStoredRankedApp], rankedAppsAvailableCount: Int ) -> OpenASOMCPRankingCrawlSnapshot { @@ -4821,12 +4767,12 @@ extension OpenASOMCPService { snapshot.resultCount = page.resultCount snapshot.errorMessage = nil - let observationKey = KeywordRankingCrawl.makeObservationKey( + let observationKey = RankingCrawlRecord.makeObservationKey( queryKey: request.queryKey, observedAt: observedAt, source: source ) - var observationDescriptor = FetchDescriptor( + var observationDescriptor = FetchDescriptor( predicate: #Predicate { crawl in crawl.observationKey == observationKey } @@ -4834,7 +4780,7 @@ extension OpenASOMCPService { observationDescriptor.fetchLimit = 1 let observation = try modelContext.fetch(observationDescriptor).first - ?? KeywordRankingCrawl( + ?? RankingCrawlRecord( keyword: request.term, storefront: request.storefront, platform: request.platform, @@ -4845,83 +4791,74 @@ extension OpenASOMCPService { ) if observation.modelContext == nil { modelContext.insert(observation) - query.observations.append(observation) } observation.resultCount = page.resultCount - var retainedAppStoreIDs: [Int64] = [] - for item in page.items { - retainedAppStoreIDs.append(item.appStoreID) - let itemKey = KeywordAppRanking.makeItemKey( - observationKey: observation.observationKey, appStoreID: item.appStoreID) - var itemDescriptor = FetchDescriptor( + let revisionPayloads = page.items.map { + RankingAppRevisionPayload( + appStoreID: $0.appStoreID, + bundleID: $0.bundleID, + name: $0.name, + subtitle: $0.subtitle, + sellerName: $0.sellerName + ) + } + let revisionsByKey = try RankingAppRevisionStore.revisions( + for: revisionPayloads, + in: modelContext + ) + let currentObservationKey = observation.observationKey + let existingRankings = try modelContext.fetch( + FetchDescriptor( predicate: #Predicate { ranking in - ranking.itemKey == itemKey + ranking.observation.observationKey == currentObservationKey } ) - itemDescriptor.fetchLimit = 1 + ) + var rankingsByAppStoreID = Dictionary( + uniqueKeysWithValues: existingRankings.map { ($0.appStoreID, $0) } + ) + + var retainedAppStoreIDs: [Int64] = [] + for (item, payload) in zip(page.items, revisionPayloads) { + retainedAppStoreIDs.append(item.appStoreID) + guard let revision = revisionsByKey[payload.revisionKey] else { + throw OpenASOError.providerUnavailable("Could not persist ranking app metadata.") + } let ranking = - try modelContext.fetch(itemDescriptor).first - ?? KeywordAppRanking( + rankingsByAppStoreID[item.appStoreID] + ?? RankingFact( position: item.position, appStoreID: item.appStoreID, - bundleID: item.bundleID, - name: item.name, - subtitle: item.subtitle, - sellerName: item.sellerName, + revision: revision, observation: observation ) if ranking.modelContext == nil { modelContext.insert(ranking) - observation.items.append(ranking) + rankingsByAppStoreID[item.appStoreID] = ranking } ranking.position = item.position - ranking.name = item.name - ranking.subtitle = item.subtitle - ranking.sellerName = item.sellerName - ranking.bundleID = item.bundleID - ranking.crawlKey = observation.observationKey - ranking.queryKey = observation.queryKey - ranking.storefront = observation.storefront - ranking.platformRaw = observation.platformRaw - ranking.observedAt = observation.observedAt - - let result = - snapshot.topResults.first { $0.appStoreID == item.appStoreID } - ?? TrackedKeywordRankedResult( - position: item.position, - appStoreID: item.appStoreID, - bundleID: item.bundleID, - name: item.name, - subtitle: item.subtitle, - sellerName: item.sellerName, - snapshot: snapshot - ) - if result.modelContext == nil { - snapshot.topResults.append(result) - modelContext.insert(result) - } - result.snapshotKey = snapshot.snapshotKey - result.position = item.position - result.appStoreID = item.appStoreID - result.bundleID = item.bundleID - result.name = item.name - result.subtitle = item.subtitle - result.sellerName = item.sellerName - if result.snapshot !== snapshot { - result.snapshot = snapshot - } + ranking.revision = revision } let retainedAppStoreIDSet = Set(retainedAppStoreIDs) - for result in snapshot.topResults where !retainedAppStoreIDSet.contains(result.appStoreID) { - modelContext.delete(result) - } - snapshot.topResults.removeAll { !retainedAppStoreIDSet.contains($0.appStoreID) } - for item in observation.items where !retainedAppStoreIDSet.contains(item.appStoreID) { + for item in rankingsByAppStoreID.values + where !retainedAppStoreIDSet.contains(item.appStoreID) + { modelContext.delete(item) } - observation.items.removeAll { !retainedAppStoreIDSet.contains($0.appStoreID) } + + var linkDescriptor = FetchDescriptor( + predicate: #Predicate { link in + link.snapshotKey == snapshotKey + } + ) + linkDescriptor.fetchLimit = 1 + if let link = try modelContext.fetch(linkDescriptor).first { + link.crawl = observation + } else { + modelContext.insert(TrackedRankingCrawlLink(snapshotKey: snapshotKey, crawl: observation)) + } try TrackedKeywordRefreshStatusStore.set( nil, @@ -4976,15 +4913,15 @@ extension OpenASOMCPService { let cutoff = Date().addingTimeInterval(-Double(max(1, lookbackDays)) * 86_400) let storefrontSet = Set(storefronts) - let descriptor = FetchDescriptor( + let descriptor = FetchDescriptor( predicate: #Predicate { ranking in - queryKeys.contains(ranking.queryKey) + queryKeys.contains(ranking.observation.queryKey) && ranking.appStoreID != appStoreID && ranking.position <= 10 - && ranking.observedAt >= cutoff + && ranking.observation.observedAt >= cutoff }, sortBy: [ - SortDescriptor(\.observedAt, order: .reverse), + SortDescriptor(\RankingFact.observation.observedAt, order: .reverse), SortDescriptor(\.position, order: .forward), ] ) @@ -5087,7 +5024,7 @@ private struct CompetitorAccumulator { var latestObservedAt = Date.distantPast var evidenceByQueryKey: [String: OpenASOMCPCompetitorKeywordEvidence] = [:] - mutating func add(row: KeywordAppRanking, keyword: String) { + mutating func add(row: RankingFact, keyword: String) { rankSum += row.position occurrenceCount += 1 bestRank = min(bestRank, row.position) diff --git a/OpenASO/Services/OpenASOError.swift b/OpenASO/Services/OpenASOError.swift index 867948a..82156d8 100644 --- a/OpenASO/Services/OpenASOError.swift +++ b/OpenASO/Services/OpenASOError.swift @@ -7,6 +7,7 @@ enum OpenASOLog { static let appDetail = Logger(subsystem: subsystem, category: "app-detail") static let ratings = Logger(subsystem: subsystem, category: "ratings") static let refresh = Logger(subsystem: subsystem, category: "refresh-observability") + static let rankingMigration = Logger(subsystem: subsystem, category: "ranking-migration") } enum OpenASOError: LocalizedError, Equatable, Sendable { diff --git a/OpenASO/Services/Persistence/ModelContainerFactory.swift b/OpenASO/Services/Persistence/ModelContainerFactory.swift index 9e4a425..4bc8a7d 100644 --- a/OpenASO/Services/Persistence/ModelContainerFactory.swift +++ b/OpenASO/Services/Persistence/ModelContainerFactory.swift @@ -1,3 +1,5 @@ +import CryptoKit +import Darwin import Foundation import SwiftData @@ -93,7 +95,11 @@ enum ModelContainerFactory { let schema = Self.schema let configuration = ModelConfiguration(schema: schema, url: storeURL) do { - return try opener(schema, configuration) + return try withExclusiveMigrationLock(for: storeURL) { + let container = try opener(schema, configuration) + try RankingSchemaV6Migrator.migrateIfNeeded(in: container) + return container + } } catch { throw PersistentStoreError.openFailed( storeURL: storeURL, @@ -119,4 +125,35 @@ enum ModelContainerFactory { configurations: [configuration] ) } + + private static func withExclusiveMigrationLock( + for storeURL: URL, + _ operation: () throws -> Value + ) throws -> Value { + let lockDirectory = FileManager.default.temporaryDirectory + .appendingPathComponent("OpenASO-Migrations", isDirectory: true) + try FileManager.default.createDirectory( + at: lockDirectory, + withIntermediateDirectories: true + ) + let digest = SHA256.hash(data: Data(storeURL.standardizedFileURL.path.utf8)) + .map { String(format: "%02x", $0) } + .joined() + let lockURL = lockDirectory.appendingPathComponent("\(digest).lock") + let descriptor = lockURL.path.withCString { + Darwin.open($0, O_CREAT | O_RDWR | O_CLOEXEC, S_IRUSR | S_IWUSR) + } + guard descriptor >= 0 else { + throw CocoaError(.fileWriteUnknown) + } + defer { Darwin.close(descriptor) } + + while flock(descriptor, LOCK_EX) != 0 { + guard errno == EINTR else { + throw CocoaError(.fileLocking) + } + } + defer { flock(descriptor, LOCK_UN) } + return try operation() + } } diff --git a/OpenASO/Services/Persistence/RankingSchemaV6Migrator.swift b/OpenASO/Services/Persistence/RankingSchemaV6Migrator.swift new file mode 100644 index 0000000..2e3d8df --- /dev/null +++ b/OpenASO/Services/Persistence/RankingSchemaV6Migrator.swift @@ -0,0 +1,712 @@ +import Foundation +import SwiftData + +enum RankingSchemaV6MigrationError: LocalizedError { + case invalidLegacyData(String) + case validationFailed(String) + + var errorDescription: String? { + switch self { + case .invalidLegacyData(let detail): + "The ranking database contains legacy data that cannot be normalized: \(detail)" + case .validationFailed(let detail): + "The normalized ranking data did not pass validation: \(detail)" + } + } +} + +/// Copies the frozen V1 ranking rows into the normalized V6 entities, validates +/// every persisted ranking payload, and only then removes redundant legacy rows. +/// Copy and link progress is committed in bounded batches, so an interrupted +/// launch resumes from the last completely saved batch. Each batch uses a fresh +/// context to keep the million-row migration's working set bounded. +enum RankingSchemaV6Migrator { + private static let crawlBatchSize = 100 + private static let snapshotBatchSize = 50 + + static func migrateIfNeeded(in container: ModelContainer) throws { + try initializeMigrationIfNeeded(in: container) + + while try autoreleasepool(invoking: { + switch try currentPhase(in: container) { + case .copyingCrawls: + try copyCanonicalBatch(in: container) + return true + case .linkingSnapshots: + try linkTrackedSnapshotBatch(in: container) + return true + case .validating: + try validateAllLegacyData(in: container) + return true + case .cleaningLegacyRows: + try removeValidatedLegacyRows(in: container) + return true + case .completed: + let context = migrationContext(for: container) + let state = try migrationState(in: context) + OpenASOLog.rankingMigration.notice( + "Normalized ranking migration completed: \(state.migratedCrawlCount) crawls, \(state.migratedFactCount) canonical facts, \(state.migratedTrackedLinkCount) tracked links, and \(state.recoveredCrawlCount) recovered crawls." + ) + return false + } + }) { + // The autorelease pool bounds Core Data's temporary objects per batch. + } + } + + private static func migrationContext(for container: ModelContainer) -> ModelContext { + let context = ModelContext(container) + context.autosaveEnabled = false + return context + } + + private static func initializeMigrationIfNeeded(in container: ModelContainer) throws { + let context = migrationContext(for: container) + let state = try migrationState(in: context) + guard state.phase != .completed else { return } + + OpenASOLog.rankingMigration.notice( + "Starting normalized ranking migration at phase \(state.phaseRaw, privacy: .public)." + ) + if state.legacyCrawlCount == 0, + state.legacyFactCount == 0, + state.legacyTrackedFactCount == 0, + state.lastObservationKey == nil, + state.lastSnapshotKey == nil { + state.legacyCrawlCount = try context.fetchCount( + FetchDescriptor() + ) + state.legacyFactCount = try context.fetchCount( + FetchDescriptor() + ) + state.legacyTrackedFactCount = try context.fetchCount( + FetchDescriptor() + ) + try context.save() + } + } + + private static func currentPhase(in container: ModelContainer) throws -> RankingMigrationPhase { + let context = migrationContext(for: container) + return try migrationState(in: context).phase + } + + private static func migrationState(in context: ModelContext) throws -> RankingMigrationState { + let key = RankingMigrationState.singletonKey + var descriptor = FetchDescriptor( + predicate: #Predicate { state in + state.migrationKey == key + } + ) + descriptor.fetchLimit = 1 + if let existing = try context.fetch(descriptor).first { + return existing + } + let state = RankingMigrationState() + context.insert(state) + try context.save() + return state + } + + // MARK: - Canonical crawl copy + + private static func copyCanonicalBatch(in container: ModelContainer) throws { + let context = migrationContext(for: container) + let state = try migrationState(in: context) + let legacyCrawls = try fetchLegacyCrawlBatch( + offset: state.migratedCrawlCount, + in: context + ) + guard !legacyCrawls.isEmpty else { + guard state.migratedCrawlCount == state.legacyCrawlCount else { + throw RankingSchemaV6MigrationError.validationFailed( + "copied \(state.migratedCrawlCount) of \(state.legacyCrawlCount) canonical crawls" + ) + } + guard state.migratedFactCount == state.legacyFactCount else { + throw RankingSchemaV6MigrationError.validationFailed( + "copied \(state.migratedFactCount) of \(state.legacyFactCount) canonical facts; orphaned facts remain" + ) + } + state.phase = .linkingSnapshots + try context.save() + return + } + + let crawlKeys = legacyCrawls.map(\.observationKey) + let legacyFacts = try context.fetch( + FetchDescriptor( + predicate: #Predicate { fact in + crawlKeys.contains(fact.crawlKey) + }, + sortBy: [ + SortDescriptor(\.crawlKey), + SortDescriptor(\.position), + SortDescriptor(\.appStoreID) + ] + ) + ) + let factsByCrawlKey = Dictionary(grouping: legacyFacts, by: \.crawlKey) + let payloads = legacyFacts.map(Self.revisionPayload) + let revisionKeyByItemKey = Dictionary( + uniqueKeysWithValues: zip(legacyFacts, payloads).map { + ($0.itemKey, $1.revisionKey) + } + ) + let revisionsByKey = try RankingAppRevisionStore.revisions( + for: payloads, + in: context + ) + + for legacyCrawl in legacyCrawls { + let normalizedCrawl = makeCrawl(from: legacyCrawl) + context.insert(normalizedCrawl) + let crawlFacts = factsByCrawlKey[legacyCrawl.observationKey, default: []] + try insertFacts( + crawlFacts, + into: normalizedCrawl, + revisionKeyByItemKey: revisionKeyByItemKey, + revisionsByKey: revisionsByKey, + in: context + ) + state.migratedCrawlCount += 1 + state.migratedFactCount += crawlFacts.count + state.lastObservationKey = legacyCrawl.observationKey + } + try context.save() + } + + private static func fetchLegacyCrawlBatch( + offset: Int, + in context: ModelContext + ) throws -> [LegacyKeywordRankingCrawl] { + var descriptor = FetchDescriptor( + sortBy: [SortDescriptor(\.observationKey)] + ) + descriptor.fetchOffset = offset + descriptor.fetchLimit = crawlBatchSize + return try context.fetch(descriptor) + } + + private static func makeCrawl( + from legacy: LegacyKeywordRankingCrawl + ) -> RankingCrawlRecord { + let crawl = RankingCrawlRecord( + keyword: legacy.keyword, + storefront: legacy.storefront, + platform: legacy.platform, + observedAt: legacy.observedAt, + source: legacy.source, + resultCount: legacy.resultCount, + observedHour: legacy.observedHour, + submissionCount: legacy.submissionCount, + winningCount: legacy.winningCount, + confidence: legacy.confidenceRaw + ) + crawl.observationKey = legacy.observationKey + crawl.queryKey = legacy.queryKey + crawl.keyword = legacy.keyword + crawl.storefront = legacy.storefront + crawl.platformRaw = legacy.platformRaw + crawl.observedAt = legacy.observedAt + crawl.observedHour = legacy.observedHour + crawl.sourceRaw = legacy.sourceRaw + crawl.resultCount = legacy.resultCount + crawl.submissionCount = legacy.submissionCount + crawl.winningCount = legacy.winningCount + crawl.confidenceRaw = legacy.confidenceRaw + return crawl + } + + private static func insertFacts( + _ legacyFacts: [LegacyKeywordAppRanking], + into crawl: RankingCrawlRecord, + revisionKeyByItemKey: [String: String], + revisionsByKey: [String: RankingAppRevision], + in context: ModelContext + ) throws { + var seenAppStoreIDs = Set() + for legacyFact in legacyFacts { + guard seenAppStoreIDs.insert(legacyFact.appStoreID).inserted else { + throw RankingSchemaV6MigrationError.invalidLegacyData( + "crawl \(legacyFact.crawlKey) contains duplicate app \(legacyFact.appStoreID)" + ) + } + guard let revisionKey = revisionKeyByItemKey[legacyFact.itemKey], + let revision = revisionsByKey[revisionKey] else { + throw RankingSchemaV6MigrationError.validationFailed( + "missing app revision for legacy fact \(legacyFact.itemKey)" + ) + } + let fact = RankingFact( + position: legacyFact.position, + appStoreID: legacyFact.appStoreID, + revision: revision, + observation: crawl + ) + context.insert(fact) + } + } + + // MARK: - Tracked snapshot links + + private static func linkTrackedSnapshotBatch(in container: ModelContainer) throws { + let context = migrationContext(for: container) + let state = try migrationState(in: context) + let snapshots = try fetchSnapshotBatch( + offset: state.processedSnapshotCount, + in: context + ) + guard !snapshots.isEmpty else { + let legacySnapshotCount = try context.fetchCount( + FetchDescriptor() + ) + guard state.processedSnapshotCount == legacySnapshotCount else { + throw RankingSchemaV6MigrationError.validationFailed( + "processed \(state.processedSnapshotCount) of \(legacySnapshotCount) tracked snapshots" + ) + } + state.phase = .validating + try context.save() + return + } + + let snapshotKeys = snapshots.map(\.snapshotKey) + let legacyResults = try context.fetch( + FetchDescriptor( + predicate: #Predicate { result in + snapshotKeys.contains(result.snapshotKey) + }, + sortBy: [ + SortDescriptor(\.snapshotKey), + SortDescriptor(\.position), + SortDescriptor(\.appStoreID) + ] + ) + ) + let resultsBySnapshotKey = Dictionary(grouping: legacyResults, by: \.snapshotKey) + + var queryKeyBySnapshotKey: [String: String] = [:] + var canonicalKeyBySnapshotKey: [String: String] = [:] + for snapshot in snapshots { + guard let queryKey = TrackedAppKeyword.queryKey( + fromIdentityKey: snapshot.trackIdentityKey + ) else { + if resultsBySnapshotKey[snapshot.snapshotKey, default: []].isEmpty { + continue + } + throw RankingSchemaV6MigrationError.invalidLegacyData( + "snapshot \(snapshot.snapshotKey) has ranked results but no valid query key" + ) + } + queryKeyBySnapshotKey[snapshot.snapshotKey] = queryKey + canonicalKeyBySnapshotKey[snapshot.snapshotKey] = RankingCrawlRecord.makeObservationKey( + queryKey: queryKey, + observedAt: snapshot.searchedAt, + source: snapshot.source + ) + } + + let canonicalKeys = Array(Set(canonicalKeyBySnapshotKey.values)) + let canonicalCrawls = canonicalKeys.isEmpty ? [] : try context.fetch( + FetchDescriptor( + predicate: #Predicate { crawl in + canonicalKeys.contains(crawl.observationKey) + } + ) + ) + let crawlsByKey = Dictionary( + canonicalCrawls.map { ($0.observationKey, $0) }, + uniquingKeysWith: { current, _ in current } + ) + let canonicalFacts = try fetchNormalizedFacts(crawlKeys: canonicalKeys, in: context) + let factsByCrawlKey = Dictionary(grouping: canonicalFacts) { + $0.observation.observationKey + } + + let existingLinks = try context.fetch( + FetchDescriptor( + predicate: #Predicate { link in + snapshotKeys.contains(link.snapshotKey) + } + ) + ) + var linksBySnapshotKey = Dictionary( + existingLinks.map { ($0.snapshotKey, $0) }, + uniquingKeysWith: { current, _ in current } + ) + + for snapshot in snapshots { + let results = resultsBySnapshotKey[snapshot.snapshotKey, default: []] + var selectedCrawl: RankingCrawlRecord? + if let canonicalKey = canonicalKeyBySnapshotKey[snapshot.snapshotKey], + let canonical = crawlsByKey[canonicalKey], + facts( + factsByCrawlKey[canonicalKey, default: []], + exactlyMatch: results + ) { + selectedCrawl = canonical + } else if !results.isEmpty { + guard let queryKey = queryKeyBySnapshotKey[snapshot.snapshotKey] else { + throw RankingSchemaV6MigrationError.invalidLegacyData( + "snapshot \(snapshot.snapshotKey) has no valid query key" + ) + } + selectedCrawl = try makeRecoveryCrawl( + for: snapshot, + queryKey: queryKey, + results: results, + state: state, + in: context + ) + } + + if let selectedCrawl { + if let link = linksBySnapshotKey[snapshot.snapshotKey] { + link.crawl = selectedCrawl + } else { + let link = TrackedRankingCrawlLink( + snapshotKey: snapshot.snapshotKey, + crawl: selectedCrawl + ) + context.insert(link) + linksBySnapshotKey[snapshot.snapshotKey] = link + } + state.migratedTrackedLinkCount += 1 + } + state.lastSnapshotKey = snapshot.snapshotKey + state.processedSnapshotCount += 1 + } + try context.save() + } + + private static func fetchSnapshotBatch( + offset: Int, + in context: ModelContext + ) throws -> [TrackedKeywordDailyRanking] { + var descriptor = FetchDescriptor( + sortBy: [SortDescriptor(\.snapshotKey)] + ) + descriptor.fetchOffset = offset + descriptor.fetchLimit = snapshotBatchSize + return try context.fetch(descriptor) + } + + private static func makeRecoveryCrawl( + for snapshot: TrackedKeywordDailyRanking, + queryKey: String, + results: [TrackedKeywordRankedResult], + state: RankingMigrationState, + in context: ModelContext + ) throws -> RankingCrawlRecord { + guard let components = KeywordQuery.components(from: queryKey) else { + throw RankingSchemaV6MigrationError.invalidLegacyData( + "query key \(queryKey) cannot be decoded" + ) + } + let recoveryKey = RankingCrawlRecord.makeTrackedRecoveryObservationKey( + snapshotKey: snapshot.snapshotKey + ) + let recovered = RankingCrawlRecord( + keyword: components.term, + storefront: components.storefront, + platform: components.platform, + observedAt: snapshot.searchedAt, + source: snapshot.source, + resultCount: snapshot.resultCount + ) + recovered.observationKey = recoveryKey + context.insert(recovered) + + let payloads = results.map(Self.revisionPayload) + let revisionsByKey = try RankingAppRevisionStore.revisions( + for: payloads, + in: context + ) + var seenAppStoreIDs = Set() + for (result, payload) in zip(results, payloads) { + guard seenAppStoreIDs.insert(result.appStoreID).inserted else { + throw RankingSchemaV6MigrationError.invalidLegacyData( + "snapshot \(snapshot.snapshotKey) contains duplicate apps" + ) + } + guard let revision = revisionsByKey[payload.revisionKey] else { + throw RankingSchemaV6MigrationError.validationFailed( + "missing recovered app revision for snapshot \(snapshot.snapshotKey)" + ) + } + let fact = RankingFact( + position: result.position, + appStoreID: result.appStoreID, + revision: revision, + observation: recovered + ) + context.insert(fact) + } + state.recoveredCrawlCount += 1 + return recovered + } + + // MARK: - Validation + + private static func validateAllLegacyData(in container: ModelContainer) throws { + var validatedCrawls = 0 + var validatedFacts = 0 + while let batch = try validateCanonicalBatch( + offset: validatedCrawls, + in: container + ) { + validatedCrawls += batch.crawlCount + validatedFacts += batch.factCount + } + + let stateContext = migrationContext(for: container) + let state = try migrationState(in: stateContext) + guard validatedCrawls == state.legacyCrawlCount, + validatedFacts == state.legacyFactCount else { + throw RankingSchemaV6MigrationError.validationFailed( + "validated \(validatedCrawls)/\(state.legacyCrawlCount) crawls and \(validatedFacts)/\(state.legacyFactCount) facts" + ) + } + + var validatedSnapshots = 0 + var validatedTrackedFacts = 0 + while let batch = try validateTrackedBatch( + offset: validatedSnapshots, + in: container + ) { + validatedSnapshots += batch.snapshotCount + validatedTrackedFacts += batch.factCount + } + guard validatedTrackedFacts == state.legacyTrackedFactCount else { + throw RankingSchemaV6MigrationError.validationFailed( + "validated \(validatedTrackedFacts)/\(state.legacyTrackedFactCount) tracked facts" + ) + } + + state.phase = .cleaningLegacyRows + try stateContext.save() + } + + private static func validateCanonicalBatch( + offset: Int, + in container: ModelContainer + ) throws -> (crawlCount: Int, factCount: Int)? { + let context = migrationContext(for: container) + let legacyCrawls = try fetchLegacyCrawlBatch(offset: offset, in: context) + guard !legacyCrawls.isEmpty else { return nil } + + let crawlKeys = legacyCrawls.map(\.observationKey) + let legacyFacts = try context.fetch( + FetchDescriptor( + predicate: #Predicate { fact in + crawlKeys.contains(fact.crawlKey) + } + ) + ) + let normalizedCrawls = try context.fetch( + FetchDescriptor( + predicate: #Predicate { crawl in + crawlKeys.contains(crawl.observationKey) + } + ) + ) + let normalizedFacts = try fetchNormalizedFacts(crawlKeys: crawlKeys, in: context) + let normalizedCrawlsByKey = Dictionary( + normalizedCrawls.map { ($0.observationKey, $0) }, + uniquingKeysWith: { current, _ in current } + ) + let legacyFactsByCrawlKey = Dictionary(grouping: legacyFacts, by: \.crawlKey) + let normalizedFactsByCrawlKey = Dictionary(grouping: normalizedFacts) { + $0.observation.observationKey + } + + for legacy in legacyCrawls { + guard let normalized = normalizedCrawlsByKey[legacy.observationKey] else { + throw RankingSchemaV6MigrationError.validationFailed( + "canonical crawl \(legacy.observationKey) is missing" + ) + } + try validateCrawlFields(legacy: legacy, normalized: normalized) + try validateFacts( + legacyItems: legacyFactsByCrawlKey[legacy.observationKey, default: []], + normalizedItems: normalizedFactsByCrawlKey[legacy.observationKey, default: []], + crawlKey: legacy.observationKey + ) + } + return (legacyCrawls.count, legacyFacts.count) + } + + private static func validateTrackedBatch( + offset: Int, + in container: ModelContainer + ) throws -> (snapshotCount: Int, factCount: Int)? { + let context = migrationContext(for: container) + let snapshots = try fetchSnapshotBatch(offset: offset, in: context) + guard !snapshots.isEmpty else { return nil } + + let snapshotKeys = snapshots.map(\.snapshotKey) + let results = try context.fetch( + FetchDescriptor( + predicate: #Predicate { result in + snapshotKeys.contains(result.snapshotKey) + } + ) + ) + let resultsBySnapshotKey = Dictionary(grouping: results, by: \.snapshotKey) + let links = try context.fetch( + FetchDescriptor( + predicate: #Predicate { link in + snapshotKeys.contains(link.snapshotKey) + } + ) + ) + let linksBySnapshotKey = Dictionary( + links.map { ($0.snapshotKey, $0) }, + uniquingKeysWith: { current, _ in current } + ) + let crawlKeys = Array(Set(links.map { $0.crawl.observationKey })) + let normalizedFacts = try fetchNormalizedFacts(crawlKeys: crawlKeys, in: context) + let factsByCrawlKey = Dictionary(grouping: normalizedFacts) { + $0.observation.observationKey + } + + for (snapshotKey, snapshotResults) in resultsBySnapshotKey { + guard let link = linksBySnapshotKey[snapshotKey], + facts( + factsByCrawlKey[link.crawl.observationKey, default: []], + exactlyMatch: snapshotResults + ) else { + throw RankingSchemaV6MigrationError.validationFailed( + "tracked snapshot \(snapshotKey) was not preserved exactly" + ) + } + } + return (snapshots.count, results.count) + } + + private static func validateCrawlFields( + legacy: LegacyKeywordRankingCrawl, + normalized: RankingCrawlRecord + ) throws { + guard normalized.observationKey == legacy.observationKey, + normalized.queryKey == legacy.queryKey, + normalized.keyword == legacy.keyword, + normalized.storefront == legacy.storefront, + normalized.platformRaw == legacy.platformRaw, + normalized.observedAt == legacy.observedAt, + normalized.observedHour == legacy.observedHour, + normalized.sourceRaw == legacy.sourceRaw, + normalized.resultCount == legacy.resultCount, + normalized.submissionCount == legacy.submissionCount, + normalized.winningCount == legacy.winningCount, + normalized.confidenceRaw == legacy.confidenceRaw else { + throw RankingSchemaV6MigrationError.validationFailed( + "canonical crawl \(legacy.observationKey) changed" + ) + } + } + + private static func validateFacts( + legacyItems: [LegacyKeywordAppRanking], + normalizedItems: [RankingFact], + crawlKey: String + ) throws { + let normalizedByAppStoreID = Dictionary( + normalizedItems.map { ($0.appStoreID, $0) }, + uniquingKeysWith: { current, _ in current } + ) + guard normalizedByAppStoreID.count == legacyItems.count else { + throw RankingSchemaV6MigrationError.validationFailed( + "canonical crawl \(crawlKey) changed item count" + ) + } + for legacy in legacyItems { + let expectedRevisionKey = revisionPayload(for: legacy).revisionKey + guard let normalized = normalizedByAppStoreID[legacy.appStoreID], + normalized.position == legacy.position, + normalized.revision.revisionKey == expectedRevisionKey else { + throw RankingSchemaV6MigrationError.validationFailed( + "canonical fact \(legacy.itemKey) changed" + ) + } + } + } + + private static func facts( + _ facts: [RankingFact], + exactlyMatch results: [TrackedKeywordRankedResult] + ) -> Bool { + guard facts.count == results.count else { return false } + let factsByAppStoreID = Dictionary( + facts.map { ($0.appStoreID, $0) }, + uniquingKeysWith: { current, _ in current } + ) + guard factsByAppStoreID.count == results.count else { return false } + return results.allSatisfy { result in + guard let fact = factsByAppStoreID[result.appStoreID] else { return false } + return fact.position == result.position + && fact.revision.revisionKey == revisionPayload(for: result).revisionKey + } + } + + private static func fetchNormalizedFacts( + crawlKeys: [String], + in context: ModelContext + ) throws -> [RankingFact] { + guard !crawlKeys.isEmpty else { return [] } + return try context.fetch( + FetchDescriptor( + predicate: #Predicate { fact in + crawlKeys.contains(fact.observation.observationKey) + } + ) + ) + } + + private static func revisionPayload( + for fact: LegacyKeywordAppRanking + ) -> RankingAppRevisionPayload { + RankingAppRevisionPayload( + appStoreID: fact.appStoreID, + bundleID: fact.bundleID, + name: fact.name, + subtitle: fact.subtitle, + sellerName: fact.sellerName + ) + } + + private static func revisionPayload( + for result: TrackedKeywordRankedResult + ) -> RankingAppRevisionPayload { + RankingAppRevisionPayload( + appStoreID: result.appStoreID, + bundleID: result.bundleID, + name: result.name, + subtitle: result.subtitle, + sellerName: result.sellerName + ) + } + + // MARK: - Cleanup + + private static func removeValidatedLegacyRows(in container: ModelContainer) throws { + let context = migrationContext(for: container) + let state = try migrationState(in: context) + try context.delete(model: TrackedKeywordRankedResult.self) + try context.delete(model: LegacyKeywordAppRanking.self) + try context.delete(model: LegacyKeywordRankingCrawl.self) + try context.save() + + guard try context.fetchCount(FetchDescriptor()) == 0, + try context.fetchCount(FetchDescriptor()) == 0, + try context.fetchCount(FetchDescriptor()) == 0 else { + throw RankingSchemaV6MigrationError.validationFailed( + "redundant legacy ranking rows were not removed" + ) + } + state.phase = .completed + state.completedAt = .now + try context.save() + } +} diff --git a/OpenASO/Services/SearchRanking/KeywordInsightsService.swift b/OpenASO/Services/SearchRanking/KeywordInsightsService.swift index b4a0557..c9d09c6 100644 --- a/OpenASO/Services/SearchRanking/KeywordInsightsService.swift +++ b/OpenASO/Services/SearchRanking/KeywordInsightsService.swift @@ -309,31 +309,31 @@ final class KeywordInsightsService { cutoffDate: Date?, in modelContext: ModelContext ) throws -> [CrawlRecord] { - let crawls: [KeywordRankingCrawl] + let crawls: [RankingCrawlRecord] if let cutoffDate { - let descriptor = FetchDescriptor( + let descriptor = FetchDescriptor( predicate: #Predicate { crawl in queryKeys.contains(crawl.queryKey) && crawl.observedAt >= cutoffDate }, sortBy: [ - SortDescriptor(\KeywordRankingCrawl.queryKey, order: .forward), - SortDescriptor(\KeywordRankingCrawl.observedAt, order: .forward) + SortDescriptor(\RankingCrawlRecord.queryKey, order: .forward), + SortDescriptor(\RankingCrawlRecord.observedAt, order: .forward) ] ) crawls = try modelContext.fetch(descriptor) } else { - let descriptor = FetchDescriptor( + let descriptor = FetchDescriptor( predicate: #Predicate { crawl in queryKeys.contains(crawl.queryKey) }, sortBy: [ - SortDescriptor(\KeywordRankingCrawl.queryKey, order: .forward), - SortDescriptor(\KeywordRankingCrawl.observedAt, order: .forward) + SortDescriptor(\RankingCrawlRecord.queryKey, order: .forward), + SortDescriptor(\RankingCrawlRecord.observedAt, order: .forward) ] ) crawls = try modelContext.fetch(descriptor) } - return crawls.map { + return crawls.filter { !$0.isTrackedRecovery }.map { CrawlRecord( queryKey: $0.queryKey, observationKey: $0.observationKey, @@ -353,14 +353,15 @@ final class KeywordInsightsService { for queryKey in queryKeys { let targetQueryKey = queryKey - var descriptor = FetchDescriptor( + let descriptor = FetchDescriptor( predicate: #Predicate { crawl in crawl.queryKey == targetQueryKey }, - sortBy: [SortDescriptor(\KeywordRankingCrawl.observedAt, order: .reverse)] + sortBy: [SortDescriptor(\RankingCrawlRecord.observedAt, order: .reverse)] ) - descriptor.fetchLimit = 1 - guard let crawl = try modelContext.fetch(descriptor).first else { continue } + guard let crawl = try modelContext.fetch(descriptor).first(where: { + !$0.isTrackedRecovery + }) else { continue } records.append( CrawlRecord( queryKey: crawl.queryKey, @@ -416,20 +417,20 @@ final class KeywordInsightsService { cutoffDate: Date?, in modelContext: ModelContext ) throws -> [RankingRecord] { - let rankings: [KeywordAppRanking] + let rankings: [RankingFact] if let cutoffDate { - let descriptor = FetchDescriptor( + let descriptor = FetchDescriptor( predicate: #Predicate { ranking in - queryKeys.contains(ranking.queryKey) + queryKeys.contains(ranking.observation.queryKey) && ranking.appStoreID == appStoreID - && ranking.observedAt >= cutoffDate + && ranking.observation.observedAt >= cutoffDate } ) rankings = try modelContext.fetch(descriptor) } else { - let descriptor = FetchDescriptor( + let descriptor = FetchDescriptor( predicate: #Predicate { ranking in - queryKeys.contains(ranking.queryKey) + queryKeys.contains(ranking.observation.queryKey) && ranking.appStoreID == appStoreID } ) @@ -452,9 +453,9 @@ final class KeywordInsightsService { Array(crawlKeys), size: 500 ) { - let descriptor = FetchDescriptor( + let descriptor = FetchDescriptor( predicate: #Predicate { ranking in - crawlKeyChunk.contains(ranking.crawlKey) + crawlKeyChunk.contains(ranking.observation.observationKey) && ranking.appStoreID == appStoreID } ) @@ -476,14 +477,14 @@ final class KeywordInsightsService { Array(crawlKeys), size: 500 ) { - let descriptor = FetchDescriptor( + let descriptor = FetchDescriptor( predicate: #Predicate { ranking in - crawlKeyChunk.contains(ranking.crawlKey) + crawlKeyChunk.contains(ranking.observation.observationKey) && ranking.position <= 5 }, sortBy: [ - SortDescriptor(\KeywordAppRanking.observedAt, order: .forward), - SortDescriptor(\KeywordAppRanking.position, order: .forward) + SortDescriptor(\RankingFact.observation.observedAt, order: .forward), + SortDescriptor(\RankingFact.position, order: .forward) ] ) for ranking in try modelContext.fetch(descriptor) { diff --git a/OpenASO/Services/SearchRanking/KeywordResearchHistoryReader.swift b/OpenASO/Services/SearchRanking/KeywordResearchHistoryReader.swift index eba4e07..7784fe4 100644 --- a/OpenASO/Services/SearchRanking/KeywordResearchHistoryReader.swift +++ b/OpenASO/Services/SearchRanking/KeywordResearchHistoryReader.swift @@ -70,9 +70,14 @@ struct KeywordResearchHistoryReader: Sendable { try Task.checkCancellation() let hasMore = rowsWithLookahead.count > limit let rows = Array(rowsWithLookahead.prefix(limit)) + let factsByObservationKey = try Self.factsByObservationKey( + rows.map(\.observationKey), + in: modelContext + ) let observations = rows.map { Self.snapshot( $0, + facts: factsByObservationKey[$0.observationKey, default: []], projectGeneration: projectGeneration, keywordGeneration: keywordGeneration ) @@ -104,11 +109,13 @@ private extension KeywordResearchHistoryReader { after cursor: KeywordResearchRankingHistoryCursor?, limit: Int, in modelContext: ModelContext - ) throws -> [KeywordRankingCrawl] { + ) throws -> [RankingCrawlRecord] { + let recoveryPrefix = RankingCrawlRecord.trackedRecoveryObservationKeyPrefix guard let cursor else { - var descriptor = FetchDescriptor( + var descriptor = FetchDescriptor( predicate: #Predicate { crawl in crawl.queryKey == queryKey + && !crawl.observationKey.starts(with: recoveryPrefix) }, sortBy: sortDescriptors ) @@ -120,11 +127,12 @@ private extension KeywordResearchHistoryReader { timeIntervalSince1970: TimeInterval(cursor.dayBucket) * 86_400 ) let boundaryEnd = boundaryStart.addingTimeInterval(86_400) - var boundaryDescriptor = FetchDescriptor( + var boundaryDescriptor = FetchDescriptor( predicate: #Predicate { crawl in crawl.queryKey == queryKey && crawl.observedAt >= boundaryStart && crawl.observedAt < boundaryEnd + && !crawl.observationKey.starts(with: recoveryPrefix) }, sortBy: sortDescriptors ) @@ -139,10 +147,11 @@ private extension KeywordResearchHistoryReader { return Array(rows.prefix(limit + 1)) } - var olderDescriptor = FetchDescriptor( + var olderDescriptor = FetchDescriptor( predicate: #Predicate { crawl in crawl.queryKey == queryKey && crawl.observedAt < boundaryStart + && !crawl.observationKey.starts(with: recoveryPrefix) }, sortBy: sortDescriptors ) @@ -151,11 +160,11 @@ private extension KeywordResearchHistoryReader { return rows } - static var sortDescriptors: [SortDescriptor] { + static var sortDescriptors: [SortDescriptor] { [ - SortDescriptor(\KeywordRankingCrawl.observedAt, order: .reverse), + SortDescriptor(\RankingCrawlRecord.observedAt, order: .reverse), SortDescriptor( - \KeywordRankingCrawl.observationKey, + \RankingCrawlRecord.observationKey, comparator: .lexical, order: .forward ), @@ -164,10 +173,10 @@ private extension KeywordResearchHistoryReader { static func cursor( after previous: KeywordResearchRankingHistoryCursor?, - returnedRows: [KeywordRankingCrawl] + returnedRows: [RankingCrawlRecord] ) -> KeywordResearchRankingHistoryCursor? { guard let boundaryRow = returnedRows.last else { return nil } - let dayBucket = KeywordRankingCrawl.utcDayBucket( + let dayBucket = RankingCrawlRecord.utcDayBucket( for: boundaryRow.observedAt ) var consumedSourceIDs = previous?.dayBucket == dayBucket @@ -176,7 +185,7 @@ private extension KeywordResearchHistoryReader { consumedSourceIDs.formUnion( returnedRows.lazy .filter { - KeywordRankingCrawl.utcDayBucket(for: $0.observedAt) + RankingCrawlRecord.utcDayBucket(for: $0.observedAt) == dayBucket } .map(\.sourceRaw) @@ -188,11 +197,12 @@ private extension KeywordResearchHistoryReader { } static func snapshot( - _ observation: KeywordRankingCrawl, + _ observation: RankingCrawlRecord, + facts: [RankingFact], projectGeneration: KeywordResearchProjectGeneration, keywordGeneration: KeywordResearchKeywordGeneration ) -> KeywordResearchRankingObservationSnapshot { - let items = observation.items + let items = facts .map { KeywordResearchRankingItemSnapshot( id: $0.itemKey, @@ -232,4 +242,19 @@ private extension KeywordResearchHistoryReader { items: items ) } + + static func factsByObservationKey( + _ observationKeys: [String], + in modelContext: ModelContext + ) throws -> [String: [RankingFact]] { + guard !observationKeys.isEmpty else { return [:] } + let facts = try modelContext.fetch( + FetchDescriptor( + predicate: #Predicate { fact in + observationKeys.contains(fact.observation.observationKey) + } + ) + ) + return Dictionary(grouping: facts) { $0.observation.observationKey } + } } diff --git a/OpenASO/Services/SearchRanking/KeywordResearchRankingWorkflow.swift b/OpenASO/Services/SearchRanking/KeywordResearchRankingWorkflow.swift index e71da1a..b421803 100644 --- a/OpenASO/Services/SearchRanking/KeywordResearchRankingWorkflow.swift +++ b/OpenASO/Services/SearchRanking/KeywordResearchRankingWorkflow.swift @@ -114,8 +114,17 @@ actor KeywordResearchRankingWorkflow { forQueryKey: currentTarget.queryKey, in: modelContext ) + let observationKey = persisted.observation.observationKey + let facts = try modelContext.fetch( + FetchDescriptor( + predicate: #Predicate { fact in + fact.observation.observationKey == observationKey + } + ) + ) let snapshot = Self.snapshot( persisted.observation, + facts: facts, projectGeneration: projectGeneration, keywordGeneration: keywordGeneration ) @@ -143,11 +152,12 @@ private extension KeywordResearchRankingWorkflow { } static func snapshot( - _ observation: KeywordRankingCrawl, + _ observation: RankingCrawlRecord, + facts: [RankingFact], projectGeneration: KeywordResearchProjectGeneration, keywordGeneration: KeywordResearchKeywordGeneration ) -> KeywordResearchRankingObservationSnapshot { - let items = observation.items + let items = facts .map { KeywordResearchRankingItemSnapshot( id: $0.itemKey, diff --git a/OpenASO/Services/SearchRanking/KeywordSuggestionService.swift b/OpenASO/Services/SearchRanking/KeywordSuggestionService.swift index 24b9137..8d84e7a 100644 --- a/OpenASO/Services/SearchRanking/KeywordSuggestionService.swift +++ b/OpenASO/Services/SearchRanking/KeywordSuggestionService.swift @@ -85,12 +85,12 @@ final class KeywordSuggestionService: Sendable { let targetAppStoreID = request.appStoreID let maximumCutoffDate = now.addingTimeInterval(-maximumSuggestionAge) let strongCutoffDate = now.addingTimeInterval(-strongSuggestionAge) - var targetItemDescriptor = FetchDescriptor( + var targetItemDescriptor = FetchDescriptor( predicate: #Predicate { item in item.appStoreID == targetAppStoreID }, sortBy: [ - SortDescriptor(\.observedAt, order: .reverse), + SortDescriptor(\RankingFact.observation.observedAt, order: .reverse), SortDescriptor(\.position, order: .forward) ] ) @@ -131,14 +131,14 @@ final class KeywordSuggestionService: Sendable { var appEvidenceByID: [Int64: AppSuggestionEvidence] = [:] for queryKey in request.trackedQueryKeys { - var competitorDescriptor = FetchDescriptor( + var competitorDescriptor = FetchDescriptor( predicate: #Predicate { item in - item.queryKey == queryKey + item.observation.queryKey == queryKey && item.appStoreID != targetAppStoreID && item.position <= 10 }, sortBy: [ - SortDescriptor(\.observedAt, order: .reverse), + SortDescriptor(\RankingFact.observation.observedAt, order: .reverse), SortDescriptor(\.position, order: .forward) ] ) @@ -200,11 +200,11 @@ final class KeywordSuggestionService: Sendable { let targetAppStoreID = request.appStoreID var latestDate: Date? - var targetDescriptor = FetchDescriptor( + var targetDescriptor = FetchDescriptor( predicate: #Predicate { item in item.appStoreID == targetAppStoreID }, - sortBy: [SortDescriptor(\.observedAt, order: .reverse)] + sortBy: [SortDescriptor(\RankingFact.observation.observedAt, order: .reverse)] ) targetDescriptor.fetchLimit = 1 if let observedAt = try modelContext.fetch(targetDescriptor).first?.observedAt, observedAt >= cutoffDate { @@ -212,13 +212,13 @@ final class KeywordSuggestionService: Sendable { } for queryKey in request.trackedQueryKeys { - var competitorDescriptor = FetchDescriptor( + var competitorDescriptor = FetchDescriptor( predicate: #Predicate { item in - item.queryKey == queryKey + item.observation.queryKey == queryKey && item.appStoreID != targetAppStoreID && item.position <= 10 }, - sortBy: [SortDescriptor(\.observedAt, order: .reverse)] + sortBy: [SortDescriptor(\RankingFact.observation.observedAt, order: .reverse)] ) competitorDescriptor.fetchLimit = 1 @@ -230,7 +230,7 @@ final class KeywordSuggestionService: Sendable { return latestDate } - private static func keyword(from item: KeywordAppRanking, queryKey: String) -> String? { + private static func keyword(from item: RankingFact, queryKey: String) -> String? { let suffix = "::\(item.storefront.lowercased())::\(item.platformRaw)" guard queryKey.hasSuffix(suffix) else { return queryKey @@ -420,8 +420,8 @@ private struct KeywordSuggestionEvidence { self.platform = platform } - mutating func add(item: KeywordAppRanking, observedAt: Date) { - let day = KeywordRankingCrawl.utcDayBucket(for: observedAt) + mutating func add(item: RankingFact, observedAt: Date) { + let day = RankingCrawlRecord.utcDayBucket(for: observedAt) let evidenceKey = "\(item.queryKey)|\(day)" guard collapsedEvidenceKeys.insert(evidenceKey).inserted else { if item.position < bestObservedRank { @@ -464,8 +464,8 @@ private struct AppSuggestionEvidence { self.sellerName = sellerName } - mutating func add(item: KeywordAppRanking, queryKey: String, observedAt: Date) { - let day = KeywordRankingCrawl.utcDayBucket(for: observedAt) + mutating func add(item: RankingFact, queryKey: String, observedAt: Date) { + let day = RankingCrawlRecord.utcDayBucket(for: observedAt) let evidenceKey = "\(queryKey)|\(day)" guard collapsedEvidenceKeys.insert(evidenceKey).inserted else { if item.position < bestObservedRank { diff --git a/OpenASO/Services/SearchRanking/RankingAppRevisionStore.swift b/OpenASO/Services/SearchRanking/RankingAppRevisionStore.swift new file mode 100644 index 0000000..75e5d8d --- /dev/null +++ b/OpenASO/Services/SearchRanking/RankingAppRevisionStore.swift @@ -0,0 +1,100 @@ +import Foundation +import SwiftData + +struct RankingAppRevisionPayload: Hashable, Sendable { + let appStoreID: Int64 + let bundleID: String? + let name: String + let subtitle: String? + let sellerName: String? + let revisionKey: String + + init(_ item: SearchRankingItem) { + self.init( + appStoreID: item.appStoreID, + bundleID: item.bundleID, + name: item.name, + subtitle: item.subtitle, + sellerName: item.sellerName + ) + } + + init( + appStoreID: Int64, + bundleID: String?, + name: String, + subtitle: String?, + sellerName: String? + ) { + self.appStoreID = appStoreID + self.bundleID = bundleID + self.name = name + self.subtitle = subtitle + self.sellerName = sellerName + self.revisionKey = RankingAppRevision.makeRevisionKey( + appStoreID: appStoreID, + bundleID: bundleID, + name: name, + subtitle: subtitle, + sellerName: sellerName + ) + } +} + +enum RankingAppRevisionStore { + private static let fetchChunkSize = 250 + + static func revisions( + for payloads: some Sequence, + in modelContext: ModelContext + ) throws -> [String: RankingAppRevision] { + let payloadsByKey = Dictionary( + payloads.map { ($0.revisionKey, $0) }, + uniquingKeysWith: { current, _ in current } + ) + guard !payloadsByKey.isEmpty else { return [:] } + + var revisionsByKey: [String: RankingAppRevision] = [:] + let revisionKeys = Array(payloadsByKey.keys) + for chunkStart in stride(from: 0, to: revisionKeys.count, by: fetchChunkSize) { + let chunkEnd = min(chunkStart + fetchChunkSize, revisionKeys.count) + let chunk = Array(revisionKeys[chunkStart..( + predicate: #Predicate { revision in + chunk.contains(revision.revisionKey) + } + ) + for revision in try modelContext.fetch(descriptor) { + revisionsByKey[revision.revisionKey] = revision + } + } + + for (revisionKey, payload) in payloadsByKey { + if let revision = revisionsByKey[revisionKey] { + guard revision.matches( + appStoreID: payload.appStoreID, + bundleID: payload.bundleID, + name: payload.name, + subtitle: payload.subtitle, + sellerName: payload.sellerName + ) else { + throw OpenASOError.providerUnavailable( + "A ranking app revision hash collision was detected." + ) + } + continue + } + + let revision = RankingAppRevision( + appStoreID: payload.appStoreID, + bundleID: payload.bundleID, + name: payload.name, + subtitle: payload.subtitle, + sellerName: payload.sellerName + ) + modelContext.insert(revision) + revisionsByKey[revisionKey] = revision + } + return revisionsByKey + } +} diff --git a/OpenASO/Services/SearchRanking/RankingRefreshCoordinator.swift b/OpenASO/Services/SearchRanking/RankingRefreshCoordinator.swift index 514a0e4..b214d2e 100644 --- a/OpenASO/Services/SearchRanking/RankingRefreshCoordinator.swift +++ b/OpenASO/Services/SearchRanking/RankingRefreshCoordinator.swift @@ -118,7 +118,7 @@ struct RankingRefreshPageResult: Sendable { /// Context-bound result from the synchronous shared-observation transaction. /// It must never cross an actor or `ModelContext` boundary. struct RankingObservationPersistenceResult { - let observation: KeywordRankingCrawl + let observation: RankingCrawlRecord let appliedIncomingPage: Bool } @@ -213,10 +213,9 @@ private struct RankingModelContextHasPendingChangesError: LocalizedError, Sendab } final class RankingRefreshCoordinator: Sendable { - // KeywordRankingCrawl is the canonical full-page history. Retain only the + // RankingCrawlRecord is the canonical full-page history. Retain only the // useful legacy preview rows on TrackedKeywordDailyRanking so each refresh // does not write the same 200-result page twice. - private static let legacySnapshotTopResultLimit = 5 private let rankingProvider: any SearchRankingProvider private let appCatalogService: AppCatalogService @@ -529,29 +528,6 @@ final class RankingRefreshCoordinator: Sendable { snapshot.errorMessage = nil snapshot.keywordTrack = track - let legacySnapshotItems = page.items.filter { - $0.position <= Self.legacySnapshotTopResultLimit - || $0.appStoreID == trackedApp.appStoreID - } - var rankedResultsByAppStoreID = snapshot.topResults.reduce( - into: [Int64: TrackedKeywordRankedResult]() - ) { result, rankedResult in - result[rankedResult.appStoreID] = rankedResult - } - for item in legacySnapshotItems { - upsertRankedResult( - from: item, - snapshot: snapshot, - snapshotKey: incomingSnapshotKey, - in: modelContext, - resultsByAppStoreID: &rankedResultsByAppStoreID - ) - } - pruneRankedResults( - for: snapshot, - keeping: legacySnapshotItems.map(\.appStoreID), - in: modelContext - ) track.rankingAppCount = page.resultCount if isNewSnapshot { track.snapshots.append(snapshot) @@ -560,6 +536,30 @@ final class RankingRefreshCoordinator: Sendable { snapshot = existingSnapshot! } + if incomingWinsTrackedSnapshot { + let linkedCrawl: RankingCrawlRecord + if sharedResult.appliedIncomingPage { + linkedCrawl = sharedResult.observation + } else if try observation( + sharedResult.observation, + exactlyMatches: pageResult.page, + in: modelContext + ) { + linkedCrawl = sharedResult.observation + } else { + linkedCrawl = try persistTrackedRecoveryObservation( + pageResult, + snapshotKey: snapshot.snapshotKey, + in: modelContext + ) + } + try upsertTrackedRankingLink( + snapshotKey: snapshot.snapshotKey, + crawl: linkedCrawl, + in: modelContext + ) + } + try persistenceMutationCheckpoint?() if rebuildDerivedStats { @@ -618,7 +618,7 @@ final class RankingRefreshCoordinator: Sendable { throw OpenASOError.unexpectedResponse } - let observationKey = KeywordRankingCrawl.makeObservationKey( + let observationKey = RankingCrawlRecord.makeObservationKey( queryKey: pageResult.request.queryKey, observedAt: pageResult.searchedAt, source: pageResult.page.source @@ -638,7 +638,7 @@ final class RankingRefreshCoordinator: Sendable { ) } - let observation = existingObservation ?? KeywordRankingCrawl( + let observation = existingObservation ?? RankingCrawlRecord( keyword: pageResult.request.term, storefront: pageResult.request.storefront, platform: pageResult.request.platform, @@ -657,7 +657,6 @@ final class RankingRefreshCoordinator: Sendable { observation.observationKey = observationKey observation.queryKey = pageResult.request.queryKey - observation.query = query observation.keyword = pageResult.request.term .trimmingCharacters(in: .whitespacesAndNewlines) observation.storefront = pageResult.request.storefront @@ -666,7 +665,7 @@ final class RankingRefreshCoordinator: Sendable { observation.platform = pageResult.request.platform observation.observedAt = pageResult.searchedAt observation.observedHour = pageResult.observedHour - ?? KeywordRankingCrawl.utcHourBucket(for: pageResult.searchedAt) + ?? RankingCrawlRecord.utcHourBucket(for: pageResult.searchedAt) observation.source = pageResult.page.source observation.resultCount = pageResult.page.resultCount observation.submissionCount = pageResult.submissionCount @@ -687,11 +686,23 @@ final class RankingRefreshCoordinator: Sendable { observedAt: pageResult.searchedAt, in: modelContext ) - var observationItemsByAppStoreID = observation.items.reduce( - into: [Int64: KeywordAppRanking]() + let currentObservationKey = observation.observationKey + let existingObservationItems = try modelContext.fetch( + FetchDescriptor( + predicate: #Predicate { fact in + fact.observation.observationKey == currentObservationKey + } + ) + ) + var observationItemsByAppStoreID = existingObservationItems.reduce( + into: [Int64: RankingFact]() ) { result, observationItem in result[observationItem.appStoreID] = observationItem } + let revisionsByKey = try RankingAppRevisionStore.revisions( + for: pageResult.page.items.map(RankingAppRevisionPayload.init), + in: modelContext + ) for item in pageResult.page.items { if catalogEvidenceAppStoreIDs?.contains(item.appStoreID) != false { let storeApp = try appCatalogService.upsertStoreApp( @@ -713,15 +724,20 @@ final class RankingRefreshCoordinator: Sendable { cache: &ratingCache ) } + let revisionKey = RankingAppRevisionPayload(item).revisionKey + guard let revision = revisionsByKey[revisionKey] else { + throw OpenASOError.unexpectedResponse + } upsertObservationItem( from: item, + revision: revision, observation: observation, in: modelContext, itemsByAppStoreID: &observationItemsByAppStoreID ) } pruneObservationItems( - for: observation, + Array(observationItemsByAppStoreID.values), keeping: pageResult.page.items.map(\.appStoreID), in: modelContext ) @@ -781,110 +797,171 @@ final class RankingRefreshCoordinator: Sendable { } } - private func upsertRankedResult( - from item: SearchRankingItem, - snapshot: TrackedKeywordDailyRanking, + private func upsertTrackedRankingLink( snapshotKey: String, - in modelContext: ModelContext, - resultsByAppStoreID: inout [Int64: TrackedKeywordRankedResult] - ) { - let storedResult = resultsByAppStoreID[item.appStoreID] ?? TrackedKeywordRankedResult( - position: item.position, - appStoreID: item.appStoreID, - bundleID: item.bundleID, - name: item.name, - subtitle: item.subtitle, - sellerName: item.sellerName, - snapshot: snapshot + crawl: RankingCrawlRecord, + in modelContext: ModelContext + ) throws { + let targetSnapshotKey = snapshotKey + var descriptor = FetchDescriptor( + predicate: #Predicate { link in + link.snapshotKey == targetSnapshotKey + } ) - if storedResult.modelContext == nil { - snapshot.topResults.append(storedResult) - modelContext.insert(storedResult) - resultsByAppStoreID[item.appStoreID] = storedResult + descriptor.fetchLimit = 1 + if let link = try modelContext.fetch(descriptor).first { + if link.crawl !== crawl { + link.crawl = crawl + } + } else { + modelContext.insert(TrackedRankingCrawlLink(snapshotKey: snapshotKey, crawl: crawl)) } - if storedResult.snapshotKey != snapshotKey { storedResult.snapshotKey = snapshotKey } - if storedResult.position != item.position { storedResult.position = item.position } - if storedResult.appStoreID != item.appStoreID { storedResult.appStoreID = item.appStoreID } - if storedResult.bundleID != item.bundleID { storedResult.bundleID = item.bundleID } - if storedResult.name != item.name { storedResult.name = item.name } - if storedResult.subtitle != item.subtitle { storedResult.subtitle = item.subtitle } - if storedResult.sellerName != item.sellerName { storedResult.sellerName = item.sellerName } - if storedResult.snapshot !== snapshot { - storedResult.snapshot = snapshot + } + + private func observation( + _ observation: RankingCrawlRecord, + exactlyMatches page: SearchRankingPage, + in modelContext: ModelContext + ) throws -> Bool { + let observationKey = observation.observationKey + let facts = try modelContext.fetch( + FetchDescriptor( + predicate: #Predicate { fact in + fact.observation.observationKey == observationKey + } + ) + ) + guard facts.count == page.items.count else { return false } + let factsByAppStoreID = Dictionary( + facts.map { ($0.appStoreID, $0) }, + uniquingKeysWith: { current, _ in current } + ) + guard factsByAppStoreID.count == page.items.count else { return false } + return page.items.allSatisfy { item in + guard let fact = factsByAppStoreID[item.appStoreID] else { return false } + return fact.position == item.position + && fact.revision.revisionKey == RankingAppRevisionPayload(item).revisionKey } } + private func persistTrackedRecoveryObservation( + _ pageResult: RankingRefreshPageResult, + snapshotKey: String, + in modelContext: ModelContext + ) throws -> RankingCrawlRecord { + let recoveryKey = RankingCrawlRecord.makeTrackedRecoveryObservationKey( + snapshotKey: snapshotKey + ) + var descriptor = FetchDescriptor( + predicate: #Predicate { crawl in + crawl.observationKey == recoveryKey + } + ) + descriptor.fetchLimit = 1 + let observation = try modelContext.fetch(descriptor).first ?? RankingCrawlRecord( + keyword: pageResult.request.term, + storefront: pageResult.request.storefront, + platform: pageResult.request.platform, + observedAt: pageResult.searchedAt, + source: pageResult.page.source, + resultCount: pageResult.page.resultCount, + observedHour: pageResult.observedHour, + submissionCount: pageResult.submissionCount, + winningCount: pageResult.winningCount, + confidence: pageResult.confidence + ) + if observation.modelContext == nil { + modelContext.insert(observation) + } + observation.observationKey = recoveryKey + observation.queryKey = pageResult.request.queryKey + observation.keyword = pageResult.request.term + .trimmingCharacters(in: .whitespacesAndNewlines) + observation.storefront = pageResult.request.storefront + .trimmingCharacters(in: .whitespacesAndNewlines) + .lowercased() + observation.platform = pageResult.request.platform + observation.observedAt = pageResult.searchedAt + observation.observedHour = pageResult.observedHour + ?? RankingCrawlRecord.utcHourBucket(for: pageResult.searchedAt) + observation.source = pageResult.page.source + observation.resultCount = pageResult.page.resultCount + observation.submissionCount = pageResult.submissionCount + observation.winningCount = pageResult.winningCount + observation.confidenceRaw = pageResult.confidence + + let currentObservationKey = observation.observationKey + let existingFacts = try modelContext.fetch( + FetchDescriptor( + predicate: #Predicate { fact in + fact.observation.observationKey == currentObservationKey + } + ) + ) + var factsByAppStoreID = Dictionary( + existingFacts.map { ($0.appStoreID, $0) }, + uniquingKeysWith: { current, _ in current } + ) + let payloads = pageResult.page.items.map(RankingAppRevisionPayload.init) + let revisionsByKey = try RankingAppRevisionStore.revisions( + for: payloads, + in: modelContext + ) + for (item, payload) in zip(pageResult.page.items, payloads) { + guard let revision = revisionsByKey[payload.revisionKey] else { + throw OpenASOError.unexpectedResponse + } + upsertObservationItem( + from: item, + revision: revision, + observation: observation, + in: modelContext, + itemsByAppStoreID: &factsByAppStoreID + ) + } + pruneObservationItems( + Array(factsByAppStoreID.values), + keeping: pageResult.page.items.map(\.appStoreID), + in: modelContext + ) + return observation + } + private func upsertObservationItem( from item: SearchRankingItem, - observation: KeywordRankingCrawl, + revision: RankingAppRevision, + observation: RankingCrawlRecord, in modelContext: ModelContext, - itemsByAppStoreID: inout [Int64: KeywordAppRanking] + itemsByAppStoreID: inout [Int64: RankingFact] ) { - let observationItem = itemsByAppStoreID[item.appStoreID] ?? KeywordAppRanking( + let observationItem = itemsByAppStoreID[item.appStoreID] ?? RankingFact( position: item.position, appStoreID: item.appStoreID, - bundleID: item.bundleID, - name: item.name, - subtitle: item.subtitle, - sellerName: item.sellerName, + revision: revision, observation: observation ) if observationItem.modelContext == nil { - observation.items.append(observationItem) modelContext.insert(observationItem) itemsByAppStoreID[item.appStoreID] = observationItem } if observationItem.position != item.position { observationItem.position = item.position } if observationItem.appStoreID != item.appStoreID { observationItem.appStoreID = item.appStoreID } - if observationItem.bundleID != item.bundleID { observationItem.bundleID = item.bundleID } - if observationItem.name != item.name { observationItem.name = item.name } - if observationItem.subtitle != item.subtitle { observationItem.subtitle = item.subtitle } - if observationItem.sellerName != item.sellerName { observationItem.sellerName = item.sellerName } - if observationItem.crawlKey != observation.observationKey { - observationItem.crawlKey = observation.observationKey - } - if observationItem.queryKey != observation.queryKey { observationItem.queryKey = observation.queryKey } - if observationItem.storefront != observation.storefront { - observationItem.storefront = observation.storefront - } - if observationItem.platform != observation.platform { observationItem.platform = observation.platform } - if observationItem.observedAt != observation.observedAt { - observationItem.observedAt = observation.observedAt - } - let itemKey = KeywordAppRanking.makeItemKey( - observationKey: observation.observationKey, - appStoreID: item.appStoreID - ) - if observationItem.itemKey != itemKey { observationItem.itemKey = itemKey } + if observationItem.revision !== revision { observationItem.revision = revision } if observationItem.observation !== observation { observationItem.observation = observation } } - private func pruneRankedResults( - for snapshot: TrackedKeywordDailyRanking, - keeping appStoreIDs: [Int64], - in modelContext: ModelContext - ) { - let retainedAppStoreIDs = Set(appStoreIDs) - let staleResults = snapshot.topResults.filter { !retainedAppStoreIDs.contains($0.appStoreID) } - for result in staleResults { - modelContext.delete(result) - } - snapshot.topResults.removeAll { !retainedAppStoreIDs.contains($0.appStoreID) } - } - private func pruneObservationItems( - for observation: KeywordRankingCrawl, + _ observationItems: [RankingFact], keeping appStoreIDs: [Int64], in modelContext: ModelContext ) { let retainedAppStoreIDs = Set(appStoreIDs) - let staleItems = observation.items.filter { !retainedAppStoreIDs.contains($0.appStoreID) } + let staleItems = observationItems.filter { !retainedAppStoreIDs.contains($0.appStoreID) } for item in staleItems { modelContext.delete(item) } - observation.items.removeAll { !retainedAppStoreIDs.contains($0.appStoreID) } } private func fetchTrackedKeywordDailyRanking( @@ -905,11 +982,11 @@ final class RankingRefreshCoordinator: Sendable { return snapshot } - let targetDayBucket = KeywordRankingCrawl.utcDayBucket(for: searchedAt) + let targetDayBucket = RankingCrawlRecord.utcDayBucket(for: searchedAt) return track.snapshots .filter { $0.source == source - && KeywordRankingCrawl.utcDayBucket(for: $0.searchedAt) == targetDayBucket + && RankingCrawlRecord.utcDayBucket(for: $0.searchedAt) == targetDayBucket } .max { $0.searchedAt < $1.searchedAt } } @@ -934,9 +1011,9 @@ final class RankingRefreshCoordinator: Sendable { observedAt: Date, source: RankingSource, in modelContext: ModelContext - ) throws -> KeywordRankingCrawl? { + ) throws -> RankingCrawlRecord? { let targetObservationKey = observationKey - var descriptor = FetchDescriptor( + var descriptor = FetchDescriptor( predicate: #Predicate { observation in observation.observationKey == targetObservationKey } @@ -947,16 +1024,16 @@ final class RankingRefreshCoordinator: Sendable { } let targetQueryKey = queryKey - let fallbackDescriptor = FetchDescriptor( + let fallbackDescriptor = FetchDescriptor( predicate: #Predicate { observation in observation.queryKey == targetQueryKey } ) - let targetDayBucket = KeywordRankingCrawl.utcDayBucket(for: observedAt) + let targetDayBucket = RankingCrawlRecord.utcDayBucket(for: observedAt) return try modelContext.fetch(fallbackDescriptor) .filter { $0.source == source - && KeywordRankingCrawl.utcDayBucket(for: $0.observedAt) == targetDayBucket + && RankingCrawlRecord.utcDayBucket(for: $0.observedAt) == targetDayBucket } .max { $0.observedAt < $1.observedAt } } @@ -1367,214 +1444,18 @@ final class RankingRefreshCoordinator: Sendable { for requests: some Sequence, in modelContext: ModelContext ) throws { - let queryKeys = Set(requests.map(\.queryKey)) - for queryKey in queryKeys { - try rebuildAppKeywordStats(queryKey: queryKey, in: modelContext) - } + // V6 computes ranking aggregates directly from indexed canonical facts. + // Keep this compatibility entry point while older workflows are updated. + _ = requests + _ = modelContext } func rebuildDerivedStats( forQueryKey queryKey: String, in modelContext: ModelContext ) throws { - try rebuildAppKeywordStats(queryKey: queryKey, in: modelContext) - } - - private func rebuildAppKeywordStats( - queryKey: String, - in modelContext: ModelContext - ) throws { - let metrics = try fetchKeywordMetrics(queryKey: queryKey, in: modelContext) - let observations = try fetchKeywordRankingCrawls(queryKey: queryKey, in: modelContext) - let existingStats = try fetchAppKeywordStats(queryKey: queryKey, in: modelContext) - - struct KeywordAggregate { - var appStoreID: Int64 - var keyword: String - var storefront: String - var platform: AppPlatform - var bestRank: Int - var latestRank: Int - var averageRank: Double - var observationCount: Int - var firstSeenAt: Date - var lastSeenAt: Date - } - - var aggregates: [Int64: KeywordAggregate] = [:] - let orderedObservations = observations.sorted { lhs, rhs in - if lhs.observedAt != rhs.observedAt { - return lhs.observedAt < rhs.observedAt - } - return lhs.observationKey < rhs.observationKey - } - for observation in orderedObservations { - for item in observation.items { - if var aggregate = aggregates[item.appStoreID] { - aggregate.bestRank = min(aggregate.bestRank, item.position) - aggregate.latestRank = item.position - aggregate.averageRank = ( - aggregate.averageRank * Double(aggregate.observationCount) - + Double(item.position) - ) / Double(aggregate.observationCount + 1) - aggregate.observationCount += 1 - aggregate.firstSeenAt = min(aggregate.firstSeenAt, observation.observedAt) - aggregate.lastSeenAt = max(aggregate.lastSeenAt, observation.observedAt) - aggregates[item.appStoreID] = aggregate - } else { - aggregates[item.appStoreID] = KeywordAggregate( - appStoreID: item.appStoreID, - keyword: observation.keyword, - storefront: observation.storefront, - platform: observation.platform, - bestRank: item.position, - latestRank: item.position, - averageRank: Double(item.position), - observationCount: 1, - firstSeenAt: observation.observedAt, - lastSeenAt: observation.observedAt - ) - } - } - } - - let existingStatsByAppStoreID = Dictionary(uniqueKeysWithValues: existingStats.map { ($0.appStoreID, $0) }) - for staleStats in existingStats where aggregates[staleStats.appStoreID] == nil { - modelContext.delete(staleStats) - } - - for aggregate in aggregates.values { - let stats = existingStatsByAppStoreID[aggregate.appStoreID] ?? AppKeywordStats( - appStoreID: aggregate.appStoreID, - queryKey: queryKey, - keyword: aggregate.keyword, - storefront: aggregate.storefront, - platform: aggregate.platform, - rank: aggregate.latestRank, - observedAt: aggregate.lastSeenAt, - popularityScore: metrics?.popularityScore, - difficultyScore: metrics?.difficultyScore - ) - if stats.modelContext == nil { - modelContext.insert(stats) - } - stats.keyword = aggregate.keyword - stats.storefront = aggregate.storefront - stats.platform = aggregate.platform - stats.bestRank = aggregate.bestRank - stats.latestRank = aggregate.latestRank - stats.averageRank = aggregate.averageRank - stats.observationCount = aggregate.observationCount - stats.firstSeenAt = aggregate.firstSeenAt - stats.lastSeenAt = aggregate.lastSeenAt - stats.popularityScore = metrics?.popularityScore - stats.difficultyScore = metrics?.difficultyScore - } - } - - private func upsertAppKeywordStats( - item: KeywordAppRanking, - observation: KeywordRankingCrawl, - metrics: KeywordDailyMetric?, - in modelContext: ModelContext - ) { - let identityKey = AppKeywordStats.makeIdentityKey( - appStoreID: item.appStoreID, - queryKey: observation.queryKey - ) - - let stats: AppKeywordStats - if let existing = try? fetchAppKeywordStats(identityKey: identityKey, in: modelContext) { - stats = existing - let previousObservationCount = max(1, stats.observationCount) - let previousAverage = stats.averageRank ?? Double(item.position) - stats.averageRank = ( - previousAverage * Double(previousObservationCount) - + Double(item.position) - ) / Double(previousObservationCount + 1) - stats.observationCount = previousObservationCount + 1 - stats.bestRank = min(stats.bestRank ?? item.position, item.position) - stats.latestRank = item.position - stats.firstSeenAt = min(stats.firstSeenAt, observation.observedAt) - stats.lastSeenAt = max(stats.lastSeenAt, observation.observedAt) - } else { - stats = AppKeywordStats( - appStoreID: item.appStoreID, - queryKey: observation.queryKey, - keyword: observation.keyword, - storefront: observation.storefront, - platform: observation.platform, - rank: item.position, - observedAt: observation.observedAt, - popularityScore: metrics?.popularityScore, - difficultyScore: metrics?.difficultyScore - ) - modelContext.insert(stats) - } - - stats.keyword = observation.keyword - stats.storefront = observation.storefront - stats.platform = observation.platform - stats.popularityScore = metrics?.popularityScore - stats.difficultyScore = metrics?.difficultyScore - } - - private func fetchKeywordMetrics(queryKey: String, in modelContext: ModelContext) throws -> KeywordDailyMetric? { - let targetQueryKey = queryKey - var descriptor = FetchDescriptor( - predicate: #Predicate { metrics in - metrics.queryKey == targetQueryKey - } - ) - descriptor.fetchLimit = 1 - return try modelContext.fetch(descriptor).first - } - - private func fetchAppKeywordStats(identityKey: String, in modelContext: ModelContext) throws -> AppKeywordStats? { - let targetIdentityKey = identityKey - var descriptor = FetchDescriptor( - predicate: #Predicate { stats in - stats.identityKey == targetIdentityKey - } - ) - descriptor.fetchLimit = 1 - return try modelContext.fetch(descriptor).first - } - - private func fetchAppKeywordStats(queryKey: String, in modelContext: ModelContext) throws -> [AppKeywordStats] { - let targetQueryKey = queryKey - let descriptor = FetchDescriptor( - predicate: #Predicate { stats in - stats.queryKey == targetQueryKey - } - ) - return try modelContext.fetch(descriptor) - } - - private func fetchKeywordRankingCrawls(queryKey: String, in modelContext: ModelContext) throws -> [KeywordRankingCrawl] { - let targetQueryKey = queryKey - let descriptor = FetchDescriptor( - predicate: #Predicate { observation in - observation.queryKey == targetQueryKey - } - ) - return try modelContext.fetch(descriptor) - } - - private func fetchKeywordRankingCrawls( - storefront: String, - platform: AppPlatform, - in modelContext: ModelContext - ) throws -> [KeywordRankingCrawl] { - let targetStorefront = storefront.lowercased() - let targetPlatformRaw = platform.rawValue - let descriptor = FetchDescriptor( - predicate: #Predicate { observation in - observation.storefront == targetStorefront - && observation.platformRaw == targetPlatformRaw - } - ) - return try modelContext.fetch(descriptor) + _ = queryKey + _ = modelContext } @discardableResult diff --git a/OpenASOTests/AppServicesDependencyTests.swift b/OpenASOTests/AppServicesDependencyTests.swift index 0626ccf..9aec4e6 100644 --- a/OpenASOTests/AppServicesDependencyTests.swift +++ b/OpenASOTests/AppServicesDependencyTests.swift @@ -7,7 +7,7 @@ import Testing @MainActor struct AppServicesDependencyTests { @Test - func modelContainerFactoryUsesAppendOnlyV5MigrationPlan() throws { + func modelContainerFactoryUsesAppendOnlyV6MigrationPlan() throws { let container = try ModelContainerFactory.makeModelContainer(isStoredInMemoryOnly: true) #expect(OpenASOSchemaV1.versionIdentifier == Schema.Version(1, 0, 0)) @@ -15,15 +15,17 @@ struct AppServicesDependencyTests { #expect(OpenASOSchemaV3.versionIdentifier == Schema.Version(3, 0, 0)) #expect(OpenASOSchemaV4.versionIdentifier == Schema.Version(4, 0, 0)) #expect(OpenASOSchemaV5.versionIdentifier == Schema.Version(5, 0, 0)) - #expect(OpenASOMigrationPlan.schemas.count == 5) + #expect(OpenASOSchemaV6.versionIdentifier == Schema.Version(6, 0, 0)) + #expect(OpenASOMigrationPlan.schemas.count == 6) #expect(OpenASOMigrationPlan.schemas.first?.versionIdentifier == OpenASOSchemaV1.versionIdentifier) - #expect(OpenASOMigrationPlan.schemas.last?.versionIdentifier == OpenASOSchemaV5.versionIdentifier) + #expect(OpenASOMigrationPlan.schemas.last?.versionIdentifier == OpenASOSchemaV6.versionIdentifier) #expect(OpenASOSchemaV1.models.count == 17) #expect(OpenASOSchemaV2.models.count == 18) #expect(OpenASOSchemaV3.models.count == 19) #expect(OpenASOSchemaV4.models.count == 21) #expect(OpenASOSchemaV5.models.count == 23) - #expect(OpenASOMigrationPlan.stages.count == 4) + #expect(OpenASOSchemaV6.models.count == 28) + #expect(OpenASOMigrationPlan.stages.count == 5) #expect(container.migrationPlan != nil) } diff --git a/OpenASOTests/ExactV4StoreFixtureTests.swift b/OpenASOTests/ExactV4StoreFixtureTests.swift index f1c8feb..c974ce8 100644 --- a/OpenASOTests/ExactV4StoreFixtureTests.swift +++ b/OpenASOTests/ExactV4StoreFixtureTests.swift @@ -127,7 +127,10 @@ struct ExactV4StoreFixtureTests { } @MainActor -func assertExactV4FixtureSentinels(in modelContext: ModelContext) throws { +func assertExactV4FixtureSentinels( + in modelContext: ModelContext, + normalizedRankings: Bool = false +) throws { let folders = try modelContext.fetch(FetchDescriptor()) let appKeywordStats = try modelContext.fetch(FetchDescriptor()) let latestRatings = try modelContext.fetch(FetchDescriptor()) @@ -138,8 +141,17 @@ func assertExactV4FixtureSentinels(in modelContext: ModelContext) throws { let screenshots = try modelContext.fetch(FetchDescriptor()) let queries = try modelContext.fetch(FetchDescriptor()) let keywordMetrics = try modelContext.fetch(FetchDescriptor()) - let crawls = try modelContext.fetch(FetchDescriptor()) - let crawlItems = try modelContext.fetch(FetchDescriptor()) + let crawls = try modelContext.fetch(FetchDescriptor()) + let crawlItems = try modelContext.fetch(FetchDescriptor()) + let normalizedCrawls = normalizedRankings + ? try modelContext.fetch(FetchDescriptor()) + : [] + let normalizedCrawlItems = normalizedRankings + ? try modelContext.fetch(FetchDescriptor()) + : [] + let rankingLinks = normalizedRankings + ? try modelContext.fetch(FetchDescriptor()) + : [] let trackedApps = try modelContext.fetch(FetchDescriptor()) let tracks = try modelContext.fetch(FetchDescriptor()) let snapshots = try modelContext.fetch(FetchDescriptor()) @@ -162,12 +174,15 @@ func assertExactV4FixtureSentinels(in modelContext: ModelContext) throws { #expect(screenshots.count == 1) #expect(queries.count == 2) #expect(keywordMetrics.count == 1) - #expect(crawls.count == 1) - #expect(crawlItems.count == 1) + #expect(crawls.count == (normalizedRankings ? 0 : 1)) + #expect(crawlItems.count == (normalizedRankings ? 0 : 1)) + #expect(normalizedCrawls.count == (normalizedRankings ? 2 : 0)) + #expect(normalizedCrawlItems.count == (normalizedRankings ? 2 : 0)) + #expect(rankingLinks.count == (normalizedRankings ? 1 : 0)) #expect(trackedApps.count == 1) #expect(tracks.count == 1) #expect(snapshots.count == 1) - #expect(rankedResults.count == 1) + #expect(rankedResults.count == (normalizedRankings ? 0 : 1)) #expect(storefronts.count == 1) #expect(attempts.count == 1) #expect(statuses.count == 2) @@ -188,12 +203,9 @@ func assertExactV4FixtureSentinels(in modelContext: ModelContext) throws { queryByKey[ExactV4FixtureSentinel.unavailableQueryKey], ) let metric = try #require(keywordMetrics.first) - let crawl = try #require(crawls.first) - let crawlItem = try #require(crawlItems.first) let trackedApp = try #require(trackedApps.first) let track = try #require(tracks.first) let snapshot = try #require(snapshots.first) - let rankedResult = try #require(rankedResults.first) let storefront = try #require(storefronts.first) let attempt = try #require(attempts.first) @@ -219,7 +231,7 @@ func assertExactV4FixtureSentinels(in modelContext: ModelContext) throws { #expect(query.queryKey == ReleasedV1FixtureSentinel.queryKey) #expect(query.tracks.map(\.identityKey) == [track.identityKey]) - #expect(query.observations.map(\.observationKey) == [crawl.observationKey]) + #expect(query.observations.count == (normalizedRankings ? 0 : 1)) #expect(unavailableQuery.term == ExactV4FixtureSentinel.unavailableKeyword) #expect(unavailableQuery.storefront == ReleasedV1FixtureSentinel.storefront) #expect(unavailableQuery.platform == .iphone) @@ -227,10 +239,25 @@ func assertExactV4FixtureSentinels(in modelContext: ModelContext) throws { #expect(unavailableQuery.observations.isEmpty) #expect(metric.queryKey == ReleasedV1FixtureSentinel.queryKey) #expect(metric.notes == "released-metric-notes") - #expect(crawl.observedHour == ReleasedV1FixtureSentinel.crawlObservedHour) - #expect(crawl.items.map(\.itemKey) == [crawlItem.itemKey]) - #expect(crawlItem.appStoreID == ReleasedV1FixtureSentinel.competitorAppStoreID) - #expect(crawlItem.observation.observationKey == crawl.observationKey) + if normalizedRankings { + let crawl = try #require(normalizedCrawls.first { $0.source == .appStoreWeb }) + let items = normalizedCrawlItems.filter { + $0.observation.observationKey == crawl.observationKey + } + let crawlItem = try #require(items.first) + #expect(crawl.observedHour == ReleasedV1FixtureSentinel.crawlObservedHour) + #expect(items.map(\.itemKey) == [crawlItem.itemKey]) + #expect(crawlItem.appStoreID == ReleasedV1FixtureSentinel.competitorAppStoreID) + #expect(crawlItem.observation.observationKey == crawl.observationKey) + } else { + let crawl = try #require(crawls.first) + let crawlItem = try #require(crawlItems.first) + #expect(query.observations.map(\.observationKey) == [crawl.observationKey]) + #expect(crawl.observedHour == ReleasedV1FixtureSentinel.crawlObservedHour) + #expect(crawl.items.map(\.itemKey) == [crawlItem.itemKey]) + #expect(crawlItem.appStoreID == ReleasedV1FixtureSentinel.competitorAppStoreID) + #expect(crawlItem.observation.observationKey == crawl.observationKey) + } #expect(trackedApp.appStoreID == ReleasedV1FixtureSentinel.appStoreID) #expect(trackedApp.folder?.id == folder.id) @@ -239,9 +266,21 @@ func assertExactV4FixtureSentinels(in modelContext: ModelContext) throws { #expect(track.statusMessage == nil) #expect(track.snapshots.map(\.snapshotKey) == [snapshot.snapshotKey]) #expect(snapshot.rank == ReleasedV1FixtureSentinel.rank) - #expect(snapshot.topResults.map(\.appStoreID) == [rankedResult.appStoreID]) - #expect(rankedResult.position == ReleasedV1FixtureSentinel.competitorPosition) - #expect(rankedResult.name == ReleasedV1FixtureSentinel.competitorName) + if normalizedRankings { + let rankingLink = try #require(rankingLinks.first) + let rankedResult = try #require(normalizedCrawlItems.first { + $0.observation.observationKey == rankingLink.crawl.observationKey + }) + #expect(snapshot.topResults.isEmpty) + #expect(rankingLink.snapshotKey == snapshot.snapshotKey) + #expect(rankedResult.position == ReleasedV1FixtureSentinel.competitorPosition) + #expect(rankedResult.name == ReleasedV1FixtureSentinel.competitorName) + } else { + let rankedResult = try #require(rankedResults.first) + #expect(snapshot.topResults.map(\.appStoreID) == [rankedResult.appStoreID]) + #expect(rankedResult.position == ReleasedV1FixtureSentinel.competitorPosition) + #expect(rankedResult.name == ReleasedV1FixtureSentinel.competitorName) + } #expect(storefront.code == ReleasedV1FixtureSentinel.storefront) #expect(storefront.title == "🇬🇧 United Kingdom") diff --git a/OpenASOTests/KeywordMetricsServiceTests.swift b/OpenASOTests/KeywordMetricsServiceTests.swift index ecda51f..c0e93af 100644 --- a/OpenASOTests/KeywordMetricsServiceTests.swift +++ b/OpenASOTests/KeywordMetricsServiceTests.swift @@ -1016,7 +1016,7 @@ struct KeywordMetricsServiceTests { modelContext.insert(trackedApp) let track = try makeTrack(term: "focus timer", trackedApp: trackedApp, in: modelContext) let observedAt = Date(timeIntervalSince1970: 2_000_000_000) - let crawl = KeywordRankingCrawl( + let crawl = RankingCrawlRecord( keyword: track.term, storefront: track.storefront, platform: track.platform, @@ -1025,13 +1025,14 @@ struct KeywordMetricsServiceTests { resultCount: 100, query: track.query ) - let ranking = KeywordAppRanking( + let ranking = makeRankingFact( position: 4, appStoreID: trackedApp.appStoreID, bundleID: trackedApp.bundleID, name: trackedApp.name, sellerName: trackedApp.sellerName, - observation: crawl + observation: crawl, + in: modelContext ) modelContext.insert(crawl) modelContext.insert(ranking) @@ -1044,8 +1045,8 @@ struct KeywordMetricsServiceTests { using: backgroundModelStore ) let stored = try await backgroundModelStore.read { context in - let crawls = try context.fetch(FetchDescriptor()) - let rankings = try context.fetch(FetchDescriptor()) + let crawls = try context.fetch(FetchDescriptor()) + let rankings = try context.fetch(FetchDescriptor()) let metrics = try context.fetch(FetchDescriptor()) return ( crawlKeys: crawls.map(\.observationKey), diff --git a/OpenASOTests/KeywordRankingHistoryTests.swift b/OpenASOTests/KeywordRankingHistoryTests.swift index b0c3ed1..debb67e 100644 --- a/OpenASOTests/KeywordRankingHistoryTests.swift +++ b/OpenASOTests/KeywordRankingHistoryTests.swift @@ -254,21 +254,24 @@ struct KeywordRankingHistoryTests { makeRanking( position: 4, appStoreID: targetAppStoreID, - observation: oldTargetCrawl + observation: oldTargetCrawl, + in: modelContext ) ) modelContext.insert( makeRanking( position: 1, appStoreID: otherAppStoreID, - observation: recentTargetCrawl + observation: recentTargetCrawl, + in: modelContext ) ) modelContext.insert( makeRanking( position: 2, appStoreID: targetAppStoreID, - observation: otherQueryCrawl + observation: otherQueryCrawl, + in: modelContext ) ) try modelContext.save() @@ -299,7 +302,8 @@ struct KeywordRankingHistoryTests { makeRanking( position: 7, appStoreID: 100, - observation: crawl + observation: crawl, + in: modelContext ) ) try modelContext.save() @@ -448,7 +452,7 @@ struct KeywordRankingHistoryTests { resultCount: 10, keywordTrack: track ) - let crawl = KeywordRankingCrawl( + let crawl = RankingCrawlRecord( keyword: query.term, storefront: query.storefront, platform: query.platform, @@ -585,8 +589,8 @@ private func makeSummary(id: String, rank: Int?, date: Date) -> KeywordRankingCr ) } -private func makeCrawl(query: KeywordQuery, date: Date) -> KeywordRankingCrawl { - KeywordRankingCrawl( +private func makeCrawl(query: KeywordQuery, date: Date) -> RankingCrawlRecord { + RankingCrawlRecord( keyword: query.term, storefront: query.storefront, platform: query.platform, @@ -597,25 +601,26 @@ private func makeCrawl(query: KeywordQuery, date: Date) -> KeywordRankingCrawl { ) } +@MainActor private func makeRanking( position: Int, appStoreID: Int64, - observation: KeywordRankingCrawl -) -> KeywordAppRanking { - KeywordAppRanking( + observation: RankingCrawlRecord, + in modelContext: ModelContext +) -> RankingFact { + makeRankingFact( position: position, appStoreID: appStoreID, bundleID: "com.example.\(appStoreID)", name: "App \(appStoreID)", sellerName: "Example", - observation: observation + observation: observation, + in: modelContext ) } private func makeInMemoryContainer() throws -> ModelContainer { - let schema = Schema(OpenASOSchemaV1.models) - let configuration = ModelConfiguration(schema: schema, isStoredInMemoryOnly: true) - return try ModelContainer(for: schema, configurations: [configuration]) + try ModelContainerFactory.makeModelContainer(isStoredInMemoryOnly: true) } private func utcCalendar() -> Calendar { diff --git a/OpenASOTests/KeywordRankingListLoaderTests.swift b/OpenASOTests/KeywordRankingListLoaderTests.swift index bd8f666..641000f 100644 --- a/OpenASOTests/KeywordRankingListLoaderTests.swift +++ b/OpenASOTests/KeywordRankingListLoaderTests.swift @@ -12,7 +12,7 @@ struct KeywordRankingListLoaderTests { let appStoreID: Int64 = 10_001 let observedAt = Date(timeIntervalSince1970: 2_000_000_000) let query = KeywordQuery(term: "focus", storefront: "us", platform: .iphone) - let crawl = KeywordRankingCrawl( + let crawl = RankingCrawlRecord( keyword: query.term, storefront: query.storefront, platform: query.platform, @@ -21,14 +21,15 @@ struct KeywordRankingListLoaderTests { resultCount: 300, query: query ) - let ranking = KeywordAppRanking( + let ranking = makeRankingFact( position: 1, appStoreID: appStoreID, bundleID: "com.example.focus", name: "Ranking Name", subtitle: "Ranking Subtitle", sellerName: "Ranking Seller", - observation: crawl + observation: crawl, + in: modelContext ) let storeApp = StoreApp( appStoreID: appStoreID, @@ -176,7 +177,7 @@ struct KeywordRankingListLoaderTests { let fallbackAppStoreID: Int64 = 101 let backgroundAppStoreID: Int64 = 202 let query = KeywordQuery(term: "focus", storefront: "us", platform: .iphone) - let crawl = KeywordRankingCrawl( + let crawl = RankingCrawlRecord( keyword: query.term, storefront: query.storefront, platform: query.platform, @@ -187,14 +188,15 @@ struct KeywordRankingListLoaderTests { ) modelContext.insert(query) modelContext.insert(crawl) - modelContext.insert(KeywordAppRanking( + modelContext.insert(makeRankingFact( position: 1, appStoreID: backgroundAppStoreID, bundleID: "com.example.background-only", name: "Background Only", subtitle: nil, sellerName: "Example", - observation: crawl + observation: crawl, + in: modelContext )) try modelContext.save() @@ -627,9 +629,7 @@ struct KeywordRankingListLoaderTests { } private func makeInMemoryContainer() throws -> ModelContainer { - let schema = Schema(OpenASOSchemaV1.models) - let configuration = ModelConfiguration(schema: schema, isStoredInMemoryOnly: true) - return try ModelContainer(for: schema, configurations: [configuration]) + try ModelContainerFactory.makeModelContainer(isStoredInMemoryOnly: true) } } diff --git a/OpenASOTests/KeywordResearchHistoryModelTests.swift b/OpenASOTests/KeywordResearchHistoryModelTests.swift index b1c0af6..a2bbe40 100644 --- a/OpenASOTests/KeywordResearchHistoryModelTests.swift +++ b/OpenASOTests/KeywordResearchHistoryModelTests.swift @@ -354,7 +354,7 @@ private func historyCursor( _ observation: KeywordResearchRankingObservationSnapshot ) -> KeywordResearchRankingHistoryCursor { KeywordResearchRankingHistoryCursor( - dayBucket: KeywordRankingCrawl.utcDayBucket( + dayBucket: RankingCrawlRecord.utcDayBucket( for: observation.observedAt ), consumedSourceIDs: [observation.source.rawValue] diff --git a/OpenASOTests/KeywordResearchHistoryReaderTests.swift b/OpenASOTests/KeywordResearchHistoryReaderTests.swift index 5bccaab..f87c2b1 100644 --- a/OpenASOTests/KeywordResearchHistoryReaderTests.swift +++ b/OpenASOTests/KeywordResearchHistoryReaderTests.swift @@ -35,7 +35,7 @@ struct KeywordResearchHistoryReaderTests { #expect(first.observations.map(\.id) == [keys[0], keys[2]].sorted()) #expect(first.observations.map(\.observedAt) == [tiedDate, tiedDate]) #expect(first.nextCursor == KeywordResearchRankingHistoryCursor( - dayBucket: KeywordRankingCrawl.utcDayBucket(for: tiedDate), + dayBucket: RankingCrawlRecord.utcDayBucket(for: tiedDate), consumedSourceIDs: Set([ RankingSource.appStoreWeb.rawValue, RankingSource.iTunesFallback.rawValue, @@ -505,7 +505,7 @@ private func seedObservations( } return seeds.map { seed in - let observation = KeywordRankingCrawl( + let observation = RankingCrawlRecord( keyword: query.term, storefront: query.storefront, platform: query.platform, @@ -514,20 +514,19 @@ private func seedObservations( resultCount: seed.items.count, query: query ) - query.observations.append(observation) modelContext.insert(observation) for itemSeed in seed.items { - let item = KeywordAppRanking( + let item = makeRankingFact( position: itemSeed.position, appStoreID: itemSeed.appStoreID, bundleID: "com.example.\(itemSeed.appStoreID)", name: itemSeed.name, subtitle: "Subtitle \(itemSeed.appStoreID)", sellerName: "Seller \(itemSeed.appStoreID)", - observation: observation + observation: observation, + in: modelContext ) - observation.items.append(item) modelContext.insert(item) } return observation.observationKey @@ -542,7 +541,7 @@ private func moveObservation( in store: BackgroundModelStore ) async throws -> String { try await store.write { modelContext in - var descriptor = FetchDescriptor( + var descriptor = FetchDescriptor( predicate: #Predicate { observation in observation.observationKey == id } @@ -552,7 +551,7 @@ private func moveObservation( throw OpenASOError.unexpectedResponse } observation.observedAt = observedAt - observation.observedHour = KeywordRankingCrawl.utcHourBucket( + observation.observedHour = RankingCrawlRecord.utcHourBucket( for: observedAt ) observation.observationKey = replacementID diff --git a/OpenASOTests/KeywordResearchMetricsWorkflowTests.swift b/OpenASOTests/KeywordResearchMetricsWorkflowTests.swift index 82b5ee2..499a783 100644 --- a/OpenASOTests/KeywordResearchMetricsWorkflowTests.swift +++ b/OpenASOTests/KeywordResearchMetricsWorkflowTests.swift @@ -675,7 +675,7 @@ struct KeywordResearchMetricsWorkflowTests { } @Test - func successfulRefreshRebuildsOnlyCanonicalDerivedRankingStats() async throws { + func successfulRefreshDoesNotMaterializeRedundantRankingStats() async throws { let client = ScriptedMetricsHTTPClient(defaultReply: .scores(defaultScore: 84)) let fixture = try await makeFixture(httpClient: client, now: metricsTestDate) let keyword = try #require(fixture.keywords.first) @@ -692,16 +692,10 @@ struct KeywordResearchMetricsWorkflowTests { policy: .requireNetwork ) - let stats = try #require(try await storedRankingStats( + #expect(try await storedRankingStats( queryKey: keyword.queryKey, in: fixture.backgroundStore - )) - #expect(stats.appStoreID == 777) - #expect(stats.bestRank == 5) - #expect(stats.latestRank == 5) - #expect(stats.observationCount == 1) - #expect(stats.popularityScore == 84) - #expect(stats.difficultyScore == nil) + ) == nil) } @Test @@ -1144,7 +1138,7 @@ private func seedRankingObservation( guard let query = try modelContext.fetch(descriptor).first else { throw OpenASOError.unexpectedResponse } - let observation = KeywordRankingCrawl( + let observation = RankingCrawlRecord( keyword: query.term, storefront: query.storefront, platform: query.platform, @@ -1153,16 +1147,15 @@ private func seedRankingObservation( resultCount: 1, query: query ) - let item = KeywordAppRanking( + let item = makeRankingFact( position: rank, appStoreID: appStoreID, bundleID: "example.derived.stats", name: "Derived stats sentinel", sellerName: "Sentinel seller", - observation: observation + observation: observation, + in: modelContext ) - observation.items.append(item) - query.observations.append(observation) modelContext.insert(observation) modelContext.insert(item) } diff --git a/OpenASOTests/KeywordResearchPersistenceTests.swift b/OpenASOTests/KeywordResearchPersistenceTests.swift index 0f4f865..a738898 100644 --- a/OpenASOTests/KeywordResearchPersistenceTests.swift +++ b/OpenASOTests/KeywordResearchPersistenceTests.swift @@ -225,7 +225,7 @@ struct KeywordResearchPersistenceTests { source: .appleAdsPopularity ) let observedAt = Date(timeIntervalSinceReferenceDate: 805_700_000) - let crawl = KeywordRankingCrawl( + let crawl = RankingCrawlRecord( keyword: query.term, storefront: query.storefront, platform: query.platform, @@ -234,13 +234,14 @@ struct KeywordResearchPersistenceTests { resultCount: 1, query: query ) - let crawlItem = KeywordAppRanking( + let crawlItem = makeRankingFact( position: 1, appStoreID: 320_000_001, bundleID: "com.example.shared", name: "Shared Result", sellerName: "Example", - observation: crawl + observation: crawl, + in: context ) let calculationID = UUID(uuidString: "32000000-0000-4000-8000-000000000210")! let difficulty = EstimatedKeywordDifficultyMetric( @@ -301,8 +302,6 @@ struct KeywordResearchPersistenceTests { project: project ) project.attachKeyword(keyword) - query.observations.append(crawl) - crawl.items.append(crawlItem) context.insert(query) context.insert(metrics) context.insert(crawl) @@ -320,8 +319,8 @@ struct KeywordResearchPersistenceTests { #expect(try context.fetch(FetchDescriptor()).isEmpty) #expect(try context.fetch(FetchDescriptor()).map(\.queryKey) == [query.queryKey]) #expect(try context.fetch(FetchDescriptor()).map(\.queryKey) == [query.queryKey]) - #expect(try context.fetch(FetchDescriptor()).map(\.queryKey) == [query.queryKey]) - #expect(try context.fetch(FetchDescriptor()).map(\.queryKey) == [query.queryKey]) + #expect(try context.fetch(FetchDescriptor()).map(\.queryKey) == [query.queryKey]) + #expect(try context.fetch(FetchDescriptor()).map(\.queryKey) == [query.queryKey]) #expect(try context.fetch(FetchDescriptor()).map(\.queryKey) == [query.queryKey]) #expect(try context.fetch( FetchDescriptor() diff --git a/OpenASOTests/KeywordResearchPresentationTestSupport.swift b/OpenASOTests/KeywordResearchPresentationTestSupport.swift index e169c3f..de5878f 100644 --- a/OpenASOTests/KeywordResearchPresentationTestSupport.swift +++ b/OpenASOTests/KeywordResearchPresentationTestSupport.swift @@ -1,8 +1,38 @@ import Foundation +import SwiftData @testable import OpenASO let testDate = Date(timeIntervalSinceReferenceDate: 807_000_000) +func makeRankingFact( + position: Int, + appStoreID: Int64, + bundleID: String?, + name: String, + subtitle: String? = nil, + sellerName: String?, + observation: RankingCrawlRecord, + in modelContext: ModelContext +) -> RankingFact { + let payload = RankingAppRevisionPayload( + appStoreID: appStoreID, + bundleID: bundleID, + name: name, + subtitle: subtitle, + sellerName: sellerName + ) + let revision = try! RankingAppRevisionStore.revisions( + for: [payload], + in: modelContext + )[payload.revisionKey]! + return RankingFact( + position: position, + appStoreID: appStoreID, + revision: revision, + observation: observation + ) +} + struct PageCall: Equatable, Sendable { let offset: Int let limit: Int diff --git a/OpenASOTests/KeywordResearchProjectCopyServiceTests.swift b/OpenASOTests/KeywordResearchProjectCopyServiceTests.swift index a2c119d..cd966bf 100644 --- a/OpenASOTests/KeywordResearchProjectCopyServiceTests.swift +++ b/OpenASOTests/KeywordResearchProjectCopyServiceTests.swift @@ -821,7 +821,7 @@ private func seedSharedEvidence( throw CopyServiceTestError.injected } let observedAt = Date(timeIntervalSinceReferenceDate: 806_400_150) - let crawl = KeywordRankingCrawl( + let crawl = RankingCrawlRecord( keyword: query.term, storefront: query.storefront, platform: query.platform, @@ -830,7 +830,6 @@ private func seedSharedEvidence( resultCount: 1, query: query ) - query.observations.append(crawl) modelContext.insert(crawl) modelContext.insert(KeywordDailyMetric( queryKey: query.queryKey, @@ -941,7 +940,7 @@ private func evidenceState(in store: BackgroundModelStore) async throws -> Evide try await store.read { modelContext in EvidenceState( queryKeys: try modelContext.fetch(FetchDescriptor()).map(\.queryKey).sorted(), - observationKeys: try modelContext.fetch(FetchDescriptor()) + observationKeys: try modelContext.fetch(FetchDescriptor()) .map(\.observationKey).sorted(), metricQueryKeys: try modelContext.fetch(FetchDescriptor()) .map(\.queryKey).sorted() diff --git a/OpenASOTests/KeywordResearchProjectStoreTests.swift b/OpenASOTests/KeywordResearchProjectStoreTests.swift index f5f9c0c..b8d5d73 100644 --- a/OpenASOTests/KeywordResearchProjectStoreTests.swift +++ b/OpenASOTests/KeywordResearchProjectStoreTests.swift @@ -718,7 +718,7 @@ struct KeywordResearchProjectStoreTests { source: .appleAdsPopularity, updatedAt: baseDate ) - let crawl = KeywordRankingCrawl( + let crawl = RankingCrawlRecord( keyword: query.term, storefront: query.storefront, platform: query.platform, @@ -727,13 +727,14 @@ struct KeywordResearchProjectStoreTests { resultCount: 1, query: query ) - let ranking = KeywordAppRanking( + let ranking = makeRankingFact( position: 1, appStoreID: 320_000_001, bundleID: "com.example.shared", name: "Shared Result", sellerName: "Example", - observation: crawl + observation: crawl, + in: modelContext ) let difficulty = EstimatedKeywordDifficultyMetric( queryKey: queryKey, @@ -785,8 +786,6 @@ struct KeywordResearchProjectStoreTests { exactTitlePhraseMatch: true, exactSubtitlePhraseMatch: false ) - query.observations.append(crawl) - crawl.items.append(ranking) modelContext.insert(metric) modelContext.insert(crawl) modelContext.insert(ranking) @@ -802,8 +801,8 @@ struct KeywordResearchProjectStoreTests { try modelContext.fetchCount(FetchDescriptor()), try modelContext.fetchCount(FetchDescriptor()), try modelContext.fetchCount(FetchDescriptor()), - try modelContext.fetchCount(FetchDescriptor()), - try modelContext.fetchCount(FetchDescriptor()), + try modelContext.fetchCount(FetchDescriptor()), + try modelContext.fetchCount(FetchDescriptor()), try modelContext.fetchCount(FetchDescriptor()), try modelContext.fetchCount(FetchDescriptor()) ] diff --git a/OpenASOTests/KeywordResearchRankingWorkflowTests.swift b/OpenASOTests/KeywordResearchRankingWorkflowTests.swift index a3c30b2..b726af3 100644 --- a/OpenASOTests/KeywordResearchRankingWorkflowTests.swift +++ b/OpenASOTests/KeywordResearchRankingWorkflowTests.swift @@ -80,7 +80,7 @@ struct KeywordResearchRankingWorkflowTests { #expect(state.screenshotCount == 2) #expect(state.latestRatingIDs == [10, 20]) #expect(state.dailyRatingIDs == [10, 20]) - #expect(state.statsAppStoreIDs == [10, 20]) + #expect(state.statsAppStoreIDs.isEmpty) #expect(state.trackedCount == 0) } @@ -299,7 +299,7 @@ struct KeywordResearchRankingWorkflowTests { let state = try await databaseState(in: fixture.backgroundStore) #expect(state.storeAppIDs == [1, 2, 3]) #expect(state.latestRatingIDs == [1, 2, 3]) - #expect(state.statsAppStoreIDs == [2, 3]) + #expect(state.statsAppStoreIDs.isEmpty) #expect(!state.storeAppIDs.contains(4)) #expect(!state.storeAppIDs.contains(5)) } @@ -343,13 +343,7 @@ struct KeywordResearchRankingWorkflowTests { ratingCount: 200, averageRating: 4.5 )) - let stats = try await statsRecords(in: fixture.backgroundStore) - #expect(stats == [StatsRecord( - appStoreID: 1, - bestRank: 1, - latestRank: 3, - observationCount: 3 - )]) + #expect(try await statsRecords(in: fixture.backgroundStore).isEmpty) } @Test @@ -803,13 +797,14 @@ struct KeywordResearchRankingWorkflowTests { } #expect(firstSnapshot.persistentModelID == secondSnapshot.persistentModelID) #expect(secondSnapshot.rank == 1) - #expect(secondSnapshot.topResults.map(\.appStoreID).sorted() == [1, 99]) - #expect(try modelContext.fetch(FetchDescriptor()).count == 1) - #expect(try modelContext.fetch(FetchDescriptor()).map(\.appStoreID).sorted() == [1, 99]) + #expect(secondSnapshot.topResults.isEmpty) + #expect(try modelContext.fetch(FetchDescriptor()).count == 1) + #expect(try modelContext.fetch(FetchDescriptor()).map(\.appStoreID).sorted() == [1, 99]) #expect(try modelContext.fetch(FetchDescriptor()).count == 1) - #expect(try modelContext.fetch(FetchDescriptor()).map(\.appStoreID).sorted() == [1, 99]) + #expect(try modelContext.fetchCount(FetchDescriptor()) == 0) + #expect(try modelContext.fetchCount(FetchDescriptor()) == 1) #expect(try modelContext.fetch(FetchDescriptor()).map(\.appStoreID).sorted() == [1, 99]) - #expect(try modelContext.fetch(FetchDescriptor()).map(\.appStoreID).sorted() == [1, 99]) + #expect(try modelContext.fetchCount(FetchDescriptor()) == 0) } } @@ -1060,15 +1055,26 @@ private func trackedSharedRaceState( } let targetQueryKey = queryKey - var crawlDescriptor = FetchDescriptor( + let recoveryPrefix = RankingCrawlRecord.trackedRecoveryObservationKeyPrefix + var crawlDescriptor = FetchDescriptor( predicate: #Predicate { crawl in crawl.queryKey == targetQueryKey + && !crawl.observationKey.starts(with: recoveryPrefix) } ) crawlDescriptor.fetchLimit = 1 guard let crawl = try modelContext.fetch(crawlDescriptor).first else { throw OpenASOError.unexpectedResponse } + let crawlKey = crawl.observationKey + let crawlItems = try modelContext.fetch( + FetchDescriptor( + predicate: #Predicate { fact in + fact.observation.observationKey == crawlKey + }, + sortBy: [SortDescriptor(\.position)] + ) + ) var snapshotDescriptor = FetchDescriptor( predicate: #Predicate { snapshot in @@ -1079,7 +1085,24 @@ private func trackedSharedRaceState( guard let snapshot = try modelContext.fetch(snapshotDescriptor).first else { throw OpenASOError.unexpectedResponse } - let rankedResults = snapshot.topResults.sorted { + let snapshotKey = snapshot.snapshotKey + var linkDescriptor = FetchDescriptor( + predicate: #Predicate { link in + link.snapshotKey == snapshotKey + } + ) + linkDescriptor.fetchLimit = 1 + guard let linkedCrawlKey = try modelContext.fetch(linkDescriptor).first?.crawl.observationKey else { + throw OpenASOError.unexpectedResponse + } + let rankedResults = try modelContext.fetch( + FetchDescriptor( + predicate: #Predicate { fact in + fact.observation.observationKey == linkedCrawlKey + }, + sortBy: [SortDescriptor(\.position)] + ) + ).sorted { if $0.position != $1.position { return $0.position < $1.position } return $0.appStoreID < $1.appStoreID } @@ -1089,7 +1112,7 @@ private func trackedSharedRaceState( ) return TrackedSharedRaceState( sharedObservedAt: crawl.observedAt, - sharedAppStoreIDs: crawl.items.sorted { $0.position < $1.position }.map(\.appStoreID), + sharedAppStoreIDs: crawlItems.map(\.appStoreID), snapshotSearchedAt: snapshot.searchedAt, snapshotRank: snapshot.rank, snapshotResultCount: snapshot.resultCount, @@ -1358,8 +1381,8 @@ private func databaseState(in store: BackgroundModelStore) async throws -> Datab + modelContext.fetchCount(FetchDescriptor()) + modelContext.fetchCount(FetchDescriptor()) return DatabaseState( - crawlCount: try modelContext.fetchCount(FetchDescriptor()), - observationItemCount: try modelContext.fetchCount(FetchDescriptor()), + crawlCount: try modelContext.fetchCount(FetchDescriptor()), + observationItemCount: try modelContext.fetchCount(FetchDescriptor()), storeAppIDs: try modelContext.fetch(FetchDescriptor()).map(\.appStoreID).sorted(), metadataCount: try modelContext.fetchCount(FetchDescriptor()), screenshotCount: try modelContext.fetchCount(FetchDescriptor()), @@ -1373,8 +1396,8 @@ private func databaseState(in store: BackgroundModelStore) async throws -> Datab private func sharedWriteCount(in store: BackgroundModelStore) async throws -> Int { try await store.read { modelContext in - try modelContext.fetchCount(FetchDescriptor()) - + modelContext.fetchCount(FetchDescriptor()) + try modelContext.fetchCount(FetchDescriptor()) + + modelContext.fetchCount(FetchDescriptor()) + modelContext.fetchCount(FetchDescriptor()) + modelContext.fetchCount(FetchDescriptor()) + modelContext.fetchCount(FetchDescriptor()) @@ -1393,9 +1416,14 @@ private struct ObservationRecord: Equatable, Sendable { private func observationRecords(in store: BackgroundModelStore) async throws -> [ObservationRecord] { try await store.read { modelContext in - try modelContext.fetch(FetchDescriptor()) + let factsByCrawlKey = Dictionary( + grouping: try modelContext.fetch(FetchDescriptor()) + ) { fact in + fact.observation.observationKey + } + return try modelContext.fetch(FetchDescriptor()) .map { observation in - let items = observation.items.sorted { + let items = factsByCrawlKey[observation.observationKey, default: []].sorted { if $0.position != $1.position { return $0.position < $1.position } return $0.appStoreID < $1.appStoreID } @@ -1493,7 +1521,7 @@ private func crossSourceWatermarkRecord( } ) storefrontMetadataDescriptor.fetchLimit = 1 - var iTunesCrawlDescriptor = FetchDescriptor( + var iTunesCrawlDescriptor = FetchDescriptor( predicate: #Predicate { crawl in crawl.queryKey == targetQueryKey && crawl.sourceRaw == iTunesSourceRaw @@ -1505,10 +1533,18 @@ private func crossSourceWatermarkRecord( let dailyRating = try modelContext.fetch(dailyRatingDescriptor).first, let storeApp = try modelContext.fetch(storeAppDescriptor).first, let storefrontMetadata = try modelContext.fetch(storefrontMetadataDescriptor).first, - let iTunesCrawl = try modelContext.fetch(iTunesCrawlDescriptor).first, - let iTunesCrawlItem = iTunesCrawl.items.first(where: { - $0.appStoreID == targetAppStoreID - }) else { + let iTunesCrawl = try modelContext.fetch(iTunesCrawlDescriptor).first else { + throw OpenASOError.unexpectedResponse + } + let iTunesCrawlKey = iTunesCrawl.observationKey + var iTunesFactDescriptor = FetchDescriptor( + predicate: #Predicate { fact in + fact.observation.observationKey == iTunesCrawlKey + && fact.appStoreID == targetAppStoreID + } + ) + iTunesFactDescriptor.fetchLimit = 1 + guard let iTunesCrawlItem = try modelContext.fetch(iTunesFactDescriptor).first else { throw OpenASOError.unexpectedResponse } diff --git a/OpenASOTests/KeywordWorkspaceProjectionTests.swift b/OpenASOTests/KeywordWorkspaceProjectionTests.swift index 15f6008..492b6c4 100644 --- a/OpenASOTests/KeywordWorkspaceProjectionTests.swift +++ b/OpenASOTests/KeywordWorkspaceProjectionTests.swift @@ -290,7 +290,7 @@ struct KeywordWorkspaceProjectionTests { trackedApp: trackedApp, query: query ) - let oldCrawl = KeywordRankingCrawl( + let oldCrawl = RankingCrawlRecord( keyword: query.term, storefront: query.storefront, platform: query.platform, @@ -299,7 +299,7 @@ struct KeywordWorkspaceProjectionTests { resultCount: 100, query: query ) - let latestCrawl = KeywordRankingCrawl( + let latestCrawl = RankingCrawlRecord( keyword: query.term, storefront: query.storefront, platform: query.platform, @@ -318,29 +318,32 @@ struct KeywordWorkspaceProjectionTests { source: .appleAdsPopularity ) let rankings = [ - KeywordAppRanking( + makeRankingFact( position: 8, appStoreID: appStoreID, bundleID: trackedApp.bundleID, name: trackedApp.name, sellerName: trackedApp.sellerName, - observation: oldCrawl + observation: oldCrawl, + in: modelContext ), - KeywordAppRanking( + makeRankingFact( position: 3, appStoreID: appStoreID, bundleID: trackedApp.bundleID, name: trackedApp.name, sellerName: trackedApp.sellerName, - observation: latestCrawl + observation: latestCrawl, + in: modelContext ), - KeywordAppRanking( + makeRankingFact( position: 1, appStoreID: 202, bundleID: "com.example.competitor", name: "Competitor", sellerName: "Example", - observation: latestCrawl + observation: latestCrawl, + in: modelContext ) ] @@ -408,7 +411,7 @@ struct KeywordWorkspaceProjectionTests { query: query ) let oldDate = Date.now.addingTimeInterval(-30 * 86_400) - let oldCrawl = KeywordRankingCrawl( + let oldCrawl = RankingCrawlRecord( keyword: query.term, storefront: query.storefront, platform: query.platform, @@ -417,13 +420,14 @@ struct KeywordWorkspaceProjectionTests { resultCount: 100, query: query ) - let ranking = KeywordAppRanking( + let ranking = makeRankingFact( position: 3, appStoreID: appStoreID, bundleID: trackedApp.bundleID, name: trackedApp.name, sellerName: trackedApp.sellerName, - observation: oldCrawl + observation: oldCrawl, + in: modelContext ) modelContext.insert(trackedApp) diff --git a/OpenASOTests/OpenASOMCPServerTests.swift b/OpenASOTests/OpenASOMCPServerTests.swift index f82ba5b..54956dc 100644 --- a/OpenASOTests/OpenASOMCPServerTests.swift +++ b/OpenASOTests/OpenASOMCPServerTests.swift @@ -859,18 +859,35 @@ private struct ServerTestContext { resultCount: 20, keywordTrack: track ) - let newerResult = TrackedKeywordRankedResult( + let snapshotCrawl = RankingCrawlRecord( + keyword: track.term, + storefront: track.storefront, + platform: track.platform, + observedAt: newerSnapshot.searchedAt, + source: newerSnapshot.source, + resultCount: 1, + query: query + ) + snapshotCrawl.observationKey = RankingCrawlRecord.makeTrackedRecoveryObservationKey( + snapshotKey: newerSnapshot.snapshotKey + ) + let newerResult = makeRankingFact( position: 1, appStoreID: 321, bundleID: "com.example.321", name: "Ranked App", sellerName: "Example Seller", - snapshot: newerSnapshot + observation: snapshotCrawl, + in: modelContext ) - newerSnapshot.topResults.append(newerResult) track.snapshots.append(newerSnapshot) modelContext.insert(newerSnapshot) + modelContext.insert(snapshotCrawl) modelContext.insert(newerResult) + modelContext.insert(TrackedRankingCrawlLink( + snapshotKey: newerSnapshot.snapshotKey, + crawl: snapshotCrawl + )) let olderSnapshot = TrackedKeywordDailyRanking( rank: 5, @@ -882,7 +899,7 @@ private struct ServerTestContext { track.snapshots.append(olderSnapshot) modelContext.insert(olderSnapshot) - let crawl = KeywordRankingCrawl( + let crawl = RankingCrawlRecord( keyword: track.term, storefront: track.storefront, platform: track.platform, @@ -891,15 +908,15 @@ private struct ServerTestContext { resultCount: 1, query: query ) - let crawlItem = KeywordAppRanking( + let crawlItem = makeRankingFact( position: 1, appStoreID: 456, bundleID: "com.example.456", name: "Crawl App", sellerName: "Example Seller", - observation: crawl + observation: crawl, + in: modelContext ) - crawl.items.append(crawlItem) modelContext.insert(crawl) modelContext.insert(crawlItem) diff --git a/OpenASOTests/OpenASOMCPServiceTests.swift b/OpenASOTests/OpenASOMCPServiceTests.swift index 3bd7b1a..2ccd5db 100644 --- a/OpenASOTests/OpenASOMCPServiceTests.swift +++ b/OpenASOTests/OpenASOMCPServiceTests.swift @@ -1897,13 +1897,26 @@ struct OpenASOMCPServiceTests { let snapshots = try context.modelContext.fetch(FetchDescriptor()) let snapshot = try #require(snapshots.first) #expect(snapshot.rank == 3) - #expect(snapshot.topResults.map(\.appStoreID).sorted() == [123, 789]) - #expect(snapshot.topResults.count == 2) - - let crawls = try context.modelContext.fetch(FetchDescriptor()) + #expect(snapshot.topResults.isEmpty) + #expect(try context.modelContext.fetchCount( + FetchDescriptor() + ) == 0) + #expect(try context.modelContext.fetchCount( + FetchDescriptor() + ) == 1) + + let crawls = try context.modelContext.fetch(FetchDescriptor()) let crawl = try #require(crawls.first) - #expect(crawl.items.map(\.appStoreID).sorted() == [123, 789]) - #expect(crawl.items.count == 2) + let crawlKey = crawl.observationKey + let facts = try context.modelContext.fetch( + FetchDescriptor( + predicate: #Predicate { fact in + fact.observation.observationKey == crawlKey + } + ) + ) + #expect(facts.map(\.appStoreID).sorted() == [123, 789]) + #expect(facts.count == 2) } @Test @@ -2532,7 +2545,7 @@ struct OpenASOMCPServiceTests { #expect(successfulDTO.source == RankingSource.iTunesFallback.rawValue) #expect(successfulDTO.rankedApps.map(\.position) == [1, 2]) #expect(successfulDTO.rankedApps.map(\.appStoreID) == ["100", "200"]) - #expect(successfulDTO.rankedAppsAvailableCount == 4) + #expect(successfulDTO.rankedAppsAvailableCount == 3) #expect(successfulDTO.rankedAppsTruncated) } @@ -2703,7 +2716,7 @@ struct OpenASOMCPServiceTests { #expect(result.submissionCount == 3) #expect(result.winningCount == 2) #expect(result.confidence == "high") - #expect(result.observedHour == KeywordRankingCrawl.utcHourBucket(for: result.observedAt)) + #expect(result.observedHour == RankingCrawlRecord.utcHourBucket(for: result.observedAt)) #expect(result.rankedApps.map(\.position) == [1, 2]) #expect(result.rankedApps.map(\.appStoreID) == ["1", "2"]) #expect(result.rankedAppsAvailableCount == 102) @@ -3614,20 +3627,37 @@ private struct MCPTestContext { modelContext.insert(track) modelContext.insert(snapshot) + let crawl = RankingCrawlRecord( + keyword: term, + storefront: "us", + platform: .iphone, + observedAt: searchedAt, + source: .iTunesFallback, + resultCount: rankedAppCount, + query: query + ) + crawl.observationKey = RankingCrawlRecord.makeTrackedRecoveryObservationKey( + snapshotKey: snapshot.snapshotKey + ) + modelContext.insert(crawl) for rankedAppIndex in 0.. KeywordRankingCrawl { - let crawl = KeywordRankingCrawl( + ) throws -> RankingCrawlRecord { + let crawl = RankingCrawlRecord( keyword: keyword, storefront: storefront, platform: platform, @@ -3661,16 +3691,16 @@ private struct MCPTestContext { ) modelContext.insert(crawl) for row in rows { - let ranking = KeywordAppRanking( + let ranking = makeRankingFact( position: row.position, appStoreID: row.appStoreID, bundleID: row.bundleID ?? "com.example.\(row.appStoreID)", name: row.name, subtitle: row.subtitle, sellerName: row.sellerName, - observation: crawl + observation: crawl, + in: modelContext ) - crawl.items.append(ranking) modelContext.insert(ranking) } try modelContext.save() @@ -3697,18 +3727,37 @@ private struct MCPTestContext { ) track.snapshots.append(snapshot) modelContext.insert(snapshot) - for row in rows { - let result = TrackedKeywordRankedResult( - position: row.position, - appStoreID: row.appStoreID, - bundleID: row.bundleID ?? "com.example.\(row.appStoreID)", - name: row.name, - subtitle: row.subtitle, - sellerName: row.sellerName, - snapshot: snapshot + if !rows.isEmpty { + let crawl = RankingCrawlRecord( + keyword: track.term, + storefront: track.storefront, + platform: track.platform, + observedAt: searchedAt, + source: source, + resultCount: resultCount, + query: track.query + ) + crawl.observationKey = RankingCrawlRecord.makeTrackedRecoveryObservationKey( + snapshotKey: snapshot.snapshotKey ) - snapshot.topResults.append(result) - modelContext.insert(result) + modelContext.insert(crawl) + for row in rows { + let result = makeRankingFact( + position: row.position, + appStoreID: row.appStoreID, + bundleID: row.bundleID ?? "com.example.\(row.appStoreID)", + name: row.name, + subtitle: row.subtitle, + sellerName: row.sellerName, + observation: crawl, + in: modelContext + ) + modelContext.insert(result) + } + modelContext.insert(TrackedRankingCrawlLink( + snapshotKey: snapshot.snapshotKey, + crawl: crawl + )) } try modelContext.save() return snapshot @@ -3806,7 +3855,7 @@ private final class MCPMetadataEnrichmentRecorder: Sendable { let record = MCPMetadataEnrichmentRecord( requests: requests, committedCrawlCount: (try? modelContext.fetchCount( - FetchDescriptor() + FetchDescriptor() )) ?? -1, committedSnapshotCount: (try? modelContext.fetchCount( FetchDescriptor() diff --git a/OpenASOTests/PersistenceMigrationTests.swift b/OpenASOTests/PersistenceMigrationTests.swift index 25990c7..19dcc40 100644 --- a/OpenASOTests/PersistenceMigrationTests.swift +++ b/OpenASOTests/PersistenceMigrationTests.swift @@ -6,7 +6,7 @@ import Testing @MainActor struct PersistenceMigrationTests { @Test - func migrationPlanAppendsThroughV5AndKeepsPriorSchemasFrozen() throws { + func migrationPlanAppendsThroughV6AndKeepsPriorSchemasFrozen() throws { let container = try ModelContainerFactory.makeModelContainer(isStoredInMemoryOnly: true) #expect(OpenASOSchemaV1.versionIdentifier == Schema.Version(1, 0, 0)) @@ -19,18 +19,21 @@ struct PersistenceMigrationTests { #expect(OpenASOSchemaV4.models.count == 21) #expect(OpenASOSchemaV5.versionIdentifier == Schema.Version(5, 0, 0)) #expect(OpenASOSchemaV5.models.count == 23) - #expect(OpenASOMigrationPlan.currentSchema.versionIdentifier == Schema.Version(5, 0, 0)) - #expect(OpenASOMigrationPlan.schemas.count == 5) + #expect(OpenASOSchemaV6.versionIdentifier == Schema.Version(6, 0, 0)) + #expect(OpenASOSchemaV6.models.count == 28) + #expect(OpenASOMigrationPlan.currentSchema.versionIdentifier == Schema.Version(6, 0, 0)) + #expect(OpenASOMigrationPlan.schemas.count == 6) #expect(OpenASOMigrationPlan.schemas[0].versionIdentifier == OpenASOSchemaV1.versionIdentifier) #expect(OpenASOMigrationPlan.schemas[1].versionIdentifier == OpenASOSchemaV2.versionIdentifier) #expect(OpenASOMigrationPlan.schemas[2].versionIdentifier == OpenASOSchemaV3.versionIdentifier) #expect(OpenASOMigrationPlan.schemas[3].versionIdentifier == OpenASOSchemaV4.versionIdentifier) #expect(OpenASOMigrationPlan.schemas[4].versionIdentifier == OpenASOSchemaV5.versionIdentifier) + #expect(OpenASOMigrationPlan.schemas[5].versionIdentifier == OpenASOSchemaV6.versionIdentifier) #expect( OpenASOMigrationPlan.currentSchema.versionIdentifier == OpenASOMigrationPlan.schemas.last?.versionIdentifier ) - #expect(OpenASOMigrationPlan.stages.count == 4) + #expect(OpenASOMigrationPlan.stages.count == 5) #expect(container.migrationPlan != nil) } @@ -226,7 +229,7 @@ struct PersistenceMigrationTests { at: materialized.storeURL ) let modelContext = ModelContext(container) - try assertExactV4FixtureSentinels(in: modelContext) + try assertExactV4FixtureSentinels(in: modelContext, normalizedRankings: true) #expect(try modelContext.fetch(FetchDescriptor()).isEmpty) #expect(try modelContext.fetch(FetchDescriptor()).isEmpty) @@ -261,7 +264,7 @@ struct PersistenceMigrationTests { at: materialized.storeURL ) let modelContext = ModelContext(container) - try assertExactV4FixtureSentinels(in: modelContext) + try assertExactV4FixtureSentinels(in: modelContext, normalizedRankings: true) let projects = try modelContext.fetch(FetchDescriptor()) let memberships = try modelContext.fetch(FetchDescriptor()) @@ -319,12 +322,12 @@ struct PersistenceMigrationTests { let configuration = ModelConfiguration(schema: schema, url: storeURL) let container = try ModelContainer(for: schema, configurations: [configuration]) let context = ModelContext(container) - let query = KeywordQuery( + let query = OpenASOSchemaV1.KeywordQuery( term: keyword, storefront: storefront, platform: platform ) - let crawl = KeywordRankingCrawl( + let crawl = LegacyKeywordRankingCrawl( keyword: keyword, storefront: storefront, platform: platform, @@ -333,7 +336,7 @@ struct PersistenceMigrationTests { resultCount: 1, query: query ) - let crawlItem = KeywordAppRanking( + let crawlItem = LegacyKeywordAppRanking( position: 1, appStoreID: 320_000_499, bundleID: "com.example.v4-result", @@ -570,8 +573,12 @@ struct PersistenceMigrationTests { let screenshots = try modelContext.fetch(FetchDescriptor()) let queries = try modelContext.fetch(FetchDescriptor()) let keywordMetrics = try modelContext.fetch(FetchDescriptor()) - let crawls = try modelContext.fetch(FetchDescriptor()) - let crawlItems = try modelContext.fetch(FetchDescriptor()) + let crawls = try modelContext.fetch(FetchDescriptor()) + let crawlItems = try modelContext.fetch(FetchDescriptor()) + let revisions = try modelContext.fetch(FetchDescriptor()) + let rankingLinks = try modelContext.fetch(FetchDescriptor()) + let legacyCrawls = try modelContext.fetch(FetchDescriptor()) + let legacyCrawlItems = try modelContext.fetch(FetchDescriptor()) let trackedApps = try modelContext.fetch(FetchDescriptor()) let tracks = try modelContext.fetch(FetchDescriptor()) let snapshots = try modelContext.fetch(FetchDescriptor()) @@ -588,12 +595,16 @@ struct PersistenceMigrationTests { #expect(screenshots.count == 1) #expect(queries.count == 1) #expect(keywordMetrics.count == 1) - #expect(crawls.count == 1) - #expect(crawlItems.count == 1) + #expect(crawls.count == 2) + #expect(crawlItems.count == 2) + #expect(revisions.count == 2) + #expect(rankingLinks.count == 1) + #expect(legacyCrawls.isEmpty) + #expect(legacyCrawlItems.isEmpty) #expect(trackedApps.count == 1) #expect(tracks.count == 1) #expect(snapshots.count == 1) - #expect(rankedResults.count == 1) + #expect(rankedResults.isEmpty) #expect(storefronts.count == 1) let folder = try #require(folders.first) @@ -606,12 +617,17 @@ struct PersistenceMigrationTests { let screenshot = try #require(screenshots.first) let query = try #require(queries.first) let metric = try #require(keywordMetrics.first) - let crawl = try #require(crawls.first) - let crawlItem = try #require(crawlItems.first) + let crawl = try #require(crawls.first { $0.source == .appStoreWeb }) + let crawlItem = try #require(crawlItems.first { + $0.observation.observationKey == crawl.observationKey + }) let trackedApp = try #require(trackedApps.first) let track = try #require(tracks.first) let snapshot = try #require(snapshots.first) - let rankedResult = try #require(rankedResults.first) + let rankingLink = try #require(rankingLinks.first) + let linkedResult = try #require(crawlItems.first { + $0.observation.observationKey == rankingLink.crawl.observationKey + }) let storefront = try #require(storefronts.first) #expect(folder.id == ReleasedV1FixtureSentinel.folderID) @@ -692,7 +708,7 @@ struct PersistenceMigrationTests { #expect(query.storefront == ReleasedV1FixtureSentinel.storefront) #expect(query.platform == .iphone) #expect(query.tracks.map(\.identityKey) == [track.identityKey]) - #expect(query.observations.map(\.observationKey) == [crawl.observationKey]) + #expect(query.observations.isEmpty) #expect(track.identityKey == ReleasedV1FixtureSentinel.trackIdentityKey) #expect(track.appStoreID == ReleasedV1FixtureSentinel.appStoreID) @@ -715,16 +731,16 @@ struct PersistenceMigrationTests { #expect(snapshot.searchedAt == ReleasedV1FixtureSentinel.fixtureDate) #expect(snapshot.source == .iTunesFallback) #expect(snapshot.errorMessage == "released-snapshot-warning") - #expect(snapshot.topResults.map(\.appStoreID) == [rankedResult.appStoreID]) + #expect(snapshot.topResults.isEmpty) - #expect(rankedResult.snapshot.snapshotKey == snapshot.snapshotKey) - #expect(rankedResult.snapshotKey == snapshot.snapshotKey) - #expect(rankedResult.position == ReleasedV1FixtureSentinel.competitorPosition) - #expect(rankedResult.appStoreID == ReleasedV1FixtureSentinel.competitorAppStoreID) - #expect(rankedResult.bundleID == ReleasedV1FixtureSentinel.competitorBundleID) - #expect(rankedResult.name == ReleasedV1FixtureSentinel.competitorName) - #expect(rankedResult.subtitle == "Fixture ranked subtitle") - #expect(rankedResult.sellerName == "Fixture Ranked Seller") + #expect(rankingLink.snapshotKey == snapshot.snapshotKey) + #expect(rankingLink.crawl.observationKey == "tracked-recovery::\(snapshot.snapshotKey)") + #expect(linkedResult.position == ReleasedV1FixtureSentinel.competitorPosition) + #expect(linkedResult.appStoreID == ReleasedV1FixtureSentinel.competitorAppStoreID) + #expect(linkedResult.bundleID == ReleasedV1FixtureSentinel.competitorBundleID) + #expect(linkedResult.name == ReleasedV1FixtureSentinel.competitorName) + #expect(linkedResult.subtitle == "Fixture ranked subtitle") + #expect(linkedResult.sellerName == "Fixture Ranked Seller") #expect(stats.identityKey == ReleasedV1FixtureSentinel.trackIdentityKey) #expect(stats.appStoreID == ReleasedV1FixtureSentinel.appStoreID) @@ -829,8 +845,8 @@ struct PersistenceMigrationTests { #expect(crawl.submissionCount == 6) #expect(crawl.winningCount == 5) #expect(crawl.confidenceRaw == "fixture-crawl-confidence") - #expect(crawl.query.queryKey == query.queryKey) - #expect(crawl.items.map(\.itemKey) == [crawlItem.itemKey]) + #expect(crawl.queryKey == query.queryKey) + #expect(crawlItem.observation.observationKey == crawl.observationKey) #expect(crawlItem.position == 2) #expect(crawlItem.appStoreID == ReleasedV1FixtureSentinel.competitorAppStoreID) @@ -859,8 +875,8 @@ struct PersistenceMigrationTests { observedAt: Date ) throws { let query = try #require(modelContext.fetch(FetchDescriptor()).first) - let crawl = try #require(modelContext.fetch(FetchDescriptor()).first) - let crawlItem = try #require(modelContext.fetch(FetchDescriptor()).first) + let crawl = try #require(modelContext.fetch(FetchDescriptor()).first) + let crawlItem = try #require(modelContext.fetch(FetchDescriptor()).first) let metrics = try #require(modelContext.fetch(FetchDescriptor()).first) let attempt = try #require( modelContext.fetch(FetchDescriptor()).first @@ -876,10 +892,10 @@ struct PersistenceMigrationTests { ).sorted { $0.position < $1.position } #expect(query.queryKey == queryKey) - #expect(query.observations.map(\.observationKey) == [crawl.observationKey]) + #expect(query.observations.isEmpty) #expect(crawl.queryKey == queryKey) #expect(crawl.observedAt == observedAt) - #expect(crawl.items.map(\.itemKey) == [crawlItem.itemKey]) + #expect(crawlItem.observation.observationKey == crawl.observationKey) #expect(crawlItem.queryKey == queryKey) #expect(crawlItem.name == "V4 Result") #expect(metrics.queryKey == queryKey) diff --git a/OpenASOTests/RankingRefreshCoordinatorTests.swift b/OpenASOTests/RankingRefreshCoordinatorTests.swift index 2cc7d20..2c5cc7f 100644 --- a/OpenASOTests/RankingRefreshCoordinatorTests.swift +++ b/OpenASOTests/RankingRefreshCoordinatorTests.swift @@ -141,7 +141,7 @@ struct RankingRefreshCoordinatorTests { #expect(ranksByTrack[fixture.secondTrack.identityKey] == 3) #expect(ranksByTrack[fixture.thirdTrack.identityKey] == 2) - let crawls = try modelContext.fetch(FetchDescriptor()) + let crawls = try modelContext.fetch(FetchDescriptor()) #expect(crawls.count == 2) #expect(crawls.allSatisfy { $0.confidenceRaw == "single_source" }) #expect(crawls.first(where: { $0.queryKey == fixture.firstTrack.queryKey })?.submissionCount == 1) @@ -214,7 +214,7 @@ struct RankingRefreshCoordinatorTests { let snapshots = try modelContext.fetch(FetchDescriptor()) #expect(snapshots.map(\.trackIdentityKey) == [fixture.thirdTrack.identityKey]) - let crawls = try modelContext.fetch(FetchDescriptor()) + let crawls = try modelContext.fetch(FetchDescriptor()) #expect(crawls.count == 1) #expect(crawls.first?.queryKey == fixture.thirdTrack.queryKey) #expect(crawls.first?.confidenceRaw == "single_source") @@ -325,7 +325,7 @@ struct RankingRefreshCoordinatorTests { switch result { case .success(let snapshot): #expect(snapshot.rank == 2) - #expect(snapshot.topResults.count == 2) + #expect(snapshot.topResults.isEmpty) #expect(track.rankingAppCount == 2) #expect(track.lastRefreshAt != nil) case .failure(let error): @@ -335,8 +335,8 @@ struct RankingRefreshCoordinatorTests { let snapshots = try modelContext.fetch(FetchDescriptor()) let rankedResults = try modelContext.fetch(FetchDescriptor()) let storeApps = try modelContext.fetch(FetchDescriptor()) - let observations = try modelContext.fetch(FetchDescriptor()) - let observationItems = try modelContext.fetch(FetchDescriptor()) + let observations = try modelContext.fetch(FetchDescriptor()) + let observationItems = try modelContext.fetch(FetchDescriptor()) let latestRatings = try modelContext.fetch(FetchDescriptor()) let ratingSnapshots = try modelContext.fetch(FetchDescriptor()) let storefrontMetadata = try modelContext.fetch(FetchDescriptor()) @@ -344,14 +344,13 @@ struct RankingRefreshCoordinatorTests { let appKeywordStats = try modelContext.fetch(FetchDescriptor()) #expect(snapshots.count == 1) - #expect(rankedResults.count == 2) + #expect(rankedResults.isEmpty) #expect(storeApps.count == 2) #expect(observations.count == 1) #expect(observationItems.count == 2) - #expect(rankedResults.first(where: { $0.appStoreID == 361309726 })?.subtitle == "Documents that stand apart") #expect(observationItems.first(where: { $0.appStoreID == 361309726 })?.subtitle == "Documents that stand apart") - #expect(appKeywordStats.count == 2) - #expect(appKeywordStats.first(where: { $0.appStoreID == 842842640 })?.bestRank == 2) + #expect(appKeywordStats.isEmpty) + #expect(try modelContext.fetchCount(FetchDescriptor()) == 1) let pagesStoreApp = storeApps.first(where: { $0.appStoreID == 361309726 }) #expect(pagesStoreApp?.iconURLString == "https://example.com/pages-100.png") #expect(pagesStoreApp?.supportedLanguageCodes == ["EN", "FR"]) @@ -439,12 +438,11 @@ struct RankingRefreshCoordinatorTests { ) try modelContext.save() - #expect(try modelContext.fetch(FetchDescriptor()).contains { - $0.appStoreID == competitorID - }) - #expect(try modelContext.fetch(FetchDescriptor()).contains { + #expect(try modelContext.fetch(FetchDescriptor()).contains { $0.appStoreID == competitorID }) + #expect(try modelContext.fetchCount(FetchDescriptor()) == 0) + #expect(try modelContext.fetchCount(FetchDescriptor()) == 1) let storedApps = try modelContext.fetch(FetchDescriptor()) let storedScreenshots = try modelContext.fetch(FetchDescriptor()) let storedRatings = try modelContext.fetch(FetchDescriptor()) @@ -461,7 +459,7 @@ struct RankingRefreshCoordinatorTests { } @Test - func largePageStoresOneCanonicalCopyAndOnlyLegacyPreviewRows() throws { + func largePageStoresOneCanonicalCopyAndNoLegacyPreviewRows() throws { let container = try makeInMemoryContainer() let modelContext = ModelContext(container) modelContext.autosaveEnabled = false @@ -514,11 +512,11 @@ struct RankingRefreshCoordinatorTests { ) try modelContext.save() - let canonicalRows = try modelContext.fetch(FetchDescriptor()) + let canonicalRows = try modelContext.fetch(FetchDescriptor()) let legacyRows = try modelContext.fetch(FetchDescriptor()) #expect(canonicalRows.count == SearchRankingCrawl.fullKeywordRankingLimit) - #expect(legacyRows.count == 6) - #expect(Set(legacyRows.map(\.position)) == Set([1, 2, 3, 4, 5, 200])) + #expect(legacyRows.isEmpty) + #expect(try modelContext.fetchCount(FetchDescriptor()) == 1) } @Test @@ -810,7 +808,7 @@ struct RankingRefreshCoordinatorTests { .first(where: { $0.code == "yy" }) ) #expect(durableStorefront.name == "Persisted stats storefront name") - #expect(try verificationContext.fetchCount(FetchDescriptor()) == 1) + #expect(try verificationContext.fetchCount(FetchDescriptor()) == 1) #expect(try verificationContext.fetchCount(FetchDescriptor()) == 1) #expect(try verificationContext.fetchCount(FetchDescriptor()) == 0) } @@ -1470,7 +1468,7 @@ struct RankingRefreshCoordinatorTests { platform: .iphone, in: modelContext ) - let observation = KeywordRankingCrawl( + let observation = RankingCrawlRecord( keyword: keyword, storefront: "us", platform: .iphone, @@ -1479,16 +1477,15 @@ struct RankingRefreshCoordinatorTests { resultCount: 10, query: query ) - let item = KeywordAppRanking( + let item = makeRankingFact( position: position, appStoreID: trackedApp.appStoreID, bundleID: trackedApp.bundleID, name: trackedApp.name, sellerName: trackedApp.sellerName, - observation: observation + observation: observation, + in: modelContext ) - observation.items.append(item) - query.observations.append(observation) modelContext.insert(observation) modelContext.insert(item) } @@ -1602,23 +1599,20 @@ struct RankingRefreshCoordinatorTests { let snapshots = try modelContext.fetch(FetchDescriptor()) let rankedResults = try modelContext.fetch(FetchDescriptor()) - let observations = try modelContext.fetch(FetchDescriptor()) - let observationItems = try modelContext.fetch(FetchDescriptor()) + let observations = try modelContext.fetch(FetchDescriptor()) + let observationItems = try modelContext.fetch(FetchDescriptor()) let appKeywordStats = try modelContext.fetch(FetchDescriptor()) #expect(snapshots.count == 1) #expect(snapshots.first?.persistentModelID == snapshot.persistentModelID) #expect(snapshot.rank == 1) - #expect(snapshot.topResults.count == 1) - #expect(rankedResults.count == 1) - #expect(rankedResults.first?.appStoreID == 842842640) + #expect(snapshot.topResults.isEmpty) + #expect(rankedResults.isEmpty) #expect(observations.count == 1) #expect(observationItems.count == 1) #expect(observationItems.first?.appStoreID == 842842640) - #expect(appKeywordStats.count == 1) - #expect(appKeywordStats.first?.appStoreID == 842842640) - #expect(appKeywordStats.first?.latestRank == 1) - #expect(appKeywordStats.first?.observationCount == 1) + #expect(appKeywordStats.isEmpty) + #expect(try modelContext.fetchCount(FetchDescriptor()) == 1) } @Test @@ -2373,8 +2367,8 @@ private func rankingPersistenceState(in modelContext: ModelContext) throws -> Ra } .sorted { $0.identityKey < $1.identityKey } return RankingPersistenceState( - crawlCount: try modelContext.fetchCount(FetchDescriptor()), - observationItemCount: try modelContext.fetchCount(FetchDescriptor()), + crawlCount: try modelContext.fetchCount(FetchDescriptor()), + observationItemCount: try modelContext.fetchCount(FetchDescriptor()), snapshotCount: try modelContext.fetchCount(FetchDescriptor()), rankedResultCount: try modelContext.fetchCount(FetchDescriptor()), storeAppIDs: try modelContext.fetch(FetchDescriptor()).map(\.appStoreID).sorted(), @@ -3016,8 +3010,10 @@ private func makeInMemoryContainer() throws -> ModelContainer { AppStoreScreenshot.self, KeywordQuery.self, KeywordDailyMetric.self, - KeywordRankingCrawl.self, - KeywordAppRanking.self, + RankingAppRevision.self, + RankingCrawlRecord.self, + RankingFact.self, + TrackedRankingCrawlLink.self, TrackedApp.self, TrackedAppKeyword.self, TrackedKeywordRefreshStatus.self, diff --git a/OpenASOTests/RankingSchemaV6MigrationTests.swift b/OpenASOTests/RankingSchemaV6MigrationTests.swift new file mode 100644 index 0000000..af27413 --- /dev/null +++ b/OpenASOTests/RankingSchemaV6MigrationTests.swift @@ -0,0 +1,230 @@ +import Foundation +import SwiftData +import Testing +@testable import OpenASO + +@MainActor +struct RankingSchemaV6MigrationTests { + @Test + func migrationPaginatesMixedUnicodeKeysWithoutSkippingOrRepeatingRows() throws { + let container = try ModelContainerFactory.makeModelContainer(isStoredInMemoryOnly: true) + let context = ModelContext(container) + let observedAt = Date(timeIntervalSince1970: 1_787_000_000) + let terms = (0..<80).flatMap { index in + [ + String(format: "unexpected free time %03d", index), + String(format: "éclair %03d", index), + String(format: "番茄 %03d", index), + ] + } + + for term in terms { + let query = KeywordQuery(term: term, storefront: "us", platform: .iphone) + let crawl = LegacyKeywordRankingCrawl( + keyword: term, + storefront: "us", + platform: .iphone, + observedAt: observedAt, + source: .appStoreWeb, + resultCount: 1, + query: query + ) + let fact = LegacyKeywordAppRanking( + position: 1, + appStoreID: 100, + bundleID: "com.example.shared", + name: "Shared App", + sellerName: "Example", + observation: crawl + ) + query.observations.append(crawl) + crawl.items.append(fact) + context.insert(query) + context.insert(crawl) + context.insert(fact) + } + try context.save() + + try RankingSchemaV6Migrator.migrateIfNeeded(in: container) + + #expect(try context.fetchCount(FetchDescriptor()) == 0) + #expect(try context.fetchCount(FetchDescriptor()) == 0) + #expect(try context.fetchCount(FetchDescriptor()) == terms.count) + #expect(try context.fetchCount(FetchDescriptor()) == terms.count) + let state = try #require(context.fetch(FetchDescriptor()).first) + #expect(state.phase == .completed) + #expect(state.migratedCrawlCount == terms.count) + #expect(state.migratedFactCount == terms.count) + } + + @Test + func migrationPreservesCanonicalAndTrackedOnlyRankingsThenIsIdempotent() throws { + let container = try ModelContainerFactory.makeModelContainer(isStoredInMemoryOnly: true) + let context = ModelContext(container) + let canonicalDate = Date(timeIntervalSince1970: 1_787_000_000) + let recoveredDate = canonicalDate.addingTimeInterval(86_400) + + let trackedApp = TrackedApp( + appStoreID: 100, + bundleID: "com.example.target", + name: "Target", + sellerName: "Example", + defaultPlatform: .iphone + ) + let query = KeywordQuery(term: "focus timer", storefront: "us", platform: .iphone) + let track = TrackedAppKeyword( + term: query.term, + storefront: query.storefront, + platform: query.platform, + trackedApp: trackedApp, + query: query + ) + let legacyCrawl = LegacyKeywordRankingCrawl( + keyword: query.term, + storefront: query.storefront, + platform: query.platform, + observedAt: canonicalDate, + source: .appStoreWeb, + resultCount: 2, + query: query, + submissionCount: 3, + winningCount: 2, + confidence: "consensus" + ) + let targetFact = LegacyKeywordAppRanking( + position: 1, + appStoreID: 100, + bundleID: "com.example.target", + name: "Target", + subtitle: nil, + sellerName: "Example", + observation: legacyCrawl + ) + let competitorFact = LegacyKeywordAppRanking( + position: 2, + appStoreID: 200, + bundleID: "com.example.competitor", + name: "Competitor", + subtitle: "Stay focused", + sellerName: "Competitor Ltd", + observation: legacyCrawl + ) + legacyCrawl.items.append(contentsOf: [targetFact, competitorFact]) + + let canonicalSnapshot = TrackedKeywordDailyRanking( + rank: 1, + searchedAt: canonicalDate, + source: .appStoreWeb, + resultCount: 2, + keywordTrack: track + ) + let canonicalResults = [targetFact, competitorFact].map { fact in + TrackedKeywordRankedResult( + position: fact.position, + appStoreID: fact.appStoreID, + bundleID: fact.bundleID, + name: fact.name, + subtitle: fact.subtitle, + sellerName: fact.sellerName, + snapshot: canonicalSnapshot + ) + } + canonicalSnapshot.topResults.append(contentsOf: canonicalResults) + + let trackedOnlySnapshot = TrackedKeywordDailyRanking( + rank: nil, + searchedAt: recoveredDate, + source: .iTunesFallback, + resultCount: 1, + keywordTrack: track + ) + let trackedOnlyResult = TrackedKeywordRankedResult( + position: 7, + appStoreID: 300, + bundleID: nil, + name: "Legacy Only", + subtitle: nil, + sellerName: nil, + snapshot: trackedOnlySnapshot + ) + trackedOnlySnapshot.topResults.append(trackedOnlyResult) + + let stats = AppKeywordStats( + appStoreID: 100, + queryKey: query.queryKey, + keyword: query.term, + storefront: query.storefront, + platform: query.platform, + rank: 1, + observedAt: canonicalDate, + popularityScore: 55, + difficultyScore: 42 + ) + + context.insert(trackedApp) + context.insert(query) + context.insert(track) + context.insert(legacyCrawl) + context.insert(targetFact) + context.insert(competitorFact) + context.insert(canonicalSnapshot) + canonicalResults.forEach(context.insert) + context.insert(trackedOnlySnapshot) + context.insert(trackedOnlyResult) + context.insert(stats) + trackedApp.keywordTracks.append(track) + query.tracks.append(track) + query.observations.append(legacyCrawl) + track.snapshots.append(contentsOf: [canonicalSnapshot, trackedOnlySnapshot]) + try context.save() + + try RankingSchemaV6Migrator.migrateIfNeeded(in: container) + + #expect(try context.fetchCount(FetchDescriptor()) == 0) + #expect(try context.fetchCount(FetchDescriptor()) == 0) + #expect(try context.fetchCount(FetchDescriptor()) == 0) + let preservedStats = try #require( + context.fetch(FetchDescriptor()).first + ) + #expect(preservedStats.identityKey == stats.identityKey) + #expect(preservedStats.bestRank == stats.bestRank) + #expect(preservedStats.latestRank == stats.latestRank) + #expect(preservedStats.popularityScore == stats.popularityScore) + #expect(preservedStats.difficultyScore == stats.difficultyScore) + + let normalizedCrawls = try context.fetch(FetchDescriptor()) + let normalizedFacts = try context.fetch(FetchDescriptor()) + let links = try context.fetch(FetchDescriptor()) + #expect(normalizedCrawls.count == 2) + #expect(normalizedFacts.count == 3) + #expect(links.count == 2) + #expect(normalizedFacts.contains { + $0.appStoreID == 200 + && $0.position == 2 + && $0.bundleID == "com.example.competitor" + && $0.name == "Competitor" + && $0.subtitle == "Stay focused" + && $0.sellerName == "Competitor Ltd" + }) + #expect(normalizedFacts.contains { + $0.appStoreID == 300 + && $0.position == 7 + && $0.name == "Legacy Only" + }) + + let state = try #require(context.fetch(FetchDescriptor()).first) + #expect(state.phase == .completed) + #expect(state.legacyCrawlCount == 1) + #expect(state.legacyFactCount == 2) + #expect(state.legacyTrackedFactCount == 3) + #expect(state.migratedCrawlCount == 1) + #expect(state.migratedFactCount == 2) + #expect(state.migratedTrackedLinkCount == 2) + #expect(state.recoveredCrawlCount == 1) + + try RankingSchemaV6Migrator.migrateIfNeeded(in: container) + #expect(try context.fetchCount(FetchDescriptor()) == 2) + #expect(try context.fetchCount(FetchDescriptor()) == 3) + #expect(try context.fetchCount(FetchDescriptor()) == 2) + } +} diff --git a/OpenASOTests/RefreshObservabilityTests.swift b/OpenASOTests/RefreshObservabilityTests.swift index 07018f0..e7ef453 100644 --- a/OpenASOTests/RefreshObservabilityTests.swift +++ b/OpenASOTests/RefreshObservabilityTests.swift @@ -464,7 +464,7 @@ struct RefreshObservabilityTests { Set(snapshots.map(\.trackIdentityKey)).count } let persistedQueryCount = try await fixture.backgroundModelStore.fetch( - FetchDescriptor() + FetchDescriptor() ) { observations in Set(observations.map(\.queryKey)).count } @@ -595,7 +595,7 @@ struct RefreshObservabilityTests { let persisted = try await fixture.backgroundModelStore.read { modelContext in let tracks = try modelContext.fetch(FetchDescriptor()) let snapshots = try modelContext.fetch(FetchDescriptor()) - let crawls = try modelContext.fetch(FetchDescriptor()) + let crawls = try modelContext.fetch(FetchDescriptor()) let statuses = try TrackedKeywordRefreshStatusStore.snapshots( for: tracks.map(\.identityKey), in: modelContext