diff --git a/InternetArchiveKit/InternetArchive.swift b/InternetArchiveKit/InternetArchive.swift index ed64690..dd1d21e 100644 --- a/InternetArchiveKit/InternetArchive.swift +++ b/InternetArchiveKit/InternetArchive.swift @@ -310,6 +310,58 @@ public final class InternetArchive: InternetArchiveProtocol, @unchecked Sendable } } + /** @inheritdoc */ + public func changes( + start: ChangesStart? = nil + ) async -> Result { + guard let credentials = credentials else { + return .failure(InternetArchiveError.missingCredentials) + } + guard let changesUrl: URL = urlGenerator.generateChangesUrl() else { + return .failure(InternetArchiveError.invalidUrl) + } + + var fields: [(String, String)] = [ + ("access", credentials.accessKey), + ("secret", credentials.secretKey), + ] + switch start { + case .coldStart: + fields.append(("start_date", "0")) + case .startDate(let date): + fields.append(("start_date", Self.changesDateFormatter.string(from: date))) + case .token(let token): + fields.append(("token", token)) + case nil: + break + } + + // built by hand rather than through makeRequest: the body carries the + // secret key, so nothing in this path is logged + var request = URLRequest(url: changesUrl) + request.httpMethod = "POST" + request.setValue( + "application/x-www-form-urlencoded", forHTTPHeaderField: "Content-Type") + request.httpBody = Self.formEncode(fields) + + do { + let (data, _) = try await urlSession.data(for: request) + let results: ChangesResponse = try decodeResponse(data) + return .success(results) + } catch { + return .failure(error) + } + } + + /// Formats `ChangesStart.startDate` dates as the API's `YYYYMMDD` + private static let changesDateFormatter: DateFormatter = { + let dateFormatter: DateFormatter = DateFormatter() + dateFormatter.locale = Locale(identifier: "en_US_POSIX") + dateFormatter.dateFormat = "yyyyMMdd" + dateFormatter.timeZone = TimeZone(secondsFromGMT: 0) + return dateFormatter + }() + /// The request for `url` with the configured credentials attached, if any private func authorizedRequest(url: URL) -> URLRequest { var request = URLRequest(url: url) diff --git a/InternetArchiveKit/InternetArchiveErrors.swift b/InternetArchiveKit/InternetArchiveErrors.swift index 8ebe5af..a37cba4 100644 --- a/InternetArchiveKit/InternetArchiveErrors.swift +++ b/InternetArchiveKit/InternetArchiveErrors.swift @@ -28,6 +28,10 @@ extension InternetArchive { /// `identifier`, if it appears in a scrape sort, to be the last sort field. /// `message` explains what to fix. case invalidSortFields(message: String) + + /// The request needs credentials and this `InternetArchive` instance was + /// created without them. Pass `Credentials` at init. + case missingCredentials } } @@ -40,6 +44,8 @@ extension InternetArchive.InternetArchiveError: LocalizedError { return "Internet Archive API error: \(message)" case .invalidSortFields(let message): return "Invalid sort fields: \(message)" + case .missingCredentials: + return "This request requires credentials" } } } diff --git a/InternetArchiveKit/InternetArchiveProtocols.swift b/InternetArchiveKit/InternetArchiveProtocols.swift index 868c0b9..036f3d8 100644 --- a/InternetArchiveKit/InternetArchiveProtocols.swift +++ b/InternetArchiveKit/InternetArchiveProtocols.swift @@ -208,6 +208,32 @@ public protocol InternetArchiveProtocol { password: String ) async -> Result + /** + Fetch items changed since a point in the Changes API's feed + + Requires credentials. Start with `.coldStart` to enumerate everything, + `.startDate` for changes since a date, or nil for the current head; then + pass each response's `nextToken` back as `.token` to keep reading. + + - parameters: + - start: Where to read from, or nil for the current head + - returns: InternetArchive.ChangesResponse + */ + func changes( + start: InternetArchive.ChangesStart? + ) async throws -> InternetArchive.ChangesResponse + + /** + Fetch items changed since a point in the Changes API's feed + + - parameters: + - start: Where to read from, or nil for the current head + - returns: Result + */ + func changes( + start: InternetArchive.ChangesStart? + ) async -> Result + /** Fetch a single item from the Internet Archive @@ -249,6 +275,7 @@ public protocol InternetArchiveURLGeneratorProtocol { func generateItemImageUrl(itemIdentifier: String) -> URL? func generateMetadataUrl(identifier: String) -> URL? func generateXauthnUrl(operation: String) -> URL? + func generateChangesUrl() -> URL? func generateDownloadUrl(itemIdentifier: String, fileName: String) -> URL? func generateSearchUrl( query: InternetArchiveURLStringProtocol, @@ -356,6 +383,21 @@ extension InternetArchiveProtocol { } } + /** @inheritdoc */ + public func changes( + start: InternetArchive.ChangesStart? + ) async throws -> InternetArchive.ChangesResponse { + let result: Result = await changes( + start: start + ) + switch result { + case .success(let success): + return success + case .failure(let error): + throw error + } + } + /** @inheritdoc */ public func itemDetail(identifier: String) async throws -> InternetArchive.Item diff --git a/InternetArchiveKit/InternetArchiveURLGenerator.swift b/InternetArchiveKit/InternetArchiveURLGenerator.swift index edd9c94..79a31ca 100644 --- a/InternetArchiveKit/InternetArchiveURLGenerator.swift +++ b/InternetArchiveKit/InternetArchiveURLGenerator.swift @@ -184,6 +184,22 @@ extension InternetArchive { return urlComponents.url } + /** + Generate a Changes API (`/changes/v1`) url + + The Changes API lives on the `be-api.us.archive.org` host. Credentials + and paging parameters travel in the POST body, so the url is static. + + - returns: Optional changes `URL` + */ + public func generateChangesUrl() -> URL? { + var urlComponents: URLComponents = URLComponents() + urlComponents.scheme = scheme + urlComponents.host = "be-api.us.archive.org" + urlComponents.path = "/changes/v1" + return urlComponents.url + } + private func getBaseUrlComponents() -> URLComponents { var urlComponents: URLComponents = URLComponents() urlComponents.scheme = scheme diff --git a/InternetArchiveKit/Models/ChangesResponse.swift b/InternetArchiveKit/Models/ChangesResponse.swift new file mode 100644 index 0000000..c1464b9 --- /dev/null +++ b/InternetArchiveKit/Models/ChangesResponse.swift @@ -0,0 +1,66 @@ +// +// ChangesResponse.swift +// InternetArchiveKit +// +// Created by Jason Buckner on 7/17/26. +// Copyright © 2026 Jason Buckner. All rights reserved. +// + +import Foundation + +extension InternetArchive { + /** + Where a `changes()` request starts reading the change feed. + + Pass nil to `changes(start:)` to read from the current head. Use + `.coldStart` to enumerate every item, `.startDate` for changes since a + date, and `.token` to continue from a previous response's `nextToken`. + */ + public enum ChangesStart: Sendable { + /// Enumerate all items from the beginning, then stream changes + case coldStart + /// Changes since the given date (sent as `YYYYMMDD`, UTC) + case startDate(Date) + /// Continue from a previous response's `nextToken` + case token(String) + } + + /** + One batch from the Changes API. + + Pass `nextToken` back to `changes(start: .token(...))` to fetch the next + batch. `doSleepBeforeReturning` is the API's advice to pause before the + next request. + */ + public struct ChangesResponse: Decodable, Sendable { + /// One changed item + public struct Change: Decodable, Sendable { + public let identifier: String + + public init(identifier: String) { + self.identifier = identifier + } + } + + /// The items that changed in this batch + public let changes: [Change] + /// The token for the next batch + public let nextToken: String? + /// Roughly how far this batch is from the head of the feed + public let estimatedDistanceFromHead: Int? + /// Whether the API advises pausing before the next request + public let doSleepBeforeReturning: Bool? + + public init( + changes: [Change], + nextToken: String? = nil, + estimatedDistanceFromHead: Int? = nil, + doSleepBeforeReturning: Bool? = nil + ) { + self.changes = changes + self.nextToken = nextToken + self.estimatedDistanceFromHead = estimatedDistanceFromHead + self.doSleepBeforeReturning = doSleepBeforeReturning + } + } +} diff --git a/InternetArchiveKitTests/ChangesTests.swift b/InternetArchiveKitTests/ChangesTests.swift new file mode 100644 index 0000000..2e4778d --- /dev/null +++ b/InternetArchiveKitTests/ChangesTests.swift @@ -0,0 +1,83 @@ +// +// ChangesTests.swift +// InternetArchiveKitTests +// +// Created by Jason Buckner on 7/17/26. +// Copyright © 2026 Jason Buckner. All rights reserved. +// + +import XCTest +import URLSessionMock +@testable import InternetArchiveKit + +class ChangesTests: XCTestCase { + + func testGenerateChangesUrl() { + let generator = InternetArchive.URLGenerator() + XCTAssertEqual( + generator.generateChangesUrl()?.absoluteString, + "https://be-api.us.archive.org/changes/v1" + ) + } + + func testChangesRequiresCredentials() async { + let archive = InternetArchive( + urlGenerator: InternetArchive.URLGenerator(), + urlSession: URLSession.mock + ) + let result = await archive.changes() + switch result { + case .success: + XCTFail("expected a failure") + case .failure(let error): + XCTAssertEqual( + error as? InternetArchive.InternetArchiveError, + InternetArchive.InternetArchiveError.missingCredentials + ) + } + } + + func testChangesDecodesBatch() async { + let json = """ + { + "changes": [{"identifier": "foo"}, {"identifier": "bar_baz"}], + "next_token": "tok123", + "estimated_distance_from_head": 5, + "do_sleep_before_returning": false + } + """ + let urlGenerator = InternetArchive.URLGenerator() + guard let url = urlGenerator.generateChangesUrl() else { + XCTFail("error generating changes url") + return + } + let endpoint = BasicEndpointMock( + status: 200, url: url, body: Data(json.utf8), headers: nil, error: nil) + URLSession.mockEndpoints = [url: endpoint] + + let archive = InternetArchive( + urlGenerator: urlGenerator, + urlSession: URLSession.mock, + credentials: InternetArchive.Credentials( + accessKey: "accessfoo", secretKey: "secretbar") + ) + let result = await archive.changes(start: .coldStart) + + switch result { + case .success(let response): + XCTAssertEqual(response.changes.map { $0.identifier }, ["foo", "bar_baz"]) + XCTAssertEqual(response.nextToken, "tok123") + XCTAssertEqual(response.estimatedDistanceFromHead, 5) + XCTAssertEqual(response.doSleepBeforeReturning, false) + case .failure(let error): + XCTFail("error, \(error.localizedDescription)") + } + + let request = endpoint.requests.first + XCTAssertEqual(request?.httpMethod, "POST") + XCTAssertEqual( + request?.value(forHTTPHeaderField: "Content-Type"), + "application/x-www-form-urlencoded" + ) + } +} diff --git a/InternetArchiveKitTests/InternetArchiveKitTests.swift b/InternetArchiveKitTests/InternetArchiveKitTests.swift index f089b81..9a4d7d0 100644 --- a/InternetArchiveKitTests/InternetArchiveKitTests.swift +++ b/InternetArchiveKitTests/InternetArchiveKitTests.swift @@ -32,6 +32,10 @@ class InternetArchiveKitTests: XCTestCase { return nil } + func generateChangesUrl() -> URL? { + return nil + } + func generateDownloadUrl(itemIdentifier: String, fileName: String) -> URL? { return nil }