From cd8dbc3d3f9f5253409ec2bb7da826e574831d09 Mon Sep 17 00:00:00 2001 From: Jason Buckner Date: Fri, 17 Jul 2026 08:28:15 -0700 Subject: [PATCH] Add IAS3 upload and file delete support upload() PUTs a file to s3.us.archive.org with the LOW header, covering auto-make-bucket, queue-derive, size-hint, and x-archive-meta-* headers (uri() encoding for non-ASCII values, double hyphens for underscores in names). deleteFile() removes a file with cascade delete. S3 XML errors surface their Message element. The body is held in memory, so this suits normal-sized files rather than multi-gigabyte ones. Closes #85 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_015u2FT1T8yYQsEztyKg6UGf --- InternetArchiveKit/InternetArchive.swift | 113 ++++++++++++ .../InternetArchiveErrors.swift | 6 + .../InternetArchiveProtocols.swift | 126 +++++++++++++ .../InternetArchiveURLGenerator.swift | 19 ++ .../InternetArchiveKitTests.swift | 4 + InternetArchiveKitTests/UploadTests.swift | 170 ++++++++++++++++++ 6 files changed, 438 insertions(+) create mode 100644 InternetArchiveKitTests/UploadTests.swift diff --git a/InternetArchiveKit/InternetArchive.swift b/InternetArchiveKit/InternetArchive.swift index ed64690..84b0b0e 100644 --- a/InternetArchiveKit/InternetArchive.swift +++ b/InternetArchiveKit/InternetArchive.swift @@ -229,6 +229,119 @@ public final class InternetArchive: InternetArchiveProtocol, @unchecked Sendable } } + /** @inheritdoc */ + public func upload( + itemIdentifier: String, + fileName: String, + data: Data, + contentType: String? = nil, + metadata: [String: String] = [:], + autoMakeBucket: Bool = true, + queueDerive: Bool = true, + sizeHint: Int? = nil + ) async -> Result { + guard credentials != nil else { + return .failure(InternetArchiveError.missingCredentials) + } + guard + let uploadUrl: URL = urlGenerator.generateUploadUrl( + itemIdentifier: itemIdentifier, + fileName: fileName + ) + else { + return .failure(InternetArchiveError.invalidUrl) + } + + var request = authorizedRequest(url: uploadUrl) + request.httpMethod = "PUT" + request.httpBody = data + if let contentType = contentType { + request.setValue(contentType, forHTTPHeaderField: "Content-Type") + } + if autoMakeBucket { + request.setValue("1", forHTTPHeaderField: "x-amz-auto-make-bucket") + } + if !queueDerive { + request.setValue("0", forHTTPHeaderField: "x-archive-queue-derive") + } + if let sizeHint = sizeHint { + request.setValue("\(sizeHint)", forHTTPHeaderField: "x-archive-size-hint") + } + for (key, value) in metadata { + // IAS3 turns double hyphens back into underscores in metadata names + let headerName = "x-archive-meta-" + + key.replacingOccurrences(of: "_", with: "--") + request.setValue( + Self.metadataHeaderValue(value), forHTTPHeaderField: headerName) + } + + return await performS3Request(request) + } + + /** @inheritdoc */ + public func deleteFile( + itemIdentifier: String, + fileName: String, + cascadeDerivatives: Bool = true + ) async -> Result { + guard credentials != nil else { + return .failure(InternetArchiveError.missingCredentials) + } + guard + let uploadUrl: URL = urlGenerator.generateUploadUrl( + itemIdentifier: itemIdentifier, + fileName: fileName + ) + else { + return .failure(InternetArchiveError.invalidUrl) + } + + var request = authorizedRequest(url: uploadUrl) + request.httpMethod = "DELETE" + if cascadeDerivatives { + request.setValue("1", forHTTPHeaderField: "x-archive-cascade-delete") + } + + return await performS3Request(request) + } + + private func performS3Request(_ request: URLRequest) async -> Result { + do { + let (data, response) = try await urlSession.data(for: request) + if let httpResponse = response as? HTTPURLResponse, + !(200..<300).contains(httpResponse.statusCode) { + // IAS3 errors are S3-style XML with a Message element + let body = String(decoding: data, as: UTF8.self) + let message = Self.s3ErrorMessage(from: body) + ?? "IAS3 request failed with HTTP \(httpResponse.statusCode)" + return .failure(InternetArchiveError.apiError(message: message)) + } + return .success(()) + } catch { + return .failure(error) + } + } + + /// Extracts the Message element from an S3-style XML error body + static func s3ErrorMessage(from body: String) -> String? { + guard + let start = body.range(of: ""), + let end = body.range(of: ""), + start.upperBound <= end.lowerBound + else { return nil } + return String(body[start.upperBound.. String { + guard !value.allSatisfy({ $0.isASCII }) else { return value } + var allowed = CharacterSet.alphanumerics + allowed.insert(charactersIn: "-._~") + let encoded = value.addingPercentEncoding(withAllowedCharacters: allowed) ?? value + return "uri(\(encoded))" + } + /** @inheritdoc */ public func itemDetail(identifier: String) async -> Result { guard diff --git a/InternetArchiveKit/InternetArchiveErrors.swift b/InternetArchiveKit/InternetArchiveErrors.swift index 8ebe5af..a37cba4 100644 --- a/InternetArchiveKit/InternetArchiveErrors.swift +++ b/InternetArchiveKit/InternetArchiveErrors.swift @@ -28,6 +28,10 @@ extension InternetArchive { /// `identifier`, if it appears in a scrape sort, to be the last sort field. /// `message` explains what to fix. case invalidSortFields(message: String) + + /// The request needs credentials and this `InternetArchive` instance was + /// created without them. Pass `Credentials` at init. + case missingCredentials } } @@ -40,6 +44,8 @@ extension InternetArchive.InternetArchiveError: LocalizedError { return "Internet Archive API error: \(message)" case .invalidSortFields(let message): return "Invalid sort fields: \(message)" + case .missingCredentials: + return "This request requires credentials" } } } diff --git a/InternetArchiveKit/InternetArchiveProtocols.swift b/InternetArchiveKit/InternetArchiveProtocols.swift index 868c0b9..2dd923f 100644 --- a/InternetArchiveKit/InternetArchiveProtocols.swift +++ b/InternetArchiveKit/InternetArchiveProtocols.swift @@ -242,6 +242,89 @@ public protocol InternetArchiveProtocol { identifier: String, completion: @escaping (InternetArchive.Item?, Error?) -> Void ) + + /** + Upload a file to an Internet Archive item over IAS3 + + Requires credentials, and the account needs write access to the item. The + file body is held in memory, so this suits metadata-sized and audio-sized + files rather than multi-gigabyte ones. + + - parameters: + - itemIdentifier: The item (bucket) identifier + - fileName: The file name to store + - data: The file contents + - contentType: The MIME type to send + - metadata: Item metadata for `x-archive-meta-*` headers, applied when + the bucket is created + - autoMakeBucket: Create the item if it doesn't exist yet + - queueDerive: Queue a derive task after the upload + - sizeHint: The expected final item size in bytes, for large items + */ + func upload( + itemIdentifier: String, + fileName: String, + data: Data, + contentType: String?, + metadata: [String: String], + autoMakeBucket: Bool, + queueDerive: Bool, + sizeHint: Int? + ) async throws + + /** + Upload a file to an Internet Archive item over IAS3 + + - parameters: + - itemIdentifier: The item (bucket) identifier + - fileName: The file name to store + - data: The file contents + - contentType: The MIME type to send + - metadata: Item metadata for `x-archive-meta-*` headers + - autoMakeBucket: Create the item if it doesn't exist yet + - queueDerive: Queue a derive task after the upload + - sizeHint: The expected final item size in bytes + - returns: Result + */ + func upload( + itemIdentifier: String, + fileName: String, + data: Data, + contentType: String?, + metadata: [String: String], + autoMakeBucket: Bool, + queueDerive: Bool, + sizeHint: Int? + ) async -> Result + + /** + Delete a file from an Internet Archive item over IAS3 + + - parameters: + - itemIdentifier: The item (bucket) identifier + - fileName: The file name to delete + - cascadeDerivatives: Also delete the file's derivatives + */ + func deleteFile( + itemIdentifier: String, + fileName: String, + cascadeDerivatives: Bool + ) async throws + + /** + Delete a file from an Internet Archive item over IAS3 + + - parameters: + - itemIdentifier: The item (bucket) identifier + - fileName: The file name to delete + - cascadeDerivatives: Also delete the file's derivatives + - returns: Result + */ + func deleteFile( + itemIdentifier: String, + fileName: String, + cascadeDerivatives: Bool + ) async -> Result } /// A protocol to which the main `InternetArchive.URLGenerator` class conforms @@ -258,6 +341,7 @@ public protocol InternetArchiveURLGeneratorProtocol { sortFields: [InternetArchiveURLQueryItemProtocol], additionalQueryParams: [URLQueryItem] ) -> URL? + func generateUploadUrl(itemIdentifier: String, fileName: String) -> URL? func generateScrapeUrl( query: InternetArchiveURLStringProtocol, fields: [String], @@ -370,4 +454,46 @@ extension InternetArchiveProtocol { throw failure } } + + /** @inheritdoc */ + public func upload( + itemIdentifier: String, + fileName: String, + data: Data, + contentType: String?, + metadata: [String: String], + autoMakeBucket: Bool, + queueDerive: Bool, + sizeHint: Int? + ) async throws { + let result: Result = await upload( + itemIdentifier: itemIdentifier, + fileName: fileName, + data: data, + contentType: contentType, + metadata: metadata, + autoMakeBucket: autoMakeBucket, + queueDerive: queueDerive, + sizeHint: sizeHint + ) + if case .failure(let error) = result { + throw error + } + } + + /** @inheritdoc */ + public func deleteFile( + itemIdentifier: String, + fileName: String, + cascadeDerivatives: Bool + ) async throws { + let result: Result = await deleteFile( + itemIdentifier: itemIdentifier, + fileName: fileName, + cascadeDerivatives: cascadeDerivatives + ) + if case .failure(let error) = result { + throw error + } + } } diff --git a/InternetArchiveKit/InternetArchiveURLGenerator.swift b/InternetArchiveKit/InternetArchiveURLGenerator.swift index edd9c94..630302d 100644 --- a/InternetArchiveKit/InternetArchiveURLGenerator.swift +++ b/InternetArchiveKit/InternetArchiveURLGenerator.swift @@ -68,6 +68,25 @@ extension InternetArchive { return urlComponents.url } + /** + Generate an IAS3 (`s3.us.archive.org`) upload url for a file + + - parameters: + - itemIdentifier: The item (bucket) identifier + - fileName: The file name (key) + + - returns: Optional upload `URL` + */ + public func generateUploadUrl(itemIdentifier: String, fileName: String) + -> URL? + { + var urlComponents: URLComponents = URLComponents() + urlComponents.scheme = scheme + urlComponents.host = "s3.us.archive.org" + urlComponents.path = "/\(itemIdentifier)/\(fileName)" + return urlComponents.url + } + public func generateSearchUrl( query: InternetArchiveURLStringProtocol, page: Int, diff --git a/InternetArchiveKitTests/InternetArchiveKitTests.swift b/InternetArchiveKitTests/InternetArchiveKitTests.swift index f089b81..1b1761e 100644 --- a/InternetArchiveKitTests/InternetArchiveKitTests.swift +++ b/InternetArchiveKitTests/InternetArchiveKitTests.swift @@ -36,6 +36,10 @@ class InternetArchiveKitTests: XCTestCase { return nil } + func generateUploadUrl(itemIdentifier: String, fileName: String) -> URL? { + return nil + } + func generateSearchUrl(query: InternetArchiveURLStringProtocol, page: Int, rows: Int, fields: [String], sortFields: [InternetArchiveURLQueryItemProtocol], additionalQueryParams: [URLQueryItem]) -> URL? { return nil } diff --git a/InternetArchiveKitTests/UploadTests.swift b/InternetArchiveKitTests/UploadTests.swift new file mode 100644 index 0000000..dae19bb --- /dev/null +++ b/InternetArchiveKitTests/UploadTests.swift @@ -0,0 +1,170 @@ +// +// UploadTests.swift +// InternetArchiveKitTests +// +// Created by Jason Buckner on 7/17/26. +// Copyright © 2026 Jason Buckner. All rights reserved. +// + +import XCTest +import URLSessionMock +@testable import InternetArchiveKit + +class UploadTests: XCTestCase { + + private let credentials = InternetArchive.Credentials( + accessKey: "accessfoo", secretKey: "secretbar") + + private func mockedArchive( + itemIdentifier: String, fileName: String, status: Int, body: String + ) -> (archive: InternetArchive, endpoint: BasicEndpointMock)? { + let urlGenerator = InternetArchive.URLGenerator() + guard + let url = urlGenerator.generateUploadUrl( + itemIdentifier: itemIdentifier, fileName: fileName) + else { return nil } + let endpoint = BasicEndpointMock( + status: status, url: url, body: Data(body.utf8), headers: nil, error: nil) + URLSession.mockEndpoints = [url: endpoint] + let archive = InternetArchive( + urlGenerator: urlGenerator, + urlSession: URLSession.mock, + credentials: credentials + ) + return (archive, endpoint) + } + + func testGenerateUploadUrl() { + let generator = InternetArchive.URLGenerator() + XCTAssertEqual( + generator.generateUploadUrl( + itemIdentifier: "foo", fileName: "track1.mp3")?.absoluteString, + "https://s3.us.archive.org/foo/track1.mp3" + ) + } + + func testUploadRequiresCredentials() async { + let archive = InternetArchive( + urlGenerator: InternetArchive.URLGenerator(), + urlSession: URLSession.mock + ) + let result = await archive.upload( + itemIdentifier: "foo", fileName: "track1.mp3", data: Data("abc".utf8)) + switch result { + case .success: + XCTFail("expected a failure") + case .failure(let error): + XCTAssertEqual( + error as? InternetArchive.InternetArchiveError, + InternetArchive.InternetArchiveError.missingCredentials + ) + } + } + + func testUploadSendsExpectedHeaders() async { + guard + let (archive, endpoint) = mockedArchive( + itemIdentifier: "foo", fileName: "track1.mp3", status: 200, body: "") + else { + XCTFail("error generating upload url") + return + } + + let result = await archive.upload( + itemIdentifier: "foo", + fileName: "track1.mp3", + data: Data("abc".utf8), + contentType: "audio/mpeg", + metadata: ["title": "Show ☃", "external_identifier": "xyz"], + queueDerive: false, + sizeHint: 12345 + ) + if case .failure(let error) = result { + XCTFail("error, \(error.localizedDescription)") + } + + let request = endpoint.requests.first + XCTAssertEqual(request?.httpMethod, "PUT") + XCTAssertEqual( + request?.value(forHTTPHeaderField: "Authorization"), + "LOW accessfoo:secretbar") + XCTAssertEqual( + request?.value(forHTTPHeaderField: "Content-Type"), "audio/mpeg") + XCTAssertEqual( + request?.value(forHTTPHeaderField: "x-amz-auto-make-bucket"), "1") + XCTAssertEqual( + request?.value(forHTTPHeaderField: "x-archive-queue-derive"), "0") + XCTAssertEqual( + request?.value(forHTTPHeaderField: "x-archive-size-hint"), "12345") + // non-ASCII metadata travels uri()-encoded + XCTAssertEqual( + request?.value(forHTTPHeaderField: "x-archive-meta-title"), + "uri(Show%20%E2%98%83)") + // underscores in metadata names travel as double hyphens + XCTAssertEqual( + request?.value(forHTTPHeaderField: "x-archive-meta-external--identifier"), + "xyz") + } + + func testUploadErrorSurfacesS3Message() async { + let xml = """ + + AccessDeniedAccess Denied + """ + guard + let (archive, _) = mockedArchive( + itemIdentifier: "foo", fileName: "track1.mp3", status: 403, body: xml) + else { + XCTFail("error generating upload url") + return + } + + let result = await archive.upload( + itemIdentifier: "foo", fileName: "track1.mp3", data: Data("abc".utf8)) + + switch result { + case .success: + XCTFail("expected a failure") + case .failure(let error): + XCTAssertEqual( + error as? InternetArchive.InternetArchiveError, + InternetArchive.InternetArchiveError.apiError(message: "Access Denied") + ) + } + } + + func testDeleteFileSendsCascadeHeader() async { + guard + let (archive, endpoint) = mockedArchive( + itemIdentifier: "foo", fileName: "track1.mp3", status: 204, body: "") + else { + XCTFail("error generating upload url") + return + } + + let result = await archive.deleteFile( + itemIdentifier: "foo", fileName: "track1.mp3") + if case .failure(let error) = result { + XCTFail("error, \(error.localizedDescription)") + } + + let request = endpoint.requests.first + XCTAssertEqual(request?.httpMethod, "DELETE") + XCTAssertEqual( + request?.value(forHTTPHeaderField: "x-archive-cascade-delete"), "1") + } + + func testS3ErrorMessageExtraction() { + XCTAssertEqual( + InternetArchive.s3ErrorMessage( + from: "Bucket not found"), + "Bucket not found") + XCTAssertNil(InternetArchive.s3ErrorMessage(from: "not xml")) + } + + func testMetadataHeaderValueEncoding() { + XCTAssertEqual(InternetArchive.metadataHeaderValue("plain ascii"), "plain ascii") + XCTAssertEqual( + InternetArchive.metadataHeaderValue("Show ☃"), "uri(Show%20%E2%98%83)") + } +}