diff --git a/app/Console/Commands/ApiClientsCreate.php b/app/Console/Commands/ApiClientsCreate.php index 3bd6252882..319ac4e38c 100644 --- a/app/Console/Commands/ApiClientsCreate.php +++ b/app/Console/Commands/ApiClientsCreate.php @@ -16,7 +16,7 @@ class ApiClientsCreate extends Command */ protected $signature = 'api-clients:create {--name= : Display name for the integration client} - {--scopes=events:read : Comma-separated scopes} + {--scopes=events:read : Comma-separated scopes (events:read, repairs:read)} {--origins= : Comma-separated allowed origins} {--networks= : Comma-separated allowed network IDs} {--rate=120 : Requests per minute} diff --git a/app/Helpers/Iso3166.php b/app/Helpers/Iso3166.php new file mode 100644 index 0000000000..801b7ba9b4 --- /dev/null +++ b/app/Helpers/Iso3166.php @@ -0,0 +1,280 @@ + 'AND', + 'AE' => 'ARE', + 'AF' => 'AFG', + 'AG' => 'ATG', + 'AI' => 'AIA', + 'AL' => 'ALB', + 'AM' => 'ARM', + 'AO' => 'AGO', + 'AQ' => 'ATA', + 'AR' => 'ARG', + 'AS' => 'ASM', + 'AT' => 'AUT', + 'AU' => 'AUS', + 'AW' => 'ABW', + 'AX' => 'ALA', + 'AZ' => 'AZE', + 'BA' => 'BIH', + 'BB' => 'BRB', + 'BD' => 'BGD', + 'BE' => 'BEL', + 'BF' => 'BFA', + 'BG' => 'BGR', + 'BH' => 'BHR', + 'BI' => 'BDI', + 'BJ' => 'BEN', + 'BL' => 'BLM', + 'BM' => 'BMU', + 'BN' => 'BRN', + 'BO' => 'BOL', + 'BQ' => 'BES', + 'BR' => 'BRA', + 'BS' => 'BHS', + 'BT' => 'BTN', + 'BV' => 'BVT', + 'BW' => 'BWA', + 'BY' => 'BLR', + 'BZ' => 'BLZ', + 'CA' => 'CAN', + 'CC' => 'CCK', + 'CD' => 'COD', + 'CF' => 'CAF', + 'CG' => 'COG', + 'CH' => 'CHE', + 'CI' => 'CIV', + 'CK' => 'COK', + 'CL' => 'CHL', + 'CM' => 'CMR', + 'CN' => 'CHN', + 'CO' => 'COL', + 'CR' => 'CRI', + 'CU' => 'CUB', + 'CV' => 'CPV', + 'CW' => 'CUW', + 'CX' => 'CXR', + 'CY' => 'CYP', + 'CZ' => 'CZE', + 'DE' => 'DEU', + 'DJ' => 'DJI', + 'DK' => 'DNK', + 'DM' => 'DMA', + 'DO' => 'DOM', + 'DZ' => 'DZA', + 'EC' => 'ECU', + 'EE' => 'EST', + 'EG' => 'EGY', + 'EH' => 'ESH', + 'ER' => 'ERI', + 'ES' => 'ESP', + 'ET' => 'ETH', + 'FI' => 'FIN', + 'FJ' => 'FJI', + 'FK' => 'FLK', + 'FM' => 'FSM', + 'FO' => 'FRO', + 'FR' => 'FRA', + 'GA' => 'GAB', + 'GB' => 'GBR', + 'GD' => 'GRD', + 'GE' => 'GEO', + 'GF' => 'GUF', + 'GG' => 'GGY', + 'GH' => 'GHA', + 'GI' => 'GIB', + 'GL' => 'GRL', + 'GM' => 'GMB', + 'GN' => 'GIN', + 'GP' => 'GLP', + 'GQ' => 'GNQ', + 'GR' => 'GRC', + 'GS' => 'SGS', + 'GT' => 'GTM', + 'GU' => 'GUM', + 'GW' => 'GNB', + 'GY' => 'GUY', + 'HK' => 'HKG', + 'HM' => 'HMD', + 'HN' => 'HND', + 'HR' => 'HRV', + 'HT' => 'HTI', + 'HU' => 'HUN', + 'ID' => 'IDN', + 'IE' => 'IRL', + 'IL' => 'ISR', + 'IM' => 'IMN', + 'IN' => 'IND', + 'IO' => 'IOT', + 'IQ' => 'IRQ', + 'IR' => 'IRN', + 'IS' => 'ISL', + 'IT' => 'ITA', + 'JE' => 'JEY', + 'JM' => 'JAM', + 'JO' => 'JOR', + 'JP' => 'JPN', + 'KE' => 'KEN', + 'KG' => 'KGZ', + 'KH' => 'KHM', + 'KI' => 'KIR', + 'KM' => 'COM', + 'KN' => 'KNA', + 'KP' => 'PRK', + 'KR' => 'KOR', + 'KW' => 'KWT', + 'KY' => 'CYM', + 'KZ' => 'KAZ', + 'LA' => 'LAO', + 'LB' => 'LBN', + 'LC' => 'LCA', + 'LI' => 'LIE', + 'LK' => 'LKA', + 'LR' => 'LBR', + 'LS' => 'LSO', + 'LT' => 'LTU', + 'LU' => 'LUX', + 'LV' => 'LVA', + 'LY' => 'LBY', + 'MA' => 'MAR', + 'MC' => 'MCO', + 'MD' => 'MDA', + 'ME' => 'MNE', + 'MF' => 'MAF', + 'MG' => 'MDG', + 'MH' => 'MHL', + 'MK' => 'MKD', + 'ML' => 'MLI', + 'MM' => 'MMR', + 'MN' => 'MNG', + 'MO' => 'MAC', + 'MP' => 'MNP', + 'MQ' => 'MTQ', + 'MR' => 'MRT', + 'MS' => 'MSR', + 'MT' => 'MLT', + 'MU' => 'MUS', + 'MV' => 'MDV', + 'MW' => 'MWI', + 'MX' => 'MEX', + 'MY' => 'MYS', + 'MZ' => 'MOZ', + 'NA' => 'NAM', + 'NC' => 'NCL', + 'NE' => 'NER', + 'NF' => 'NFK', + 'NG' => 'NGA', + 'NI' => 'NIC', + 'NL' => 'NLD', + 'NO' => 'NOR', + 'NP' => 'NPL', + 'NR' => 'NRU', + 'NU' => 'NIU', + 'NZ' => 'NZL', + 'OM' => 'OMN', + 'PA' => 'PAN', + 'PE' => 'PER', + 'PF' => 'PYF', + 'PG' => 'PNG', + 'PH' => 'PHL', + 'PK' => 'PAK', + 'PL' => 'POL', + 'PM' => 'SPM', + 'PN' => 'PCN', + 'PR' => 'PRI', + 'PS' => 'PSE', + 'PT' => 'PRT', + 'PW' => 'PLW', + 'PY' => 'PRY', + 'QA' => 'QAT', + 'RE' => 'REU', + 'RO' => 'ROU', + 'RS' => 'SRB', + 'RU' => 'RUS', + 'RW' => 'RWA', + 'SA' => 'SAU', + 'SB' => 'SLB', + 'SC' => 'SYC', + 'SD' => 'SDN', + 'SE' => 'SWE', + 'SG' => 'SGP', + 'SH' => 'SHN', + 'SI' => 'SVN', + 'SJ' => 'SJM', + 'SK' => 'SVK', + 'SL' => 'SLE', + 'SM' => 'SMR', + 'SN' => 'SEN', + 'SO' => 'SOM', + 'SR' => 'SUR', + 'SS' => 'SSD', + 'ST' => 'STP', + 'SV' => 'SLV', + 'SX' => 'SXM', + 'SY' => 'SYR', + 'SZ' => 'SWZ', + 'TC' => 'TCA', + 'TD' => 'TCD', + 'TF' => 'ATF', + 'TG' => 'TGO', + 'TH' => 'THA', + 'TJ' => 'TJK', + 'TK' => 'TKL', + 'TL' => 'TLS', + 'TM' => 'TKM', + 'TN' => 'TUN', + 'TO' => 'TON', + 'TR' => 'TUR', + 'TT' => 'TTO', + 'TV' => 'TUV', + 'TW' => 'TWN', + 'TZ' => 'TZA', + 'UA' => 'UKR', + 'UG' => 'UGA', + 'UM' => 'UMI', + 'US' => 'USA', + 'UY' => 'URY', + 'UZ' => 'UZB', + 'VA' => 'VAT', + 'VC' => 'VCT', + 'VE' => 'VEN', + 'VG' => 'VGB', + 'VI' => 'VIR', + 'VN' => 'VNM', + 'VU' => 'VUT', + 'WF' => 'WLF', + 'WS' => 'WSM', + 'YE' => 'YEM', + 'YT' => 'MYT', + 'ZA' => 'ZAF', + 'ZM' => 'ZMB', + 'ZW' => 'ZWE', + ]; + + public static function alpha3(?string $alpha2): ?string + { + if ($alpha2 === null) { + return null; + } + + $key = strtoupper(trim($alpha2)); + + return self::ALPHA2_TO_ALPHA3[$key] ?? null; + } +} diff --git a/app/Http/Controllers/API/PublicRepairController.php b/app/Http/Controllers/API/PublicRepairController.php new file mode 100644 index 0000000000..330348e7dd --- /dev/null +++ b/app/Http/Controllers/API/PublicRepairController.php @@ -0,0 +1,260 @@ +guardExportConfig()) { + return $guard; + } + + $this->normalisePoweredInput($request); + + $validated = $request->validate([ + 'format' => ['nullable', 'string', 'in:json,csv'], + 'updated_since' => ['nullable', 'date'], + 'event_start' => ['nullable', 'date'], + 'event_end' => ['nullable', 'date'], + 'powered' => ['nullable', 'boolean'], + 'page' => ['nullable', 'integer', 'min:1'], + // Cast with a literal fallback: a null config value would build the rule "max:" and 500 every request. + 'per_page' => ['nullable', 'integer', 'min:1', 'max:' . (int) config('ords.pagination.max_per_page', 1000)], + ]); + + $query = $this->buildBaseRepairQuery(); + $this->applyClientRestrictions($query, $request); + $this->applyFilters($query, $validated); + + $maxUpdatedAt = (clone $query)->max('devices.updated_at'); + + $perPage = (int) ($validated['per_page'] ?? config('ords.pagination.default_per_page')); + $paginator = $query->paginate($perPage); + + $this->mapper->scrubber()->reset(); + $records = $paginator->getCollection() + ->map(fn (Device $device) => $this->mapper->map($device)) + ->values(); + + $this->logRedactions($request, $paginator); + + if (($validated['format'] ?? 'json') === 'csv') { + return $this->csvResponse($records->all(), $paginator, $maxUpdatedAt); + } + + return $this->jsonResponse($records->all(), $paginator, $maxUpdatedAt); + } + + /** Laravel's `boolean` rule rejects "true"/"false"; normalise those before validation. */ + private function normalisePoweredInput(Request $request): void + { + if (!$request->has('powered')) { + return; + } + + $raw = $request->input('powered'); + + // filter_var maps "" and null to false rather than firing + // FILTER_NULL_ON_FAILURE, so an empty `?powered=` would narrow the + // export to unpowered items. Null instead, which applyFilters skips. + if ($raw === null || $raw === '') { + $request->merge(['powered' => null]); + + return; + } + + $normalised = filter_var($raw, FILTER_VALIDATE_BOOL, FILTER_NULL_ON_FAILURE); + + if ($normalised !== null) { + $request->merge(['powered' => $normalised]); + } + } + + /** + * `id_prefix` and `data_provider` are instance-specific with no safe + * default. Both are checked for emptiness, not just presence, because an + * env var set to "" yields an empty string rather than falling back to + * any default. + */ + private function guardExportConfig(): ?JsonResponse + { + $prefix = trim((string) config('ords.id_prefix')); + + if ($prefix === '' || $prefix === OrdsRecordMapper::UNASSIGNED_ID_PREFIX) { + return response()->json([ + 'message' => 'ORDS export is not configured: no partner id namespace has been assigned.', + ], 503); + } + + if (trim((string) config('ords.data_provider')) === '') { + return response()->json([ + 'message' => 'ORDS export is not configured: no data provider name has been set.', + ], 503); + } + + return null; + } + + private function jsonResponse(array $records, LengthAwarePaginator $paginator, $maxUpdatedAt): JsonResponse + { + return response()->json([ + 'data' => $records, + 'meta' => [ + 'page' => $paginator->currentPage(), + 'per_page' => $paginator->perPage(), + 'total' => $paginator->total(), + 'last_page' => $paginator->lastPage(), + 'standard' => OrdsRecordMapper::STANDARD, + 'columns' => OrdsRecordMapper::COLUMNS, + ], + 'sync' => [ + 'generated_at' => Carbon::now()->toIso8601String(), + 'max_updated_at' => $maxUpdatedAt ? Carbon::parse($maxUpdatedAt)->toIso8601String() : null, + ], + ]); + } + + /** No envelope for pagination/sync metadata in CSV, so it travels in headers instead. */ + private function csvResponse(array $records, LengthAwarePaginator $paginator, $maxUpdatedAt): StreamedResponse + { + return response()->streamDownload(function () use ($records) { + $csv = Writer::createFromStream(fopen('php://output', 'w')); + // `problem` is free text; a cell opening with = + - @ executes as a formula on open. + $csv->addFormatter([new EscapeFormula(), 'escapeRecord']); + $csv->insertOne(OrdsRecordMapper::COLUMNS); + + foreach ($records as $record) { + // ORDS uses an empty string for every missing value. + $csv->insertOne(array_map( + fn ($value) => $value === null ? '' : $value, + $record + )); + } + }, 'ords-repairs.csv', [ + 'Content-Type' => 'text/csv; charset=UTF-8', + 'X-Total-Count' => (string) $paginator->total(), + 'X-Page' => (string) $paginator->currentPage(), + 'X-Per-Page' => (string) $paginator->perPage(), + 'X-Last-Page' => (string) $paginator->lastPage(), + 'X-Max-Updated-At' => $maxUpdatedAt ? Carbon::parse($maxUpdatedAt)->toIso8601String() : '', + ]); + } + + private function buildBaseRepairQuery(): Builder + { + return Device::query() + ->join('events', 'events.idevents', '=', 'devices.event') + ->join('groups', 'groups.idgroups', '=', 'events.group') + ->join('categories', 'categories.idcategories', '=', 'devices.category') + ->whereNull('events.deleted_at') + ->whereNull('groups.deleted_at') + ->where('events.approved', true) + ->where('groups.approved', true) + // Ordered: the mapper emits only the first barrier, and an unordered + // relation would let a device publish a different one between exports. + ->with(['barriers' => fn ($q) => $q->orderBy('barriers.id')]) + ->select( + 'devices.*', + 'categories.name as ords_category_name', + 'categories.powered as ords_category_powered', + 'groups.name as ords_group_name', + 'groups.country_code as ords_country_code', + 'events.event_start_utc as ords_event_start_utc', + 'events.timezone as ords_event_timezone', + ) + // Stable ordering so pagination can't skip or repeat rows between pages. + ->orderBy('devices.iddevices', 'asc'); + } + + /** + * Subquery, not a join: a group in several allowed networks would + * otherwise multiply its devices across the result. + */ + private function applyClientRestrictions(Builder $query, Request $request): void + { + $client = $request->attributes->get('apiClient'); + $allowedNetworkIds = $client?->allowed_network_ids ?: []; + + if (!empty($allowedNetworkIds)) { + $query->whereIn('groups.idgroups', function ($sub) use ($allowedNetworkIds) { + $sub->select('group_id') + ->from('group_network') + ->whereIn('network_id', $allowedNetworkIds); + }); + } + } + + private function applyFilters(Builder $query, array $validated): void + { + if (!empty($validated['updated_since'])) { + $updatedSince = Carbon::parse($validated['updated_since'])->setTimezone('UTC')->toDateTimeString(); + $query->where('devices.updated_at', '>=', $updatedSince); + } + + if (!empty($validated['event_start'])) { + $start = Carbon::parse($validated['event_start'])->setTimezone('UTC')->toIso8601String(); + $query->where('events.event_start_utc', '>=', $start); + } + + if (!empty($validated['event_end'])) { + $end = Carbon::parse($validated['event_end']); + + // A date-only bound reads as "include that day"; left at 00:00 it'd exclude it entirely. + if ($end->format('H:i:s') === '00:00:00') { + $end = $end->endOfDay(); + } + + $query->where('events.event_start_utc', '<=', $end->setTimezone('UTC')->toIso8601String()); + } + + // Unfiltered by default: ORA publishes powered/unpowered as separate datasets. + if (array_key_exists('powered', $validated) && $validated['powered'] !== null) { + $query->where('categories.powered', (bool) $validated['powered']); + } + } + + private function logRedactions(Request $request, LengthAwarePaginator $paginator): void + { + $scrubber = $this->mapper->scrubber(); + + // Also covers scrubbing-disabled: the count is 0 when the scrubber never ran. + if ($scrubber->totalRedactions() === 0) { + return; + } + + $client = $request->attributes->get('apiClient'); + + Log::info('ORDS export problem-text redactions', [ + 'api_client_id' => $client?->id, + 'page' => $paginator->currentPage(), + 'records' => $paginator->count(), + 'redactions' => $scrubber->counts(), + ]); + } +} diff --git a/app/Http/Middleware/EnsurePublicApiEnabled.php b/app/Http/Middleware/EnsurePublicApiEnabled.php new file mode 100644 index 0000000000..3660d837ba --- /dev/null +++ b/app/Http/Middleware/EnsurePublicApiEnabled.php @@ -0,0 +1,21 @@ +headers->set('Access-Control-Allow-Origin', $allowOrigin); $response->headers->set('Access-Control-Allow-Methods', 'GET, OPTIONS'); $response->headers->set('Access-Control-Allow-Headers', 'Authorization, Content-Type'); + // Without this, cross-origin JavaScript receives the CSV body but none of + // the pagination headers, so it cannot tell when it has hit the last page. + $response->headers->set( + 'Access-Control-Expose-Headers', + 'X-Total-Count, X-Page, X-Per-Page, X-Last-Page, X-Max-Updated-At' + ); $response->headers->set('Access-Control-Max-Age', '3600'); $response->headers->set('Vary', 'Origin', false); diff --git a/app/Services/Ords/OrdsRecordMapper.php b/app/Services/Ords/OrdsRecordMapper.php new file mode 100644 index 0000000000..3145f26fc3 --- /dev/null +++ b/app/Services/Ords/OrdsRecordMapper.php @@ -0,0 +1,219 @@ +scrubber; + } + + /** @return array keyed in ORDS column order */ + public function map(Device $device): array + { + $eventDate = $this->eventDate($device); + $productAge = $this->productAge($device); + [$productCategory, $productCategoryId] = $this->productCategory($device); + + return [ + // Trimmed to match the controller's guard: untrimmed, " ifixit_" would + // pass validation and emit ids with a leading space. + 'id' => trim((string) config('ords.id_prefix')) . $device->iddevices, + 'data_provider' => config('ords.data_provider'), + 'country' => Iso3166::alpha3($device->ords_country_code), + 'partner_product_category' => $this->partnerProductCategory($device), + 'product_category' => $productCategory, + 'product_category_id' => $productCategoryId, + 'brand' => $this->nullIfBlank($device->brand), + 'year_of_manufacture' => $this->yearOfManufacture($eventDate, $productAge), + 'product_age' => $productAge, + 'repair_status' => $this->repairStatus($device), + 'repair_barrier_if_end_of_life' => $this->repairBarrier($device), + 'group_identifier' => $this->nullIfBlank($device->ords_group_name), + 'event_date' => $eventDate?->toDateString(), + 'problem' => $this->problem($device), + ]; + } + + /** + * " ~ " when an item type is present, bare category + * otherwise -- follows The Restart Project's own published row convention. + */ + private function partnerProductCategory(Device $device): ?string + { + $category = $this->nullIfBlank($device->ords_category_name); + $itemType = $this->nullIfBlank($device->item_type); + + if ($category === null) { + return $itemType; + } + + return $itemType === null ? $category : "{$category} ~ {$itemType}"; + } + + /** @return array{0: ?string, 1: ?int} [product_category, product_category_id] */ + private function productCategory(Device $device): array + { + $name = $this->nullIfBlank($device->ords_category_name); + + if (! $device->ords_category_powered) { + // ORA's unpowered dataset carries no product_category_id. + return [ + config('ords.categories_unpowered')[$name] ?? config('ords.categories_unpowered_fallback'), + null, + ]; + } + + // Unmapped powered category is a vocabulary gap, not a data error: fall + // back to our own name with a null id rather than dropping the record. + return config('ords.categories_powered')[$name] ?? [$name, null]; + } + + /** + * `devices.age` is DECIMAL(5,2) UNSIGNED ZEROFILL NOT NULL DEFAULT 0, so + * MySQL returns it zero-padded ("005.00") and 0 means "not recorded", not + * a real age. is_numeric also covers instances still on the old free-text + * VARCHAR column. + */ + private function productAge(Device $device): int|float|null + { + $age = $device->age; + + if ($age === null || ! is_numeric($age)) { + return null; + } + + $age = (float) $age; + + if ($age <= 0) { + return null; + } + + return $age == (int) $age ? (int) $age : $age; + } + + /** + * Not stored, so derived: the year the event ran minus the item's age. + * ORDS wants a 4-digit string; anything we cannot derive is omitted. + */ + private function yearOfManufacture(?Carbon $eventDate, int|float|null $productAge): ?string + { + if ($eventDate === null || $productAge === null) { + return null; + } + + $year = (int) round($eventDate->year - $productAge); + + // ORDS constrains this to ^[0-9]{4}$; clamp rather than emit a rejected value. + if ($year < 1000 || $year > 9999) { + return null; + } + + return (string) $year; + } + + private function eventDate(Device $device): ?Carbon + { + $startUtc = $device->ords_event_start_utc; + + if (empty($startUtc)) { + return null; + } + + $date = Carbon::parse($startUtc, 'UTC'); + $timezone = $this->nullIfBlank($device->ords_event_timezone); + + if ($timezone !== null) { + try { + $date = $date->setTimezone($timezone); + } catch (\Throwable) { + // Unrecognised timezone: keep UTC rather than drop a required column. + } + } + + return $date; + } + + private function repairStatus(Device $device): string + { + return config('ords.repair_status')[$device->repair_status] + ?? config('ords.repair_status_unknown'); + } + + /** Devices can carry several barriers; ORDS has one column, so the first wins. */ + private function repairBarrier(Device $device): ?string + { + if ((int) $device->repair_status !== Device::REPAIR_STATUS_ENDOFLIFE) { + return null; + } + + $barrier = $device->barriers->first(); + + if ($barrier === null) { + return null; + } + + return config('ords.barriers')[$barrier->barrier] ?? null; + } + + private function problem(Device $device): ?string + { + if (! config('ords.problem.include')) { + return null; + } + + $problem = config('ords.problem.scrub') + ? $this->scrubber->scrub($device->problem) + : (string) $device->problem; + + return $this->nullIfBlank($problem); + } + + private function nullIfBlank(?string $value): ?string + { + if ($value === null) { + return null; + } + + $trimmed = trim($value); + + return $trimmed === '' ? null : $trimmed; + } +} diff --git a/app/Services/Ords/ProblemTextScrubber.php b/app/Services/Ords/ProblemTextScrubber.php new file mode 100644 index 0000000000..bf2dae6b71 --- /dev/null +++ b/app/Services/Ords/ProblemTextScrubber.php @@ -0,0 +1,186 @@ + '[email removed]', + self::PHONE => '[phone removed]', + self::LONG_DIGITS => '[number removed]', + ]; + + /** @var array */ + private array $counts = []; + + public function __construct() + { + $this->reset(); + } + + public function reset(): void + { + $this->counts = [ + self::HTML => 0, + self::URL_QUERY => 0, + self::EMAIL => 0, + self::PHONE => 0, + self::LONG_DIGITS => 0, + ]; + } + + /** @return array */ + public function counts(): array + { + return $this->counts; + } + + public function totalRedactions(): int + { + return array_sum($this->counts); + } + + public function scrub(?string $text): string + { + if ($text === null || trim($text) === '') { + return ''; + } + + $text = $this->stripHtml($text); + // Must run before the digit/phone passes, or they chew through tracking params. + $text = $this->stripUrlQueryStrings($text); + $text = $this->redactEmails($text); + $text = $this->redactPhoneNumbers($text); + $text = $this->redactLongDigitRuns($text); + + return $this->normaliseWhitespace($text); + } + + /** + * Not strip_tags(): it truncates after an unterminated "<". Replaced with + * a space, not deleted, so tags don't fuse adjacent words. + */ + private function stripHtml(string $text): string + { + $decoded = html_entity_decode($text, ENT_QUOTES | ENT_HTML5, 'UTF-8'); + + $count = 0; + $result = preg_replace('~<\s*/?\s*[a-z!][^>]*>~iu', ' ', $decoded, -1, $count); + + return $this->record($result, $count > 0 ? 1 : 0, self::HTML); + } + + private function stripUrlQueryStrings(string $text): string + { + return $this->replace( + '~(https?://[^\s<>"\']+?)[?#][^\s<>"\']*~iu', + '$1', + $text, + self::URL_QUERY + ); + } + + /** + * Exclusion-based match, not an ASCII allow-list, so international addresses redact whole. + */ + private function redactEmails(string $text): string + { + // Fullwidth (U+FF20)/small (U+FE6B) commercial-at also render as "@" and must be normalised first. + $normalised = str_replace(["\u{FF20}", "\u{FE6B}"], '@', $text); + + return $this->replace( + '~[^\s@<>"\'()\[\],;:]+@[^\s@<>"\'()\[\],;:]+\.[^\s@<>"\'()\[\],;:.]{2,}~u', + self::PLACEHOLDERS[self::EMAIL], + $normalised, + self::EMAIL + ); + } + + /** + * Separator-bearing sequences only; bare runs go to the digit pass below, + * so an IMEI isn't mislabelled as a phone. Nine-digit floor: rpm ranges, + * part numbers, firmware versions and year ranges sit at 8 digits or + * fewer, while a dialable number needs 9+ once an area/country code is + * present. A 7-digit local number is missed by design -- cheaper than + * destroying ranges and part numbers. + */ + private function redactPhoneNumbers(string $text): string + { + $count = 0; + + $result = preg_replace_callback( + // Bounded on digits only: \b would let "phone-555-123-4567" through untouched. + '~(?record($result, $count, self::PHONE); + } + + private function redactLongDigitRuns(string $text): string + { + return $this->replace( + '~(?record($result, $count, $countKey); + } + + /** Null result = regex engine failed; must never leak the original, so text is dropped and the failure counted. */ + private function record(?string $result, int $count, string $countKey): string + { + if ($result === null) { + $this->counts[$countKey]++; + + return ''; + } + + $this->counts[$countKey] += $count; + + return $result; + } + + private function normaliseWhitespace(string $text): string + { + return trim(preg_replace('/\s+/u', ' ', $text) ?? $text); + } +} diff --git a/bootstrap/app.php b/bootstrap/app.php index c4af8f3cf5..692ca816ac 100644 --- a/bootstrap/app.php +++ b/bootstrap/app.php @@ -99,7 +99,9 @@ 'localeViewPath' => \Mcamara\LaravelLocalization\Middleware\LaravelLocalizationViewPath::class, 'localizationRedirect' => \Mcamara\LaravelLocalization\Middleware\LaravelLocalizationRedirectFilter::class, 'localize' => \Mcamara\LaravelLocalization\Middleware\LaravelLocalizationRoutes::class, + 'publicApiEnabled' => \App\Http\Middleware\EnsurePublicApiEnabled::class, 'publicEventsApiEnabled' => \App\Http\Middleware\EnsurePublicEventsApiEnabled::class, + 'publicRepairsApiEnabled' => \App\Http\Middleware\EnsurePublicRepairsApiEnabled::class, 'publicApiCors' => \App\Http\Middleware\PublicApiCors::class, 'centralizedAuth' => \App\Http\Middleware\CentralizedAuth::class, ]); diff --git a/charts/restarters/templates/env-config.yaml b/charts/restarters/templates/env-config.yaml index 7bf4407a0c..257077502c 100644 --- a/charts/restarters/templates/env-config.yaml +++ b/charts/restarters/templates/env-config.yaml @@ -66,3 +66,6 @@ data: # Meta Configuration {{- include "restarters.envGroup" (dict "groupName" "meta" "context" .) | nindent 8 }} + + # Open Repair Data Standard export + {{- include "restarters.envGroup" (dict "groupName" "ords" "context" .) | nindent 8 }} diff --git a/charts/restarters/values.yaml b/charts/restarters/values.yaml index a3ffb3dbb6..f5980648ea 100644 --- a/charts/restarters/values.yaml +++ b/charts/restarters/values.yaml @@ -238,6 +238,7 @@ envGroups: FEATURE__AUTO_APPROVE_GROUPS: "false" FEATURE__AUTO_APPROVE_EVENTS: "false" FEATURE__PUBLIC_EVENTS_API: "false" + FEATURE__PUBLIC_REPAIRS_API: "false" # Logging configuration logging: @@ -361,3 +362,18 @@ envGroups: REPAIRDIRECTORY_URL: "" META_TWITTER_SITE: "" META_TWITTER_IMAGE_ALT: "" + + # Open Repair Data Standard export (/api/public/v2/repairs) + # + # ORDS_ID_PREFIX is the ORA-assigned partner namespace for this instance. It + # is intentionally blank here: the exporter refuses to serve (503) unless an + # instance sets it explicitly, because ORDS ids are a stable key that ORA + # upserts on, so a wrong prefix silently overwrites another provider's rows. + ords: + ORDS_ID_PREFIX: "" + ORDS_DATA_PROVIDER: "" + # Off by default so turning the endpoint on and publishing the free-text + # column stay two separate decisions. The scrubber removes contact details + # and identifiers but cannot remove names, so this needs a deliberate opt-in. + ORDS_INCLUDE_PROBLEM: "false" + ORDS_SCRUB_PROBLEM: "true" diff --git a/config/ords.php b/config/ords.php new file mode 100644 index 0000000000..f4bb84116f --- /dev/null +++ b/config/ords.php @@ -0,0 +1,141 @@ + env('ORDS_ID_PREFIX', OrdsRecordMapper::UNASSIGNED_ID_PREFIX), + + // Organisation name emitted on every record. Undefaulted for the same + // reason as the prefix: a blank credits the data to nobody. + 'data_provider' => env('ORDS_DATA_PROVIDER', ''), + + // `include` drops the column for a structured-fields-only export. `scrub` + // is the redaction pass. Personal names are not pattern-detectable and are + // not removed, so `include` defaults off: enabling the endpoint and + // publishing volunteer free text are two decisions, not one. + 'problem' => [ + 'include' => env('ORDS_INCLUDE_PROBLEM', false), + 'scrub' => env('ORDS_SCRUB_PROBLEM', true), + ], + + // Higher ceiling than the events endpoint's 100: this is a bulk export. + 'pagination' => [ + 'default_per_page' => 100, + 'max_per_page' => 1000, + ], + + // Unknown is a real value in the published data, not a blank, and is what + // the 0 default on `devices.repair_status` maps to. + 'repair_status' => [ + Device::REPAIR_STATUS_FIXED => 'Fixed', + Device::REPAIR_STATUS_REPAIRABLE => 'Repairable', + Device::REPAIR_STATUS_ENDOFLIFE => 'End of life', + ], + + 'repair_status_unknown' => 'Unknown', + + // Our seeded barriers match the published vocabulary except for the "the" + // in "No way to open the product". "Item too worn out" is published but not + // seeded here; mapped so an instance that adds it is covered. + 'barriers' => [ + 'Spare parts not available' => 'Spare parts not available', + 'Spare parts too expensive' => 'Spare parts too expensive', + 'No way to open the product' => 'No way to open product', + 'No way to open product' => 'No way to open product', + 'Repair information not available' => 'Repair information not available', + 'Lack of equipment' => 'Lack of equipment', + 'Item too worn out' => 'Item too worn out', + 'Too worn out' => 'Item too worn out', + ], + + // Keyed on `categories.name` where `categories.powered` is true. + // + // Names mostly pass straight through, but the ids do not line up (our + // "Desktop computer" is idcategories 11, product_category_id 4) and the + // standard collapses our screen-size and laptop-size splits into one + // category each, which is confirmed against The Restart Project's own + // published rows. + 'categories_powered' => [ + 'Desktop computer' => ['Desktop computer', 4], + 'Flat screen 15-17"' => ['Flat screen', 8], + 'Flat screen 19-20"' => ['Flat screen', 8], + 'Flat screen 22-24"' => ['Flat screen', 8], + 'Flat screen 26-30"' => ['Flat screen', 8], + 'Flat screen 32-37"' => ['Flat screen', 8], + 'Laptop large' => ['Laptop', 16], + 'Laptop medium' => ['Laptop', 16], + 'Laptop small' => ['Laptop', 16], + 'Paper shredder' => ['Paper shredder', 21], + 'PC accessory' => ['PC accessory', 22], + 'Printer/scanner' => ['Printer/scanner', 25], + 'Digital compact camera' => ['Digital compact camera', 5], + 'DSLR/video camera' => ['DSLR/video camera', 6], + 'Handheld entertainment device' => ['Handheld entertainment device', 10], + 'Headphones' => ['Headphones', 11], + 'Mobile' => ['Mobile', 19], + 'Tablet' => ['Tablet', 30], + 'Hi-Fi integrated' => ['Hi-Fi integrated', 12], + 'Hi-Fi separates' => ['Hi-Fi separates', 13], + 'Musical instrument' => ['Musical instrument', 20], + 'Portable radio' => ['Portable radio', 23], + 'Projector' => ['Projector', 26], + 'TV and gaming-related accessories' => ['TV and gaming-related accessories', 33], + 'Aircon/dehumidifier' => ['Aircon/dehumidifier', 1], + 'Decorative or safety lights' => ['Decorative or safety lights', 3], + 'Fan' => ['Fan', 7], + 'Hair & beauty item' => ['Hair & beauty item', 9], + 'Kettle' => ['Kettle', 14], + 'Lamp' => ['Lamp', 15], + 'Power tool' => ['Power tool', 24], + 'Small kitchen item' => ['Small kitchen item', 29], + 'Toaster' => ['Toaster', 31], + 'Toy' => ['Toy', 32], + 'Vacuum' => ['Vacuum', 34], + 'Misc' => ['Misc', 18], + + // Added by 2021_08_13_000439_update_lca_unpowered_categories. + 'Games console' => ['Games console', 38], + 'Watch/clock' => ['Watch/clock', 35], + 'Sewing machine' => ['Sewing machine', 27], + 'Iron' => ['Iron', 40], + 'Coffee maker' => ['Coffee maker', 36], + + // Pre-2021_10_04_124708_fix_category_names aliases, for instances that + // never ran that migration. + 'PC Accessory' => ['PC accessory', 22], + 'Digital Compact Camera' => ['Digital compact camera', 5], + 'DLSR / Video Camera' => ['DSLR/video camera', 6], + 'Aircon/Dehumidifier' => ['Aircon/dehumidifier', 1], + 'Hair & Beauty item' => ['Hair & beauty item', 9], + ], + + // Keyed on `categories.name` where `categories.powered` is false. Unpowered + // repairs are published as a separate four-column dataset with no + // product_category_id, so these carry the "Unpowered - X" name and a null + // id. Callers wanting the 14-column aggregate shape should pass powered=1. + 'categories_unpowered' => [ + 'Furniture' => 'Unpowered - Furniture', + 'Bicycle' => 'Unpowered - Bicycle', + 'Clothing/textile' => 'Unpowered - Textile', + 'Jewellery' => 'Unpowered - Jewellery', + 'Misc' => 'Unpowered - Other', + // No published equivalent; "Household" is the nearest but overstates it. + 'Hand tool' => 'Unpowered - Other', + ], + + 'categories_unpowered_fallback' => 'Unpowered - Other', + +]; diff --git a/config/restarters.php b/config/restarters.php index e7cc396533..b87a729af5 100644 --- a/config/restarters.php +++ b/config/restarters.php @@ -8,6 +8,7 @@ 'image_upload_enabled' => env('FEATURE__IMAGE_UPLOAD', false), 'matomo_integration' => env('FEATURE__MATOMO_INTEGRATION', false), 'public_events_api' => env('FEATURE__PUBLIC_EVENTS_API', false), + 'public_repairs_api' => env('FEATURE__PUBLIC_REPAIRS_API', false), ], 'auth' => [ diff --git a/docs/public-repairs-api.md b/docs/public-repairs-api.md new file mode 100644 index 0000000000..79771340db --- /dev/null +++ b/docs/public-repairs-api.md @@ -0,0 +1,126 @@ +# Public Repairs API + +Exports repair records as [Open Repair Data Standard](https://openrepair.org/open-data/open-standard/) +v0.3, under `/api/public/v2`. Intended for bulk ingestion by the Open Repair Alliance rather than +for display. + +## Feature flag + +Enable with: + +`FEATURE__PUBLIC_REPAIRS_API=true` + +Independent of `FEATURE__PUBLIC_EVENTS_API`; either scope can be enabled without the other. + +## Required configuration + +The endpoint returns `503` until both are set: + +- `ORDS_ID_PREFIX` — the partner namespace assigned by the Open Repair Alliance, e.g. `ifixit_` +- `ORDS_DATA_PROVIDER` — the organisation name shown on every published row + +Neither has a default. The identifier is a stable key the consumer upserts on across releases, so +publishing under an unassigned or borrowed namespace overwrites another provider's records. Once +records have been published under a prefix it must not change. + +Optional: + +- `ORDS_INCLUDE_PROBLEM` (default `false`) — set `true` to add the `problem` column; the export is + structured fields only until you do +- `ORDS_SCRUB_PROBLEM` (default `true`) — redaction pass over the `problem` column + +Publishing free text is a separate decision from enabling the endpoint, so the column is opt-in. +The scrub cannot remove personal names, which is why the default is off. + +## Authentication + +`Authorization: Bearer `, with the `repairs:read` scope: + +- `php artisan api-clients:create --name="Partner Name" --scopes=repairs:read` +- `php artisan api-clients:revoke ` +- `php artisan api-clients:rotate ` + +An `events:read` token is rejected with `403`, and vice versa. Scopes are comma-separated if one +client needs both. + +## Endpoint + +- `GET /api/public/v2/repairs` + +## Query params + +- `format` — `json` (default) or `csv` +- `updated_since` (ISO8601) — devices modified at or after this time +- `event_start` (ISO8601 or date) — events starting at or after +- `event_end` (ISO8601 or date) — events starting at or before; a date-only value covers that whole day +- `powered` — `1`/`true` for powered categories, `0`/`false` for unpowered; omit for both +- `page` (default `1`) +- `per_page` (default `100`, max `1000`) + +## Output + +JSON carries the records in `data`, with `meta` (pagination, column list) and `sync` +(`generated_at`, `max_updated_at`) alongside. + +CSV has no envelope, so the same metadata travels in headers: `X-Total-Count`, `X-Page`, +`X-Per-Page`, `X-Last-Page`, `X-Max-Updated-At`. Cells opening with `=`, `+`, `-` or `@` are +escaped, since spreadsheet software executes them on open. + +## Columns + +Fourteen columns in the order the standard defines, of which `problem` is emitted only when +`ORDS_INCLUDE_PROBLEM` is on. Expect thirteen by default. + +| Column | Source | +| --- | --- | +| `id` | `ORDS_ID_PREFIX` + `devices.iddevices` | +| `data_provider` | `ORDS_DATA_PROVIDER` | +| `country` | `groups.country_code`, translated alpha-2 to alpha-3 | +| `partner_product_category` | `categories.name ~ devices.item_type` | +| `product_category` | `categories.name`, mapped to the standard's vocabulary | +| `product_category_id` | name lookup; our `idcategories` do not match the published ids | +| `brand` | `devices.brand` | +| `year_of_manufacture` | derived: event year minus `devices.age` | +| `product_age` | `devices.age`, omitted when `0` | +| `repair_status` | `devices.repair_status`, `Unknown` when unset | +| `repair_barrier_if_end_of_life` | first `barriers` row, only when end of life | +| `group_identifier` | `groups.name` | +| `event_date` | `events.event_start_utc` as a local date in the event's timezone | +| `problem` | `devices.problem`, scrubbed; only when `ORDS_INCLUDE_PROBLEM` is on | + +## Vocabulary notes + +The maps in `config/ords.php` follow the Open Repair Alliance's published data, not its +`tableschema.json`, which is stale in two places. Where they disagree: + +- the schema's `id` pattern uses a hyphen; every published row uses an underscore +- the schema's barrier enum reads `Too worn out`; the published data reads `Item too worn out` + +Other differences worth knowing: + +- the standard collapses our screen-size and laptop-size splits into `Flat screen` and `Laptop` +- unpowered repairs are published separately with no `product_category_id`, so those records carry + an `Unpowered - X` name and a null id +- a category with no mapping falls back to our own name with a null id rather than being dropped + +Re-check the published category list when a new release lands; unmapped categories degrade quietly. + +## Redaction + +`devices.problem` is unsanitised free text. The scrub strips HTML and redacts email addresses, +phone numbers, digit runs of eight or more, and URL query strings, keeping the bare URL. Counts by +type are logged per request so an export can be checked before handover. + +Personal names are not pattern-detectable and are not removed. + +## Defaults and visibility rules + +- Returns only devices from approved events on approved groups. +- Excludes soft-deleted events and groups. Devices are hard-deleted, so there is no deleted-device case. +- Honours the client's `allowed_network_ids` when set. +- No date filter by default; the full history is returned. + +## CORS/origin behavior + +Same as the events API: CORS headers are returned for public API routes, and a client with +`allowed_origins` configured rejects a non-matching `Origin` with `403`. diff --git a/routes/api.php b/routes/api.php index 411d80c4d2..063ed4835f 100644 --- a/routes/api.php +++ b/routes/api.php @@ -43,17 +43,25 @@ Route::get('timezone', [API\TimeZoneController::class, 'lookup']); }); +// Two layers of flag. The group gate takes the whole prefix off the map when no +// scope is live, and has to precede publicApiCors because that middleware answers +// OPTIONS without calling $next. The per-scope gates then let the repairs export +// ship dark while the events API stays live. Route::prefix('public/v2') ->withoutMiddleware('customApiAuth') - ->middleware(['publicEventsApiEnabled', 'publicApiCors']) + ->middleware(['publicApiEnabled', 'publicApiCors']) ->group(function () { Route::options('{any}', fn () => response()->noContent())->where('any', '.*'); - Route::middleware(['apiClient:events:read', 'apiClientOrigin', 'throttle:public-api'])->group(function () { + Route::middleware(['publicEventsApiEnabled', 'apiClient:events:read', 'apiClientOrigin', 'throttle:public-api'])->group(function () { Route::get('events', [API\PublicEventController::class, 'listEvents']); Route::get('events/{id}', [API\PublicEventController::class, 'showEvent']); Route::get('groups/{id}/events', [API\PublicEventController::class, 'listGroupEvents']); }); + + Route::middleware(['publicRepairsApiEnabled', 'apiClient:repairs:read', 'apiClientOrigin', 'throttle:public-api'])->group(function () { + Route::get('repairs', [API\PublicRepairController::class, 'listRepairs']); + }); }); // ============================================================================= diff --git a/tests/Feature/Repairs/PublicRepairsApiTest.php b/tests/Feature/Repairs/PublicRepairsApiTest.php new file mode 100644 index 0000000000..91dd8c4720 --- /dev/null +++ b/tests/Feature/Repairs/PublicRepairsApiTest.php @@ -0,0 +1,974 @@ + true, + 'restarters.features.public_repairs_api' => true, + // Production ships with the placeholder, which the controller + // refuses to serve under; tests need a usable namespace. + 'ords.id_prefix' => 'testinstance_', + 'ords.data_provider' => 'Test Repair Org', + 'ords.problem.include' => true, + 'ords.problem.scrub' => true, + ]); + } + + // ---------------------------------------------------------------- auth + + public function test_requires_bearer_token(): void + { + $this->get('/api/public/v2/repairs')->assertStatus(401); + } + + public function test_ignores_query_token_auth(): void + { + $this->get('/api/public/v2/repairs?api_token=not_a_valid_public_key')->assertStatus(401); + } + + public function test_events_scope_is_forbidden_on_repairs(): void + { + $token = $this->createPublicApiToken(['scopes' => ['events:read']]); + + $this->withHeader('Authorization', 'Bearer ' . $token) + ->get('/api/public/v2/repairs') + ->assertStatus(403); + } + + public function test_repairs_scope_is_forbidden_on_events(): void + { + $token = $this->createPublicApiToken(['scopes' => ['repairs:read']]); + + $this->withHeader('Authorization', 'Bearer ' . $token) + ->get('/api/public/v2/events') + ->assertStatus(403); + } + + public function test_enforces_allowed_origins_when_configured(): void + { + $this->seedRepair(); + $token = $this->createPublicApiToken(['allowed_origins' => ['https://allowed.example']]); + + $this->withHeaders([ + 'Authorization' => 'Bearer ' . $token, + 'Origin' => 'https://disallowed.example', + ])->get('/api/public/v2/repairs')->assertStatus(403); + + $this->withHeaders([ + 'Authorization' => 'Bearer ' . $token, + 'Origin' => 'https://allowed.example', + ])->get('/api/public/v2/repairs')->assertSuccessful(); + } + + // ------------------------------------------------------- feature flags + + public function test_route_is_absent_when_repairs_flag_is_off(): void + { + $this->withExceptionHandling(); + config(['restarters.features.public_repairs_api' => false]); + + $token = $this->createPublicApiToken(); + + $this->withHeader('Authorization', 'Bearer ' . $token) + ->get('/api/public/v2/repairs') + ->assertStatus(404); + } + + public function test_events_api_still_works_when_repairs_flag_is_off(): void + { + // The flags are split so repairs can ship dark without taking the live + // events API down with it. + config(['restarters.features.public_repairs_api' => false]); + + $token = $this->createPublicApiToken(['scopes' => ['events:read']]); + + $this->withHeader('Authorization', 'Bearer ' . $token) + ->get('/api/public/v2/events') + ->assertSuccessful(); + } + + public function test_events_route_is_absent_when_events_flag_is_off_but_repairs_is_on(): void + { + $this->withExceptionHandling(); + config(['restarters.features.public_events_api' => false]); + + $token = $this->createPublicApiToken(['scopes' => ['events:read', 'repairs:read']]); + + $this->withHeader('Authorization', 'Bearer ' . $token) + ->get('/api/public/v2/events') + ->assertStatus(404); + + $this->withHeader('Authorization', 'Bearer ' . $token) + ->get('/api/public/v2/repairs') + ->assertSuccessful(); + } + + public function test_preflight_is_absent_when_both_flags_are_off(): void + { + // A dark instance should not answer preflight with CORS headers. + $this->withExceptionHandling(); + config([ + 'restarters.features.public_events_api' => false, + 'restarters.features.public_repairs_api' => false, + ]); + + $this->options('/api/public/v2/repairs')->assertStatus(404); + $this->options('/api/public/v2/events')->assertStatus(404); + } + + public function test_preflight_answers_when_a_scope_is_live(): void + { + config(['restarters.features.public_repairs_api' => false]); + + $this->options('/api/public/v2/repairs') + ->assertStatus(204) + ->assertHeader('Access-Control-Allow-Methods', 'GET, OPTIONS'); + } + + public function test_csv_pagination_headers_are_exposed_to_browsers(): void + { + // The CSV metadata travels in headers, and cross-origin JavaScript can + // read none of it without an explicit expose list. + $this->seedRepair(); + + $token = $this->createPublicApiToken(); + + $response = $this->withHeader('Authorization', 'Bearer ' . $token) + ->get('/api/public/v2/repairs?format=csv'); + + $exposed = $response->headers->get('Access-Control-Expose-Headers'); + + foreach (['X-Total-Count', 'X-Page', 'X-Per-Page', 'X-Last-Page', 'X-Max-Updated-At'] as $header) { + $this->assertStringContainsString($header, (string) $exposed); + } + } + + // ------------------------------------------------------ id namespace + + public function test_refuses_to_serve_under_an_unassigned_id_namespace(): void + { + // ORDS ids are a stable key that ORA updates in place across releases, + // and `fixitclinic_` already holds published rows overlapping our + // auto-increment range. Guessing a prefix would overwrite real data. + config(['ords.id_prefix' => OrdsRecordMapper::UNASSIGNED_ID_PREFIX]); + + $this->seedRepair(); + $token = $this->createPublicApiToken(); + + $response = $this->withHeader('Authorization', 'Bearer ' . $token) + ->get('/api/public/v2/repairs'); + + $response->assertStatus(503); + $this->assertStringContainsString('id namespace', $response->json('message')); + } + + public function test_refuses_to_serve_under_a_blank_id_namespace(): void + { + // An env var set to "" yields an empty string rather than falling back + // to the config default, so the chart shipping ORDS_ID_PREFIX="" must + // not be mistaken for a configured export. + config(['ords.id_prefix' => '']); + + $this->seedRepair(); + $token = $this->createPublicApiToken(); + + $this->withHeader('Authorization', 'Bearer ' . $token) + ->get('/api/public/v2/repairs') + ->assertStatus(503); + } + + public function test_refuses_to_serve_without_a_data_provider(): void + { + // data_provider is the attribution carried on every published row. + config(['ords.data_provider' => '']); + + $this->seedRepair(); + $token = $this->createPublicApiToken(); + + $response = $this->withHeader('Authorization', 'Bearer ' . $token) + ->get('/api/public/v2/repairs'); + + $response->assertStatus(503); + $this->assertStringContainsString('data provider', $response->json('message')); + } + + public function test_emits_ids_under_the_assigned_ora_namespace(): void + { + // The Open Repair Alliance assigned this instance `ifixit_`. Set in + // deployment config, not defaulted in code, but pinned here so a change + // to the emitted id shape cannot pass silently: ORA upserts on this id, + // so it must stay stable once we have published under it. + config(['ords.id_prefix' => 'ifixit_']); + + $device = $this->seedRepair(); + + $this->assertEquals( + 'ifixit_' . $device->iddevices, + $this->fetchRecords()[0]['id'] + ); + } + + public function test_a_padded_id_prefix_is_trimmed_before_emission(): void + { + // The config guard validates the trimmed prefix, so the emitted id has + // to be trimmed too or " ifixit_ " passes the guard and publishes ids + // carrying a leading space. + config(['ords.id_prefix' => ' ifixit_ ']); + + $device = $this->seedRepair(); + + $this->assertSame('ifixit_' . $device->iddevices, $this->fetchRecords()[0]['id']); + } + + // ------------------------------------------------------------ columns + + public function test_emits_the_fourteen_ords_columns_in_spec_order(): void + { + $this->seedRepair(); + + $record = $this->fetchRecords()[0]; + + $this->assertEquals([ + 'id', + 'data_provider', + 'country', + 'partner_product_category', + 'product_category', + 'product_category_id', + 'brand', + 'year_of_manufacture', + 'product_age', + 'repair_status', + 'repair_barrier_if_end_of_life', + 'group_identifier', + 'event_date', + 'problem', + ], array_keys($record)); + } + + public function test_maps_every_column_from_the_fixture(): void + { + $device = $this->seedRepair([ + 'brand' => 'Acme', + 'item_type' => 'Tower PC', + 'age' => 5, + 'problem' => 'Would not power on. Replaced the PSU.', + 'repair_status' => Device::REPAIR_STATUS_FIXED, + ]); + + $record = $this->fetchRecords()[0]; + + $this->assertEquals('testinstance_' . $device->iddevices, $record['id']); + $this->assertEquals('Test Repair Org', $record['data_provider']); + // groups.country_code is alpha-2; ORDS requires alpha-3. + $this->assertEquals('GBR', $record['country']); + // Restart's own published rows use " ~ ". + $this->assertEquals('Desktop computer ~ Tower PC', $record['partner_product_category']); + $this->assertEquals('Desktop computer', $record['product_category']); + // Our idcategories is 11; ORA publishes Desktop computer as 4. + $this->assertEquals(4, $record['product_category_id']); + $this->assertEquals('Acme', $record['brand']); + // 2024 event minus a 5 year old item. + $this->assertEquals('2019', $record['year_of_manufacture']); + $this->assertEquals(5, $record['product_age']); + $this->assertEquals('Fixed', $record['repair_status']); + $this->assertNull($record['repair_barrier_if_end_of_life']); + $this->assertEquals('ORDS Group', $record['group_identifier']); + $this->assertEquals('2024-06-15', $record['event_date']); + $this->assertEquals('Would not power on. Replaced the PSU.', $record['problem']); + } + + public function test_event_date_uses_the_events_local_timezone(): void + { + // 2024-06-16 06:00 UTC is still 2024-06-15 in Los Angeles. + $this->seedRepair([], ['start_utc' => '2024-06-16 06:00:00', 'timezone' => 'America/Los_Angeles']); + + $this->assertEquals('2024-06-15', $this->fetchRecords()[0]['event_date']); + } + + public function test_repair_status_falls_back_to_unknown(): void + { + // `devices.repair_status` is NOT NULL DEFAULT 0, and ORDS carries + // Unknown as a real enum value rather than a blank. + $this->seedRepair(['repair_status' => 0]); + + $this->assertEquals('Unknown', $this->fetchRecords()[0]['repair_status']); + } + + public function test_year_of_manufacture_and_age_are_omitted_when_age_is_not_recorded(): void + { + // `devices.age` is DECIMAL NOT NULL DEFAULT 0, so 0 is how "we did not + // record an age" is stored rather than a real age of zero. + $this->seedRepair(['age' => 0]); + + $record = $this->fetchRecords()[0]; + $this->assertNull($record['year_of_manufacture']); + $this->assertNull($record['product_age']); + } + + public function test_fractional_age_is_preserved(): void + { + $this->seedRepair(['age' => 2.5]); + + $record = $this->fetchRecords()[0]; + $this->assertEquals(2.5, $record['product_age']); + // 2024 minus 2.5 years, rounded to a whole year. + $this->assertEquals('2022', $record['year_of_manufacture']); + } + + public function test_barrier_is_emitted_only_for_end_of_life_and_uses_ords_wording(): void + { + // We seed "No way to open the product"; ORDS publishes it without "the". + $barrier = Barrier::where('barrier', 'No way to open the product')->firstOrFail(); + + $device = $this->seedRepair(['repair_status' => Device::REPAIR_STATUS_ENDOFLIFE]); + $device->barriers()->attach($barrier->id); + + $this->assertEquals( + 'No way to open product', + $this->fetchRecords()[0]['repair_barrier_if_end_of_life'] + ); + } + + public function test_barrier_choice_is_stable_across_exports(): void + { + // ORDS has one barrier column but a device can carry several, so the + // mapper takes the first. The record id is a stable key ORA upserts on, + // so an unordered relation would republish a different barrier at random. + $first = Barrier::where('barrier', 'Lack of equipment')->firstOrFail(); + $second = Barrier::where('barrier', 'Spare parts too expensive')->firstOrFail(); + [$lower, $higher] = $first->id < $second->id ? [$first, $second] : [$second, $first]; + + $device = $this->seedRepair(['repair_status' => Device::REPAIR_STATUS_ENDOFLIFE]); + // Attached highest-first so insertion order cannot be what makes this pass. + $device->barriers()->attach($higher->id); + $device->barriers()->attach($lower->id); + + $expected = config('ords.barriers')[$lower->barrier]; + + $this->assertEquals($expected, $this->fetchRecords()[0]['repair_barrier_if_end_of_life']); + $this->assertEquals($expected, $this->fetchRecords()[0]['repair_barrier_if_end_of_life']); + } + + public function test_an_unmapped_barrier_is_omitted(): void + { + // A barrier outside the ORDS vocabulary is a mapping gap; emitting our + // own wording would put an invalid value in a constrained column. + $barrier = Barrier::create(['barrier' => 'Not an ORDS barrier']); + + $device = $this->seedRepair(['repair_status' => Device::REPAIR_STATUS_ENDOFLIFE]); + $device->barriers()->attach($barrier->id); + + $this->assertNull($this->fetchRecords()[0]['repair_barrier_if_end_of_life']); + } + + public function test_barrier_is_suppressed_when_the_item_was_repaired(): void + { + $barrier = Barrier::where('barrier', 'Lack of equipment')->firstOrFail(); + + $device = $this->seedRepair(['repair_status' => Device::REPAIR_STATUS_FIXED]); + $device->barriers()->attach($barrier->id); + + $this->assertNull($this->fetchRecords()[0]['repair_barrier_if_end_of_life']); + } + + /** + * The vocabulary maps were originally built from the 2018 seed and missed + * the seven categories added in 2021, which then exported with a null + * product_category_id. Reads the category names straight out of the + * migrations so a future addition fails here rather than degrading quietly. + */ + public function test_every_seeded_category_has_a_vocabulary_mapping(): void + { + $powered = array_keys(config('ords.categories_powered')); + $unpowered = array_keys(config('ords.categories_unpowered')); + + $seeded = []; + foreach (glob(database_path('migrations/*.php')) as $migration) { + $source = file_get_contents($migration); + + // Names from the initialise migration's bulk INSERT. + if (preg_match_all('/\(\d+,\s*"([^"]+)",/', $source, $m)) { + $seeded = array_merge($seeded, $m[1]); + } + // Names from the later insert()/update() calls. + if (preg_match_all("/'name'\s*=>\s*'([^']+)'/", $source, $m)) { + $seeded = array_merge($seeded, $m[1]); + } + } + + // Only names that are actually category rows on a migrated database. + $known = array_merge($powered, $unpowered); + $categoryNames = array_intersect(array_unique($seeded), $known); + + $this->assertNotEmpty($categoryNames, 'no category names parsed from migrations'); + + foreach (['Games console', 'Watch/clock', 'Sewing machine', 'Iron', 'Coffee maker'] as $late) { + $this->assertContains($late, $powered, "{$late} is seeded but has no powered mapping"); + } + + foreach (['Jewellery', 'Hand tool'] as $late) { + $this->assertContains($late, $unpowered, "{$late} is seeded but has no unpowered mapping"); + } + } + + public function test_unpowered_categories_use_the_ords_unpowered_vocabulary(): void + { + Category::factory()->create([ + 'idcategories' => 501, + 'name' => 'Clothing/textile', + 'revision' => 2, + 'aggregate' => 0, + 'powered' => 0, + ]); + + $this->seedRepair(['category' => 501, 'category_creation' => 501, 'item_type' => null]); + + $record = $this->fetchRecords()[0]; + $this->assertEquals('Unpowered - Textile', $record['product_category']); + // ORA's unpowered dataset carries no product_category_id. + $this->assertNull($record['product_category_id']); + $this->assertEquals('Clothing/textile', $record['partner_product_category']); + } + + public function test_an_unmapped_powered_category_falls_back_to_our_own_name(): void + { + // A category outside the ORDS vocabulary is a mapping gap, not a data + // error. The record stays usable and the gap is visible in the export. + Category::factory()->create([ + 'idcategories' => 504, + 'name' => '3D printer', + 'revision' => 2, + 'aggregate' => 0, + 'powered' => 1, + ]); + + $this->seedRepair(['category' => 504, 'category_creation' => 504, 'item_type' => null]); + + $record = $this->fetchRecords()[0]; + $this->assertEquals('3D printer', $record['product_category']); + $this->assertNull($record['product_category_id']); + $this->assertEquals('3D printer', $record['partner_product_category']); + } + + public function test_country_is_null_when_it_cannot_be_mapped_to_alpha3(): void + { + // ORDS requires alpha-3. `groups.country_code` is alpha-2 and nullable, + // and nothing constrains it to a real code. + $this->seedRepair([], ['country_code' => null]); + $this->assertNull($this->fetchRecords()[0]['country']); + + $this->seedRepair([], ['group' => 'Unknown Country Group', 'country_code' => 'ZZ']); + $countries = array_column($this->fetchRecords(), 'country'); + $this->assertEquals([null, null], $countries); + } + + // ---------------------------------------------------------- visibility + + public function test_excludes_unapproved_events_unapproved_groups_and_soft_deletes(): void + { + $visible = $this->seedRepair(); + + $unapprovedEventDevice = $this->seedRepair([], ['group' => 'Second Group', 'approve_event' => false]); + $unapprovedGroupDevice = $this->seedRepair([], ['group' => 'Third Group', 'approve_group' => false]); + + $deletedEventDevice = $this->seedRepair([], ['group' => 'Fourth Group']); + Party::findOrFail($deletedEventDevice->event)->delete(); + + $deletedGroupDevice = $this->seedRepair([], ['group' => 'Fifth Group']); + Group::findOrFail(Party::findOrFail($deletedGroupDevice->event)->group)->delete(); + + $ids = array_column($this->fetchRecords(), 'id'); + + $this->assertContains('testinstance_' . $visible->iddevices, $ids); + $this->assertNotContains('testinstance_' . $unapprovedEventDevice->iddevices, $ids); + $this->assertNotContains('testinstance_' . $unapprovedGroupDevice->iddevices, $ids); + $this->assertNotContains('testinstance_' . $deletedEventDevice->iddevices, $ids); + $this->assertNotContains('testinstance_' . $deletedGroupDevice->iddevices, $ids); + } + + public function test_respects_allowed_network_restrictions(): void + { + $allowedDevice = $this->seedRepair([], ['group' => 'Allowed Group']); + $blockedDevice = $this->seedRepair([], ['group' => 'Blocked Group']); + + $allowedNetwork = Network::factory()->create(); + $blockedNetwork = Network::factory()->create(); + $allowedNetwork->addGroup(Group::findOrFail(Party::findOrFail($allowedDevice->event)->group)); + $blockedNetwork->addGroup(Group::findOrFail(Party::findOrFail($blockedDevice->event)->group)); + + $ids = array_column( + $this->fetchRecords(['allowed_network_ids' => [$allowedNetwork->id]]), + 'id' + ); + + $this->assertContains('testinstance_' . $allowedDevice->iddevices, $ids); + $this->assertNotContains('testinstance_' . $blockedDevice->iddevices, $ids); + } + + public function test_a_group_in_several_allowed_networks_is_not_duplicated(): void + { + $device = $this->seedRepair(); + $group = Group::findOrFail(Party::findOrFail($device->event)->group); + + $networkOne = Network::factory()->create(); + $networkTwo = Network::factory()->create(); + $networkOne->addGroup($group); + $networkTwo->addGroup($group); + + $records = $this->fetchRecords(['allowed_network_ids' => [$networkOne->id, $networkTwo->id]]); + + $this->assertCount(1, $records); + } + + // ------------------------------------------------------------ filters + + public function test_powered_filter_selects_each_dataset(): void + { + Category::factory()->create([ + 'idcategories' => 502, + 'name' => 'Bicycle', + 'revision' => 2, + 'aggregate' => 0, + 'powered' => 0, + ]); + + $powered = $this->seedRepair(); + $unpowered = $this->seedRepair(['category' => 502, 'category_creation' => 502], ['reuse' => true]); + + $poweredIds = array_column($this->fetchRecords([], ['powered' => 1]), 'id'); + $this->assertContains('testinstance_' . $powered->iddevices, $poweredIds); + $this->assertNotContains('testinstance_' . $unpowered->iddevices, $poweredIds); + + $unpoweredIds = array_column($this->fetchRecords([], ['powered' => 0]), 'id'); + $this->assertContains('testinstance_' . $unpowered->iddevices, $unpoweredIds); + $this->assertNotContains('testinstance_' . $powered->iddevices, $unpoweredIds); + + // Unfiltered returns both: ORA publishes them as separate datasets, but + // the endpoint does not silently drop half the data. + $this->assertCount(2, $this->fetchRecords()); + } + + public function test_powered_filter_accepts_true_and_false_spellings(): void + { + // Laravel's `boolean` rule takes only true/false/0/1, and a rejected + // value surfaces as a 500 here, so `?powered=true` used to break. + Category::factory()->create([ + 'idcategories' => 503, + 'name' => 'Bicycle', + 'revision' => 2, + 'aggregate' => 0, + 'powered' => 0, + ]); + + $powered = $this->seedRepair(); + $unpowered = $this->seedRepair(['category' => 503, 'category_creation' => 503], ['reuse' => true]); + + $trueIds = array_column($this->fetchRecords([], ['powered' => 'true']), 'id'); + $this->assertContains('testinstance_' . $powered->iddevices, $trueIds); + $this->assertNotContains('testinstance_' . $unpowered->iddevices, $trueIds); + + $falseIds = array_column($this->fetchRecords([], ['powered' => 'false']), 'id'); + $this->assertContains('testinstance_' . $unpowered->iddevices, $falseIds); + $this->assertNotContains('testinstance_' . $powered->iddevices, $falseIds); + } + + public function test_an_empty_powered_param_returns_both_datasets(): void + { + // An empty param is how plenty of clients spell an unset one. + Category::factory()->create([ + 'idcategories' => 504, + 'name' => 'Bicycle', + 'revision' => 2, + 'aggregate' => 0, + 'powered' => 0, + ]); + + $powered = $this->seedRepair(); + $unpowered = $this->seedRepair(['category' => 504, 'category_creation' => 504], ['reuse' => true]); + + $ids = array_column($this->fetchRecords([], ['powered' => '']), 'id'); + + $this->assertContains('testinstance_' . $powered->iddevices, $ids); + $this->assertContains('testinstance_' . $unpowered->iddevices, $ids); + } + + public function test_a_date_only_event_end_includes_that_whole_day(): void + { + // The fixture event runs at 18:00 on 2024-06-15. A caller asking for a + // window ending on that date means to include it. + $this->seedRepair(); + + $this->assertCount(1, $this->fetchRecords([], ['event_end' => '2024-06-15'])); + $this->assertEmpty($this->fetchRecords([], ['event_end' => '2024-06-14'])); + } + + public function test_updated_since_filter(): void + { + $device = $this->seedRepair(); + $device->timestamps = false; + $device->updated_at = '2000-01-01 00:00:00'; + $device->save(); + + $this->assertEmpty($this->fetchRecords([], ['updated_since' => '2010-01-01T00:00:00+00:00'])); + $this->assertCount(1, $this->fetchRecords([], ['updated_since' => '1999-01-01T00:00:00+00:00'])); + } + + public function test_event_window_filters(): void + { + $device = $this->seedRepair(); + + $this->assertCount(1, $this->fetchRecords([], [ + 'event_start' => '2024-01-01T00:00:00+00:00', + 'event_end' => '2024-12-31T00:00:00+00:00', + ])); + + $this->assertEmpty($this->fetchRecords([], ['event_start' => '2025-01-01T00:00:00+00:00'])); + $this->assertEmpty($this->fetchRecords([], ['event_end' => '2023-01-01T00:00:00+00:00'])); + + $this->assertNotNull($device->iddevices); + } + + /** + * Note: this codebase surfaces a failed validation as a 500 carrying the + * raw translation key rather than a 422 — the already-live events endpoint + * does the same for per_page > 100. These tests assert that the input is + * rejected rather than asserting the status, so they cover the ceiling + * without baking in that pre-existing behaviour. + */ + public function test_per_page_ceiling_is_one_thousand(): void + { + $this->withExceptionHandling(); + $this->seedRepair(); + $token = $this->createPublicApiToken(); + + $this->withHeader('Authorization', 'Bearer ' . $token) + ->getJson('/api/public/v2/repairs?per_page=1000') + ->assertSuccessful() + ->assertJsonPath('meta.per_page', 1000); + + // The sibling events endpoint caps at 100, which is too low for a bulk + // export, but the ceiling still has to hold. + $rejected = $this->withHeader('Authorization', 'Bearer ' . $token) + ->getJson('/api/public/v2/repairs?per_page=1001'); + + $this->assertFalse($rejected->isSuccessful()); + $this->assertStringContainsString('validation.max', $rejected->json('message')); + } + + public function test_pages_cover_every_record_exactly_once(): void + { + // A bulk consumer walks this endpoint page by page, so the pages must + // partition the result set: no row skipped, none served twice. + $devices = [ + $this->seedRepair(), + $this->seedRepair([], ['reuse' => true]), + $this->seedRepair([], ['reuse' => true]), + ]; + + $expected = array_map(fn (Device $d) => 'testinstance_' . $d->iddevices, $devices); + + $token = $this->createPublicApiToken(); + + $first = $this->withHeader('Authorization', 'Bearer ' . $token) + ->getJson('/api/public/v2/repairs?per_page=2'); + $first->assertSuccessful() + ->assertJsonPath('meta.total', 3) + ->assertJsonPath('meta.last_page', 2); + + $second = $this->withHeader('Authorization', 'Bearer ' . $token) + ->getJson('/api/public/v2/repairs?per_page=2&page=2'); + $second->assertSuccessful()->assertJsonPath('meta.page', 2); + + $firstIds = array_column($first->json('data'), 'id'); + $secondIds = array_column($second->json('data'), 'id'); + $all = array_merge($firstIds, $secondIds); + + $this->assertCount(2, $firstIds); + $this->assertCount(1, $secondIds); + $this->assertSame($all, array_unique($all)); + $this->assertEqualsCanonicalizing($expected, $all); + } + + public function test_rejects_an_unknown_format(): void + { + $this->withExceptionHandling(); + $token = $this->createPublicApiToken(); + + $rejected = $this->withHeader('Authorization', 'Bearer ' . $token) + ->getJson('/api/public/v2/repairs?format=xml'); + + $this->assertFalse($rejected->isSuccessful()); + $this->assertStringContainsString('validation.in', $rejected->json('message')); + } + + public function test_reports_the_standard_and_columns_in_meta(): void + { + $this->seedRepair(); + $token = $this->createPublicApiToken(); + + $this->withHeader('Authorization', 'Bearer ' . $token) + ->getJson('/api/public/v2/repairs') + ->assertSuccessful() + // Pinned as a literal: this is the standard version the record + // shape implements, so a change to the constant should fail here. + ->assertJsonPath('meta.standard', 'Open Repair Data Standard v0.3') + ->assertJsonPath('meta.columns', OrdsRecordMapper::COLUMNS) + ->assertJsonStructure(['sync' => ['generated_at', 'max_updated_at']]); + } + + // ---------------------------------------------------------------- CSV + + public function test_csv_output_matches_the_ords_column_order(): void + { + $device = $this->seedRepair([ + 'brand' => 'Acme', + 'item_type' => 'Tower PC', + 'age' => 5, + 'repair_status' => Device::REPAIR_STATUS_FIXED, + 'problem' => 'Would not power on.', + ]); + + $rows = $this->fetchCsvRows(); + + $this->assertEquals(OrdsRecordMapper::COLUMNS, $rows[0]); + $this->assertEquals([ + 'testinstance_' . $device->iddevices, + 'Test Repair Org', + 'GBR', + 'Desktop computer ~ Tower PC', + 'Desktop computer', + '4', + 'Acme', + '2019', + '5', + 'Fixed', + '', + 'ORDS Group', + '2024-06-15', + 'Would not power on.', + ], $rows[1]); + } + + public function test_csv_writes_empty_strings_for_missing_values(): void + { + // ORDS declares "" as the missing value for every optional column. + $this->seedRepair(['brand' => null, 'age' => 0, 'problem' => '']); + + $row = $this->fetchCsvRows()[1]; + $columns = array_combine(OrdsRecordMapper::COLUMNS, $row); + + $this->assertSame('', $columns['brand']); + $this->assertSame('', $columns['year_of_manufacture']); + $this->assertSame('', $columns['product_age']); + $this->assertSame('', $columns['problem']); + $this->assertSame('', $columns['repair_barrier_if_end_of_life']); + } + + public function test_csv_neutralises_spreadsheet_formulas(): void + { + // `problem` is volunteer free text, and Excel and Sheets execute a cell + // opening with = + - @ as a formula the moment the file is opened. + $this->seedRepair(['problem' => '=HYPERLINK("http://example.com","click")']); + + $row = $this->fetchCsvRows()[1]; + $columns = array_combine(OrdsRecordMapper::COLUMNS, $row); + + $this->assertSame( + '\'=HYPERLINK("http://example.com","click")', + $columns['problem'] + ); + } + + public function test_csv_carries_the_pagination_metadata_in_headers(): void + { + // CSV has no envelope for `meta`/`sync`, so without these a bulk + // consumer cannot tell that a second page exists. + $this->seedRepair(); + $this->seedRepair([], ['reuse' => true]); + $this->seedRepair([], ['reuse' => true]); + + $token = $this->createPublicApiToken(); + $response = $this->withHeader('Authorization', 'Bearer ' . $token) + ->get('/api/public/v2/repairs?format=csv&per_page=2'); + $response->assertSuccessful(); + + $this->assertSame('3', $response->headers->get('X-Total-Count')); + $this->assertSame('1', $response->headers->get('X-Page')); + $this->assertSame('2', $response->headers->get('X-Per-Page')); + $this->assertSame('2', $response->headers->get('X-Last-Page')); + $this->assertNotEmpty($response->headers->get('X-Max-Updated-At')); + } + + // ---------------------------------------------------------- redaction + + public function test_problem_text_is_scrubbed(): void + { + $this->seedRepair([ + 'problem' => '

Owner jane@example.com, call 020 7946 0958.

' + . 'Serial 123456789012. Part https://example.com/p?gclid=ABC123', + ]); + + $problem = $this->fetchRecords()[0]['problem']; + + $this->assertStringNotContainsString('jane@example.com', $problem); + $this->assertStringNotContainsString('020 7946 0958', $problem); + $this->assertStringNotContainsString('123456789012', $problem); + $this->assertStringNotContainsString('gclid', $problem); + $this->assertStringNotContainsString('

', $problem); + $this->assertStringContainsString('https://example.com/p', $problem); + } + + public function test_scrubbing_can_be_disabled(): void + { + config(['ords.problem.scrub' => false]); + $this->seedRepair(['problem' => 'Owner jane@example.com']); + + $this->assertEquals('Owner jane@example.com', $this->fetchRecords()[0]['problem']); + } + + public function test_problem_can_be_withheld_entirely(): void + { + // Supports a structured-fields-only export with no free text at all. + config(['ords.problem.include' => false]); + $this->seedRepair(['problem' => 'Owner jane@example.com']); + + $record = $this->fetchRecords()[0]; + $this->assertArrayHasKey('problem', $record); + $this->assertNull($record['problem']); + } + + // ------------------------------------------------------------ helpers + + /** + * @param array $deviceAttributes + * @param array $context + */ + private function seedRepair(array $deviceAttributes = [], array $context = []): Device + { + // TestCase::setUp truncates categories, so the fixture owns its own. + if (!Category::find(11)) { + Category::factory()->desktopComputer()->create(); + } + + if (!empty($context['reuse'])) { + $eventId = Party::query()->orderBy('idevents', 'desc')->firstOrFail()->idevents; + } else { + // Groups and events are built straight from the factories rather + // than through TestCase::createGroup/createEvent, which post to the + // v2 API and geocode the location against the live Google Maps + // service. This fixture needs an exact country_code, event start and + // timezone anyway, so the round trip would only be overwritten. + $group = Group::factory()->create([ + 'name' => $context['group'] ?? 'ORDS Group', + // array_key_exists, not ??, so a test can ask for a null code. + 'country_code' => array_key_exists('country_code', $context) ? $context['country_code'] : 'GB', + 'approved' => $context['approve_group'] ?? true, + 'latitude' => 51.5074, + 'longitude' => -0.1278, + 'location' => 'London', + ]); + + $startUtc = $context['start_utc'] ?? self::EVENT_START_UTC; + + $event = Party::factory()->create([ + 'group' => $group->idgroups, + 'approved' => $context['approve_event'] ?? true, + 'event_start_utc' => $startUtc, + 'event_end_utc' => Carbon::parse($startUtc)->addHours(2)->toDateTimeString(), + 'timezone' => $context['timezone'] ?? 'Europe/London', + ]); + + $eventId = $event->idevents; + } + + return Device::create(array_merge([ + 'event' => $eventId, + 'category' => 11, + 'category_creation' => 11, + 'brand' => 'Acme', + 'item_type' => 'Tower PC', + 'age' => 5, + 'problem' => 'Would not power on.', + 'repair_status' => Device::REPAIR_STATUS_FIXED, + ], $deviceAttributes)); + } + + /** + * @param array $clientAttributes + * @param array $query + * @return array> + */ + private function fetchRecords(array $clientAttributes = [], array $query = []): array + { + $token = $this->createPublicApiToken($clientAttributes); + $url = '/api/public/v2/repairs' . ($query ? '?' . http_build_query($query) : ''); + + $response = $this->withHeader('Authorization', 'Bearer ' . $token)->getJson($url); + $response->assertSuccessful(); + + return $response->json('data'); + } + + /** + * @return array> + */ + private function fetchCsvRows(): array + { + $token = $this->createPublicApiToken(); + + $response = $this->withHeader('Authorization', 'Bearer ' . $token) + ->get('/api/public/v2/repairs?format=csv'); + $response->assertSuccessful(); + + $body = trim($response->streamedContent()); + $handle = fopen('php://temp', 'r+'); + fwrite($handle, $body); + rewind($handle); + + $rows = []; + while (($row = fgetcsv($handle, 0, ',', '"', '')) !== false) { + $rows[] = $row; + } + fclose($handle); + + return $rows; + } + + private function createPublicApiToken(array $attributes = []): string + { + $token = 'public_api_token_' . uniqid(); + + ApiClient::factory()->create(array_merge([ + 'token_hash' => hash('sha256', $token), + 'scopes' => ['repairs:read'], + 'active' => true, + 'expires_at' => null, + ], $attributes)); + + return $token; + } +} diff --git a/tests/Unit/Ords/ProblemTextScrubberTest.php b/tests/Unit/Ords/ProblemTextScrubberTest.php new file mode 100644 index 0000000000..d1a9ab8a31 --- /dev/null +++ b/tests/Unit/Ords/ProblemTextScrubberTest.php @@ -0,0 +1,237 @@ +scrubber = new ProblemTextScrubber(); + } + + public function test_strips_html_tags(): void + { + // ORA's own published data contains

tags, and `devices.problem` has + // no Purify mutator so anything a volunteer pastes lands raw. + $this->assertEquals( + 'Screen flickers Replaced the inverter', + $this->scrubber->scrub('

Screen flickers

Replaced the inverter

') + ); + } + + public function test_strips_escaped_html_tags(): void + { + $this->assertEquals( + 'Broken hinge', + $this->scrubber->scrub('<b>Broken hinge</b>') + ); + } + + /** + * strip_tags() discards everything after an unterminated "<", and repair + * notes routinely compare against a threshold, a price or a value. + */ + public function test_keeps_text_containing_a_bare_less_than(): void + { + $this->assertEquals( + 'temp <100C and rising, unit dead', + $this->scrubber->scrub('temp <100C and rising, unit dead') + ); + + $this->assertEquals( + 'price was <5 pounds', + $this->scrubber->scrub('price was <5 pounds') + ); + + $this->assertEquals( + 'reads ascrubber->scrub('reads aassertEquals('phone-[phone removed]', $this->scrubber->scrub('phone-555-123-4567')); + $this->assertEquals('mob-[phone removed]', $this->scrubber->scrub('mob-07700-900123')); + } + + public function test_leaves_iso_dates_alone(): void + { + // An ISO date carries enough digits and separators to look like a phone + // number, and dates are substance rather than personal data. + $this->assertEquals( + 'serviced on 2024-06-15 by the owner', + $this->scrubber->scrub('serviced on 2024-06-15 by the owner') + ); + $this->assertEquals( + 'logged 2024-06-15 14:30 at the bench', + $this->scrubber->scrub('logged 2024-06-15 14:30 at the bench') + ); + $this->assertEquals(0, $this->scrubber->counts()[ProblemTextScrubber::PHONE]); + } + + /** + * Widening the phone pattern to catch "phone-555-123-4567" also made it + * greedy enough to eat hyphenated numeric substance. The nine-digit floor + * is what holds these apart, so both sides of it are pinned here. + */ + public function test_leaves_hyphenated_numeric_substance_alone(): void + { + foreach ([ + 'spins at 1000-2000 rpm now', + 'firmware 1.2.3-4567 installed', + 'part no. 12-345-678 ordered', + 'fault seen 2023-2024 repeatedly', + 'drop from 240-110 volts', + ] as $input) { + $this->scrubber->reset(); + $this->assertEquals($input, $this->scrubber->scrub($input)); + $this->assertEquals(0, $this->scrubber->counts()[ProblemTextScrubber::PHONE], $input); + } + } + + public function test_still_redacts_numbers_long_enough_to_dial(): void + { + foreach ([ + 'phone-555-123-4567' => 'phone-[phone removed]', + 'mob-07700-900123' => 'mob-[phone removed]', + 'call 555-123-4567' => 'call [phone removed]', + 'ring +1 (555) 123-4567' => 'ring [phone removed]', + 'owner 020 7946 0958' => 'owner [phone removed]', + ] as $input => $expected) { + $this->scrubber->reset(); + $this->assertEquals($expected, $this->scrubber->scrub($input)); + } + } + + public function test_redacts_non_ascii_and_homoglyph_email_addresses(): void + { + // An ASCII-only pattern left the first two untouched or, worse, redacted + // only the ASCII tail and published the name fragment ahead of it. + $this->assertEquals('[email removed]', $this->scrubber->scrub('john@münchen.de')); + $this->assertEquals('[email removed]', $this->scrubber->scrub('josé.garcía@example.com')); + // Fullwidth commercial at (U+FF20). + $this->assertEquals('[email removed]', $this->scrubber->scrub("john\u{FF20}example.com")); + + $this->assertEquals(3, $this->scrubber->counts()[ProblemTextScrubber::EMAIL]); + } + + public function test_redacts_email_addresses(): void + { + $result = $this->scrubber->scrub('Owner is jane.doe+repairs@example.co.uk, will follow up'); + + $this->assertEquals('Owner is [email removed], will follow up', $result); + $this->assertEquals(1, $this->scrubber->counts()[ProblemTextScrubber::EMAIL]); + } + + public function test_strips_url_query_strings_but_keeps_the_bare_url(): void + { + // Modelled on published record fixitclinic_584, which carries a full + // affiliate URL with gclid and sfdr_ptcid tracking parameters. + $result = $this->scrubber->scrub( + 'Part at https://www.example.com/parts/motor?gclid=ABC123xyz&sfdr_ptcid=99887766 ordered' + ); + + $this->assertEquals('Part at https://www.example.com/parts/motor ordered', $result); + $this->assertEquals(1, $this->scrubber->counts()[ProblemTextScrubber::URL_QUERY]); + } + + public function test_strips_url_fragments(): void + { + $this->assertEquals( + 'See https://example.com/guide', + $this->scrubber->scrub('See https://example.com/guide#step-4-user-jane') + ); + } + + public function test_redacts_phone_numbers(): void + { + $result = $this->scrubber->scrub('Call the owner on +1 (555) 123-4567 before scrapping'); + + $this->assertEquals('Call the owner on [phone removed] before scrapping', $result); + $this->assertEquals(1, $this->scrubber->counts()[ProblemTextScrubber::PHONE]); + } + + public function test_redacts_uk_style_phone_numbers(): void + { + $this->assertEquals( + 'Owner [phone removed] collecting', + $this->scrubber->scrub('Owner 020 7946 0958 collecting') + ); + } + + public function test_redacts_long_digit_runs(): void + { + $result = $this->scrubber->scrub('IMEI 356938035643809 does not match the label'); + + $this->assertEquals('IMEI [number removed] does not match the label', $result); + } + + public function test_leaves_short_numbers_alone(): void + { + // Model numbers, years, capacities and measurements are the substance + // of a repair note; over-redacting would gut the dataset. + $this->assertEquals( + 'Bosch WAE24166GB from 2011, 1200 rpm, blew a 13 amp fuse', + $this->scrubber->scrub('Bosch WAE24166GB from 2011, 1200 rpm, blew a 13 amp fuse') + ); + } + + public function test_handles_multiple_redaction_types_in_one_string(): void + { + $result = $this->scrubber->scrub( + '

Contact me@example.com or 555-123-4567.

Serial 123456789012. ' + . 'Guide https://example.com/x?utm_source=email' + ); + + $this->assertStringNotContainsString('me@example.com', $result); + $this->assertStringNotContainsString('555-123-4567', $result); + $this->assertStringNotContainsString('123456789012', $result); + $this->assertStringNotContainsString('utm_source', $result); + $this->assertStringContainsString('https://example.com/x', $result); + + $counts = $this->scrubber->counts(); + $this->assertEquals(1, $counts[ProblemTextScrubber::EMAIL]); + $this->assertEquals(1, $counts[ProblemTextScrubber::PHONE]); + $this->assertEquals(1, $counts[ProblemTextScrubber::LONG_DIGITS]); + $this->assertEquals(1, $counts[ProblemTextScrubber::URL_QUERY]); + $this->assertEquals(1, $counts[ProblemTextScrubber::HTML]); + $this->assertEquals(5, $this->scrubber->totalRedactions()); + } + + public function test_counts_accumulate_across_records_until_reset(): void + { + $this->scrubber->scrub('a@example.com'); + $this->scrubber->scrub('b@example.com'); + + $this->assertEquals(2, $this->scrubber->counts()[ProblemTextScrubber::EMAIL]); + + $this->scrubber->reset(); + + $this->assertEquals(0, $this->scrubber->counts()[ProblemTextScrubber::EMAIL]); + $this->assertEquals(0, $this->scrubber->totalRedactions()); + } + + public function test_handles_null_and_blank_input(): void + { + $this->assertEquals('', $this->scrubber->scrub(null)); + $this->assertEquals('', $this->scrubber->scrub('')); + $this->assertEquals('', $this->scrubber->scrub(' ')); + $this->assertEquals(0, $this->scrubber->totalRedactions()); + } + + public function test_collapses_whitespace(): void + { + $this->assertEquals( + 'Fixed the switch', + $this->scrubber->scrub(" Fixed the\n\tswitch ") + ); + } +}