From d39a4f440a76037768ff91a9b4afb667fc87b446 Mon Sep 17 00:00:00 2001 From: Thomas Skerbis Date: Fri, 7 Aug 2026 13:57:38 +0200 Subject: [PATCH] 3.6.0 --- Changelog.md | 4 + lib/DataTypeAggregationRepository.php | 60 +++++++++ lib/Visit.php | 186 ++++++++++++++++++-------- lib/api/EventRequest.php | 92 ++++++++++--- lib/data/Brand.php | 7 +- lib/data/Browser.php | 7 +- lib/data/Browsertype.php | 7 +- lib/data/Country.php | 7 +- lib/data/Hour.php | 7 +- lib/data/Model.php | 7 +- lib/data/OS.php | 7 +- lib/data/Weekday.php | 7 +- 12 files changed, 277 insertions(+), 121 deletions(-) create mode 100644 lib/DataTypeAggregationRepository.php diff --git a/Changelog.md b/Changelog.md index 9d0b1db..8326f29 100644 --- a/Changelog.md +++ b/Changelog.md @@ -5,6 +5,10 @@ - Unbenutzte Abhängigkeiten und Frontend-Assets bereinigt: `exceljs.min.js` und `jspdf.umd.min.js` entfernt sowie die nicht mehr benötigten Composer-Abhängigkeiten `phpoffice/phpspreadsheet` und `matomo/referrer-spam-blacklist` aus dem Addon entfernt - Release-Paketierung ergänzt: Entwicklungsordner `.tools` und `.github` werden über `installer_ignore` sowie per `.gitattributes` (`export-ignore`) aus Release-Archiven ausgeschlossen - Frontend-Vendor-Assets aktualisiert (DataTables auf 1.13.11, ECharts auf 5.6.1) und automatischen GitHub-Workflow für regelmäßige Asset-Update-PRs ergänzt (`.github/workflows/update-frontend-assets.yml`) +- Tracking-SQL gehärtet und vereinheitlicht: bisherige String-SQL mit `addslashes` in `Visit` auf parametrisierte Upsert-Queries umgestellt, wiederholte Counter-Upserts in `Visit`/`EventRequest` über interne Helper konsolidiert und den `pagestats_data`-Write-Path per Bulk-Upsert reduziert +- Data-Aggregationen für Browser/Brand/Browsertype/OS/Model/Country/Hour/Weekday zusammengeführt: statt mehrerer Einzelabfragen je Klasse werden die Typen zentral über eine gemeinsame Query geladen und intern wiederverwendet +- REDAXO-Härtung ergänzt: direkte `$_SERVER`-Zugriffe in `Visit`/`EventRequest` durch REDAXO-Serverzugriff ersetzt (u. a. Client-Hints und `HTTP_VIA`) +- Wartungs- und Analyseballast reduziert: veraltete `psalm.xml` entfernt und README um eine Maintainer-Sektion für `.tools` erweitert ## [3.5.2] - 04.08.2026 diff --git a/lib/DataTypeAggregationRepository.php b/lib/DataTypeAggregationRepository.php new file mode 100644 index 0000000..e384998 --- /dev/null +++ b/lib/DataTypeAggregationRepository.php @@ -0,0 +1,60 @@ +> + */ + private static ?array $cache = null; + + /** + * @param string $type + * @return array + */ + public static function getRowsByType(string $type): array + { + self::ensureLoaded(); + + return self::$cache[$type] ?? []; + } + + private static function ensureLoaded(): void + { + if (null !== self::$cache) { + return; + } + + self::$cache = []; + $types = ['browser', 'brand', 'browsertype', 'os', 'model', 'country', 'hour', 'weekday']; + $quotedTypes = array_map(static fn(string $type): string => '"' . $type . '"', $types); + + $sql = rex_sql::factory(); + $rows = $sql->getArray( + 'SELECT type, name, count' + . ' FROM ' . rex::getTable('pagestats_data') + . ' WHERE type IN (' . implode(', ', $quotedTypes) . ')' + . ' ORDER BY type ASC, count DESC' + ); + + foreach ($types as $type) { + self::$cache[$type] = []; + } + + foreach ($rows as $row) { + $type = (string) ($row['type'] ?? ''); + if (!isset(self::$cache[$type])) { + continue; + } + + self::$cache[$type][] = [ + 'name' => (string) ($row['name'] ?? ''), + 'count' => (int) ($row['count'] ?? 0), + ]; + } + } +} diff --git a/lib/Visit.php b/lib/Visit.php index 2f521d1..15c7dc5 100644 --- a/lib/Visit.php +++ b/lib/Visit.php @@ -445,8 +445,10 @@ private function normalizePathFromUrl(string $rawUrl): string public function isChromeDataSaverUsed(IP $ip): bool { // see https://github.com/piwik/piwik/issues/7733 - return !empty($_SERVER['HTTP_VIA']) - && false !== strpos(strtolower($_SERVER['HTTP_VIA']), 'chrome-compression-proxy') + $httpVia = rex_server('HTTP_VIA', 'string', ''); + + return '' !== $httpVia + && false !== strpos(strtolower($httpVia), 'chrome-compression-proxy') && $ip->isInRanges(self::BOTIPS); } @@ -475,28 +477,26 @@ public function persist(): void $this->brand = trim($brandInfo) != '' ? ucfirst($brandInfo) : 'Undefiniert'; $this->model = trim($modelInfo) != '' ? ucfirst($modelInfo) : 'Undefiniert'; - - $sql = rex_sql::factory(); - - $sql_insert = 'INSERT INTO ' . rex::getTable('pagestats_data') . ' (type,name,count) VALUES - ("browser","' . addslashes($this->browser) . '",1), - ("os","' . addslashes($this->os) . ' ' . addslashes($this->osVer) . '",1), - ("browsertype","' . addslashes($this->device_type) . '",1), - ("brand","' . addslashes($this->brand) . '",1), - ("model","' . addslashes($this->brand) . ' - ' . addslashes($this->model) . '",1), - ("hour","' . $this->datetime_now->format('H') . '",1), - ("weekday","' . $this->datetime_now->format('N') . '",1), - ("country","' . $this->country . '",1) - ON DUPLICATE KEY UPDATE count = count + 1;'; - - $sql->setQuery($sql_insert); - - - $sql_insert = 'INSERT INTO ' . rex::getTable('pagestats_visits_per_day') . ' (date,domain,count) VALUES - ("' . $this->datetime_now->format('Y-m-d') . '","' . addslashes($this->domain) . '",1) - ON DUPLICATE KEY UPDATE count = count + 1;'; - - $sql->setQuery($sql_insert); + $counterRows = [ + ['type' => 'browser', 'name' => $this->browser], + ['type' => 'os', 'name' => $this->os . ' ' . $this->osVer], + ['type' => 'browsertype', 'name' => $this->device_type], + ['type' => 'brand', 'name' => $this->brand], + ['type' => 'model', 'name' => $this->brand . ' - ' . $this->model], + ['type' => 'hour', 'name' => $this->datetime_now->format('H')], + ['type' => 'weekday', 'name' => $this->datetime_now->format('N')], + ['type' => 'country', 'name' => $this->country], + ]; + + $this->incrementCounterRows(rex::getTable('pagestats_data'), $counterRows); + + $this->incrementCounterRow( + rex::getTable('pagestats_visits_per_day'), + [ + 'date' => $this->datetime_now->format('Y-m-d'), + 'domain' => $this->domain, + ] + ); } @@ -509,11 +509,14 @@ public function persist(): void */ public function updateVisitsPerUrl(): void { - $sql_insert = 'INSERT INTO ' . rex::getTable('pagestats_visits_per_url') . ' (hash,date,url,count) VALUES - ("' . md5($this->datetime_now->format('Y-m-d') . $this->url) . '","' . $this->datetime_now->format('Y-m-d') . '","' . addslashes($this->url) . '",1) - ON DUPLICATE KEY UPDATE count = count + 1;'; - - $this->executeWriteWithRetry($sql_insert); + $this->incrementCounterRow( + rex::getTable('pagestats_visits_per_url'), + [ + 'hash' => md5($this->datetime_now->format('Y-m-d') . $this->url), + 'date' => $this->datetime_now->format('Y-m-d'), + 'url' => $this->url, + ] + ); // save url http status @@ -590,13 +593,13 @@ public function getCountry(): void */ public function persistVisitor(): void { - $sql = rex_sql::factory(); - - $sql_insert = 'INSERT INTO ' . rex::getTable('pagestats_visitors_per_day') . ' (date,domain,count) VALUES - ("' . $this->datetime_now->format('Y-m-d') . '","' . addslashes($this->domain) . '",1) - ON DUPLICATE KEY UPDATE count = count + 1;'; - - $sql->setQuery($sql_insert); + $this->incrementCounterRow( + rex::getTable('pagestats_visitors_per_day'), + [ + 'date' => $this->datetime_now->format('Y-m-d'), + 'domain' => $this->domain, + ] + ); } /** @@ -725,7 +728,7 @@ public function shouldSaveVisitor(): bool public function parseUA(): void { $cache = new StaticCache(); - $clientHints = ClientHints::factory($_SERVER); + $clientHints = ClientHints::factory(self::buildClientHintsServerBag()); $this->DeviceDetector = new DeviceDetector($this->userAgent, $clientHints); // $this->DeviceDetector = new DeviceDetector($this->userAgent); $this->DeviceDetector->setYamlParser(new DeviceDetectorSymfonyYamlParser()); @@ -733,6 +736,36 @@ public function parseUA(): void $this->DeviceDetector->parse(); } + /** + * @return array + */ + private static function buildClientHintsServerBag(): array + { + $keys = [ + 'HTTP_USER_AGENT', + 'HTTP_SEC_CH_UA', + 'HTTP_SEC_CH_UA_MOBILE', + 'HTTP_SEC_CH_UA_PLATFORM', + 'HTTP_SEC_CH_UA_PLATFORM_VERSION', + 'HTTP_SEC_CH_UA_MODEL', + 'HTTP_SEC_CH_UA_FULL_VERSION', + 'HTTP_SEC_CH_UA_FULL_VERSION_LIST', + 'HTTP_SEC_CH_UA_ARCH', + 'HTTP_SEC_CH_UA_BITNESS', + ]; + + $server = []; + + foreach ($keys as $key) { + $value = rex_server($key, 'string', ''); + if ('' !== $value) { + $server[$key] = $value; + } + } + + return $server; + } + /** * @@ -749,12 +782,14 @@ public function saveBot(): void $botcategory = $botInfo['category'] ?? '-'; $botproducer = $botInfo['producer']['name'] ?? '-'; - $sql = rex_sql::factory(); - - $sql->setQuery(' - INSERT INTO ' . rex::getTable('pagestats_bot') . ' (name,category,producer,count) VALUES - (:botname,:botcategory,:botproducer,1) - ON DUPLICATE KEY UPDATE count = count + 1;', ['botname' => $botname, 'botcategory' => $botcategory, 'botproducer' => $botproducer]); + $this->incrementCounterRow( + rex::getTable('pagestats_bot'), + [ + 'name' => $botname, + 'category' => $botcategory, + 'producer' => $botproducer, + ] + ); } @@ -767,12 +802,14 @@ public function saveBot(): void */ public function saveCrawlerDetect(string $name): void { - $sql = rex_sql::factory(); - - $sql->setQuery(' - INSERT INTO ' . rex::getTable('pagestats_bot') . ' (name,category,producer,count) VALUES - (:botname,:botcategory,:botproducer,1) - ON DUPLICATE KEY UPDATE count = count + 1;', ['botname' => $name, 'botcategory' => "Crawler", 'botproducer' => "-"]); + $this->incrementCounterRow( + rex::getTable('pagestats_bot'), + [ + 'name' => $name, + 'category' => 'Crawler', + 'producer' => '-', + ] + ); } @@ -786,12 +823,53 @@ public function saveCrawlerDetect(string $name): void */ public function saveReferer(string $referer): void { - $sql = rex_sql::factory(); + $this->incrementCounterRow( + rex::getTable('pagestats_referer'), + [ + 'hash' => md5($this->datetime_now->format('Y-m-d') . $referer), + 'referer' => $referer, + 'date' => $this->datetime_now->format('Y-m-d'), + ] + ); + } + + /** + * @param array $keyValues + */ + private function incrementCounterRow(string $table, array $keyValues): void + { + $this->incrementCounterRows($table, [$keyValues]); + } + + /** + * @param list> $rows + */ + private function incrementCounterRows(string $table, array $rows): void + { + if ([] === $rows) { + return; + } + + $columns = array_keys($rows[0]); + $params = []; + $valueGroups = []; + + foreach ($rows as $index => $row) { + $rowPlaceholders = []; + foreach ($columns as $column) { + $placeholder = ':' . $column . '_' . $index; + $rowPlaceholders[] = $placeholder; + $params[$placeholder] = $row[$column] ?? null; + } + $valueGroups[] = '(' . implode(',', $rowPlaceholders) . ',1)'; + } + + $query = 'INSERT INTO ' . $table + . ' (' . implode(',', $columns) . ',count) VALUES ' + . implode(',', $valueGroups) + . ' ON DUPLICATE KEY UPDATE count = count + 1;'; - $sql->setQuery(' - INSERT INTO ' . rex::getTable('pagestats_referer') . ' (hash,referer,date,count) VALUES - (:hash,:referer,:date,1) - ON DUPLICATE KEY UPDATE count = count + 1;', ['hash' => md5($this->datetime_now->format('Y-m-d') . $referer), 'referer' => $referer, 'date' => $this->datetime_now->format('Y-m-d')]); + $this->executeWriteWithRetry($query, $params); } diff --git a/lib/api/EventRequest.php b/lib/api/EventRequest.php index 16c461c..89f0646 100644 --- a/lib/api/EventRequest.php +++ b/lib/api/EventRequest.php @@ -11,7 +11,6 @@ use rex; use rex_addon; use rex_addon_interface; -use rex_path; use rex_sql; use InvalidArgumentException; use rex_sql_exception; @@ -81,24 +80,28 @@ public function shouldSave(): bool $max_visit_length = intval($this->addon->getConfig('statistics_visit_duration')); if ($minute_diff > $max_visit_length) { - // update set last visit to now - $sql->setQuery('UPDATE ' . rex::getTable('pagestats_hash') . ' SET datetime = :datetime WHERE hash = :hash ', ['hash' => $hash, 'datetime' => $this->datetime_now->format('Y-m-d H:i:s')]); + $this->touchHashEntry($hash, $this->datetime_now->format('Y-m-d H:i:s')); return true; } else { return false; } } else { // hash was not found, save hash with current datetime, then save visit - $sql = rex_sql::factory(); - $sql->setTable(rex::getTable('pagestats_hash')); - $sql->setValue('hash', $hash); - $sql->setValue('datetime', $this->datetime_now->format('Y-m-d H:i:s')); - $sql->insert(); + $this->touchHashEntry($hash, $this->datetime_now->format('Y-m-d H:i:s')); return true; } } + private function touchHashEntry(string $hash, string $datetime): void + { + $sql = rex_sql::factory(); + $sql->setTable(rex::getTable('pagestats_hash')); + $sql->setValue('hash', $hash); + $sql->setValue('datetime', $datetime); + $sql->insertOrUpdate(); + } + /** * @@ -109,13 +112,43 @@ public function shouldSave(): bool public function parseUA(): void { $cache = new StaticCache(); - $clientHints = ClientHints::factory($_SERVER); + $clientHints = ClientHints::factory(self::buildClientHintsServerBag()); $this->DeviceDetector = new DeviceDetector($this->userAgent, $clientHints); $this->DeviceDetector->setYamlParser(new DeviceDetectorSymfonyYamlParser()); $this->DeviceDetector->setCache($cache); $this->DeviceDetector->parse(); } + /** + * @return array + */ + private static function buildClientHintsServerBag(): array + { + $keys = [ + 'HTTP_USER_AGENT', + 'HTTP_SEC_CH_UA', + 'HTTP_SEC_CH_UA_MOBILE', + 'HTTP_SEC_CH_UA_PLATFORM', + 'HTTP_SEC_CH_UA_PLATFORM_VERSION', + 'HTTP_SEC_CH_UA_MODEL', + 'HTTP_SEC_CH_UA_FULL_VERSION', + 'HTTP_SEC_CH_UA_FULL_VERSION_LIST', + 'HTTP_SEC_CH_UA_ARCH', + 'HTTP_SEC_CH_UA_BITNESS', + ]; + + $server = []; + + foreach ($keys as $key) { + $value = rex_server($key, 'string', ''); + if ('' !== $value) { + $server[$key] = $value; + } + } + + return $server; + } + /** * @@ -126,17 +159,38 @@ public function parseUA(): void */ public function save(): void { - $sql = rex_sql::factory(); - $result = $sql->setQuery('UPDATE ' . rex::getTable('pagestats_api') . ' SET count = count + 1 WHERE name = :name AND date = :date', ['name' => $this->name, 'date' => $this->datetime_now->format('Y-m-d')]); - - if ($result->getRows() === 0) { - $bot = rex_sql::factory(); - $bot->setTable(rex::getTable('pagestats_api')); - $bot->setValue('name', $this->name); - $bot->setValue('date', $this->datetime_now->format('Y-m-d')); - $bot->setValue('count', 1); - $bot->insert(); + $this->incrementCounterRow( + rex::getTable('pagestats_api'), + [ + 'name' => $this->name, + 'date' => $this->datetime_now->format('Y-m-d'), + ] + ); + } + + /** + * @param array $keyValues + */ + private function incrementCounterRow(string $table, array $keyValues): void + { + $columns = []; + $placeholders = []; + $params = []; + + foreach ($keyValues as $column => $value) { + $columns[] = $column; + $placeholder = ':' . $column; + $placeholders[] = $placeholder; + $params[$placeholder] = $value; } + + $query = 'INSERT INTO ' . $table + . ' (' . implode(',', $columns) . ',count)' + . ' VALUES (' . implode(',', $placeholders) . ',1)' + . ' ON DUPLICATE KEY UPDATE count = count + 1;'; + + $sql = rex_sql::factory(); + $sql->setQuery($query, $params); } diff --git a/lib/data/Brand.php b/lib/data/Brand.php index 43e44da..6f4b40f 100644 --- a/lib/data/Brand.php +++ b/lib/data/Brand.php @@ -4,7 +4,6 @@ use rex; use rex_addon; -use rex_sql; use rex_view; use InvalidArgumentException; use rex_sql_exception; @@ -31,11 +30,7 @@ private function getRows(): array return $this->rows; } - $sql = rex_sql::factory(); - $this->rows = array_map( - static fn(array $row): array => ['name' => (string) $row['name'], 'count' => (int) $row['count']], - $sql->getArray('SELECT name, count FROM ' . rex::getTable('pagestats_data') . ' WHERE type = "brand" ORDER BY count DESC') - ); + $this->rows = DataTypeAggregationRepository::getRowsByType('brand'); return $this->rows; } diff --git a/lib/data/Browser.php b/lib/data/Browser.php index af67a05..2f1bee5 100644 --- a/lib/data/Browser.php +++ b/lib/data/Browser.php @@ -4,7 +4,6 @@ use rex; use rex_addon; -use rex_sql; use rex_view; use InvalidArgumentException; use rex_sql_exception; @@ -31,11 +30,7 @@ private function getRows(): array return $this->rows; } - $sql = rex_sql::factory(); - $this->rows = array_map( - static fn(array $row): array => ['name' => (string) $row['name'], 'count' => (int) $row['count']], - $sql->getArray('SELECT name, count FROM ' . rex::getTable('pagestats_data') . ' WHERE type = "browser" ORDER BY count DESC') - ); + $this->rows = DataTypeAggregationRepository::getRowsByType('browser'); return $this->rows; } diff --git a/lib/data/Browsertype.php b/lib/data/Browsertype.php index 22111d1..6845c56 100644 --- a/lib/data/Browsertype.php +++ b/lib/data/Browsertype.php @@ -4,7 +4,6 @@ use rex; use rex_addon; -use rex_sql; use rex_view; use InvalidArgumentException; use rex_sql_exception; @@ -32,11 +31,7 @@ private function getRows(): array return $this->rows; } - $sql = rex_sql::factory(); - $this->rows = array_map( - static fn(array $row): array => ['name' => (string) $row['name'], 'count' => (int) $row['count']], - $sql->getArray('SELECT name, count FROM ' . rex::getTable('pagestats_data') . ' WHERE type = "browsertype" ORDER BY count DESC') - ); + $this->rows = DataTypeAggregationRepository::getRowsByType('browsertype'); return $this->rows; } diff --git a/lib/data/Country.php b/lib/data/Country.php index 2e9db31..c2ab442 100644 --- a/lib/data/Country.php +++ b/lib/data/Country.php @@ -6,7 +6,6 @@ use rex_addon; use InvalidArgumentException; use rex_exception; -use rex_sql; use rex_view; class Country @@ -23,11 +22,7 @@ private function getRows(): array return $this->rows; } - $sql = rex_sql::factory(); - $this->rows = array_map( - static fn(array $row): array => ['name' => (string) $row['name'], 'count' => (int) $row['count']], - $sql->getArray('SELECT name, count FROM ' . rex::getTable('pagestats_data') . ' where type = "country" ORDER BY count DESC') - ); + $this->rows = DataTypeAggregationRepository::getRowsByType('country'); return $this->rows; } diff --git a/lib/data/Hour.php b/lib/data/Hour.php index b41a05e..0c84485 100644 --- a/lib/data/Hour.php +++ b/lib/data/Hour.php @@ -4,7 +4,6 @@ use rex; use rex_addon; -use rex_sql; use rex_view; use InvalidArgumentException; use rex_sql_exception; @@ -31,11 +30,7 @@ private function getRows(): array return $this->rows; } - $sql = rex_sql::factory(); - $this->rows = array_map( - static fn(array $row): array => ['name' => (string) $row['name'], 'count' => (int) $row['count']], - $sql->getArray('SELECT name, count FROM ' . rex::getTable('pagestats_data') . ' WHERE type = "hour" ORDER BY count DESC') - ); + $this->rows = DataTypeAggregationRepository::getRowsByType('hour'); return $this->rows; } diff --git a/lib/data/Model.php b/lib/data/Model.php index 30bc59c..b1423bf 100644 --- a/lib/data/Model.php +++ b/lib/data/Model.php @@ -4,7 +4,6 @@ use rex; use rex_addon; -use rex_sql; use rex_view; use InvalidArgumentException; use rex_sql_exception; @@ -32,11 +31,7 @@ private function getRows(): array return $this->rows; } - $sql = rex_sql::factory(); - $this->rows = array_map( - static fn(array $row): array => ['name' => (string) $row['name'], 'count' => (int) $row['count']], - $sql->getArray('SELECT name, count FROM ' . rex::getTable('pagestats_data') . ' WHERE type = "model" ORDER BY count DESC') - ); + $this->rows = DataTypeAggregationRepository::getRowsByType('model'); return $this->rows; } diff --git a/lib/data/OS.php b/lib/data/OS.php index d880d19..dd786e6 100644 --- a/lib/data/OS.php +++ b/lib/data/OS.php @@ -4,7 +4,6 @@ use rex; use rex_addon; -use rex_sql; use rex_view; use InvalidArgumentException; use rex_sql_exception; @@ -31,11 +30,7 @@ private function getRows(): array return $this->rows; } - $sql = rex_sql::factory(); - $this->rows = array_map( - static fn(array $row): array => ['name' => (string) $row['name'], 'count' => (int) $row['count']], - $sql->getArray('SELECT name, count FROM ' . rex::getTable('pagestats_data') . ' WHERE type = "os" ORDER BY count DESC') - ); + $this->rows = DataTypeAggregationRepository::getRowsByType('os'); return $this->rows; } diff --git a/lib/data/Weekday.php b/lib/data/Weekday.php index d50f707..378e7e7 100644 --- a/lib/data/Weekday.php +++ b/lib/data/Weekday.php @@ -5,7 +5,6 @@ use rex; use rex_addon; use rex_addon_interface; -use rex_sql; use rex_view; use InvalidArgumentException; use rex_sql_exception; @@ -48,11 +47,7 @@ private function getRows(): array return $this->rows; } - $sql = rex_sql::factory(); - $this->rows = array_map( - static fn(array $row): array => ['name' => (string) $row['name'], 'count' => (int) $row['count']], - $sql->getArray('SELECT name, count FROM ' . rex::getTable('pagestats_data') . ' WHERE type = "weekday" ORDER BY count DESC') - ); + $this->rows = DataTypeAggregationRepository::getRowsByType('weekday'); return $this->rows; }