From f9ca39e42d13bb3d8b377c770e2028ebfd5b4d71 Mon Sep 17 00:00:00 2001 From: Jacob Ransom Date: Tue, 11 Aug 2026 13:49:45 +1200 Subject: [PATCH 1/4] Generate min-Matomo compatibility checks into plugin test runs Plugin CI already runs PluginTests against the minimum Matomo declared in plugin.json, but nothing in that suite compiles assets or templates, so a plugin using a core Less mixin or Twig function newer than its declared minimum shipped undetected (PG-5029). --- action.yml | 16 ++++ scripts/bash/generate_compatibility_checks.sh | 44 +++++++++ .../GeneratedAssetCompilationTest.php.tpl | 47 ++++++++++ .../GeneratedTwigCompilationTest.php.tpl | 91 +++++++++++++++++++ 4 files changed, 198 insertions(+) create mode 100755 scripts/bash/generate_compatibility_checks.sh create mode 100644 scripts/php/templates/GeneratedAssetCompilationTest.php.tpl create mode 100644 scripts/php/templates/GeneratedTwigCompilationTest.php.tpl diff --git a/action.yml b/action.yml index bdbf78a..7a80ba2 100644 --- a/action.yml +++ b/action.yml @@ -99,6 +99,11 @@ inputs: description: "Flag for whether to force uploading a report to Testomat.io instead of just push events. Only applies if the 'testomatio' token is set." required: false default: false + skip-generated-checks: + type: boolean + description: "If true, the compatibility checks this action generates into the plugin's integration tests are not written. Escape hatch only." + required: false + default: false runs: using: "composite" @@ -332,6 +337,17 @@ runs: run: sudo apt-get install ripgrep working-directory: ${{ github.workspace }}/matomo + - name: Generate plugin compatibility checks + if: inputs.plugin-name != '' + working-directory: ${{ github.workspace }}/matomo + shell: bash + run: ${{ github.action_path }}/scripts/bash/generate_compatibility_checks.sh + env: + TEST_SUITE: ${{ inputs.test-type }} + PLUGIN_NAME: ${{ inputs.plugin-name }} + ACTION_PATH: ${{ github.action_path }} + SKIP_GENERATED_CHECKS: ${{ inputs.skip-generated-checks }} + - name: Run tests working-directory: ${{ github.workspace }}/matomo shell: bash diff --git a/scripts/bash/generate_compatibility_checks.sh b/scripts/bash/generate_compatibility_checks.sh new file mode 100755 index 0000000..cb4aff1 --- /dev/null +++ b/scripts/bash/generate_compatibility_checks.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +GREEN='\033[0;32m' +YELLOW='\033[0;33m' +SET='\033[0m' + +if [ -z "$PLUGIN_NAME" ]; then + exit 0 +fi + +if [ "$SKIP_GENERATED_CHECKS" == "true" ]; then + echo -e "${YELLOW}skip-generated-checks is set, not generating compatibility checks${SET}" + exit 0 +fi + +# only these suites include plugins/*/tests/Integration, see tests/PHPUnit/phpunit.xml.dist +case "$TEST_SUITE" in + PluginTests|IntegrationTestsPlugins) ;; + *) + echo "Test suite '$TEST_SUITE' does not run plugin integration tests, not generating compatibility checks" + exit 0 + ;; +esac + +# same precedence as run_tests.sh uses to pick the directory it hands to phpunit +if [ -d "plugins/$PLUGIN_NAME/Test" ]; then + TARGET_DIR="plugins/$PLUGIN_NAME/Test/Integration" +elif [ -d "plugins/$PLUGIN_NAME/tests" ]; then + TARGET_DIR="plugins/$PLUGIN_NAME/tests/Integration" +else + echo "Plugin $PLUGIN_NAME has no test directory, not generating compatibility checks" + exit 0 +fi + +mkdir -p "$TARGET_DIR" + +for template in "$ACTION_PATH"/scripts/php/templates/Generated*.php.tpl; do + target="$TARGET_DIR/$(basename "$template" .tpl)" + + sed "s/{{PLUGIN_NAME}}/$PLUGIN_NAME/g" "$template" > "$target" + + echo -e "${GREEN}Generated $target${SET}" + cat "$target" +done diff --git a/scripts/php/templates/GeneratedAssetCompilationTest.php.tpl b/scripts/php/templates/GeneratedAssetCompilationTest.php.tpl new file mode 100644 index 0000000..59bdc8d --- /dev/null +++ b/scripts/php/templates/GeneratedAssetCompilationTest.php.tpl @@ -0,0 +1,47 @@ +isPluginLoaded(self::PLUGIN_NAME), + self::PLUGIN_NAME . ' is not loaded, so this check would not cover the plugin.' + ); + + $assetManager = AssetManager::getInstance(); + $assetManager->removeMergedAssets(); + + self::assertNotEmpty($assetManager->getMergedStylesheet()->getContent()); + } +} diff --git a/scripts/php/templates/GeneratedTwigCompilationTest.php.tpl b/scripts/php/templates/GeneratedTwigCompilationTest.php.tpl new file mode 100644 index 0000000..90e070b --- /dev/null +++ b/scripts/php/templates/GeneratedTwigCompilationTest.php.tpl @@ -0,0 +1,91 @@ +getTemplateNames($templateDir) : []; + + if (empty($templates)) { + self::markTestSkipped(self::PLUGIN_NAME . ' ships no templates.'); + } + + // the plugin's own Twig extensions are only registered while it is loaded, and the test + // framework decides what to load from this class' namespace + self::assertTrue( + Manager::getInstance()->isPluginLoaded(self::PLUGIN_NAME), + self::PLUGIN_NAME . ' is not loaded, so this check would not cover the plugin.' + ); + + $twig = new Twig(); + $environment = $twig->getTwigEnvironment(); + + foreach ($templates as $template) { + $environment->load('@' . self::PLUGIN_NAME . '/' . $template); + } + + self::assertNotEmpty($templates); + } + + /** + * @return string[] template names relative to the plugin's templates directory + */ + private function getTemplateNames(string $templateDir): array + { + $iterator = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator($templateDir, \FilesystemIterator::SKIP_DOTS) + ); + + $names = []; + + foreach ($iterator as $file) { + if ($file->getExtension() !== 'twig') { + continue; + } + + $name = str_replace('\\', '/', substr($file->getPathname(), strlen($templateDir) + 1)); + + // templates/plugins// holds theme overrides that Twig registers under the + // other plugin's namespace, so they are not this plugin's to compile + if (strpos($name, 'plugins/') === 0) { + continue; + } + + $names[] = $name; + } + + sort($names); + + return $names; + } +} From 035bc96d59b0fac0cdd360e9c56e08353d16488c Mon Sep 17 00:00:00 2001 From: Jacob Ransom Date: Tue, 11 Aug 2026 14:22:38 +1200 Subject: [PATCH 2/4] Keep the generated Twig check out of the vendor dependency grep tests:check-direct-dependency-use matches a vendor namespace preceded by a space, and most plugins assert an exact list of the files it finds, so the generated file must not name the template engine that way. --- .../GeneratedTwigCompilationTest.php.tpl | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/scripts/php/templates/GeneratedTwigCompilationTest.php.tpl b/scripts/php/templates/GeneratedTwigCompilationTest.php.tpl index 90e070b..6d2b02e 100644 --- a/scripts/php/templates/GeneratedTwigCompilationTest.php.tpl +++ b/scripts/php/templates/GeneratedTwigCompilationTest.php.tpl @@ -15,15 +15,19 @@ namespace Piwik\Plugins\{{PLUGIN_NAME}}\tests\Integration; use Piwik\Plugin\Manager; use Piwik\Tests\Framework\TestCase\IntegrationTestCase; -use Piwik\Twig; /** - * Compiles every template the plugin ships, so that a template using a core Twig function, - * filter or tag that does not exist yet in the Matomo version under test fails here rather than - * when a user opens the page on an install running that version. + * Compiles every template the plugin ships, so that one using a core template function, filter + * or tag that does not exist yet in the Matomo version under test fails here rather than when a + * user opens the page on an install running that version. * * Only compilation is covered. Undefined variables are a runtime concern and are not reported. * + * Core's tests:check-direct-dependency-use greps plugin files for a vendor namespace preceded by + * a space, and most plugins assert an exact list of the files it matches. Nothing in this file + * may therefore write the template engine's namespace with a space in front of it, which is why + * the environment is built from an inline \Piwik\Twig below rather than an imported short name. + * * @group {{PLUGIN_NAME}} * @group Plugins */ @@ -40,15 +44,14 @@ class GeneratedTwigCompilationTest extends IntegrationTestCase self::markTestSkipped(self::PLUGIN_NAME . ' ships no templates.'); } - // the plugin's own Twig extensions are only registered while it is loaded, and the test - // framework decides what to load from this class' namespace + // the plugin's own template extensions are only registered while it is loaded, and the + // test framework decides what to load from this class' namespace self::assertTrue( Manager::getInstance()->isPluginLoaded(self::PLUGIN_NAME), self::PLUGIN_NAME . ' is not loaded, so this check would not cover the plugin.' ); - $twig = new Twig(); - $environment = $twig->getTwigEnvironment(); + $environment = (new \Piwik\Twig())->getTwigEnvironment(); foreach ($templates as $template) { $environment->load('@' . self::PLUGIN_NAME . '/' . $template); @@ -75,7 +78,7 @@ class GeneratedTwigCompilationTest extends IntegrationTestCase $name = str_replace('\\', '/', substr($file->getPathname(), strlen($templateDir) + 1)); - // templates/plugins// holds theme overrides that Twig registers under the + // templates/plugins// holds theme overrides that are registered under the // other plugin's namespace, so they are not this plugin's to compile if (strpos($name, 'plugins/') === 0) { continue; From 3c611a9bdcfe8c5de70810583b3840862968a34c Mon Sep 17 00:00:00 2001 From: Jacob Ransom Date: Tue, 11 Aug 2026 14:57:40 +1200 Subject: [PATCH 3/4] Drop the redundant plugin-name group from the generated checks The test framework already derives it from the class namespace, and a plugin whose name matches a vendor prefix (GeoIp2, Monolog) would otherwise land in the direct-dependency grep via the annotation. --- scripts/php/templates/GeneratedAssetCompilationTest.php.tpl | 1 - scripts/php/templates/GeneratedTwigCompilationTest.php.tpl | 1 - 2 files changed, 2 deletions(-) diff --git a/scripts/php/templates/GeneratedAssetCompilationTest.php.tpl b/scripts/php/templates/GeneratedAssetCompilationTest.php.tpl index 59bdc8d..3fd647a 100644 --- a/scripts/php/templates/GeneratedAssetCompilationTest.php.tpl +++ b/scripts/php/templates/GeneratedAssetCompilationTest.php.tpl @@ -22,7 +22,6 @@ use Piwik\Tests\Framework\TestCase\IntegrationTestCase; * mixin or variable that does not exist yet in the Matomo version under test fails here rather * than fatally on every page of an install running that version. * - * @group {{PLUGIN_NAME}} * @group Plugins */ class GeneratedAssetCompilationTest extends IntegrationTestCase diff --git a/scripts/php/templates/GeneratedTwigCompilationTest.php.tpl b/scripts/php/templates/GeneratedTwigCompilationTest.php.tpl index 6d2b02e..c25b20d 100644 --- a/scripts/php/templates/GeneratedTwigCompilationTest.php.tpl +++ b/scripts/php/templates/GeneratedTwigCompilationTest.php.tpl @@ -28,7 +28,6 @@ use Piwik\Tests\Framework\TestCase\IntegrationTestCase; * may therefore write the template engine's namespace with a space in front of it, which is why * the environment is built from an inline \Piwik\Twig below rather than an imported short name. * - * @group {{PLUGIN_NAME}} * @group Plugins */ class GeneratedTwigCompilationTest extends IntegrationTestCase From 29d5558b10bbec57b81490ff7f8e1c7368da0171 Mon Sep 17 00:00:00 2001 From: Jacob Ransom Date: Tue, 11 Aug 2026 15:53:06 +1200 Subject: [PATCH 4/4] Compile theme override templates and document skip-generated-checks Overrides under templates/plugins// are reachable through the plugin's own namespace, so they are compiled rather than skipped. --- README.md | 7 +++++++ .../GeneratedTwigCompilationTest.php.tpl | 15 ++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index dfa3d19..1f56111 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,13 @@ This action is able to run certain test suites for Matomo or any Matomo plugin. For security reasons this option should not be provided in plain text, but using a repository secret instead. + * **skip-generated-checks** + + Before running plugin integration tests, this action generates compatibility checks into the plugin's test directory that compile the plugin's stylesheets and templates against the Matomo version under test. That catches a plugin using a core Less mixin or Twig function newer than the minimum Matomo its plugin.json declares, which would otherwise only surface once users on an older Matomo install the release. + + Set this to true to skip generating them. This is an escape hatch, not a way to keep a real incompatibility unfixed: the two valid fixes are to stop using the newer core API, or to raise the required Matomo version in plugin.json. + + * **setup-script** This option can contain the path to a bash script that should be executed before running the tests. diff --git a/scripts/php/templates/GeneratedTwigCompilationTest.php.tpl b/scripts/php/templates/GeneratedTwigCompilationTest.php.tpl index c25b20d..d8be821 100644 --- a/scripts/php/templates/GeneratedTwigCompilationTest.php.tpl +++ b/scripts/php/templates/GeneratedTwigCompilationTest.php.tpl @@ -75,15 +75,12 @@ class GeneratedTwigCompilationTest extends IntegrationTestCase continue; } - $name = str_replace('\\', '/', substr($file->getPathname(), strlen($templateDir) + 1)); - - // templates/plugins// holds theme overrides that are registered under the - // other plugin's namespace, so they are not this plugin's to compile - if (strpos($name, 'plugins/') === 0) { - continue; - } - - $names[] = $name; + // theme overrides under templates/plugins// are included: the plugin's own + // namespace is rooted at templates/, so this reaches the override file itself. Their + // registered namespace is only added for the theme that is currently enabled, which + // the plugin under test is not, so addressing them that way would compile the + // overridden plugin's copy instead of the file shipped here. + $names[] = str_replace('\\', '/', substr($file->getPathname(), strlen($templateDir) + 1)); } sort($names);