Generate minimum-Matomo compatibility checks into plugin test runs

README.md

@@ -110,6 +110,13 @@ This action is able to run certain test suites for Matomo or any Matomo plugin.
 
     For security reasons this option should not be provided in plain text, but using a repository secret instead.
 
+  * **skip-generated-checks**
+
+    Before running plugin integration tests, this action generates compatibility checks into the plugin's test directory that compile the plugin's stylesheets and templates against the Matomo version under test. That catches a plugin using a core Less mixin or Twig function newer than the minimum Matomo its plugin.json declares, which would otherwise only surface once users on an older Matomo install the release.
+
+    Set this to true to skip generating them. This is an escape hatch, not a way to keep a real incompatibility unfixed: the two valid fixes are to stop using the newer core API, or to raise the required Matomo version in plugin.json.
+
+
   * **setup-script**
 
     This option can contain the path to a bash script that should be executed before running the tests.

action.yml

@@ -99,6 +99,11 @@ inputs:
     description: "Flag for whether to force uploading a report to Testomat.io instead of just push events. Only applies if the 'testomatio' token is set."
     required: false
     default: false
+  skip-generated-checks:
+    type: boolean
+    description: "If true, the compatibility checks this action generates into the plugin's integration tests are not written. Escape hatch only."
+    required: false
+    default: false
 
 runs:
   using: "composite"
@@ -332,6 +337,17 @@ runs:
       run: sudo apt-get install ripgrep
       working-directory: ${{ github.workspace }}/matomo
 
+    - name: Generate plugin compatibility checks
+      if: inputs.plugin-name != ''
+      working-directory: ${{ github.workspace }}/matomo
+      shell: bash
+      run: ${{ github.action_path }}/scripts/bash/generate_compatibility_checks.sh
+      env:
+        TEST_SUITE: ${{ inputs.test-type }}
+        PLUGIN_NAME: ${{ inputs.plugin-name }}
+        ACTION_PATH: ${{ github.action_path }}
+        SKIP_GENERATED_CHECKS: ${{ inputs.skip-generated-checks }}
+
     - name: Run tests
       working-directory: ${{ github.workspace }}/matomo
       shell: bash

scripts/bash/generate_compatibility_checks.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+GREEN='\033[0;32m'
+YELLOW='\033[0;33m'
+SET='\033[0m'
+
+if [ -z "$PLUGIN_NAME" ]; then
+  exit 0
+fi
+
+if [ "$SKIP_GENERATED_CHECKS" == "true" ]; then
+  echo -e "${YELLOW}skip-generated-checks is set, not generating compatibility checks${SET}"
+  exit 0
+fi
+
+# only these suites include plugins/*/tests/Integration, see tests/PHPUnit/phpunit.xml.dist
+case "$TEST_SUITE" in
+  PluginTests|IntegrationTestsPlugins) ;;
+  *)
+    echo "Test suite '$TEST_SUITE' does not run plugin integration tests, not generating compatibility checks"
+    exit 0
+    ;;
+esac
+
+# same precedence as run_tests.sh uses to pick the directory it hands to phpunit
+if [ -d "plugins/$PLUGIN_NAME/Test" ]; then
+  TARGET_DIR="plugins/$PLUGIN_NAME/Test/Integration"
+elif [ -d "plugins/$PLUGIN_NAME/tests" ]; then
+  TARGET_DIR="plugins/$PLUGIN_NAME/tests/Integration"
+else
+  echo "Plugin $PLUGIN_NAME has no test directory, not generating compatibility checks"
+  exit 0
+fi
+
+mkdir -p "$TARGET_DIR"
+
+for template in "$ACTION_PATH"/scripts/php/templates/Generated*.php.tpl; do
+  target="$TARGET_DIR/$(basename "$template" .tpl)"
+
+  sed "s/{{PLUGIN_NAME}}/$PLUGIN_NAME/g" "$template" > "$target"
+
+  echo -e "${GREEN}Generated $target${SET}"
+  cat "$target"
+done

scripts/php/templates/GeneratedAssetCompilationTest.php.tpl

@@ -0,0 +1,46 @@
+<?php
+
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link    https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ * GENERATED FILE - do not commit this to the plugin repository.
+ * It is written into the plugin checkout during CI by matomo-org/github-action-tests.
+ * To change it, edit scripts/php/templates/GeneratedAssetCompilationTest.php.tpl in that repository.
+ */
+
+namespace Piwik\Plugins\{{PLUGIN_NAME}}\tests\Integration;
+
+use Piwik\AssetManager;
+use Piwik\Plugin\Manager;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+
+/**
+ * Compiles the merged stylesheet with the plugin loaded, so that a plugin using a core Less
+ * mixin or variable that does not exist yet in the Matomo version under test fails here rather
+ * than fatally on every page of an install running that version.
+ *
+ * @group Plugins
+ */
+class GeneratedAssetCompilationTest extends IntegrationTestCase
+{
+    private const PLUGIN_NAME = '{{PLUGIN_NAME}}';
+
+    public function testStylesheetsCompileAgainstTheMatomoVersionUnderTest()
+    {
+        // the test framework decides which plugin to load from this class' namespace, so a
+        // wrongly generated namespace would leave the plugin out and make the merge below pass
+        // no matter what the plugin ships
+        self::assertTrue(
+            Manager::getInstance()->isPluginLoaded(self::PLUGIN_NAME),
+            self::PLUGIN_NAME . ' is not loaded, so this check would not cover the plugin.'
+        );
+
+        $assetManager = AssetManager::getInstance();
+        $assetManager->removeMergedAssets();
+
+        self::assertNotEmpty($assetManager->getMergedStylesheet()->getContent());
+    }
+}

scripts/php/templates/GeneratedTwigCompilationTest.php.tpl

@@ -0,0 +1,90 @@
+<?php
+
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link    https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ * GENERATED FILE - do not commit this to the plugin repository.
+ * It is written into the plugin checkout during CI by matomo-org/github-action-tests.
+ * To change it, edit scripts/php/templates/GeneratedTwigCompilationTest.php.tpl in that repository.
+ */
+
+namespace Piwik\Plugins\{{PLUGIN_NAME}}\tests\Integration;
+
+use Piwik\Plugin\Manager;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+
+/**
+ * Compiles every template the plugin ships, so that one using a core template function, filter
+ * or tag that does not exist yet in the Matomo version under test fails here rather than when a
+ * user opens the page on an install running that version.
+ *
+ * Only compilation is covered. Undefined variables are a runtime concern and are not reported.
+ *
+ * Core's tests:check-direct-dependency-use greps plugin files for a vendor namespace preceded by
+ * a space, and most plugins assert an exact list of the files it matches. Nothing in this file
+ * may therefore write the template engine's namespace with a space in front of it, which is why
+ * the environment is built from an inline \Piwik\Twig below rather than an imported short name.
+ *
+ * @group Plugins
+ */
+class GeneratedTwigCompilationTest extends IntegrationTestCase
+{
+    private const PLUGIN_NAME = '{{PLUGIN_NAME}}';
+
+    public function testTemplatesCompileAgainstTheMatomoVersionUnderTest()
+    {
+        $templateDir = PIWIK_DOCUMENT_ROOT . '/plugins/' . self::PLUGIN_NAME . '/templates';
+        $templates   = is_dir($templateDir) ? $this->getTemplateNames($templateDir) : [];
+
+        if (empty($templates)) {
+            self::markTestSkipped(self::PLUGIN_NAME . ' ships no templates.');
+        }
+
+        // the plugin's own template extensions are only registered while it is loaded, and the
+        // test framework decides what to load from this class' namespace
+        self::assertTrue(
+            Manager::getInstance()->isPluginLoaded(self::PLUGIN_NAME),
+            self::PLUGIN_NAME . ' is not loaded, so this check would not cover the plugin.'
+        );
+
+        $environment = (new \Piwik\Twig())->getTwigEnvironment();
+
+        foreach ($templates as $template) {
+            $environment->load('@' . self::PLUGIN_NAME . '/' . $template);
+        }
+
+        self::assertNotEmpty($templates);
+    }
+
+    /**
+     * @return string[] template names relative to the plugin's templates directory
+     */
+    private function getTemplateNames(string $templateDir): array
+    {
+        $iterator = new \RecursiveIteratorIterator(
+            new \RecursiveDirectoryIterator($templateDir, \FilesystemIterator::SKIP_DOTS)
+        );
+
+        $names = [];
+
+        foreach ($iterator as $file) {
+            if ($file->getExtension() !== 'twig') {
+                continue;
+            }
+
+            // theme overrides under templates/plugins/<Other>/ are included: the plugin's own
+            // namespace is rooted at templates/, so this reaches the override file itself. Their
+            // registered namespace is only added for the theme that is currently enabled, which
+            // the plugin under test is not, so addressing them that way would compile the
+            // overridden plugin's copy instead of the file shipped here.
+            $names[] = str_replace('\\', '/', substr($file->getPathname(), strlen($templateDir) + 1));
+        }
+
+        sort($names);
+
+        return $names;
+    }
+}