[TOOLS][AYY1] Improve accesibility testing to save images and compare the differences against a reference (tests/screenshots/

Makefile

@@ -37,7 +37,7 @@ database-force-schema-update:
 	docker exec -it $(call translate-container-name,$(strip $(DIR))_php_1) sh -c "/var/www/social/bin/console doctrine:schema:update --dump-sql --force"
 
 tooling-docker: .PHONY
-	@cd docker/tooling && docker-compose up -d > /dev/null 2>&1
+	@cd docker/tooling && docker-compose up -d --build > /dev/null 2>&1
 
 stop-tooling: .PHONY
 	cd docker/tooling && docker-compose down
@@ -46,7 +46,7 @@ tooling-php-shell: tooling-docker
 	docker exec -it $(call translate-container-name,tooling_php_1) sh
 
 test-accesibility: tooling-docker
-	cd docker/tooling && docker-compose run pa11y /usr/local/bin/pa11y-ci -c /pa11y/config.json
+	cd docker/tooling && docker-compose run pa11y /accessibility.sh
 
 test: tooling-docker
 	docker exec $(call translate-container-name,tooling_php_1) /var/tooling/coverage.sh $(call args,'')

docker/tooling/accessibility.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+rm -rf /screenshots/diff
+mv -fn /screenshots/new /screenshots/old
+mkdir -p /screenshots/diff
+mkdir -p /screenshots/new
+chmod 777 -R /screenshots
+
+/generate_pa11y-ci-config.php
+
+su puppet -c '/usr/local/bin/pa11y-ci -c /pa11y/config.json'
+
+cd /screenshots/new || exit 1
+
+for f in *; do
+    XC=$(compare -metric NCC "/screenshots/old/${f}" "${f}" "/screenshots/diff/${f}" 2>&1)
+    if [ 1 -eq "$(echo "${XC} < 0.999" | bc)" ]; then
+        printf '\e[33mCheck file for differences: \e]8;;%s\e\\%s\e]8;;\e\\\e[0m\n' "file:tests/screenshots/diff/${f}" "tests/screenshots/diff/${f}"
+    fi
+done

docker/tooling/docker-compose.yaml

@@ -13,7 +13,7 @@ services:
             - ../social/install.sh:/var/entrypoint.d/0_social_install.sh
             - ./coverage.sh:/var/tooling/coverage.sh
             - ./phpstan.sh:/var/tooling/phpstan.sh
-            - ./acceptance_and_accessibility.sh:/var/tooling/acceptance_and_accessibility.sh
+            - ./acceptance.sh:/var/tooling/acceptance.sh
             # Main files
             - ../../:/var/www/social
             - /var/www/social/docker # exclude docker folder
@@ -47,7 +47,10 @@ services:
         depends_on:
             - nginx
         volumes:
-          - ../../tests/pa11y-ci-config.json:/pa11y/config.json
+          - ../../tests/screenshots:/screenshots
+          - ./accessibility.sh:/accessibility.sh
+          - ./generate_pa11y-ci-config.php:/generate_pa11y-ci-config.php
+          - /pa11y
         cap_add:
           - SYS_ADMIN
 

docker/tooling/generate_pa11y-ci-config.php

@@ -0,0 +1,70 @@
+#!/usr/bin/env php
+<?php
+
+declare(strict_types = 1);
+
+$urls = [];
+foreach ([[360, 640, true], [1280, 720, true], [1280, 720, false], [2560, 1080, false]] as $viewport) {
+    [$x, $y, $is_mobile] = $viewport;
+    $gen = function (string $url, string $actions = "") use ($x, $y, $is_mobile) {
+        $path = "/screenshots/new/{$x}x{$y}" . ($is_mobile ? '-mobile' : '') . '-' . ($url === '' ? 'root' : str_replace('/', '-', $url)) . ".png";
+        $is_mobile = $is_mobile ? 'true' : 'false';
+        return <<<EOU
+        {
+            "url": "https://nginx/{$url}",
+            "screenCapture": "{$path}",
+            "viewport": {
+                "width": {$x},
+                "height": {$y},
+                "isMobile": {$is_mobile}
+            }{$actions}
+        }
+EOU;
+    };
+
+    foreach ([
+        '', 'feed/public',
+        'doc/faq', 'doc/tos', 'doc/privacy', 'doc/source', 'doc/version',
+        'main/login', 'main/register',
+    ] as $url) {
+        $urls[] = $gen($url);
+    }
+
+    $urls[] = $gen('main/login', <<<EOA
+,
+            "actions": [
+                "navigate to https://nginx/main/login",
+                "set field #inputNicknameOrEmail to taken_user",
+                "set field #inputPassword to foobar",
+                "click element #signIn",
+                "wait for path to not be /login"
+            ]
+EOA);
+
+    foreach (['feed/public', 'feed/home', '@taken_user/circles',
+              'feed/network', 'feed/clique', 'feed/federated', 'feed/notifications',
+              '@taken_user/collections', '@taken_user/favourites', '@taken_user/reverse_favourites',
+              'directory/people', 'directory/groups', 'settings', 'main/logout'
+    ] as $url) {
+        $urls[] = $gen($url);
+    }
+}
+
+$urls = implode(",\n", $urls);
+$config = <<<EOF
+{
+    "defaults": {
+        "chromeLaunchConfig": {
+            "ignoreHTTPSErrors": true
+        },
+        "standard": "WCAG2AAA",
+        "timeout": 10000
+    },
+    "concurrency": 4,
+    "urls": [
+{$urls}
+    ]
+}
+EOF;
+
+file_put_contents('/pa11y/config.json', $config);

docker/tooling/pa11y/Dockerfile

@@ -9,10 +9,9 @@ RUN apt-get update && apt-get install -y \
     libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 libpangocairo-1.0-0 \
     libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 \
     libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 \
-    libxtst6 locales lsb-release unzip xdg-utils wget \
+    libxtst6 locales lsb-release unzip xdg-utils wget imagemagick php bc \
     && apt-get clean \
     && apt-get autoremove -q \
     && npm install -g pa11y-ci
 
 RUN useradd -ms /bin/bash puppet
-USER puppet

tests/pa11y-ci-config.json

@@ -1,48 +0,0 @@
-{
-    "defaults": {
-        "chromeLaunchConfig": {
-            "ignoreHTTPSErrors": true
-        },
-        "standard": "WCAG2AAA",
-        "timeout": 10000,
-        "viewport": {
-            "width": 1920,
-            "height": 1080
-        }
-    },
-    "concurrency": 4,
-    "urls": [
-        "https://nginx/",
-        "https://nginx/feed/public",
-        "https://nginx/doc/faq",
-        "https://nginx/doc/tos",
-        "https://nginx/doc/privacy",
-        "https://nginx/doc/source",
-        "https://nginx/doc/version",
-        "https://nginx/main/login",
-        "https://nginx/main/register",
-        {
-            "url": "http://nginx/main/login",
-            "actions": [
-                "navigate to https://nginx/main/login",
-                "set field #inputNicknameOrEmail to taken_user",
-                "set field #inputPassword to foobar",
-                "click element #signIn",
-                "wait for path to not be /login"
-            ]
-        },
-        "https://nginx/feed/public",
-        "https://nginx/feed/home",
-        "https://nginx/@taken_user/circles",
-        "https://nginx/feed/network",
-        "https://nginx/feed/clique",
-        "https://nginx/feed/federated",
-        "https://nginx/feed/notifications",
-        "https://nginx/@taken_user/collections",
-        "https://nginx/@taken_user/favourites",
-        "https://nginx/@taken_user/reverse_favourites",
-        "https://nginx/directory/people",
-        "https://nginx/directory/groups",
-        "https://nginx/settings"
-    ]
-}

tests/screenshots/.gitignore

@@ -0,0 +1 @@
+*