Run vtests on PRs and commits to master branch and compare to parent commit vtest results; output a comment and upload a report when differences are found

.github/workflows/vtests.yml

@@ -0,0 +1,117 @@
+name: CI_vtests
+
+on:
+  pull_request:
+    branches:
+    - master
+  push:
+    branches:
+    - master
+
+jobs:
+  run_vtests:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Clone repository
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+    - name: Retrieve base commit for PR
+      if: github.event_name == 'pull_request'
+      run: |
+        export BASEREF=${{ github.event.pull_request.base.sha }}
+        if [ -z "$BASEREF" ]; then
+        export found=0
+        else
+        export found=1
+        fi
+        echo "::set-env name=BASEREF::${BASEREF}"
+        echo "::set-env name=found::${found}"
+    - name: Retrieve base commit for push commit
+      if: github.event_name == 'push'
+      run: |
+        export BASEREF=$( git show -s --pretty=%P ${{ github.sha }} | head -c 10 )
+        if [ -z "$BASEREF" ]; then
+        export found=0
+        else
+        export found=1
+        fi
+        echo "::set-env name=BASEREF::${BASEREF}"
+        echo "::set-env name=found::${found}"
+    - name: Install dependencies
+      if: contains( env.found, '1')
+      run: |
+        sudo apt-get update
+        sudo apt-get install libasound2-dev portaudio19-dev libmp3lame-dev libsndfile1-dev libportmidi-dev
+        sudo apt-get install libssl-dev libpulse-dev libfreetype6-dev libfreetype6
+        sudo apt-get install libdrm-dev libgl1-mesa-dev libegl1-mesa-dev
+    - name: Setup the environment
+      if: contains( env.found, '1')
+      run: |
+        sed -i 's/travis_retry//g' build/travis/job1_Tests/environment.sh
+        source build/travis/job1_Tests/environment.sh
+        echo "::set-env name=PATH::${PATH}"
+        echo "::set-env name=QT_PLUGIN_PATH::${QT_PLUGIN_PATH}"
+        echo "::set-env name=QML2_IMPORT_PATH::${QML2_IMPORT_PATH}"
+    - name: Build
+      if: contains( env.found, '1')
+      run: |
+        make installdebug CPUS=2 PREFIX="$HOME/software" COVERAGE=ON
+    - name: Build reference branch
+      if: contains( env.found, '1')
+      run: |
+        mkdir -p "$HOME/softwarebase"
+        git checkout $BASEREF
+        make clean
+        make installdebug CPUS=2 PREFIX="$HOME/softwarebase" COVERAGE=ON 2>/dev/null
+    - name: Run reference vtests
+      if: contains( env.found, '1')
+      run: |
+        cd vtest
+        export VTEST_MSCORE="$HOME/softwarebase/bin/mscore"
+        export VTEST_BROWSER=ls
+        export VTEST_DIR=$(pwd)
+        xvfb-run ./gen
+    - name: Run vtests
+      if: contains( env.found, '1')
+      run: |
+        git checkout -
+        cd vtest
+        mkdir -p compare
+        find ./html -name "*-1.png" -exec mv '{}' ./compare \;
+        rm -rf html 2>/dev/null
+        export VTEST_MSCORE="$HOME/software/bin/mscore"
+        export VTEST_BROWSER=ls
+        export VTEST_DIR=$(pwd)
+        xvfb-run ./gen
+    - name: Compare vtests
+      if: contains( env.found, '1')
+      run: |
+        cd vtest
+        export VTEST_DIR=$(pwd)
+        ./gen_compare
+    - name: Upload artifact
+      if: contains( env.found, '1') && contains( env.VTEST_DIFF_FOUND, 'true')
+      uses: actions/upload-artifact@v1
+      with:
+        name: compare
+        path: ./vtest/compare
+    - name: Comment PR
+      if: github.event_name == 'pull_request' && contains( env.found, '1') && contains( env.VTEST_DIFF_FOUND, 'true')
+      uses: thollander/actions-comment-pull-request@1.0.0
+      with:
+        message: 'This is an automatic message. This PR appears to change some of the visual tests.
+          Please carefully review the new visual test results in the uploaded artifact that can be found
+          [here](https://github.com/musescore/MuseScore/actions/runs/${{ github.run_id }})'
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Comment push commit
+      if: github.event_name == 'push' && contains( env.found, '1') && contains( env.VTEST_DIFF_FOUND, 'true')
+      uses: peter-evans/commit-comment@v1.1.0
+      with:
+        body: |
+          This is an automatic message. This commit appears to change some of the visual tests.
+          Please carefully review the new visual test results in the uploaded artifact that can be found
+          [here][1]
+          
+          [1]: https://github.com/musescore/MuseScore/actions/runs/${{ github.run_id }}
+        token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -32,6 +32,7 @@ CMakeLists.txt.user.*
 *.qmlc
 *.jsc
 vtest/html
+vtest/compare
 thirdparty/mupdf-qt
 vtest/LOG
 vtest/META-INF

vtest/gen

@@ -78,7 +78,13 @@ fi
 
 DPI=130
 
-DIR="$(cd "$(dirname "$0")" && pwd)"
+if [ -n "$VTEST_DIR" ]; then
+# SRC env variable needed for Github action vtests
+      echo "::set-env name=SRC::${SRC}"
+      DIR="$VTEST_DIR"
+else
+      DIR="$(cd "$(dirname "$0")" && pwd)"
+fi
 mkdir -p $DIR/html
 cd $DIR/html
 rm -f *.png

vtest/gen_compare

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+echo "Run vtest comparison"
+
+# Create "compare" folder
+if [ -n "$VTEST_DIR" ]; then
+      DIR="$VTEST_DIR"
+else
+      DIR="$(cd "$(dirname "$0")" && pwd)"
+fi
+mkdir -p $DIR/compare
+cd $DIR/compare
+
+# rm -f *.png
+# Download reference images
+# wget -nd -r -P . -A png http://vtest.musescore.org/${BASEREF}/vtest.html
+# rm -f *-ref.png
+# rm -f *-diff.png
+
+# This flag will become true only if differences are found
+export VTEST_DIFF_FOUND=false
+
+# Generate first part of the html report
+F=vtest_compare.html
+rm -f $F
+cp ../style.css .
+echo "<html>" >> $F
+echo "  <head>" >> $F
+echo "   <link rel=\"stylesheet\" type=\"text/css\" href=\"style.css\">" >> $F
+echo "  </head>" >> $F
+echo "  <body>" >> $F
+echo "    <div id=\"topbar\">" >> $F
+echo "      <span>Current</span>" >> $F
+echo "      <span>Reference</span>" >> $F
+echo "      <span>Comparison</span>" >> $F
+echo "    </div>" >> $F
+echo "    <div id=\"topmargin\"></div>" >> $F
+
+# Compare current PNG results and reference files from parent commit
+echo "Compare PNG files and references"
+for src in $SRC; do
+      if test -f ../html/$src-1.png; then
+            LOG=$(compare -metric AE -fuzz 0.0% ../html/$src-1.png $src-1.png $src-diff.png 2>&1)
+            if (( $LOG > 0 )); then
+                  echo "Comparison of $src gave result $LOG"
+                  mv $src-1.png $src-ref.png 2>/dev/null
+                  cp ../html/$src-1.png . 2>/dev/null
+                  export VTEST_DIFF_FOUND=true
+                  echo "    <h2 id=\"$src\">$src <a class=\"toc-anchor\" href=\"#$src\">#</a></h2>" >> $F
+                  echo "    <div>" >> $F
+                  echo "      <img src=\"$src-1.png\">" >> $F
+                  echo "      <img src=\"$src-ref.png\">" >> $F
+                  echo "      <img src=\"$src-diff.png\">" >> $F
+                  echo "    </div>" >> $F
+            else
+                  rm $src-diff.png 2>/dev/null
+                  rm $src-1.png 2>/dev/null
+                  fi
+      else
+            echo "There were errors generating image $src"
+            export VTEST_DIFF_FOUND=true
+            echo "    <h2 id=\"$src\">$src <a class=\"toc-anchor\" href=\"#$src\">#</a></h2>" >> $F
+            echo "    <div>" >> $F
+            echo "      <img src=\"$src-1.png\">" >> $F
+            echo "      <img src=\"$src-ref.png\">" >> $F
+            echo "      <img src=\"$src-diff.png\">" >> $F
+            echo "    </div>" >> $F
+            fi
+      done
+# Close html file tags
+echo "  </body>" >> $F
+echo "</html>" >> $F
+
+# Remove "compare" folder if no differences were detected
+cd ..
+if [ "$VTEST_DIFF_FOUND" = false ]; then
+      rm -rf ./compare
+fi
+echo "::set-env name=VTEST_DIFF_FOUND::${VTEST_DIFF_FOUND}"

vtest/style.css

@@ -15,5 +15,5 @@
 }
 
 #topmargin {
-  height:20px; 
+  height:30px; 
 }