Run vtests on PRs and commits to master branch and compare to parent commit vtest results; output a comment and upload a report when differences are found
This commit is contained in:
AntonioBL
parent
66c38772dc
commit
b63eb79610
117
.github/workflows/vtests.yml
vendored
Normal file
117
.github/workflows/vtests.yml
vendored
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
name: CI_vtests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
|
||||
jobs:
|
||||
run_vtests:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Clone repository
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Retrieve base commit for PR
|
||||
if: github.event_name == 'pull_request'
|
||||
run: |
|
||||
export BASEREF=${{ github.event.pull_request.base.sha }}
|
||||
if [ -z "$BASEREF" ]; then
|
||||
export found=0
|
||||
else
|
||||
export found=1
|
||||
fi
|
||||
echo "::set-env name=BASEREF::${BASEREF}"
|
||||
echo "::set-env name=found::${found}"
|
||||
- name: Retrieve base commit for push commit
|
||||
if: github.event_name == 'push'
|
||||
run: |
|
||||
export BASEREF=$( git show -s --pretty=%P ${{ github.sha }} | head -c 10 )
|
||||
if [ -z "$BASEREF" ]; then
|
||||
export found=0
|
||||
else
|
||||
export found=1
|
||||
fi
|
||||
echo "::set-env name=BASEREF::${BASEREF}"
|
||||
echo "::set-env name=found::${found}"
|
||||
- name: Install dependencies
|
||||
if: contains( env.found, '1')
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install libasound2-dev portaudio19-dev libmp3lame-dev libsndfile1-dev libportmidi-dev
|
||||
sudo apt-get install libssl-dev libpulse-dev libfreetype6-dev libfreetype6
|
||||
sudo apt-get install libdrm-dev libgl1-mesa-dev libegl1-mesa-dev
|
||||
- name: Setup the environment
|
||||
if: contains( env.found, '1')
|
||||
run: |
|
||||
sed -i 's/travis_retry//g' build/travis/job1_Tests/environment.sh
|
||||
source build/travis/job1_Tests/environment.sh
|
||||
echo "::set-env name=PATH::${PATH}"
|
||||
echo "::set-env name=QT_PLUGIN_PATH::${QT_PLUGIN_PATH}"
|
||||
echo "::set-env name=QML2_IMPORT_PATH::${QML2_IMPORT_PATH}"
|
||||
- name: Build
|
||||
if: contains( env.found, '1')
|
||||
run: |
|
||||
make installdebug CPUS=2 PREFIX="$HOME/software" COVERAGE=ON
|
||||
- name: Build reference branch
|
||||
if: contains( env.found, '1')
|
||||
run: |
|
||||
mkdir -p "$HOME/softwarebase"
|
||||
git checkout $BASEREF
|
||||
make clean
|
||||
make installdebug CPUS=2 PREFIX="$HOME/softwarebase" COVERAGE=ON 2>/dev/null
|
||||
- name: Run reference vtests
|
||||
if: contains( env.found, '1')
|
||||
run: |
|
||||
cd vtest
|
||||
export VTEST_MSCORE="$HOME/softwarebase/bin/mscore"
|
||||
export VTEST_BROWSER=ls
|
||||
export VTEST_DIR=$(pwd)
|
||||
xvfb-run ./gen
|
||||
- name: Run vtests
|
||||
if: contains( env.found, '1')
|
||||
run: |
|
||||
git checkout -
|
||||
cd vtest
|
||||
mkdir -p compare
|
||||
find ./html -name "*-1.png" -exec mv '{}' ./compare \;
|
||||
rm -rf html 2>/dev/null
|
||||
export VTEST_MSCORE="$HOME/software/bin/mscore"
|
||||
export VTEST_BROWSER=ls
|
||||
export VTEST_DIR=$(pwd)
|
||||
xvfb-run ./gen
|
||||
- name: Compare vtests
|
||||
if: contains( env.found, '1')
|
||||
run: |
|
||||
cd vtest
|
||||
export VTEST_DIR=$(pwd)
|
||||
./gen_compare
|
||||
- name: Upload artifact
|
||||
if: contains( env.found, '1') && contains( env.VTEST_DIFF_FOUND, 'true')
|
||||
uses: actions/upload-artifact@v1
|
||||
with:
|
||||
name: compare
|
||||
path: ./vtest/compare
|
||||
- name: Comment PR
|
||||
if: github.event_name == 'pull_request' && contains( env.found, '1') && contains( env.VTEST_DIFF_FOUND, 'true')
|
||||
uses: thollander/actions-comment-pull-request@1.0.0
|
||||
with:
|
||||
message: 'This is an automatic message. This PR appears to change some of the visual tests.
|
||||
Please carefully review the new visual test results in the uploaded artifact that can be found
|
||||
[here](https://github.com/musescore/MuseScore/actions/runs/${{ github.run_id }})'
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Comment push commit
|
||||
if: github.event_name == 'push' && contains( env.found, '1') && contains( env.VTEST_DIFF_FOUND, 'true')
|
||||
uses: peter-evans/commit-comment@v1.1.0
|
||||
with:
|
||||
body: |
|
||||
This is an automatic message. This commit appears to change some of the visual tests.
|
||||
Please carefully review the new visual test results in the uploaded artifact that can be found
|
||||
[here][1]
|
||||
|
||||
[1]: https://github.com/musescore/MuseScore/actions/runs/${{ github.run_id }}
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -32,6 +32,7 @@ CMakeLists.txt.user.*
|
|||
*.qmlc
|
||||
*.jsc
|
||||
vtest/html
|
||||
vtest/compare
|
||||
thirdparty/mupdf-qt
|
||||
vtest/LOG
|
||||
vtest/META-INF
|
||||
|
|
|
|||
|
|
@ -78,7 +78,13 @@ fi
|
|||
|
||||
DPI=130
|
||||
|
||||
DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
if [ -n "$VTEST_DIR" ]; then
|
||||
# SRC env variable needed for Github action vtests
|
||||
echo "::set-env name=SRC::${SRC}"
|
||||
DIR="$VTEST_DIR"
|
||||
else
|
||||
DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
fi
|
||||
mkdir -p $DIR/html
|
||||
cd $DIR/html
|
||||
rm -f *.png
|
||||
|
|
|
|||
79
vtest/gen_compare
Executable file
79
vtest/gen_compare
Executable file
|
|
@ -0,0 +1,79 @@
|
|||
#!/bin/bash
|
||||
|
||||
echo "Run vtest comparison"
|
||||
|
||||
# Create "compare" folder
|
||||
if [ -n "$VTEST_DIR" ]; then
|
||||
DIR="$VTEST_DIR"
|
||||
else
|
||||
DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
fi
|
||||
mkdir -p $DIR/compare
|
||||
cd $DIR/compare
|
||||
|
||||
# rm -f *.png
|
||||
# Download reference images
|
||||
# wget -nd -r -P . -A png http://vtest.musescore.org/${BASEREF}/vtest.html
|
||||
# rm -f *-ref.png
|
||||
# rm -f *-diff.png
|
||||
|
||||
# This flag will become true only if differences are found
|
||||
export VTEST_DIFF_FOUND=false
|
||||
|
||||
# Generate first part of the html report
|
||||
F=vtest_compare.html
|
||||
rm -f $F
|
||||
cp ../style.css .
|
||||
echo "<html>" >> $F
|
||||
echo " <head>" >> $F
|
||||
echo " <link rel=\"stylesheet\" type=\"text/css\" href=\"style.css\">" >> $F
|
||||
echo " </head>" >> $F
|
||||
echo " <body>" >> $F
|
||||
echo " <div id=\"topbar\">" >> $F
|
||||
echo " <span>Current</span>" >> $F
|
||||
echo " <span>Reference</span>" >> $F
|
||||
echo " <span>Comparison</span>" >> $F
|
||||
echo " </div>" >> $F
|
||||
echo " <div id=\"topmargin\"></div>" >> $F
|
||||
|
||||
# Compare current PNG results and reference files from parent commit
|
||||
echo "Compare PNG files and references"
|
||||
for src in $SRC; do
|
||||
if test -f ../html/$src-1.png; then
|
||||
LOG=$(compare -metric AE -fuzz 0.0% ../html/$src-1.png $src-1.png $src-diff.png 2>&1)
|
||||
if (( $LOG > 0 )); then
|
||||
echo "Comparison of $src gave result $LOG"
|
||||
mv $src-1.png $src-ref.png 2>/dev/null
|
||||
cp ../html/$src-1.png . 2>/dev/null
|
||||
export VTEST_DIFF_FOUND=true
|
||||
echo " <h2 id=\"$src\">$src <a class=\"toc-anchor\" href=\"#$src\">#</a></h2>" >> $F
|
||||
echo " <div>" >> $F
|
||||
echo " <img src=\"$src-1.png\">" >> $F
|
||||
echo " <img src=\"$src-ref.png\">" >> $F
|
||||
echo " <img src=\"$src-diff.png\">" >> $F
|
||||
echo " </div>" >> $F
|
||||
else
|
||||
rm $src-diff.png 2>/dev/null
|
||||
rm $src-1.png 2>/dev/null
|
||||
fi
|
||||
else
|
||||
echo "There were errors generating image $src"
|
||||
export VTEST_DIFF_FOUND=true
|
||||
echo " <h2 id=\"$src\">$src <a class=\"toc-anchor\" href=\"#$src\">#</a></h2>" >> $F
|
||||
echo " <div>" >> $F
|
||||
echo " <img src=\"$src-1.png\">" >> $F
|
||||
echo " <img src=\"$src-ref.png\">" >> $F
|
||||
echo " <img src=\"$src-diff.png\">" >> $F
|
||||
echo " </div>" >> $F
|
||||
fi
|
||||
done
|
||||
# Close html file tags
|
||||
echo " </body>" >> $F
|
||||
echo "</html>" >> $F
|
||||
|
||||
# Remove "compare" folder if no differences were detected
|
||||
cd ..
|
||||
if [ "$VTEST_DIFF_FOUND" = false ]; then
|
||||
rm -rf ./compare
|
||||
fi
|
||||
echo "::set-env name=VTEST_DIFF_FOUND::${VTEST_DIFF_FOUND}"
|
||||
|
|
@ -15,5 +15,5 @@
|
|||
}
|
||||
|
||||
#topmargin {
|
||||
height:20px;
|
||||
height:30px;
|
||||
}
|
||||
Loading…
Reference in a new issue