return old vtest system
This commit is contained in:
Igor Korsukov
parent
85c969d7b8
commit
bf8372368d
|
|
@ -4,47 +4,239 @@ on:
|
|||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- 4.0.2
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
name: "VTests workflow"
|
||||
name: "Setup VTests workflow"
|
||||
runs-on: ubuntu-20.04
|
||||
outputs:
|
||||
do_run: ${{ steps.output_data.outputs.do_run }}
|
||||
reference_sha: ${{ steps.output_data.outputs.reference_sha }}
|
||||
artifact_name: ${{ steps.output_data.outputs.artifact_name }}
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.11.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- name: Clone repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v3
|
||||
- name: "Configure workflow"
|
||||
env:
|
||||
pull_request_title: ${{ github.event.pull_request.title }}
|
||||
run: |
|
||||
bash ./build/ci/tools/make_build_number.sh
|
||||
BUILD_NUMBER=$(cat ./build.artifacts/env/build_number.env)
|
||||
|
||||
ADD_INFO="_${GITHUB_REF#refs/heads/}"
|
||||
if [ "${{ github.event_name }}" == "pull_request" ]; then ADD_INFO="_${{ github.event.pull_request.number }}_${pull_request_title}"; fi
|
||||
UPLOAD_ARTIFACT_NAME="$(tr '":<>|*?/\\' '_' <<<"VTests_Comparison_${BUILD_NUMBER}_${ADD_INFO}")"
|
||||
sudo bash ./build/ci/tools/make_build_number.sh
|
||||
BUILD_NUMBER=$(cat ./build.artifacts/env/build_number.env)
|
||||
|
||||
echo "BUILD_NUMBER=$BUILD_NUMBER" >> $GITHUB_ENV
|
||||
echo "BUILD_NUMBER: $BUILD_NUMBER"
|
||||
echo "UPLOAD_ARTIFACT_NAME=$UPLOAD_ARTIFACT_NAME" >> $GITHUB_ENV
|
||||
echo "UPLOAD_ARTIFACT_NAME: $UPLOAD_ARTIFACT_NAME"
|
||||
- name: Get reference commit for PR
|
||||
if: github.event_name == 'pull_request'
|
||||
env:
|
||||
pull_request_title: ${{ github.event.pull_request.title }}
|
||||
run: |
|
||||
REFERENCE_SHA=${{ github.event.pull_request.base.sha }}
|
||||
if [ -z "$REFERENCE_SHA" ]; then DO_RUN='false'; else DO_RUN='true'; fi
|
||||
echo "REFERENCE_SHA=$REFERENCE_SHA" >> $GITHUB_ENV
|
||||
echo "DO_RUN=$DO_RUN" >> $GITHUB_ENV
|
||||
echo "PR_INFO= ${{ github.event.pull_request.number }} ${pull_request_title}" >> $GITHUB_ENV
|
||||
- name: Get reference commit for push commit
|
||||
if: github.event_name == 'push'
|
||||
run: |
|
||||
REFERENCE_SHA=$( git show -s --pretty=%P ${{ github.sha }} | head -c 10 )
|
||||
if [ -z "$REFERENCE_SHA" ]; then DO_RUN='false'; else DO_RUN='true'; fi
|
||||
echo "REFERENCE_SHA=$REFERENCE_SHA" >> $GITHUB_ENV
|
||||
echo "DO_RUN=$DO_RUN" >> $GITHUB_ENV
|
||||
echo "PR_INFO=" >> $GITHUB_ENV
|
||||
- id: output_data
|
||||
name: Output workflow data
|
||||
env:
|
||||
pr_info: ${{ env.PR_INFO }}
|
||||
run: |
|
||||
echo "::set-output name=do_run::${{ env.DO_RUN }}"
|
||||
echo "DO_RUN=${{ env.DO_RUN }}"
|
||||
echo "::set-output name=reference_sha::${{ env.REFERENCE_SHA }}"
|
||||
echo "REFERENCE_SHA=${{ env.REFERENCE_SHA }}"
|
||||
UPLOAD_ARTIFACT_NAME="$(tr '":<>|*?/\\' '_' <<<"VTests Comparison ${{ env.BUILD_NUMBER }}${pr_info}")"
|
||||
echo "::set-output name=artifact_name::$UPLOAD_ARTIFACT_NAME"
|
||||
echo "UPLOAD_ARTIFACT_NAME=$UPLOAD_ARTIFACT_NAME"
|
||||
|
||||
generate_current_pngs:
|
||||
name: "Generate current PNGs"
|
||||
runs-on: ubuntu-20.04
|
||||
needs: setup
|
||||
if: needs.setup.outputs.do_run == 'true' # Can't use env: see https://github.com/actions/runner/issues/480
|
||||
steps:
|
||||
- name: Clone repository
|
||||
uses: actions/checkout@v3
|
||||
- name: Ccache cache files
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.ccache
|
||||
key: ${{github.workflow}}-ccache-$(date -u +"%F-%T")
|
||||
restore-keys: ${{github.workflow}}-ccache- # restore ccache from either previous build on this branch or on master
|
||||
- name: Setup ccache
|
||||
run: |
|
||||
bash ./build/ci/tools/setup_ccache.sh
|
||||
- name: Setup environment
|
||||
run: |
|
||||
bash ./build/ci/linux/setup.sh
|
||||
- name: Build
|
||||
run: |
|
||||
source $HOME/build_tools/environment.sh
|
||||
bash ./vtest/build.sh
|
||||
- name: Run test
|
||||
mkdir -p "$HOME/musescore_install"
|
||||
bash ninja_build.sh -t clean
|
||||
MUSESCORE_INSTALL_DIR="$HOME/musescore_install" bash ninja_build.sh -t installdebug
|
||||
- name: Generate current PNGs
|
||||
run: |
|
||||
bash ./vtest/vtest.sh
|
||||
- name: Upload comparison
|
||||
if: ${{ always() }}
|
||||
xvfb-run ./vtest/vtest-generate-pngs.sh -o ./current_pngs -m $HOME/musescore_install/bin/mscore
|
||||
- name: Upload PNGs
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ${{ env.UPLOAD_ARTIFACT_NAME }}
|
||||
path: ./vtest/temp_comparison
|
||||
name: Current PNGs
|
||||
path: ./current_pngs
|
||||
|
||||
generate_reference_pngs:
|
||||
name: "Generate reference PNGs"
|
||||
runs-on: ubuntu-20.04
|
||||
needs: setup
|
||||
if: needs.setup.outputs.do_run == 'true'
|
||||
steps:
|
||||
- name: Clone repository and checkout reference commit
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
ref: ${{ needs.setup.outputs.reference_sha }}
|
||||
- name: Ccache cache files
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.ccache
|
||||
key: ${{github.workflow}}-ccache-$(date -u +"%F-%T")
|
||||
restore-keys: ${{github.workflow}}-ccache- # restore ccache from either previous build on this branch or on master
|
||||
- name: Setup ccache
|
||||
run: |
|
||||
bash ./build/ci/tools/setup_ccache.sh
|
||||
- name: Setup environment
|
||||
run: |
|
||||
bash ./build/ci/linux/setup.sh
|
||||
- name: Build
|
||||
run: |
|
||||
source $HOME/build_tools/environment.sh
|
||||
mkdir -p "$HOME/musescore_install"
|
||||
bash ninja_build.sh -t clean
|
||||
MUSESCORE_INSTALL_DIR="$HOME/musescore_install" bash ninja_build.sh -t installdebug
|
||||
- name: Checkout current commit
|
||||
uses: actions/checkout@v3
|
||||
- name: Generate reference PNGs
|
||||
run: |
|
||||
xvfb-run ./vtest/vtest-generate-pngs.sh -o ./reference_pngs -m $HOME/musescore_install/bin/mscore
|
||||
- name: Upload PNGs
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: Reference PNGs
|
||||
path: ./reference_pngs
|
||||
|
||||
compare_pngs:
|
||||
name: "Compare PNGs"
|
||||
runs-on: ubuntu-20.04
|
||||
needs: [setup, generate_current_pngs, generate_reference_pngs]
|
||||
if: needs.setup.outputs.do_run == 'true'
|
||||
steps:
|
||||
- name: Clone repository
|
||||
uses: actions/checkout@v3
|
||||
- name: Download current PNGs
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: Current PNGs
|
||||
path: ./current_pngs
|
||||
- name: Download reference PNGs
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: Reference PNGs
|
||||
path: ./reference_pngs
|
||||
- name: Compare PNGs
|
||||
run: |
|
||||
echo "VTEST_DIFF_FOUND=false" >> $GITHUB_ENV
|
||||
./vtest/vtest-compare-pngs.sh
|
||||
- name: Upload comparison
|
||||
if: env.VTEST_DIFF_FOUND == 'true'
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ${{ needs.setup.outputs.artifact_name }}
|
||||
path: ./comparison
|
||||
- name: Skip failure signal if PR is labeled 'vtests'
|
||||
if: github.event_name == 'pull_request' && contains( env.VTEST_DIFF_FOUND, 'true') && contains(github.event.pull_request.labels.*.name, 'vtests')
|
||||
run: |
|
||||
echo "This PR appears to change some of the visual tests."
|
||||
echo "Please carefully review the new visual test results in the uploaded artifact that can be found here: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "Failure signal skipped because 'vtests' label is applied to the PR"
|
||||
export VTEST_DIFF_FOUND=false
|
||||
echo "VTEST_DIFF_FOUND=$VTEST_DIFF_FOUND" >> $GITHUB_ENV
|
||||
- name: Emit failure signal for PR if differences are found
|
||||
if: github.event_name == 'pull_request' && contains( env.VTEST_DIFF_FOUND, 'true')
|
||||
run: |
|
||||
echo "This PR appears to change some of the visual tests."
|
||||
echo "Please carefully review the new visual test results in the uploaded artifact that can be found here: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
exit 1
|
||||
- name: Comment push commit
|
||||
if: github.event_name == 'push' && contains( env.VTEST_DIFF_FOUND, 'true')
|
||||
uses: peter-evans/commit-comment@v1.1.0
|
||||
with:
|
||||
body: |
|
||||
This is an automatic message. This commit appears to change some of the visual tests.
|
||||
Please carefully review the new visual test results in the uploaded artifact that can be found
|
||||
[here][1]
|
||||
|
||||
[1]: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
|
||||
|
||||
|
||||
# name: CI_vtests
|
||||
|
||||
# on:
|
||||
# pull_request:
|
||||
# branches:
|
||||
# - master
|
||||
|
||||
# jobs:
|
||||
# setup:
|
||||
# name: "VTests workflow"
|
||||
# runs-on: ubuntu-20.04
|
||||
# steps:
|
||||
# - name: Cancel Previous Runs
|
||||
# uses: styfle/cancel-workflow-action@0.11.0
|
||||
# with:
|
||||
# access_token: ${{ github.token }}
|
||||
# - name: Clone repository
|
||||
# uses: actions/checkout@v3
|
||||
# - name: "Configure workflow"
|
||||
# env:
|
||||
# pull_request_title: ${{ github.event.pull_request.title }}
|
||||
# run: |
|
||||
# bash ./build/ci/tools/make_build_number.sh
|
||||
# BUILD_NUMBER=$(cat ./build.artifacts/env/build_number.env)
|
||||
|
||||
# ADD_INFO="_${GITHUB_REF#refs/heads/}"
|
||||
# if [ "${{ github.event_name }}" == "pull_request" ]; then ADD_INFO="_${{ github.event.pull_request.number }}_${pull_request_title}"; fi
|
||||
# UPLOAD_ARTIFACT_NAME="$(tr '":<>|*?/\\' '_' <<<"VTests_Comparison_${BUILD_NUMBER}_${ADD_INFO}")"
|
||||
|
||||
# echo "BUILD_NUMBER=$BUILD_NUMBER" >> $GITHUB_ENV
|
||||
# echo "BUILD_NUMBER: $BUILD_NUMBER"
|
||||
# echo "UPLOAD_ARTIFACT_NAME=$UPLOAD_ARTIFACT_NAME" >> $GITHUB_ENV
|
||||
# echo "UPLOAD_ARTIFACT_NAME: $UPLOAD_ARTIFACT_NAME"
|
||||
|
||||
# - name: Setup environment
|
||||
# run: |
|
||||
# bash ./build/ci/linux/setup.sh
|
||||
# - name: Build
|
||||
# run: |
|
||||
# source $HOME/build_tools/environment.sh
|
||||
# bash ./vtest/build.sh
|
||||
# - name: Run test
|
||||
# run: |
|
||||
# bash ./vtest/vtest.sh
|
||||
# - name: Upload comparison
|
||||
# if: ${{ always() }}
|
||||
# uses: actions/upload-artifact@v3
|
||||
# with:
|
||||
# name: ${{ env.UPLOAD_ARTIFACT_NAME }}
|
||||
# path: ./vtest/temp_comparison
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
#topbar {
|
||||
border-bottom:1px solid #eb7429;
|
||||
background:#fc9453;
|
||||
padding:10px 20px;
|
||||
position:fixed;
|
||||
top:0;
|
||||
left:0;
|
||||
z-index:2000;
|
||||
width:100%;
|
||||
}
|
||||
|
||||
#topbar span {
|
||||
float:left;
|
||||
width:512px;
|
||||
}
|
||||
|
||||
#topmargin {
|
||||
height:30px;
|
||||
}
|
||||
|
|
@ -0,0 +1,110 @@
|
|||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: GPL-3.0-only
|
||||
# MuseScore-CLA-applies
|
||||
#
|
||||
# MuseScore
|
||||
# Music Composition & Notation
|
||||
#
|
||||
# Copyright (C) 2021 MuseScore BVBA and others
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
echo "MuseScore VTest Compare PNGs"
|
||||
|
||||
HERE="$(dirname ${BASH_SOURCE[0]})"
|
||||
CURRENT_DIR="./current_pngs"
|
||||
REFERENCE_DIR="./reference_pngs"
|
||||
OUTPUT_DIR="./comparison"
|
||||
|
||||
while [[ "$#" -gt 0 ]]; do
|
||||
case $1 in
|
||||
-c|--current-dir) CURRENT_DIR="$2"; shift ;;
|
||||
-r|--reference-dir) REFERENCE_DIR="$2"; shift ;;
|
||||
-o|--output-dir) OUTPUT_DIR="$2"; shift ;;
|
||||
*) echo "Unknown parameter passed: $1"; exit 1 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
echo "::group::Configuration:"
|
||||
echo "CURRENT_DIR: $CURRENT_DIR"
|
||||
echo "REFERENCE_DIR: $REFERENCE_DIR"
|
||||
echo "OUTPUT_DIR: $OUTPUT_DIR"
|
||||
echo "::endgroup::"
|
||||
|
||||
rm -rf $OUTPUT_DIR
|
||||
mkdir $OUTPUT_DIR
|
||||
|
||||
PNG_REF_LIST=$(ls $REFERENCE_DIR/*.png)
|
||||
DIFF_NAME_LIST=""
|
||||
for PNG_REF_FILE in $PNG_REF_LIST ; do
|
||||
png_file_name=$(basename $PNG_REF_FILE)
|
||||
FILE_NAME=${png_file_name%.png}
|
||||
PNG_CUR_FILE=$CURRENT_DIR/${FILE_NAME}.png
|
||||
PNG_DIFF_FILE=$OUTPUT_DIR/${FILE_NAME}.diff.png
|
||||
GIF_DIFF_FILE=$OUTPUT_DIR/${FILE_NAME}.diff.gif
|
||||
|
||||
if test -f $PNG_CUR_FILE; then
|
||||
code=$(compare -metric AE -fuzz 0.0% $PNG_REF_FILE $PNG_CUR_FILE $PNG_DIFF_FILE 2>&1)
|
||||
if (( $code > 0 )); then
|
||||
echo "Different: ref: $PNG_REF_FILE, current: $PNG_CUR_FILE, code: $code"
|
||||
export VTEST_DIFF_FOUND=true
|
||||
echo "VTEST_DIFF_FOUND=$VTEST_DIFF_FOUND" >> $GITHUB_ENV
|
||||
DIFF_NAME_LIST+=" "$FILE_NAME
|
||||
|
||||
cp $PNG_REF_FILE $OUTPUT_DIR/$FILE_NAME.ref.png
|
||||
cp $PNG_CUR_FILE $OUTPUT_DIR
|
||||
|
||||
# generate comparison gif
|
||||
convert -delay 80 -loop 0 $PNG_CUR_FILE $PNG_REF_FILE $GIF_DIFF_FILE
|
||||
else
|
||||
echo "Equal: ref: $PNG_REF_FILE, current: $PNG_CUR_FILE"
|
||||
rm -f $PNG_DIFF_FILE 2>/dev/null
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Generate html report
|
||||
if [ "$VTEST_DIFF_FOUND" == "true" ]; then
|
||||
|
||||
echo "Generate html report"
|
||||
HTML=$OUTPUT_DIR/vtest_compare.html
|
||||
rm -f $HTML
|
||||
cp $HERE/style.css $OUTPUT_DIR
|
||||
echo "<html>" >> $HTML
|
||||
echo " <head>" >> $HTML
|
||||
echo " <link rel=\"stylesheet\" type=\"text/css\" href=\"style.css\">" >> $HTML
|
||||
echo " </head>" >> $HTML
|
||||
echo " <body>" >> $HTML
|
||||
echo " <div id=\"topbar\">" >> $HTML
|
||||
echo " <span>Reference</span>" >> $HTML
|
||||
echo " <span>Current</span>" >> $HTML
|
||||
echo " <span>Diff</span>" >> $HTML
|
||||
echo " </div>" >> $HTML
|
||||
echo " <div id=\"topmargin\"></div>" >> $HTML
|
||||
|
||||
for DIFF_NAME in $DIFF_NAME_LIST ; do
|
||||
echo " <h2 id=\"$DIFF_NAME\">$DIFF_NAME <a class=\"toc-anchor\" href=\"#$DIFF_NAME\">#</a></h2>" >> $HTML
|
||||
echo " <div>" >> $HTML
|
||||
echo " <img src=\"$DIFF_NAME.ref.png\">" >> $HTML
|
||||
echo " <img src=\"$DIFF_NAME.png\">" >> $HTML
|
||||
echo " <img src=\"$DIFF_NAME.diff.png\">" >> $HTML
|
||||
echo " <img src=\"$DIFF_NAME.diff.gif\">" >> $HTML
|
||||
echo " </div>" >> $HTML
|
||||
done
|
||||
|
||||
echo " </body>" >> $HTML
|
||||
echo "</html>" >> $HTML
|
||||
|
||||
else
|
||||
rm -rf $OUTPUT_DIR
|
||||
fi
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: GPL-3.0-only
|
||||
# MuseScore-CLA-applies
|
||||
#
|
||||
# MuseScore
|
||||
# Music Composition & Notation
|
||||
#
|
||||
# Copyright (C) 2021 MuseScore BVBA and others
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License version 3 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
echo "MuseScore VTest Generate PNGs"
|
||||
|
||||
set -o pipefail
|
||||
|
||||
HERE="$(dirname ${BASH_SOURCE[0]})"
|
||||
SCORES_DIR="$HERE/scores"
|
||||
OUTPUT_DIR="./vtest_pngs"
|
||||
MSCORE_BIN=build.debug/install/bin/mscore
|
||||
DPI=180
|
||||
|
||||
while [[ "$#" -gt 0 ]]; do
|
||||
case $1 in
|
||||
-s|--scores) SCORES_DIR="$2"; shift ;;
|
||||
-o|--output-dir) OUTPUT_DIR="$2"; shift ;;
|
||||
-m|--mscore) MSCORE_BIN="$2"; shift ;;
|
||||
*) echo "Unknown parameter passed: $1"; exit 1 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
echo "::group::Configuration:"
|
||||
echo "SCORES_DIR: $SCORES_DIR"
|
||||
echo "OUTPUT_DIR: $OUTPUT_DIR"
|
||||
echo "MSCORE_BIN: $MSCORE_BIN"
|
||||
echo "DPI: $DPI"
|
||||
echo "::endgroup::"
|
||||
|
||||
rm -rf $OUTPUT_DIR
|
||||
mkdir -p $OUTPUT_DIR
|
||||
|
||||
LOG_FILE=$OUTPUT_DIR/convert.log
|
||||
JSON_FILE=$OUTPUT_DIR/vtestjob.json
|
||||
|
||||
echo "::group::Generating JSON job file"
|
||||
echo "[" >> $JSON_FILE
|
||||
SCORES_LIST=$(ls -p $SCORES_DIR | grep -v /)
|
||||
for score in $SCORES_LIST ; do
|
||||
OUT_FILE=$OUTPUT_DIR/${score%.*}.png
|
||||
echo "{ \"in\" : \"$SCORES_DIR/$score\", \"out\" : \"$OUT_FILE\" }," >> $JSON_FILE;
|
||||
done
|
||||
echo "{}]" >> $JSON_FILE
|
||||
cat $JSON_FILE
|
||||
echo "::endgroup::"
|
||||
|
||||
echo "::group::Generating PNG files"
|
||||
$MSCORE_BIN -j $JSON_FILE -r $DPI 2>&1 | tee $LOG_FILE && SUCCESS="true"
|
||||
echo "::endgroup::"
|
||||
|
||||
if [ -z "$SUCCESS" ]; then
|
||||
echo -e "\033[0;31mGenerating PNGs failed!\033[0m"
|
||||
fi
|
||||
|
||||
echo "::group::Generated files:"
|
||||
ls $OUTPUT_DIR
|
||||
echo "::endgroup::"
|
||||
|
||||
if [ -z "$SUCCESS" ]; then
|
||||
exit 1
|
||||
fi
|
||||
Loading…
Reference in New Issue