From b633d54d2c51935c49cb463dc491023f1120a710 Mon Sep 17 00:00:00 2001 From: John Barker Date: Mon, 8 Feb 2021 10:55:29 +0000 Subject: [PATCH] Tidy up files --- .azure-pipelines/azure-pipelines.yml | 6 -- tests/utils/gitlab/gitlab.sh | 9 ++- tests/utils/gitlab/integration.sh | 7 ++- tests/utils/shippable/shippable.sh | 87 +++++++++++++++++++++++++++- 4 files changed, 96 insertions(+), 13 deletions(-) diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml index 7e57a54..f52b72b 100644 --- a/.azure-pipelines/azure-pipelines.yml +++ b/.azure-pipelines/azure-pipelines.yml @@ -80,8 +80,6 @@ stages: nameFormat: Python 3.8 {0} testFormat: devel/hcloud/3.8/{0} targets: - - test: '' - groups: - 1 - 2 - stage: Hetzner_2_10 @@ -93,8 +91,6 @@ stages: nameFormat: Python 3.8 {0} testFormat: 2.10/hcloud/3.8/{0} targets: - - test: '' - groups: - 1 - 2 - 3 @@ -107,8 +103,6 @@ stages: nameFormat: Python 3.8 {0} testFormat: 2.9/hcloud/3.8/{0} targets: - - test: '' - groups: - 1 - 2 - 3 diff --git a/tests/utils/gitlab/gitlab.sh b/tests/utils/gitlab/gitlab.sh index 33c3af8..2ae1c41 100755 --- a/tests/utils/gitlab/gitlab.sh +++ b/tests/utils/gitlab/gitlab.sh @@ -7,6 +7,7 @@ declare -a args IFS='/:' read -ra args <<< "$1" ansible_version="${args[0]}" +# shellcheck disable=SC2034 script="${args[1]}" function join { @@ -21,6 +22,7 @@ python -V function retry { + # shellcheck disable=SC2034 for repetition in 1 2 3; do set +e "$@" @@ -29,9 +31,9 @@ function retry if [ ${result} == 0 ]; then return ${result} fi - echo "$@ -> ${result}" + echo "@* -> ${result}" done - echo "Command '$@' failed 3 times!" + echo "Command '@*' failed 3 times!" exit -1 } @@ -44,6 +46,7 @@ else retry pip install "https://github.com/ansible/ansible/archive/stable-${ansible_version}.tar.gz" --disable-pip-version-check fi export ANSIBLE_COLLECTIONS_PATHS="${HOME}/.ansible" +# shellcheck disable=SC2034 SHIPPABLE_RESULT_DIR="$(pwd)/shippable" TEST_DIR="${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/hetzner/hcloud" rm -rf "${TEST_DIR}" @@ -87,7 +90,7 @@ find plugins -type d -empty -print -delete ansible-test env --dump --show --timeout "50" --color -v group="${args[1]}" -echo $test +echo "$test" if [[ "${test}" =~ hcloud ]]; then group="${args[3]}" bash tests/utils/gitlab/integration.sh "shippable/hcloud/group${group}/" diff --git a/tests/utils/gitlab/integration.sh b/tests/utils/gitlab/integration.sh index 2d660c8..c94392b 100755 --- a/tests/utils/gitlab/integration.sh +++ b/tests/utils/gitlab/integration.sh @@ -3,13 +3,18 @@ target="$1" HCLOUD_TOKEN=$(cat hcloud_token.txt) +# shellcheck disable=SC2034,SC2154 changed_all_target="shippable/${cloud}/smoketest/" +# shellcheck disable=SC2046 echo "[default] hcloud_api_token=${HCLOUD_TOKEN} " >> $(pwd)/tests/integration/cloud-config-hcloud.ini -# shellcheck disable=SC2086 export SHIPPABLE="true" + +# shellcheck disable=SC2155 export SHIPPABLE_BUILD_NUMBER="gl-$(cat prefix.txt)" + +# shellcheck disable=SC2155,SC2002 export SHIPPABLE_JOB_NUMBER="$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 2 | head -n 1)" ansible-test integration --color --local -vv "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} diff --git a/tests/utils/shippable/shippable.sh b/tests/utils/shippable/shippable.sh index f10ae7f..fea6b17 100755 --- a/tests/utils/shippable/shippable.sh +++ b/tests/utils/shippable/shippable.sh @@ -18,6 +18,7 @@ function join { sudo chown "$(whoami)" "${PWD}/../../" test="$(join / "${args[@]:1}")" + docker images ansible/ansible docker images quay.io/ansible/* docker ps @@ -48,8 +49,8 @@ function retry fi echo "@* -> ${result}" done - echo "Command '$@' failed 3 times!" - exit -1 + echo "Command '@*' failed 3 times!" + exit 1 } command -v pip @@ -60,6 +61,7 @@ if [ "${ansible_version}" == "devel" ]; then else retry pip install "https://github.com/ansible/ansible/archive/stable-${ansible_version}.tar.gz" --disable-pip-version-check fi + if [ "${SHIPPABLE_BUILD_ID:-}" ]; then export ANSIBLE_COLLECTIONS_PATHS="${HOME}/.ansible" SHIPPABLE_RESULT_DIR="$(pwd)/shippable" @@ -83,9 +85,20 @@ retry ansible-galaxy -vvv collection install community.internal_test_tools export PYTHONIOENCODING='utf-8' if [ "${JOB_TRIGGERED_BY_NAME:-}" == "nightly-trigger" ]; then + COVERAGE=yes COMPLETE=yes fi +if [ -n "${COVERAGE:-}" ]; then + # on-demand coverage reporting triggered by setting the COVERAGE environment variable to a non-empty value + export COVERAGE="--coverage" +elif [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]]; then + # on-demand coverage reporting triggered by having 'ci_coverage' in the latest commit message + export COVERAGE="--coverage" +else + # on-demand coverage reporting disabled (default behavior, always-on coverage reporting remains enabled) + export COVERAGE="--coverage-check" +fi if [ -n "${COMPLETE:-}" ]; then # disable change detection triggered by setting the COMPLETE environment variable to a non-empty value @@ -111,6 +124,68 @@ find plugins -type d -empty -print -delete function cleanup { + # for complete on-demand coverage generate a report for all files with no coverage on the "sanity/5" job so we only have one copy + if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ] && [ "${test}" == "sanity/5" ]; then + stub="--stub" + # trigger coverage reporting for stubs even if no other coverage data exists + mkdir -p tests/output/coverage/ + else + stub="" + fi + + if [ -d tests/output/coverage/ ]; then + if find tests/output/coverage/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then + process_coverage='yes' # process existing coverage files + elif [ "${stub}" ]; then + process_coverage='yes' # process coverage when stubs are enabled + else + process_coverage='' + fi + + if [ "${process_coverage}" ]; then + # use python 3.7 for coverage to avoid running out of memory during coverage xml processing + # only use it for coverage to avoid the additional overhead of setting up a virtual environment for a potential no-op job + virtualenv --python /usr/bin/python3.7 ~/ansible-venv + set +ux + . ~/ansible-venv/bin/activate + set -ux + + # shellcheck disable=SC2086 + ansible-test coverage xml --color -v --requirements --group-by command --group-by version ${stub:+"$stub"} + cp -a tests/output/reports/coverage=*.xml "$SHIPPABLE_RESULT_DIR/codecoverage/" + + if [ "${ansible_version}" != "2.9" ]; then + # analyze and capture code coverage aggregated by integration test target + ansible-test coverage analyze targets generate -v "$SHIPPABLE_RESULT_DIR/testresults/coverage-analyze-targets.json" + fi + + # upload coverage report to codecov.io only when using complete on-demand coverage + if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ]; then + for file in tests/output/reports/coverage=*.xml; do + flags="${file##*/coverage=}" + flags="${flags%-powershell.xml}" + flags="${flags%.xml}" + # remove numbered component from stub files when converting to tags + flags="${flags//stub-[0-9]*/stub}" + flags="${flags//=/,}" + flags="${flags//[^a-zA-Z0-9_,]/_}" + + bash <(curl -s https://codecov.io/bash) \ + -f "${file}" \ + -F "${flags}" \ + -n "${test}" \ + -t 8a86e979-f37b-4d5d-95a4-960c280d5eaa \ + -X coveragepy \ + -X gcov \ + -X fix \ + -X search \ + -X xcode \ + || echo "Failed to upload code coverage report to codecov.io: ${file}" + done + fi + fi + fi + if [ -d tests/output/junit/ ]; then cp -aT tests/output/junit/ "$SHIPPABLE_RESULT_DIR/testresults/" fi @@ -126,7 +201,13 @@ function cleanup if [ "${SHIPPABLE_BUILD_ID:-}" ]; then trap cleanup EXIT; fi -ansible-test env --dump --show --timeout "50" --color -v +if [[ "${COVERAGE:-}" == "--coverage" ]]; then + timeout=60 +else + timeout=45 +fi + +ansible-test env --dump --show --timeout "${timeout}" --color -v if [ "${SHIPPABLE_BUILD_ID:-}" ]; then "tests/utils/shippable/check_matrix.py"; fi "tests/utils/shippable/${script}.sh" "${test}"