[stable-11] AZP: Generate CI matrix and run tests with antsibull-nox (#12330)

* AZP: Generate CI matrix and run tests with antsibull-nox.

* Update aliases test.

* Fix wrong remote name.

.azure-pipelines/azure-pipelines.yml

@@ -44,7 +44,9 @@ variables:
   - name: coverageBranches
     value: main
   - name: entryPoint
-    value: tests/utils/shippable/shippable.sh
+    value: tests/utils/shippable/nox.sh
+  - name: prepareEntryPoint
+    value: tests/utils/shippable/nox-prepare.sh
   - name: fetchDepth
     value: 0
 
@@ -56,298 +58,270 @@ resources:
 pool: Standard
 
 stages:
-### Sanity
-  - stage: Sanity_2_21
-    displayName: Sanity 2.21
+  - stage: sanity
+    displayName: Sanity
     dependsOn: []
+    variables:
+      entryPoint: tests/utils/shippable/nox.sh
     jobs:
       - template: templates/matrix.yml
         parameters:
-          nameFormat: Test {0}
-          testFormat: 2.21/sanity/{0}
           targets:
-            - test: 1
-            - test: 2
-            - test: 3
-            - test: 4
-  - stage: Sanity_2_20
-    displayName: Sanity 2.20
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          nameFormat: Test {0}
-          testFormat: 2.20/sanity/{0}
-          targets:
-            - test: 1
-            - test: 2
-            - test: 3
-            - test: 4
-  - stage: Sanity_2_19
-    displayName: Sanity 2.19
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          nameFormat: Test {0}
-          testFormat: 2.19/sanity/{0}
-          targets:
-            - test: 1
-            - test: 2
-            - test: 3
-            - test: 4
-### Units
-  - stage: Units_2_21
-    displayName: Units 2.21
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          nameFormat: Python {0}
-          testFormat: 2.21/units/{0}/1
-          targets:
-            - test: 3.9
-            - test: '3.10'
-            - test: '3.11'
-            - test: '3.12'
-            - test: '3.13'
-            - test: '3.14'
-  - stage: Units_2_20
-    displayName: Units 2.20
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          nameFormat: Python {0}
-          testFormat: 2.20/units/{0}/1
-          targets:
-            - test: 3.9
-            - test: "3.12"
-            - test: "3.14"
-  - stage: Units_2_19
+            - name: ansible-core 2.19
+              test: ansible-test-sanity-2.19
+            - name: ansible-core 2.20
+              test: ansible-test-sanity-2.20
+            - name: ansible-core 2.21
+              test: ansible-test-sanity-2.21
+            - name: Extra sanity tests
+              test: extra-sanity-tests
+
+  - stage: units_2_19
     displayName: Units 2.19
     dependsOn: []
+    variables:
+      entryPoint: tests/utils/shippable/nox.sh
     jobs:
       - template: templates/matrix.yml
         parameters:
-          nameFormat: Python {0}
-          testFormat: 2.19/units/{0}/1
           targets:
-            - test: 3.8
-            - test: "3.11"
-            - test: "3.13"
+            - name: Python 3.8
+              test: ansible-test-units-2.19-3.8
+            - name: Python 3.11
+              test: ansible-test-units-2.19-3.11
+            - name: Python 3.13
+              test: ansible-test-units-2.19-3.13
 
-## Remote
-  - stage: Remote_2_21_extra_vms
-    displayName: Remote 2.21 extra VMs
+  - stage: units_2_20
+    displayName: Units 2.20
     dependsOn: []
+    variables:
+      entryPoint: tests/utils/shippable/nox.sh
     jobs:
       - template: templates/matrix.yml
         parameters:
-          testFormat: 2.21/{0}
           targets:
-            - name: Alpine 3.23
-              test: alpine/3.23
-            # - name: Fedora 43
-            #   test: fedora/43
-            - name: Ubuntu 22.04
-              test: ubuntu/22.04
-            - name: Ubuntu 24.04
-              test: ubuntu/24.04
-          groups:
-            - vm
-  - stage: Remote_2_21
-    displayName: Remote 2.21
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          testFormat: 2.21/{0}
-          targets:
-            - name: macOS 26.3
-              test: macos/26.3
-            - name: RHEL 10.1
-              test: rhel/10.1
-            - name: RHEL 9.7
-              test: rhel/9.7
-            # TODO: enable this ASAP!
-            # - name: FreeBSD 15.0
-            #   test: freebsd/15.0
-            # TODO: enable this ASAP!
-            # - name: FreeBSD 14.4
-            #   test: freebsd/14.4
-          groups:
-            - 1
-            - 2
-            - 3
-  - stage: Remote_2_20
-    displayName: Remote 2.20
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          testFormat: 2.20/{0}
-          targets:
-            - name: macOS 15.3
-              test: macos/15.3
-            - name: RHEL 10.1
-              test: rhel/10.1
-            - name: FreeBSD 14.3
-              test: freebsd/14.3
-          groups:
-            - 1
-            - 2
-            - 3
-  - stage: Remote_2_19
-    displayName: Remote 2.19
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          testFormat: 2.19/{0}
-          targets:
-            - name: RHEL 10.1
-              test: rhel/10.1
-            # - name: FreeBSD 14.2
-            #   test: freebsd/14.2
-          groups:
-            - 1
-            - 2
-            - 3
+            - name: Python 3.9
+              test: ansible-test-units-2.20-3.9
+            - name: Python 3.12
+              test: ansible-test-units-2.20-3.12
+            - name: Python 3.14
+              test: ansible-test-units-2.20-3.14
 
-### Docker
-  - stage: Docker_2_21
-    displayName: Docker 2.21
+  - stage: units_2_21
+    displayName: Units 2.21
     dependsOn: []
+    variables:
+      entryPoint: tests/utils/shippable/nox.sh
     jobs:
       - template: templates/matrix.yml
         parameters:
-          testFormat: 2.21/linux/{0}
           targets:
-            - name: Fedora 43
-              test: fedora43
-            - name: Alpine 3.23
-              test: alpine323
-            - name: Ubuntu 22.04
-              test: ubuntu2204
-            - name: Ubuntu 24.04
-              test: ubuntu2404
-          groups:
-            - 1
-            - 2
-            - 3
-  - stage: Docker_2_20
-    displayName: Docker 2.20
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          testFormat: 2.20/linux/{0}
-          targets:
-            - name: Fedora 42
-              test: fedora42
-            - name: Alpine 3.22
-              test: alpine322
-          groups:
-            - 1
-            - 2
-            - 3
-  - stage: Docker_2_19
+            - name: Python 3.9
+              test: ansible-test-units-2.21-3.9
+            - name: Python 3.10
+              test: ansible-test-units-2.21-3.10
+            - name: Python 3.11
+              test: ansible-test-units-2.21-3.11
+            - name: Python 3.12
+              test: ansible-test-units-2.21-3.12
+            - name: Python 3.13
+              test: ansible-test-units-2.21-3.13
+            - name: Python 3.14
+              test: ansible-test-units-2.21-3.14
+
+  - stage: docker_2_19
     displayName: Docker 2.19
     dependsOn: []
+    variables:
+      entryPoint: tests/utils/shippable/nox.sh
     jobs:
       - template: templates/matrix.yml
         parameters:
-          testFormat: 2.19/linux/{0}
           targets:
-            - name: Fedora 41
-              test: fedora41
-            - name: Alpine 3.21
-              test: alpine321
-          groups:
-            - 1
-            - 2
-            - 3
+            - name: Alpine 3.21 + group 1
+              test: ansible-test-integration-2.19-alpine321-azp-posix-1
+            - name: Alpine 3.21 + group 2
+              test: ansible-test-integration-2.19-alpine321-azp-posix-2
+            - name: Alpine 3.21 + group 3
+              test: ansible-test-integration-2.19-alpine321-azp-posix-3
+            - name: Fedora 41 + group 1
+              test: ansible-test-integration-2.19-fedora41-azp-posix-1
+            - name: Fedora 41 + group 2
+              test: ansible-test-integration-2.19-fedora41-azp-posix-2
+            - name: Fedora 41 + group 3
+              test: ansible-test-integration-2.19-fedora41-azp-posix-3
 
-### Community Docker
-  - stage: Docker_community_2_21
-    displayName: Docker (community images) 2.21
+  - stage: docker_2_20
+    displayName: Docker 2.20
     dependsOn: []
+    variables:
+      entryPoint: tests/utils/shippable/nox.sh
     jobs:
       - template: templates/matrix.yml
         parameters:
-          testFormat: 2.21/linux-community/{0}
           targets:
-            - name: Debian 11 Bullseye
-              test: debian-bullseye/3.9
-            - name: Debian 12 Bookworm
-              test: debian-bookworm/3.11
-            - name: Debian 13 Trixie
-              test: debian-13-trixie/3.13
-            - name: ArchLinux
-              test: archlinux/3.14
-          groups:
-            - 1
-            - 2
-            - 3
+            - name: Alpine 3.22 + group 1
+              test: ansible-test-integration-2.20-alpine322-azp-posix-1
+            - name: Alpine 3.22 + group 2
+              test: ansible-test-integration-2.20-alpine322-azp-posix-2
+            - name: Alpine 3.22 + group 3
+              test: ansible-test-integration-2.20-alpine322-azp-posix-3
+            - name: Fedora 42 + group 1
+              test: ansible-test-integration-2.20-fedora42-azp-posix-1
+            - name: Fedora 42 + group 2
+              test: ansible-test-integration-2.20-fedora42-azp-posix-2
+            - name: Fedora 42 + group 3
+              test: ansible-test-integration-2.20-fedora42-azp-posix-3
 
-### Generic
-# Right now all generic tests are disabled. Uncomment when at least one of them is re-enabled.
-#  - stage: Generic_2_21
-#    displayName: Generic 2.21
-#    dependsOn: []
-#    jobs:
-#      - template: templates/matrix.yml
-#        parameters:
-#          nameFormat: Python {0}
-#          testFormat: 2.21/generic/{0}/1
-#          targets:
-#            - test: '3.9'
-#            - test: '3.12'
-#            - test: '3.14'
-#  - stage: Generic_2_20
-#    displayName: Generic 2.20
-#    dependsOn: []
-#    jobs:
-#      - template: templates/matrix.yml
-#        parameters:
-#          nameFormat: Python {0}
-#          testFormat: 2.20/generic/{0}/1
-#          targets:
-#            - test: '3.10'
-#            - test: '3.14'
-#  - stage: Generic_2_19
-#    displayName: Generic 2.19
-#    dependsOn: []
-#    jobs:
-#      - template: templates/matrix.yml
-#        parameters:
-#          nameFormat: Python {0}
-#          testFormat: 2.19/generic/{0}/1
-#          targets:
-#            - test: '3.9'
-#            - test: '3.13'
+  - stage: docker_2_21
+    displayName: Docker 2.21
+    dependsOn: []
+    variables:
+      entryPoint: tests/utils/shippable/nox.sh
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          targets:
+            - name: Alpine 3.23 + group 1
+              test: ansible-test-integration-2.21-alpine323-azp-posix-1
+            - name: Alpine 3.23 + group 2
+              test: ansible-test-integration-2.21-alpine323-azp-posix-2
+            - name: Alpine 3.23 + group 3
+              test: ansible-test-integration-2.21-alpine323-azp-posix-3
+            - name: Arch Linux + py3.14 + group 1
+              test: ansible-test-integration-2.21-archlinux-3.14-azp-posix-1
+            - name: Arch Linux + py3.14 + group 2
+              test: ansible-test-integration-2.21-archlinux-3.14-azp-posix-2
+            - name: Arch Linux + py3.14 + group 3
+              test: ansible-test-integration-2.21-archlinux-3.14-azp-posix-3
+            - name: Debian 13 + py3.13 + group 1
+              test: ansible-test-integration-2.21-debian-13-trixie-3.13-azp-posix-1
+            - name: Debian 13 + py3.13 + group 2
+              test: ansible-test-integration-2.21-debian-13-trixie-3.13-azp-posix-2
+            - name: Debian 13 + py3.13 + group 3
+              test: ansible-test-integration-2.21-debian-13-trixie-3.13-azp-posix-3
+            - name: Debian 12 + py3.11 + group 1
+              test: ansible-test-integration-2.21-debian-bookworm-3.11-azp-posix-1
+            - name: Debian 12 + py3.11 + group 2
+              test: ansible-test-integration-2.21-debian-bookworm-3.11-azp-posix-2
+            - name: Debian 12 + py3.11 + group 3
+              test: ansible-test-integration-2.21-debian-bookworm-3.11-azp-posix-3
+            - name: Debian 11 + py3.9 + group 1
+              test: ansible-test-integration-2.21-debian-bullseye-3.9-azp-posix-1
+            - name: Debian 11 + py3.9 + group 2
+              test: ansible-test-integration-2.21-debian-bullseye-3.9-azp-posix-2
+            - name: Debian 11 + py3.9 + group 3
+              test: ansible-test-integration-2.21-debian-bullseye-3.9-azp-posix-3
+            - name: Fedora 43 + group 1
+              test: ansible-test-integration-2.21-fedora43-azp-posix-1
+            - name: Fedora 43 + group 2
+              test: ansible-test-integration-2.21-fedora43-azp-posix-2
+            - name: Fedora 43 + group 3
+              test: ansible-test-integration-2.21-fedora43-azp-posix-3
+            - name: Ubuntu 22.04 + group 1
+              test: ansible-test-integration-2.21-ubuntu2204-azp-posix-1
+            - name: Ubuntu 22.04 + group 2
+              test: ansible-test-integration-2.21-ubuntu2204-azp-posix-2
+            - name: Ubuntu 22.04 + group 3
+              test: ansible-test-integration-2.21-ubuntu2204-azp-posix-3
+            - name: Ubuntu 24.04 + group 1
+              test: ansible-test-integration-2.21-ubuntu2404-azp-posix-1
+            - name: Ubuntu 24.04 + group 2
+              test: ansible-test-integration-2.21-ubuntu2404-azp-posix-2
+            - name: Ubuntu 24.04 + group 3
+              test: ansible-test-integration-2.21-ubuntu2404-azp-posix-3
+
+  - stage: remote_2_19
+    displayName: Remote 2.19
+    dependsOn: []
+    variables:
+      entryPoint: tests/utils/shippable/nox.sh
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          targets:
+            - name: RHEL 10.1 + group 1
+              test: ansible-test-integration-2.19-rhel-10.1-azp-posix-1
+            - name: RHEL 10.1 + group 2
+              test: ansible-test-integration-2.19-rhel-10.1-azp-posix-2
+            - name: RHEL 10.1 + group 3
+              test: ansible-test-integration-2.19-rhel-10.1-azp-posix-3
+
+  - stage: remote_2_20
+    displayName: Remote 2.20
+    dependsOn: []
+    variables:
+      entryPoint: tests/utils/shippable/nox.sh
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          targets:
+            - name: FreeBSD 14.3 + group 1
+              test: ansible-test-integration-2.20-freebsd-14.3-azp-posix-1
+            - name: FreeBSD 14.3 + group 2
+              test: ansible-test-integration-2.20-freebsd-14.3-azp-posix-2
+            - name: FreeBSD 14.3 + group 3
+              test: ansible-test-integration-2.20-freebsd-14.3-azp-posix-3
+            - name: macOS 15.3 + group 1
+              test: ansible-test-integration-2.20-macos-15.3-azp-posix-1
+            - name: macOS 15.3 + group 2
+              test: ansible-test-integration-2.20-macos-15.3-azp-posix-2
+            - name: macOS 15.3 + group 3
+              test: ansible-test-integration-2.20-macos-15.3-azp-posix-3
+            - name: RHEL 10.1 + group 1
+              test: ansible-test-integration-2.20-rhel-10.1-azp-posix-1
+            - name: RHEL 10.1 + group 2
+              test: ansible-test-integration-2.20-rhel-10.1-azp-posix-2
+            - name: RHEL 10.1 + group 3
+              test: ansible-test-integration-2.20-rhel-10.1-azp-posix-3
+
+  - stage: remote_2_21
+    displayName: Remote 2.21
+    dependsOn: []
+    variables:
+      entryPoint: tests/utils/shippable/nox.sh
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          targets:
+            - name: Alpine 3.23 + extra VMs
+              test: ansible-test-integration-2.21-alpine-3.23-azp-posix-vm
+            - name: macOS 26.3 + group 1
+              test: ansible-test-integration-2.21-macos-26.3-azp-posix-1
+            - name: macOS 26.3 + group 2
+              test: ansible-test-integration-2.21-macos-26.3-azp-posix-2
+            - name: macOS 26.3 + group 3
+              test: ansible-test-integration-2.21-macos-26.3-azp-posix-3
+            - name: RHEL 10.1 + group 1
+              test: ansible-test-integration-2.21-rhel-10.1-azp-posix-1
+            - name: RHEL 10.1 + group 2
+              test: ansible-test-integration-2.21-rhel-10.1-azp-posix-2
+            - name: RHEL 10.1 + group 3
+              test: ansible-test-integration-2.21-rhel-10.1-azp-posix-3
+            - name: RHEL 9.7 + group 1
+              test: ansible-test-integration-2.21-rhel-9.7-azp-posix-1
+            - name: RHEL 9.7 + group 2
+              test: ansible-test-integration-2.21-rhel-9.7-azp-posix-2
+            - name: RHEL 9.7 + group 3
+              test: ansible-test-integration-2.21-rhel-9.7-azp-posix-3
+            - name: Ubuntu 22.04 + extra VMs
+              test: ansible-test-integration-2.21-ubuntu-22.04-azp-posix-vm
+            - name: Ubuntu 24.04 + extra VMs
+              test: ansible-test-integration-2.21-ubuntu-24.04-azp-posix-vm
 
   - stage: Summary
     condition: succeededOrFailed()
     dependsOn:
-      - Sanity_2_21
-      - Sanity_2_20
-      - Sanity_2_19
-      - Units_2_21
-      - Units_2_20
-      - Units_2_19
-      - Remote_2_21_extra_vms
-      - Remote_2_21
-      - Remote_2_20
-      - Remote_2_19
-      - Docker_2_21
-      - Docker_2_20
-      - Docker_2_19
-      - Docker_community_2_21
-# Right now all generic tests are disabled. Uncomment when at least one of them is re-enabled.
-#      - Generic_2_21
-#      - Generic_2_20
-#      - Generic_2_19
+      - sanity
+      - units_2_19
+      - units_2_20
+      - units_2_21
+      - docker_2_19
+      - docker_2_20
+      - docker_2_21
+      - remote_2_19
+      - remote_2_20
+      - remote_2_21
     jobs:
       - template: templates/coverage.yml

.azure-pipelines/scripts/run-tests.sh

@@ -10,11 +10,13 @@ set -o pipefail -eu
 entry_point="$1"
 test="$2"
 read -r -a coverage_branches <<< "$3"  # space separated list of branches to run code coverage on for scheduled builds
+agent_temp_directory="$4"
 
 export COMMIT_MESSAGE
 export COMPLETE
 export COVERAGE
 export IS_PULL_REQUEST
+export COVERAGE_DESTINATION_DIRECTORY="${agent_temp_directory}/coverage"
 
 if [ "${SYSTEM_PULLREQUEST_TARGETBRANCH:-}" ]; then
     IS_PULL_REQUEST=true

.azure-pipelines/templates/test.yml

@@ -23,14 +23,16 @@ jobs:
           - checkout: self
             fetchDepth: $(fetchDepth)
             path: $(checkoutPath)
-          - bash: .azure-pipelines/scripts/run-tests.sh "$(entryPoint)" "${{ job.test }}" "$(coverageBranches)"
+          - bash: .azure-pipelines/scripts/run-tests.sh "$(prepareEntryPoint)" "${{ job.test }}" "$(coverageBranches)" "$(Agent.TempDirectory)"
+            displayName: Prepare Tests
+          - bash: .azure-pipelines/scripts/run-tests.sh "$(entryPoint)" "${{ job.test }}" "$(coverageBranches)" "$(Agent.TempDirectory)"
             displayName: Run Tests
           - bash: .azure-pipelines/scripts/process-results.sh
             condition: succeededOrFailed()
             displayName: Process Results
-          - bash: .azure-pipelines/scripts/aggregate-coverage.sh "$(Agent.TempDirectory)"
-            condition: eq(variables.haveCoverageData, 'true')
-            displayName: Aggregate Coverage Data
+          # - bash: .azure-pipelines/scripts/aggregate-coverage.sh "$(Agent.TempDirectory)"
+          #   condition: eq(variables.haveCoverageData, 'true')
+          #   displayName: Aggregate Coverage Data
           - task: PublishTestResults@2
             condition: eq(variables.haveTestResults, 'true')
             inputs:

.github/workflows/nox.yml

@@ -21,11 +21,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  nox:
-    uses: ansible-community/antsibull-nox/.github/workflows/reusable-nox-run.yml@main
-    with:
-      session-name: Run extra sanity tests
-      change-detection-in-prs: true
   ansible-test:
     uses: ansible-community/antsibull-nox/.github/workflows/reusable-nox-matrix.yml@main
     with:

antsibull-nox.toml

@@ -251,7 +251,7 @@ target = [ "azp/posix/vm/" ]
 remote = [
     "alpine/3.23",
     # "fedora/43",
-    "ubuntu/26.04",
+    "ubuntu/22.04",
     "ubuntu/24.04",
 ]
 

noxfile.py

@@ -13,6 +13,7 @@ import nox
 
 try:
     import antsibull_nox
+    from antsibull_nox.cli import run as run_antsibull_nox
 except ImportError:
     print("You need to install antsibull-nox in the same Python environment as nox.")
     sys.exit(1)
@@ -32,6 +33,24 @@ def botmeta(session: nox.Session) -> None:
     session.run("python", "tests/sanity/extra/botmeta.py")
 
 
+@nox.session(name="update-azp-config", python=False)
+def update_azp_config(session: nox.Session) -> None:
+    command = [
+        "antsibull-nox",
+        "update-azp-config",
+        "--min-ansible-core",
+        "2.19",
+        "--extra-session",
+        '{"group": "Sanity", "title": "Extra sanity tests", "session": "extra-sanity-tests"}',
+    ]
+    if antsibull_nox.IN_CI:
+        command.extend(["--show-diff", "--fail-on-change"])
+    session.debug(" ".join(command))
+    result = run_antsibull_nox(command)
+    if result != 0:
+        session.error(f"Execution failed with status code {result}")
+
+
 # Allow to run the noxfile with `python noxfile.py`, `pipx run noxfile.py`, or similar.
 # Requires nox >= 2025.02.09
 if __name__ == "__main__":

tests/sanity/extra/aliases.py

@@ -19,10 +19,11 @@ def main():
 
     allowed_targets = set(['azp/generic/1'])
     for stage in azp['stages']:
-        if stage['stage'].startswith(('Sanity', 'Unit', 'Generic', 'Summary')):
+        if stage['stage'].startswith(('sanity', 'unit', 'Summary')):
             continue
         for job in stage['jobs']:
-            for group in job['parameters']['groups']:
+            for param in job['parameters']['targets']:
+                group = param['test'].rsplit('-', 1)[-1]
                 allowed_targets.add('azp/posix/{0}'.format(group))
 
     paths = glob.glob("tests/integration/targets/*/aliases")

tests/utils/shippable/nox-prepare.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+set -o pipefail -eux
+
+# Fix for https://github.com/ansible-community/antsibull-nox/issues/222#issuecomment-4778928615
+# caused by https://github.com/ansible/azure-pipelines-test-container/blob/7714d81f64f268bbb10779e1265d312128607b76/Containerfile#L4
+export PATH="${PATH//:~\//:${HOME}/}"
+
+nox_session="$1"
+
+docker images ansible/ansible
+docker images quay.io/ansible/*
+docker ps
+
+for container in $(docker ps --format '{{.Image}} {{.ID}}' | grep -v -e '^drydock/' -e '^quay.io/ansible/azure-pipelines-test-container:' | sed 's/^.* //'); do
+    docker rm -f "${container}" || true  # ignore errors
+done
+
+docker ps
+command -v python
+python -V
+
+function retry
+{
+    # shellcheck disable=SC2034
+    for repetition in 1 2 3; do
+        set +e
+        "$@"
+        result=$?
+        set -e
+        if [ ${result} == 0 ]; then
+            return ${result}
+        fi
+        echo "@* -> ${result}"
+    done
+    echo "Command '@*' failed 3 times!"
+    exit 255
+}
+
+command -v pip
+pip --version
+pip list --disable-pip-version-check
+retry pip install https://github.com/ansible-community/antsibull-nox/archive/main.tar.gz --disable-pip-version-check
+
+export PYTHONIOENCODING='utf-8'
+
+export FORCE_COLOR=1
+export ANTSIBULL_NOX_IGNORE_INSTALLED_COLLECTIONS="true"
+
+if [ "${nox_session}" == "extra-sanity-tests" ]; then
+    # We need the ansible-galaxy CLI tool to install collection dependencies
+    retry pip install ansible-core --disable-pip-version-check
+    nox --verbose --install-only
+else
+    nox --verbose --install-only -e "${nox_session}"
+fi

tests/utils/shippable/nox.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+set -o pipefail -eux
+
+# Fix for https://github.com/ansible-community/antsibull-nox/issues/222#issuecomment-4778928615
+# caused by https://github.com/ansible/azure-pipelines-test-container/blob/7714d81f64f268bbb10779e1265d312128607b76/Containerfile#L4
+export PATH="${PATH//:~\//:${HOME}/}"
+
+nox_session="$1"
+
+export PYTHONIOENCODING='utf-8'
+
+if [ -n "${COVERAGE:-}" ]; then
+    # on-demand coverage reporting triggered by setting the COVERAGE environment variable to a non-empty value
+    export COVERAGE="--coverage"
+elif [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]]; then
+    # on-demand coverage reporting triggered by having 'ci_coverage' in the latest commit message
+    export COVERAGE="--coverage"
+else
+    # on-demand coverage reporting disabled (default behavior, always-on coverage reporting remains enabled)
+    export COVERAGE="--coverage-check"
+fi
+
+if [ -n "${COMPLETE:-}" ]; then
+    # disable change detection triggered by setting the COMPLETE environment variable to a non-empty value
+    export ANTSIBULL_CHANGE_DETECTION=""
+elif [[ "${COMMIT_MESSAGE}" =~ ci_complete ]]; then
+    # disable change detection triggered by having 'ci_complete' in the latest commit message
+    export ANTSIBULL_CHANGE_DETECTION=""
+elif [ "${IS_PULL_REQUEST:-}" == "true" ]; then
+    # enable change detection for PRs (default behavior)
+    export ANTSIBULL_CHANGE_DETECTION="true"
+    export ANTSIBULL_BASE_BRANCH="${SYSTEM_PULLREQUEST_TARGETBRANCH}"
+    # Create a branch for the current HEAD, which happens to be a merge commit
+    git checkout -b "pull-request-branch"
+    # Name the target branch
+    git branch "${SYSTEM_PULLREQUEST_TARGETBRANCH}" --track "origin/${SYSTEM_PULLREQUEST_TARGETBRANCH}"
+    # Show branches
+    git branch -vv
+else
+    # disable change detection for pushes and scheduled runs
+    export ANTSIBULL_CHANGE_DETECTION=""
+fi
+
+if [[ "${COVERAGE:-}" == "--coverage" ]]; then
+    export ANTSIBULL_NOX_TIMEOUT=60
+else
+    export ANTSIBULL_NOX_TIMEOUT=50
+fi
+
+if [ "${IS_PULL_REQUEST:-}" == "true" ]; then
+    export ANTSIBULL_NOX_INTEGRATION_ALLOW_UNSTABLE_CHANGED="true"
+fi
+
+export FORCE_COLOR=1
+export ANTSIBULL_NOX_IGNORE_INSTALLED_COLLECTIONS="true"
+export ANTSIBULL_NOX_COVERAGE_DESTINATION="${COVERAGE_DESTINATION_DIRECTORY}"
+export ANTSIBULL_NOX_COVERAGE_ANALYSIS_FILE="${COVERAGE_DESTINATION_DIRECTORY}/coverage-analyze-targets.json"
+export ANTSIBULL_NOX_COVERAGE_NO_XML="true"
+
+if [ "${nox_session}" == "extra-sanity-tests" ]; then
+    nox --reuse-existing-virtualenvs --no-install
+else
+    nox --reuse-existing-virtualenvs --no-install -e "${nox_session}" -- ${COVERAGE}
+fi