# Regression Detection Suite
#
# This workflow runs under the following conditions:
#   - Once per day, based on a schedule
#   - On demand, triggered from https://github.com/vectordotdev/vector/actions/workflows/regression_v2.yml
#
# The workflow accepts two optional inputs:
#   - The baseline SHA:
#     - If not specified, the SHA from 7 days ago on origin/master is used.
#   - The comparison SHA:
#     - If not specified, the current HEAD of origin/master is used.
#
# This workflow runs regression detection experiments, performing relative
# evaluations of the baseline SHA and comparison SHA. The exact SHAs are determined
# by how the workflow is triggered.
#
# The goal is to provide quick feedback on Vector's performance across a variety
# of configurations, checking if throughput performance has degraded or become
# more variable in the comparison SHA relative to the baseline SHA.
#
# Docker image tags are based on the resolved SHAs.

name: Regression Detection Suite

on:
  workflow_dispatch:
    inputs:
      baseline-sha:
        description: "The SHA to use as the baseline (optional). If not provided, it defaults to the SHA from 7 days ago."
        required: false
      comparison-sha:
        description: "The SHA to use for comparison (optional). If not provided, it defaults to the current HEAD of the origin/master branch."
        required: false
  schedule:
    - cron: "0 7 * * 1" # Runs at 7 AM UTC on Mondays

# Workflow-level permissions - read access to repository contents
permissions:
  contents: read # Required to checkout code

env:
  SINGLE_MACHINE_PERFORMANCE_API: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_API }}
  SMP_WARMUP_SECONDS: 70 # default is 45 seconds
  SMP_REPLICAS: 100 # default is 10

jobs:
  resolve-inputs:
    runs-on: ubuntu-24.04
    outputs:
      baseline-sha: ${{ steps.set_and_validate_shas.outputs.BASELINE_SHA }}
      comparison-sha: ${{ steps.set_and_validate_shas.outputs.COMPARISON_SHA }}
      baseline-tag: ${{ steps.set_and_validate_shas.outputs.BASELINE_TAG }}
      comparison-tag: ${{ steps.set_and_validate_shas.outputs.COMPARISON_TAG }}
      smp-version: ${{ steps.experimental-meta.outputs.SMP_CRATE_VERSION }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0 # need to pull repository history to find merge bases

      - name: Set and Validate SHAs
        id: set_and_validate_shas
        run: |
          # Set baseline SHA
          if [ -z "${{ github.event.inputs.baseline-sha }}" ]; then
            BASELINE_SHA=$(git rev-list -n 1 --before="7 days ago" origin/master)
            echo "Using baseline SHA from 7 days ago: ${BASELINE_SHA}"
          else
            BASELINE_SHA="${{ github.event.inputs.baseline-sha }}"
            echo "Using provided baseline SHA: ${BASELINE_SHA}"
          fi

          # Validate baseline SHA
          if [ -n "${BASELINE_SHA}" ] && git cat-file -e "${BASELINE_SHA}^{commit}"; then
            echo "Baseline SHA is valid."
          else
            echo "Invalid baseline SHA: ${BASELINE_SHA}."
            exit 1
          fi

          # Set comparison SHA
          if [ -z "${{ github.event.inputs.comparison-sha }}" ]; then
            COMPARISON_SHA=$(git rev-parse origin/master)
            echo "Using current HEAD for comparison SHA: ${COMPARISON_SHA}"
          else
            COMPARISON_SHA="${{ github.event.inputs.comparison-sha }}"
            echo "Using provided comparison SHA: ${COMPARISON_SHA}"
          fi

          # Validate comparison SHA
          if [ -n "${COMPARISON_SHA}" ] && git cat-file -e "${COMPARISON_SHA}^{commit}"; then
            echo "Comparison SHA is valid."
          else
            echo "Invalid comparison SHA: ${COMPARISON_SHA}."
            exit 1
          fi

          # Set tags and export them
          BASELINE_TAG="workflow_dispatch-${COMPARISON_SHA}-${BASELINE_SHA}"
          COMPARISON_TAG="workflow_dispatch-${COMPARISON_SHA}-${COMPARISON_SHA}"

          echo "BASELINE_SHA=${BASELINE_SHA}" >> $GITHUB_OUTPUT
          echo "COMPARISON_SHA=${COMPARISON_SHA}" >> $GITHUB_OUTPUT

          echo "BASELINE_TAG=${BASELINE_TAG}" >> $GITHUB_OUTPUT
          echo "COMPARISON_TAG=${COMPARISON_TAG}" >> $GITHUB_OUTPUT

      - name: Set SMP version
        id: experimental-meta
        run: |
          export SMP_CRATE_VERSION="0.26.1"
          echo "smp crate version: ${SMP_CRATE_VERSION}"
          echo "SMP_CRATE_VERSION=${SMP_CRATE_VERSION}" >> $GITHUB_OUTPUT

  # Only run this workflow if files changed in areas that could possibly introduce a regression.
  check-source-changed:
    runs-on: ubuntu-24.04
    timeout-minutes: 5
    needs: resolve-inputs
    outputs:
      source_changed: ${{ steps.filter.outputs.SOURCE_CHANGED }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Collect file changes
        id: changes
        uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
        with:
          base: ${{ needs.resolve-inputs.outputs.baseline-sha }}
          ref: ${{ needs.resolve-inputs.outputs.comparison-sha }}
          list-files: shell
          filters: |
            all_changed:
              - added|deleted|modified: "**"
            ignore:
              - "./.github/**/!(regression.yml)"
              - "./.gitignore"
              - "distribution/**"
              - "rust-doc/**"
              - "docs/**"
              - "rfcs/**"
              - "testing/**"
              - "tilt/**"
              - "website/**"
              - "*.md"
              - "Tiltfile"
              - "NOTICE"
              - "LICENSE-3rdparty.csv"
              - "LICENSE"
              - "lib/codecs/tests/data/**"

      # This step allows us to conservatively run the tests if we added a new
      # file or directory for source code, but forgot to add it to this workflow.
      # Instead, we may unnecessarily run the test on new file or dir additions that
      # wouldn't likely introduce regressions.
      - name: Determine if should not run due to irrelevant file changes
        id: filter
        env:
          ALL: ${{ steps.changes.outputs.all_changed_files }}
          IGNORE: ${{ steps.changes.outputs.ignore_files }}
        run: |
          echo "ALL='${{ env.ALL }}'"
          echo "IGNORE='${{ env.IGNORE }}'"
          export SOURCE_CHANGED=$(comm -2 -3 <(printf "%s\n" "${{ env.ALL }}") <(printf "%s\n" "${{ env.IGNORE }}"))
          echo "SOURCE_CHANGED='${SOURCE_CHANGED}'"

          if [ "${SOURCE_CHANGED}" == "" ]; then
            export SOURCE_CHANGED="false"
          else
            export SOURCE_CHANGED="true"
          fi

          echo "SOURCE_CHANGED='${SOURCE_CHANGED}'"
          echo "SOURCE_CHANGED=${SOURCE_CHANGED}" >> $GITHUB_OUTPUT

  should-run-gate:
    runs-on: ubuntu-24.04
    needs: check-source-changed
    if: ${{ needs.check-source-changed.outputs.source_changed }}
    steps:
      - name: Gate check passed
        run: echo "Source code changes detected, proceeding with regression tests"

  ##
  ## BUILD
  ##

  build-baseline:
    name: Build baseline Vector container
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    needs:
      - should-run-gate
      - resolve-inputs
    # Job-level permissions for artifact upload
    permissions:
      contents: read # Required to checkout code
      actions: write # Required to upload artifacts
    steps:
      - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1

      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ needs.resolve-inputs.outputs.baseline-sha }}
          path: baseline-vector

      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0

      - name: Build 'vector' target image
        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2
        with:
          context: baseline-vector/
          cache-from: type=gha
          cache-to: type=gha,mode=max
          file: regression/Dockerfile
          builder: ${{ steps.buildx.outputs.name }}
          outputs: type=docker,dest=${{ runner.temp }}/baseline-image.tar
          tags: |
            vector:${{ needs.resolve-inputs.outputs.baseline-tag }}

      - name: Upload image as artifact
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: baseline-image
          path: "${{ runner.temp }}/baseline-image.tar"

  build-comparison:
    name: Build comparison Vector container
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    needs:
      - should-run-gate
      - resolve-inputs
    # Job-level permissions for artifact upload
    permissions:
      contents: read # Required to checkout code
      actions: write # Required to upload artifacts
    steps:
      - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1

      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ needs.resolve-inputs.outputs.comparison-sha }}
          path: comparison-vector

      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0

      - name: Build 'vector' target image
        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2
        with:
          context: comparison-vector/
          cache-from: type=gha
          cache-to: type=gha,mode=max
          file: regression/Dockerfile
          builder: ${{ steps.buildx.outputs.name }}
          outputs: type=docker,dest=${{ runner.temp }}/comparison-image.tar
          tags: |
            vector:${{ needs.resolve-inputs.outputs.comparison-tag }}

      - name: Upload image as artifact
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: comparison-image
          path: "${{ runner.temp }}/comparison-image.tar"

  confirm-valid-credentials:
    name: Confirm AWS credentials are minimally valid
    runs-on: ubuntu-22.04
    timeout-minutes: 5
    needs:
      - should-run-gate
      - resolve-inputs
    # Job level permissions for downloading SMP binary
    permissions:
      id-token: write # Required for GitHub OIDC token exchange
    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
        with:
          role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc
          aws-region: us-west-2
          role-duration-seconds: 14400 # 4 hours

      - name: Download SMP binary
        run: |
          aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-musl/smp ${{ runner.temp }}/bin/smp

  ##
  ## SUBMIT
  ##

  upload-baseline-image-to-ecr:
    name: Upload baseline images to ECR
    runs-on: ubuntu-22.04
    timeout-minutes: 5
    needs:
      - should-run-gate
      - resolve-inputs
      - confirm-valid-credentials
      - build-baseline
    # Job level permissions for uploading baseline image to SMP ECR
    permissions:
      id-token: write # Required for GitHub OIDC token exchange
    steps:
      - name: "Download baseline image"
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: baseline-image

      - name: Load baseline image
        run: |
          docker load --input baseline-image.tar

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
        with:
          role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc
          aws-region: us-west-2
          role-duration-seconds: 14400 # 4 hours

      - name: Login to Amazon ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1

      - name: Docker Login to ECR
        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
        with:
          registry: ${{ steps.login-ecr.outputs.registry }}

      - name: Tag & push baseline image
        run: |
          docker tag vector:${{ needs.resolve-inputs.outputs.baseline-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.baseline-tag }}
          docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.baseline-tag }}

  upload-comparison-image-to-ecr:
    name: Upload comparison images to ECR
    runs-on: ubuntu-22.04
    timeout-minutes: 5
    needs:
      - should-run-gate
      - resolve-inputs
      - confirm-valid-credentials
      - build-comparison
    # Job level permissions for uploading comparison image to SMP ECR
    permissions:
      id-token: write # Required for GitHub OIDC token exchange
    steps:
      - name: "Download comparison image"
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: comparison-image

      - name: Load comparison image
        run: |
          docker load --input comparison-image.tar

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
        with:
          role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc
          aws-region: us-west-2
          role-duration-seconds: 14400 # 4 hours

      - name: Login to Amazon ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1

      - name: Docker Login to ECR
        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
        with:
          registry: ${{ steps.login-ecr.outputs.registry }}

      - name: Tag & push comparison image
        run: |
          docker tag vector:${{ needs.resolve-inputs.outputs.comparison-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.comparison-tag }}
          docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.comparison-tag }}

  submit-job:
    name: Submit regression job
    runs-on: ubuntu-22.04
    timeout-minutes: 120
    needs:
      - should-run-gate
      - resolve-inputs
      - upload-baseline-image-to-ecr
      - upload-comparison-image-to-ecr
    # Job-level permissions for artifact upload
    permissions:
      contents: read # Required to checkout code
      actions: write # Required to upload artifacts
      id-token: write # Required for GitHub OIDC token exchange
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ needs.resolve-inputs.outputs.comparison-sha }}

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
        with:
          role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc
          aws-region: us-west-2
          role-duration-seconds: 14400 # 4 hours

      - name: Login to Amazon ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1

      - name: Download SMP binary
        run: |
          aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-musl/smp ${{ runner.temp }}/bin/smp

      - name: Submit job
        env:
          RUST_LOG: info
        run: |
          chmod +x ${{ runner.temp }}/bin/smp

          ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job submit \
            --baseline-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.baseline-tag }} \
            --comparison-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.comparison-tag }} \
            --baseline-sha ${{ needs.resolve-inputs.outputs.baseline-sha }} \
            --comparison-sha ${{ needs.resolve-inputs.outputs.comparison-sha }} \
            --target-config-dir ${{ github.workspace }}/regression/ \
            --warmup-seconds ${{ env.SMP_WARMUP_SECONDS }} \
            --submission-metadata ${{ runner.temp }}/submission-metadata \
            --replicas ${{ env.SMP_REPLICAS }}

      - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: vector-submission-metadata
          path: ${{ runner.temp }}/submission-metadata

      - name: Await job
        timeout-minutes: 120
        env:
          RUST_LOG: info
        run: |
          chmod +x ${{ runner.temp }}/bin/smp

          ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job status \
            --wait \
            --wait-delay-seconds 60 \
            --wait-timeout-minutes 120 \
            --submission-metadata ${{ runner.temp }}/submission-metadata

      - name: Handle cancellation if necessary
        if: ${{ cancelled() }}
        env:
          RUST_LOG: info
        run: |
          chmod +x ${{ runner.temp }}/bin/smp
          ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job cancel \
            --submission-metadata ${{ runner.temp }}/submission-metadata

  ##
  ## ANALYZE
  ##

  detect-regression:
    name: Determine regression status
    runs-on: ubuntu-22.04
    timeout-minutes: 5
    needs:
      - submit-job
      - should-run-gate
      - resolve-inputs
    # Job level permissions for downloading SMP results
    permissions:
      contents: read # Required to checkout code
      id-token: write # Required for GitHub OIDC token exchange
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
        with:
          role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc
          aws-region: us-west-2
          role-duration-seconds: 14400 # 4 hours

      - name: Download SMP binary
        run: |
          aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-musl/smp ${{ runner.temp }}/bin/smp

      - name: Download submission metadata
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: vector-submission-metadata
          path: ${{ runner.temp }}/

      - name: Determine SMP job result
        env:
          RUST_LOG: info
        run: |
          chmod +x ${{ runner.temp }}/bin/smp

          ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job result \
            --submission-metadata ${{ runner.temp }}/submission-metadata

  analyze-experiment:
    name: Download regression analysis & upload report
    runs-on: ubuntu-22.04
    timeout-minutes: 5
    needs:
      - should-run-gate
      - submit-job
      - resolve-inputs
    # Job-level permissions for artifact upload
    permissions:
      contents: read # Required to checkout code
      actions: write # Required to upload artifacts
      id-token: write # Required for GitHub OIDC token exchange
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ needs.resolve-inputs.outputs.comparison-sha }}

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
        with:
          role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc
          aws-region: us-west-2
          role-duration-seconds: 14400 # 4 hours

      - name: Download SMP binary
        run: |
          aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-musl/smp ${{ runner.temp }}/bin/smp

      - name: Download submission metadata
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: vector-submission-metadata
          path: ${{ runner.temp }}/

      - name: Sync regression report to local system
        env:
          RUST_LOG: info
        run: |
          chmod +x ${{ runner.temp }}/bin/smp

          ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job sync \
            --submission-metadata ${{ runner.temp }}/submission-metadata \
            --output-path "${{ runner.temp }}/outputs"

      - name: Read regression report
        id: read-analysis
        uses: juliangruber/read-file-action@271ff311a4947af354c6abcd696a306553b9ec18 # v1.1.8
        with:
          path: ${{ runner.temp }}/outputs/report.md

      - name: Upload regression report to artifacts
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: capture-artifacts
          path: ${{ runner.temp }}/outputs/*

  regression-detection-suite:
    name: Set final result
    runs-on: ubuntu-latest
    timeout-minutes: 5
    if: always()
    needs:
      - should-run-gate
      - resolve-inputs
      - build-baseline
      - build-comparison
      - confirm-valid-credentials
      - upload-baseline-image-to-ecr
      - upload-comparison-image-to-ecr
      - submit-job
      - detect-regression
      - analyze-experiment
    steps:
      - name: Download capture-artifacts
        continue-on-error: true
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: capture-artifacts

      - name: Display Markdown Summary
        continue-on-error: true
        run: |
          REPORT_MD=report.md
          if [ -f ${REPORT_MD} ]; then
            cat ${REPORT_MD} >> $GITHUB_STEP_SUMMARY
          else
            echo "Did not find ${REPORT_MD} file."
          fi

      - name: Check all jobs status
        run: |
          if [[ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
            echo "One or more jobs failed or were cancelled"
            exit 1
          else
            echo "All jobs completed successfully"
          fi