diff --git a/.codspeed-runner-version b/.codspeed-runner-version index f280719..fcdb2e1 100644 --- a/.codspeed-runner-version +++ b/.codspeed-runner-version @@ -1 +1 @@ -3.8.1 +4.0.0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 869e8c3..bd86a22 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,9 @@ jobs: - ubuntu-latest - codspeedhq-arm64-ubuntu-22.04 - codspeedhq-arm64-ubuntu-24.04 + mode: + - instrumentation + - walltime runs-on: ${{ matrix.os }} env: @@ -31,12 +34,14 @@ jobs: - name: Check basic action execution uses: ./ with: + mode: ${{ matrix.mode }} run: echo "Working!" - name: Check action execution with env variables uses: ./ env: MY_ENV_VAR: "Hello" with: + mode: ${{ matrix.mode }} run: | output=$(echo "$MY_ENV_VAR") if [ "$output" != "Hello" ]; then @@ -46,12 +51,14 @@ jobs: - name: Check action in a custom directory uses: ./ with: + mode: ${{ matrix.mode }} working-directory: examples # Check that the directory is actually changed run: if [ $(basename $(pwd)) != "examples" ]; then exit 1; fi - name: Check action with multiline command uses: ./ with: + mode: ${{ matrix.mode }} run: | echo "Working"; echo "with"; @@ -74,4 +81,5 @@ jobs: uses: ./ with: runner-version: ${{ matrix.version }} + mode: instrumentation run: echo "Testing version format ${{ matrix.version }}!" diff --git a/README.md b/README.md index f9faf6d..e4cee57 100644 --- a/README.md +++ b/README.md @@ -12,18 +12,23 @@ GitHub Actions for running [CodSpeed](https://codspeed.io) in your CI. # Usage ```yaml -- uses: CodSpeedHQ/action@v3 +- uses: CodSpeedHQ/action@v4 with: + # [REQUIRED] + # The command used to run your CodSpeed benchmarks + run: "" + + # [REQUIRED] + # The measurement mode to use, either: "instrumentation" or "walltime". + # More details on the instruments at https://docs.codspeed.io/instruments/ + mode: "instrumentation" + # [REQUIRED for private repositories] # The CodSpeed upload token: can be found at https://codspeed.io///settings # It's strongly recommended to use a secret for this value # If you're instrumenting a public repository, you can omit this value token: "" - # [REQUIRED] - # The command used to run your codspeed benchmarks - run: "" - # [OPTIONAL] # The directory where the `run` command will be executed. # ⚠️ WARNING: if you use `defaults.run.working-directory`, you must still set this parameter. @@ -78,8 +83,9 @@ jobs: run: pip install -r requirements.txt - name: Run benchmarks - uses: CodSpeedHQ/action@v3 + uses: CodSpeedHQ/action@v4 with: + mode: instrumentation token: ${{ secrets.CODSPEED_TOKEN }} run: pytest tests/ --codspeed ``` @@ -120,8 +126,9 @@ jobs: run: cargo codspeed build - name: Run the benchmarks - uses: CodSpeedHQ/action@v3 + uses: CodSpeedHQ/action@v4 with: + mode: instrumentation run: cargo codspeed run token: ${{ secrets.CODSPEED_TOKEN }} ``` @@ -157,8 +164,9 @@ jobs: run: npm install - name: Run benchmarks - uses: CodSpeedHQ/action@v3 + uses: CodSpeedHQ/action@v4 with: + mode: instrumentation run: npx vitest bench token: ${{ secrets.CODSPEED_TOKEN }} ``` diff --git a/action.yml b/action.yml index ede13ee..ec08c9f 100644 --- a/action.yml +++ b/action.yml @@ -6,18 +6,33 @@ branding: author: "Arthur Pastel" inputs: - token: - description: "CodSpeed upload token" - required: false run: description: "The command to run the benchmarks" required: true + mode: + description: | + The mode to to run the benchmarks in. The following modes are available: + - `instrumentation`: Run the benchmarks with CPU instrumentation measurements. + - `walltime`: Run the benchmarks with walltime measurement. + + We strongly recommend starting with the `instrumentation` mode. + + Using the `walltime` mode on traditional VMs/Hosted Runners might lead to inconsistent data. For the best results, we recommend using CodSpeed Hosted Macro Runners, which are fine-tuned for performance measurement consistency. + Check out the [Walltime Instrument Documentation](https://docs.codspeed.io/instruments/walltime/) for more details. + required: true + + token: + description: | + CodSpeed upload token. Only required for private repositories. + required: false + working-directory: description: | The directory where the `run` command will be executed. Warning: if you use defaults.working-directory, you must still set this parameter. required: false + upload-url: description: "The upload endpoint (for on-premise deployments)" required: false @@ -26,18 +41,6 @@ inputs: description: "The version of the runner to use. Use 'latest' to automatically fetch the latest release version from GitHub, or specify a version like '3.5.0' or 'v3.5.0'." required: false - mode: - description: | - The mode to to run the benchmarks in. The following modes are available: - - `instrumentation` (default): Run the benchmarks with instrumentation enabled. - - `walltime`: Run the benchmarks with walltime enabled. - - We strongly recommend not changing this mode unless you know what you are doing. - - Using the `walltime` mode on traditional VMs/Hosted Runners will lead to inconsistent data. For the best results, we recommend using CodSpeed Hosted Macro Runners, which are fine-tuned for performance measurement consistency. - Check out the [Walltime Instrument Documentation](https://docs.codspeed.io/instruments/walltime/) for more details. - required: false - instruments: description: | Comma separated list of instruments to enable. The following instruments are available: @@ -57,6 +60,14 @@ runs: GH_MATRIX: "${{ toJson(matrix) }}" GH_STRATEGY: "${{ toJson(strategy) }}" run: | + # Validate required inputs (custom message for smoother v4 migration) + if [ -z "${{ inputs.mode }}" ]; then + echo "::error title=Missing required input 'mode'::The 'mode' input is required as of CodSpeed Action v4. Please explicitly set 'mode' to 'instrumentation' or 'walltime'.= + Before, this variable was automatically set to instrumentation on every runner except for CodSpeed macro runners where it was set to walltime by default. + See https://codspeed.io/docs/instruments for details." + exit 1 + fi + # Configure and run codspeed-runner RUNNER_VERSION="${{ inputs.runner-version }}" if [ -z "$RUNNER_VERSION" ]; then diff --git a/examples/nodejs-typescript-codspeed.yml b/examples/nodejs-typescript-codspeed.yml index ffcd0e7..d6b5c5d 100644 --- a/examples/nodejs-typescript-codspeed.yml +++ b/examples/nodejs-typescript-codspeed.yml @@ -22,7 +22,8 @@ jobs: run: npm install - name: Run benchmarks - uses: CodSpeedHQ/action@v3 + uses: CodSpeedHQ/action@v4 with: + mode: instrumentation run: node -r esbuild-register benches/bench.ts token: ${{ secrets.CODSPEED_TOKEN }} diff --git a/examples/python-pytest-codspeed.yml b/examples/python-pytest-codspeed.yml index fc22a6f..5a5ab38 100644 --- a/examples/python-pytest-codspeed.yml +++ b/examples/python-pytest-codspeed.yml @@ -24,7 +24,8 @@ jobs: run: pip install -r requirements.txt - name: Run benchmarks - uses: CodSpeedHQ/action@v3 + uses: CodSpeedHQ/action@v4 with: - token: ${{ secrets.CODSPEED_TOKEN }} + mode: instrumentation run: pytest tests/ --codspeed + token: ${{ secrets.CODSPEED_TOKEN }} diff --git a/examples/rust-cargo-codspeed.yml b/examples/rust-cargo-codspeed.yml index 4ba458e..f7da761 100644 --- a/examples/rust-cargo-codspeed.yml +++ b/examples/rust-cargo-codspeed.yml @@ -27,7 +27,8 @@ jobs: run: cargo codspeed build - name: Run the benchmarks - uses: CodSpeedHQ/action@v3 + uses: CodSpeedHQ/action@v4 with: + mode: instrumentation run: cargo codspeed run token: ${{ secrets.CODSPEED_TOKEN }}