diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 81f123610c4..74d71636d70 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,16 +1,19 @@ include: - local: ".gitlab/one-pipeline.locked.yml" - local: ".gitlab/benchmarks.yml" - - local: ".gitlab/macrobenchmarks.yml" - local: ".gitlab/exploration-tests.yml" - local: ".gitlab/ci-visibility-tests.yml" + - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' + file: '.gitlab/ci-java-spring-petclinic.yml' + ref: 'sarahchen6/migrate-java-macrobenchmarks' stages: - build - publish - shared-pipeline - benchmarks - - macrobenchmarks + - java-spring-petclinic-tests + - java-spring-petclinic-macrobenchmarks - tests - exploration-tests - ci-visibility-tests diff --git a/.gitlab/benchmarks/bp-runner.fail-on-breach.yml b/.gitlab/benchmarks/bp-runner.fail-on-breach.yml index eaea9830029..ce23ca5ef60 100644 --- a/.gitlab/benchmarks/bp-runner.fail-on-breach.yml +++ b/.gitlab/benchmarks/bp-runner.fail-on-breach.yml @@ -35,14 +35,15 @@ experiments: thresholds: - throughput > 1100.0 op/s + # TODO: Startup benchmarks need to be moved from dd-trace-java to apm-sdks-benchmarks (https://github.com/DataDog/dd-trace-java/blob/master/benchmark/startup/petclinic/benchmark.json) # Startup macrobenchmarks # https://benchmarking.us1.prod.dog/trends?projectId=4&branch=master&trendsTab=per_scenario&scenario=startup%3Apetclinic%3Atracing%3AAgent.start&trendsType=scenario - - name: "startup:petclinic:tracing:Agent.start" - thresholds: - - execution_time < 1120ms + # - name: "startup:petclinic:tracing:Agent.start" + # thresholds: + # - execution_time < 1120ms # https://benchmarking.us1.prod.dog/trends?projectId=4&branch=master&trendsTab=per_scenario&scenario=startup%3Apetclinic%3Aprofiling%3AAgent.start&trendsType=scenario # https://benchmarking.us1.prod.dog/trends?projectId=4&branch=master&trendsTab=per_scenario&scenario=startup%3Apetclinic%3Aappsec%3AAgent.start&trendsType=scenario # https://benchmarking.us1.prod.dog/trends?projectId=4&branch=master&trendsTab=per_scenario&scenario=startup%3Apetclinic%3Aiast%3AAgent.start&trendsType=scenario - - name: "startup:petclinic:(profiling|appsec|iast):Agent.start" - thresholds: - - execution_time < 1300 ms + # - name: "startup:petclinic:(profiling|appsec|iast):Agent.start" + # thresholds: + # - execution_time < 1300 ms diff --git a/.gitlab/macrobenchmarks.yml b/.gitlab/macrobenchmarks.yml deleted file mode 100644 index 9c7a9cd4cad..00000000000 --- a/.gitlab/macrobenchmarks.yml +++ /dev/null @@ -1,146 +0,0 @@ -include: - - project: 'DataDog/benchmarking-platform-tools' - file: 'images/templates/gitlab/notify-slo-breaches.template.yml' - - project: 'DataDog/benchmarking-platform-tools' - file: 'images/templates/gitlab/check-slo-breaches.template.yml' - -.macrobenchmarks: - stage: macrobenchmarks - rules: - - if: $POPULATE_CACHE - when: never - - if: ($NIGHTLY_BENCHMARKS || $CI_PIPELINE_SOURCE != "schedule") && $CI_COMMIT_REF_NAME == "master" - when: always - interruptible: false - - when: manual - interruptible: true - allow_failure: true - tags: ["runner:apm-k8s-same-cpu"] - needs: ["build"] - timeout: 1h - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:dd-trace-java-petclinic - script: - - git clone --branch java/petclinic https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform platform && cd platform - - ./steps/run-benchmarks.sh - artifacts: - name: "artifacts" - when: always - paths: - - platform/artifacts/ - expire_in: 3 months - variables: - K6_OPTIONS_WARMUP_RATE: 2000 - K6_OPTIONS_WARMUP_DURATION: 5m - K6_OPTIONS_WARMUP_GRACEFUL_STOP: 10s - K6_OPTIONS_WARMUP_PRE_ALLOCATED_VUS: 4 - K6_OPTIONS_WARMUP_MAX_VUS: 4 - - K6_OPTIONS_NORMAL_OPERATION_RATE: 1500 - K6_OPTIONS_NORMAL_OPERATION_DURATION: 10m - K6_OPTIONS_NORMAL_OPERATION_GRACEFUL_STOP: 10s - K6_OPTIONS_NORMAL_OPERATION_PRE_ALLOCATED_VUS: 4 - K6_OPTIONS_NORMAL_OPERATION_MAX_VUS: 4 - - K6_OPTIONS_HIGH_LOAD_RATE: 4000 - K6_OPTIONS_HIGH_LOAD_DURATION: 5m - K6_OPTIONS_HIGH_LOAD_GRACEFUL_STOP: 10s - K6_OPTIONS_HIGH_LOAD_PRE_ALLOCATED_VUS: 4 - K6_OPTIONS_HIGH_LOAD_MAX_VUS: 4 - retry: - max: 2 - when: - - unknown_failure - - data_integrity_failure - - runner_system_failure - - scheduler_failure - - api_failure - -baseline: - extends: .macrobenchmarks - variables: - BP_BENCHMARKS_CONFIGURATION: baseline - TRACER_OPTS: -Ddd.service=bp-java-petclinic - JAVA_OPTS: -javaagent:/app/memcheck/stability-testing-memwatch.jar -Xmx128M - -only-tracing: - extends: .macrobenchmarks - variables: - BP_BENCHMARKS_CONFIGURATION: only-tracing - TRACER_OPTS: -javaagent:/app/dd-java-agent.jar -Ddd.env=${BP_BENCHMARKS_CONFIGURATION} -Ddd.service=bp-java-petclinic - JAVA_OPTS: -javaagent:/app/memcheck/stability-testing-memwatch.jar -Xmx128M - -otel-latest: - extends: .macrobenchmarks - variables: - BP_BENCHMARKS_CONFIGURATION: otel-latest - TRACER_OPTS: -javaagent:/app/otel-java-agent.jar -Ddd.env=otel-latest -Ddd.service=bp-java-petclinic - JAVA_OPTS: -javaagent:/app/memcheck/stability-testing-memwatch.jar -Xmx128M - - -check-slo-breaches: - extends: .check-slo-breaches - stage: macrobenchmarks - interruptible: true - rules: - - if: $POPULATE_CACHE - when: never - - when: on_success - needs: - - job: baseline - artifacts: true - - job: only-tracing - artifacts: true - - job: otel-latest - artifacts: true - - job: benchmarks-startup - artifacts: true - - job: benchmarks-load - artifacts: true - - job: benchmarks-dacapo - artifacts: true - script: - # macrobenchmarks are located here, files are already in "converted" format - - export ARTIFACTS_DIR="$(pwd)/platform/artifacts/" && mkdir -p "${ARTIFACTS_DIR}" - - # Need to move the artifacts the benchmarks-* job - - | - export BENCHMARKS_ARTIFACTS_DIR="$(pwd)/reports" && mkdir -p "${BENCHMARKS_ARTIFACTS_DIR}" - for benchmarkType in startup load dacapo; do - find "$BENCHMARKS_ARTIFACTS_DIR/$benchmarkType" -name "benchmark-baseline.json" -o -name "benchmark-candidate.json" | while read file; do - relpath="${file#$BENCHMARKS_ARTIFACTS_DIR/$benchmarkType/}" - prefix="${relpath%/benchmark-*}" # Remove the trailing /benchmark-(baseline|candidate).json - prefix="${prefix#./}" # Remove any leading ./ - prefix="${prefix//\//-}" # Replace / with - - case "$file" in - *benchmark-baseline.json) type="baseline" ;; - *benchmark-candidate.json) type="candidate" ;; - esac - echo "Moving $file to $ARTIFACTS_DIR/${type}-${benchmarkType}-${prefix}.converted.json" - cp "$file" "$ARTIFACTS_DIR/${type}-${benchmarkType}-${prefix}.converted.json" - done - done - - ls -lah "$ARTIFACTS_DIR" - - bp-runner .gitlab/benchmarks/bp-runner.fail-on-breach.yml - artifacts: - name: "artifacts" - when: always - paths: - - platform/artifacts/ - expire_in: 1 week - variables: - DDOCTOSTS_POLICY: "self.gitlab.github-access.read" - UPSTREAM_PROJECT_ID: $CI_PROJECT_ID # The ID of the current project. This ID is unique across all projects on the GitLab instance. - UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME # "dd-trace-java" - UPSTREAM_BRANCH: $CI_COMMIT_REF_NAME # The branch or tag name for which project is built. - UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA # The commit revision the project is built for. - -notify-slo-breaches: - extends: .notify-slo-breaches - stage: macrobenchmarks - needs: ["check-slo-breaches"] - rules: - - if: $POPULATE_CACHE - when: never - - when: always - variables: - CHANNEL: "apm-java"