chore(ci_visibility): add pytest-benchmark support to new plugin

Author: vitor-de-araujo

ddtrace/testing/internal/pytest/benchmark.py

@@ -0,0 +1,65 @@
+from dataclasses import dataclass
+import typing as t
+
+import pytest
+
+
+BENCHMARK_INFO_TAG = "benchmark.duration.info"
+
+PYTEST_BENCHMARK_KEYS_TO_DATADOG_TAGS = {
+    "outliers": "benchmark.duration.statistics.outliers",
+}
+
+PYTEST_BENCHMARK_KEYS_TO_DATADOG_METRICS = {
+    "hd15iqr": "benchmark.duration.statistics.hd15iqr",
+    "iqr": "benchmark.duration.statistics.iqr",
+    "iqr_outliers": "benchmark.duration.statistics.iqr_outliers",
+    "ld15iqr": "benchmark.duration.statistics.ld15iqr",
+    "max": "benchmark.duration.statistics.max",
+    "mean": "benchmark.duration.statistics.mean",
+    "median": "benchmark.duration.statistics.median",
+    "min": "benchmark.duration.statistics.min",
+    "ops": "benchmark.duration.statistics.ops",
+    "q1": "benchmark.duration.statistics.q1",
+    "q3": "benchmark.duration.statistics.q3",
+    "rounds": "benchmark.duration.statistics.n",
+    "stddev": "benchmark.duration.statistics.std_dev",
+    "stddev_outliers": "benchmark.duration.statistics.std_dev_outliers",
+    "total": "benchmark.duration.statistics.total",
+}
+
+
+@dataclass
+class BenchmarkData:
+    tags: t.Dict[str, str]
+    metrics: t.Dict[str, float]
+
+
+def get_benchmark_tags_and_metrics(item: pytest.Item) -> t.Optional[BenchmarkData]:
+    if not item.config.pluginmanager.hasplugin("benchmark"):
+        return None
+
+    funcargs = getattr(item, "funcargs", None)
+    if not funcargs:
+        return None
+
+    benchmark_fixture = item.funcargs.get("benchmark")
+    if not benchmark_fixture or not benchmark_fixture.stats:
+        return None
+
+    stats = item.funcargs.get("benchmark").stats.stats
+
+    data = BenchmarkData(tags={}, metrics={})
+    data.tags[BENCHMARK_INFO_TAG] = "Time"
+
+    for stats_attr, tag_name in PYTEST_BENCHMARK_KEYS_TO_DATADOG_TAGS.items():
+        value = getattr(stats, stats_attr, None)
+        if value is not None:
+            data.tags[tag_name] = value
+
+    for stats_attr, metric_name in PYTEST_BENCHMARK_KEYS_TO_DATADOG_METRICS.items():
+        value = getattr(stats, stats_attr, None)
+        if value is not None:
+            data.metrics[metric_name] = value
+
+    return data

ddtrace/testing/internal/pytest/plugin.py

@@ -18,6 +18,8 @@
 from ddtrace.testing.internal.git import get_workspace_path
 from ddtrace.testing.internal.logging import catch_and_log_exceptions
 from ddtrace.testing.internal.logging import setup_logging
+from ddtrace.testing.internal.pytest.benchmark import BenchmarkData
+from ddtrace.testing.internal.pytest.benchmark import get_benchmark_tags_and_metrics
 from ddtrace.testing.internal.retry_handlers import RetryHandler
 from ddtrace.testing.internal.session_manager import SessionManager
 from ddtrace.testing.internal.telemetry import TelemetryAPI
@@ -139,6 +141,7 @@ def __init__(self, session_manager: SessionManager) -> None:
         self.enable_ddtrace = False
         self.reports_by_nodeid: t.Dict[str, _ReportGroup] = defaultdict(lambda: {})
         self.excinfo_by_report: t.Dict[pytest.TestReport, t.Optional[pytest.ExceptionInfo[t.Any]]] = {}
+        self.benchmark_data_by_nodeid: t.Dict[str, BenchmarkData] = {}
         self.tests_by_nodeid: t.Dict[str, Test] = {}
         self.is_xdist_worker = False
 
@@ -282,10 +285,7 @@ def pytest_runtest_protocol_wrapper(
             )
             test_run = test.make_test_run()
             test_run.start(start_ns=test.start_ns)
-            status, tags = self._get_test_outcome(item.nodeid)
-            test_run.set_status(status)
-            test_run.set_tags(tags)
-            test_run.set_context(context)
+            self._set_test_run_data(test_run, item, context)
             test_run.finish()
             test.set_status(test_run.get_status())  # TODO: this should be automatic?
             self.manager.writer.put_item(test_run)
@@ -323,10 +323,7 @@ def _do_one_test_run(
         TelemetryAPI.get().record_test_created(test_framework=TEST_FRAMEWORK, test_run=test_run)
 
         reports = _make_reports_dict(runtestprotocol(item, nextitem=nextitem, log=False))
-        status, tags = self._get_test_outcome(item.nodeid)
-        test_run.set_status(status)
-        test_run.set_tags(tags)
-        test_run.set_context(context)
+        self._set_test_run_data(test_run, item, context)
 
         TelemetryAPI.get().record_test_finished(
             test_framework=TEST_FRAMEWORK,
@@ -354,6 +351,17 @@ def _do_test_runs(self, item: pytest.Item, nextitem: t.Optional[pytest.Item]) ->
             test.set_status(test_run.get_status())  # TODO: this should be automatic?
             self.manager.writer.put_item(test_run)
 
+    def _set_test_run_data(self, test_run: TestRun, item: pytest.Item, context: TestContext) -> None:
+        status, tags = self._get_test_outcome(item.nodeid)
+        test_run.set_status(status)
+        test_run.set_tags(tags)
+        test_run.set_context(context)
+
+        if benchmark_data := self.benchmark_data_by_nodeid.pop(item.nodeid):
+            test_run.set_tags(benchmark_data.tags)
+            test_run.set_metrics(benchmark_data.metrics)
+            test_run.mark_benchmark()
+
     def _do_retries(
         self,
         item: pytest.Item,
@@ -526,6 +534,11 @@ def pytest_runtest_makereport(
         self.reports_by_nodeid[item.nodeid][call.when] = report
         self.excinfo_by_report[report] = call.excinfo
 
+        if call.when == TestPhase.TEARDOWN:
+            # We need to extract pytest-benchmark data _before_ the fixture teardown.
+            if benchmark_data := get_benchmark_tags_and_metrics(item):
+                self.benchmark_data_by_nodeid[item.nodeid] = benchmark_data
+
     def pytest_report_teststatus(self, report: pytest.TestReport) -> t.Optional[_ReportTestStatus]:
         if retry_outcome := _get_user_property(report, "dd_retry_outcome"):
             retry_reason = _get_user_property(report, "dd_retry_reason")

ddtrace/testing/internal/test_data.py

@@ -154,6 +154,8 @@ def __init__(self, name: str, parent: Test) -> None:
         self.module = self.suite.parent
         self.session = self.module.parent
 
+        self.tags[TestTag.TEST_TYPE] = "test"
+
     def __str__(self) -> str:
         return f"{self.test} #{self.attempt_number}"
 
@@ -169,8 +171,11 @@ def is_retry(self) -> bool:
     def has_failed_all_retries(self) -> bool:
         return self.tags.get(TestTag.HAS_FAILED_ALL_RETRIES) == TAG_TRUE
 
+    def mark_benchmark(self) -> None:
+        self.tags[TestTag.TEST_TYPE] = "benchmark"
+
     def is_benchmark(self) -> bool:
-        return False  # TODO: change when benchmark tests are implemented
+        return self.tags.get(TestTag.TEST_TYPE) == "benchmark"
 
     # Selenium / RUM functionality. These tags are only available after the test has finished and ddtrace span tags have
     # been copied over to the test run object.
@@ -348,6 +353,7 @@ class TestTag:
 
     SKIP_REASON = "test.skip_reason"
 
+    TEST_TYPE = "test.type"
     IS_NEW = "test.is_new"
     IS_QUARANTINED = "test.test_management.is_quarantined"
     IS_DISABLED = "test.test_management.is_test_disabled"

ddtrace/testing/internal/writer.py

@@ -223,7 +223,6 @@ def serialize_test_run(test_run: TestRun) -> Event:
                 "test.name": test_run.name,
                 "test.status": test_run.get_status().value,
                 "test.suite": test_run.suite.name,
-                "test.type": "test",
                 "type": "test",
             },
             "metrics": {