Support pytest-benchmark marker attributes

.github/workflows/ci.yml

@@ -21,7 +21,7 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: "3.11"
-      - uses: pre-commit/action@v3.0.0
+      - uses: pre-commit/action@v3.0.1
         with:
           extra_args: --all-files
 

.pre-commit-config.yaml

@@ -13,8 +13,8 @@ repos:
     hooks:
       - id: mypy
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.5
+    rev: v0.11.12
     hooks:
-      - id: ruff
+      - id: ruff-check
         args: [--fix]
       - id: ruff-format

pyproject.toml

@@ -37,7 +37,7 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-lint = ["mypy ~= 1.11.2", "ruff ~= 0.6.5"]
+lint = ["mypy ~= 1.11.2", "ruff ~= 0.11.12"]
 compat = [
     "pytest-benchmark ~= 5.0.0",
     "pytest-xdist ~= 3.6.1",

src/pytest_codspeed/config.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import pytest
+
+
+@dataclass(frozen=True)
+class CodSpeedConfig:
+    """
+    The configuration for the codspeed plugin.
+    Usually created from the command line arguments.
+    """
+
+    warmup_time_ns: int | None = None
+    max_time_ns: int | None = None
+    max_rounds: int | None = None
+
+    @classmethod
+    def from_pytest_config(cls, config: pytest.Config) -> CodSpeedConfig:
+        warmup_time = config.getoption("--codspeed-warmup-time", None)
+        warmup_time_ns = (
+            int(warmup_time * 1_000_000_000) if warmup_time is not None else None
+        )
+        max_time = config.getoption("--codspeed-max-time", None)
+        max_time_ns = int(max_time * 1_000_000_000) if max_time is not None else None
+        return cls(
+            warmup_time_ns=warmup_time_ns,
+            max_rounds=config.getoption("--codspeed-max-rounds", None),
+            max_time_ns=max_time_ns,
+        )
+
+
+@dataclass(frozen=True)
+class BenchmarkMarkerOptions:
+    group: str | None = None
+    """The group name to use for the benchmark."""
+    min_time: int | None = None
+    """
+    The minimum time of a round (in seconds).
+    Only available in walltime mode.
+    """
+    max_time: int | None = None
+    """
+    The maximum time to run the benchmark for (in seconds).
+    Only available in walltime mode.
+    """
+    max_rounds: int | None = None
+    """
+    The maximum number of rounds to run the benchmark for.
+    Takes precedence over max_time. Only available in walltime mode.
+    """
+
+    @classmethod
+    def from_pytest_item(cls, item: pytest.Item) -> BenchmarkMarkerOptions:
+        marker = item.get_closest_marker(
+            "codspeed_benchmark"
+        ) or item.get_closest_marker("benchmark")
+        if marker is None:
+            return cls()
+        if len(marker.args) > 0:
+            raise ValueError(
+                "Positional arguments are not allowed in the benchmark marker"
+            )
+
+        options = cls(
+            group=marker.kwargs.pop("group", None),
+            min_time=marker.kwargs.pop("min_time", None),
+            max_time=marker.kwargs.pop("max_time", None),
+            max_rounds=marker.kwargs.pop("max_rounds", None),
+        )
+
+        if len(marker.kwargs) > 0:
+            raise ValueError(
+                "Unknown kwargs passed to benchmark marker: "
+                + ", ".join(marker.kwargs.keys())
+            )
+        return options

src/pytest_codspeed/instruments/__init__.py

@@ -9,6 +9,7 @@
 
     import pytest
 
+    from pytest_codspeed.config import BenchmarkMarkerOptions
     from pytest_codspeed.plugin import CodSpeedConfig
 
     T = TypeVar("T")
@@ -27,6 +28,7 @@ def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]: ...
     @abstractmethod
     def measure(
         self,
+        marker_options: BenchmarkMarkerOptions,
         name: str,
         uri: str,
         fn: Callable[P, T],

src/pytest_codspeed/instruments/hooks/__init__.py

@@ -6,7 +6,7 @@
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from .dist_instrument_hooks import lib as LibType
+    from .dist_instrument_hooks import InstrumentHooksPointer, LibType
 
 SUPPORTS_PERF_TRAMPOLINE = sys.version_info >= (3, 12)
 
@@ -15,7 +15,7 @@ class InstrumentHooks:
     """Zig library wrapper class providing benchmark measurement functionality."""
 
     lib: LibType
-    instance: int
+    instance: InstrumentHooksPointer
 
     def __init__(self) -> None:
         if os.environ.get("CODSPEED_ENV") is None:
@@ -28,17 +28,15 @@ def __init__(self) -> None:
             from .dist_instrument_hooks import lib  # type: ignore
         except ImportError as e:
             raise RuntimeError(f"Failed to load instrument hooks library: {e}") from e
+        self.lib = lib
 
-        instance = lib.instrument_hooks_init()
-        if instance == 0:
+        self.instance = self.lib.instrument_hooks_init()
+        if self.instance == 0:
             raise RuntimeError("Failed to initialize CodSpeed instrumentation library.")
 
         if SUPPORTS_PERF_TRAMPOLINE:
             sys.activate_stack_trampoline("perf")  # type: ignore
 
-        self.lib = lib
-        self.instance = instance
-
     def __del__(self):
         if hasattr(self, "lib") and hasattr(self, "instance"):
             self.lib.instrument_hooks_deinit(self.instance)

src/pytest_codspeed/instruments/hooks/dist_instrument_hooks.pyi

@@ -0,0 +1,27 @@
+InstrumentHooksPointer = object
+
+class lib:
+    @staticmethod
+    def instrument_hooks_init() -> InstrumentHooksPointer: ...
+    @staticmethod
+    def instrument_hooks_deinit(hooks: InstrumentHooksPointer) -> None: ...
+    @staticmethod
+    def instrument_hooks_is_instrumented(hooks: InstrumentHooksPointer) -> bool: ...
+    @staticmethod
+    def instrument_hooks_start_benchmark(hooks: InstrumentHooksPointer) -> int: ...
+    @staticmethod
+    def instrument_hooks_stop_benchmark(hooks: InstrumentHooksPointer) -> int: ...
+    @staticmethod
+    def instrument_hooks_executed_benchmark(
+        hooks: InstrumentHooksPointer, pid: int, uri: bytes
+    ) -> int: ...
+    @staticmethod
+    def instrument_hooks_set_integration(
+        hooks: InstrumentHooksPointer, name: bytes, version: bytes
+    ) -> int: ...
+    @staticmethod
+    def callgrind_start_instrumentation() -> int: ...
+    @staticmethod
+    def callgrind_stop_instrumentation() -> int: ...
+
+LibType = type[lib]

src/pytest_codspeed/instruments/valgrind.py

@@ -13,7 +13,7 @@
     from pytest import Session
 
     from pytest_codspeed.instruments import P, T
-    from pytest_codspeed.plugin import CodSpeedConfig
+    from pytest_codspeed.plugin import BenchmarkMarkerOptions, CodSpeedConfig
 
 SUPPORTS_PERF_TRAMPOLINE = sys.version_info >= (3, 12)
 
@@ -35,7 +35,7 @@ def __init__(self, config: CodSpeedConfig) -> None:
     def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]:
         config = (
             f"mode: instrumentation, "
-            f"callgraph: {'enabled' if SUPPORTS_PERF_TRAMPOLINE  else 'not supported'}"
+            f"callgraph: {'enabled' if SUPPORTS_PERF_TRAMPOLINE else 'not supported'}"
         )
         warnings = []
         if not self.should_measure:
@@ -49,6 +49,7 @@ def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]:
 
     def measure(
         self,
+        marker_options: BenchmarkMarkerOptions,
         name: str,
         uri: str,
         fn: Callable[P, T],

src/pytest_codspeed/instruments/walltime.py

@@ -23,12 +23,12 @@
     from pytest import Session
 
     from pytest_codspeed.instruments import P, T
-    from pytest_codspeed.plugin import CodSpeedConfig
+    from pytest_codspeed.plugin import BenchmarkMarkerOptions, CodSpeedConfig
 
 DEFAULT_WARMUP_TIME_NS = 1_000_000_000
 DEFAULT_MAX_TIME_NS = 3_000_000_000
 TIMER_RESOLUTION_NS = get_clock_info("perf_counter").resolution * 1e9
-DEFAULT_MIN_ROUND_TIME_NS = TIMER_RESOLUTION_NS * 1_000_000
+DEFAULT_MIN_ROUND_TIME_NS = int(TIMER_RESOLUTION_NS * 1_000_000)
 
 IQR_OUTLIER_FACTOR = 1.5
 STDEV_OUTLIER_FACTOR = 3
@@ -42,16 +42,35 @@ class BenchmarkConfig:
     max_rounds: int | None
 
     @classmethod
-    def from_codspeed_config(cls, config: CodSpeedConfig) -> BenchmarkConfig:
+    def from_codspeed_config_and_marker_data(
+        cls, config: CodSpeedConfig, marker_data: BenchmarkMarkerOptions
+    ) -> BenchmarkConfig:
+        if marker_data.max_time is not None:
+            max_time_ns = int(marker_data.max_time * 1e9)
+        elif config.max_time_ns is not None:
+            max_time_ns = config.max_time_ns
+        else:
+            max_time_ns = DEFAULT_MAX_TIME_NS
+
+        if marker_data.max_rounds is not None:
+            max_rounds = marker_data.max_rounds
+        elif config.max_rounds is not None:
+            max_rounds = config.max_rounds
+        else:
+            max_rounds = None
+
+        if marker_data.min_time is not None:
+            min_round_time_ns = int(marker_data.min_time * 1e9)
+        else:
+            min_round_time_ns = DEFAULT_MIN_ROUND_TIME_NS
+
         return cls(
             warmup_time_ns=config.warmup_time_ns
             if config.warmup_time_ns is not None
             else DEFAULT_WARMUP_TIME_NS,
-            min_round_time_ns=DEFAULT_MIN_ROUND_TIME_NS,
-            max_time_ns=config.max_time_ns
-            if config.max_time_ns is not None
-            else DEFAULT_MAX_TIME_NS,
-            max_rounds=config.max_rounds,
+            min_round_time_ns=min_round_time_ns,
+            max_time_ns=max_time_ns,
+            max_rounds=max_rounds,
         )
 
 
@@ -231,6 +250,7 @@ def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]:
 
     def measure(
         self,
+        marker_options: BenchmarkMarkerOptions,
         name: str,
         uri: str,
         fn: Callable[P, T],
@@ -244,7 +264,9 @@ def measure(
             fn=fn,
             args=args,
             kwargs=kwargs,
-            config=BenchmarkConfig.from_codspeed_config(self.config),
+            config=BenchmarkConfig.from_codspeed_config_and_marker_data(
+                self.config, marker_options
+            ),
         )
         self.benchmarks.append(bench)
         return out