diff --git a/pyproject.toml b/pyproject.toml
index 01acf03..a4c4257 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -18,7 +18,7 @@ dynamic = ["version"]
 requires-python = ">=3.10"
 dependencies = [
     "pytest",
-    "pytest-benchmark",
+    "pytest-benchmark>=5.3.0",
     "typer",
     "dask[complete]",
     "dash",
diff --git a/src/lbench/pytest/plugin.py b/src/lbench/pytest/plugin.py
index fd8c7c4..b98ba51 100644
--- a/src/lbench/pytest/plugin.py
+++ b/src/lbench/pytest/plugin.py
@@ -39,7 +39,7 @@ def pytest_configure(config: pytest.Config):
 
     # configure pytest-benchmark
     config.option.benchmark_only = True
-    config.option.benchmark_json = (run_dir / "pytest-benchmark.json").open("wb")  # kinda hacky
+    config.option.benchmark_json = run_dir / "pytest-benchmark.json"
 
     terminal = config.pluginmanager.get_plugin("terminalreporter")
     if terminal: