benchmarking
This commit is contained in:
Maxime Van Hees
222
scripts/parse_results.py
Executable file
222
scripts/parse_results.py
Executable file
@@ -0,0 +1,222 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Parse Criterion benchmark results and export to CSV/JSON formats.
|
||||
"""
|
||||
|
||||
import json
|
||||
import csv
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Any
|
||||
|
||||
def parse_criterion_json(criterion_dir: str) -> List[Dict[str, Any]]:
|
||||
"""Parse Criterion benchmark results from the target directory."""
|
||||
results = []
|
||||
criterion_path = Path(criterion_dir)
|
||||
|
||||
if not criterion_path.exists():
|
||||
print(f"Error: Criterion directory not found: {criterion_dir}", file=sys.stderr)
|
||||
return results
|
||||
|
||||
# Find all benchmark.json files
|
||||
for benchmark_file in criterion_path.rglob("new/benchmark.json"):
|
||||
try:
|
||||
with open(benchmark_file, 'r') as f:
|
||||
data = json.load(f)
|
||||
|
||||
# Extract benchmark name from path
|
||||
bench_name = str(benchmark_file.parent.parent.name)
|
||||
|
||||
# Extract metrics
|
||||
result = {
|
||||
'name': bench_name,
|
||||
'mean_ns': data.get('mean', {}).get('point_estimate', 0),
|
||||
'median_ns': data.get('median', {}).get('point_estimate', 0),
|
||||
'std_dev_ns': data.get('std_dev', {}).get('point_estimate', 0),
|
||||
}
|
||||
|
||||
# Calculate throughput
|
||||
if result['mean_ns'] > 0:
|
||||
result['throughput_ops_sec'] = 1_000_000_000 / result['mean_ns']
|
||||
else:
|
||||
result['throughput_ops_sec'] = 0
|
||||
|
||||
results.append(result)
|
||||
except Exception as e:
|
||||
print(f"Warning: Failed to parse {benchmark_file}: {e}", file=sys.stderr)
|
||||
|
||||
return results
|
||||
|
||||
def parse_benchmark_name(name: str) -> Dict[str, str]:
|
||||
"""Parse benchmark name into components."""
|
||||
parts = name.split('/')
|
||||
|
||||
result = {
|
||||
'suite': parts[0] if len(parts) > 0 else '',
|
||||
'category': parts[1] if len(parts) > 1 else '',
|
||||
'operation': parts[2] if len(parts) > 2 else '',
|
||||
'backend': '',
|
||||
'parameter': ''
|
||||
}
|
||||
|
||||
# Try to extract backend name
|
||||
for part in parts:
|
||||
if 'redb' in part.lower():
|
||||
result['backend'] = 'redb'
|
||||
break
|
||||
elif 'sled' in part.lower():
|
||||
result['backend'] = 'sled'
|
||||
break
|
||||
|
||||
# Extract parameter (size, clients, etc.)
|
||||
if len(parts) > 3:
|
||||
result['parameter'] = parts[3]
|
||||
|
||||
return result
|
||||
|
||||
def export_to_csv(results: List[Dict[str, Any]], output_file: str):
|
||||
"""Export results to CSV format."""
|
||||
if not results:
|
||||
print("No results to export", file=sys.stderr)
|
||||
return
|
||||
|
||||
fieldnames = ['name', 'backend', 'operation', 'mean_ns', 'median_ns',
|
||||
'std_dev_ns', 'throughput_ops_sec']
|
||||
|
||||
with open(output_file, 'w', newline='') as f:
|
||||
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
|
||||
for result in results:
|
||||
parsed = parse_benchmark_name(result['name'])
|
||||
row = {
|
||||
'name': result['name'],
|
||||
'backend': parsed['backend'],
|
||||
'operation': parsed['operation'],
|
||||
'mean_ns': int(result['mean_ns']),
|
||||
'median_ns': int(result['median_ns']),
|
||||
'std_dev_ns': int(result['std_dev_ns']),
|
||||
'throughput_ops_sec': f"{result['throughput_ops_sec']:.2f}"
|
||||
}
|
||||
writer.writerow(row)
|
||||
|
||||
print(f"Exported {len(results)} results to {output_file}")
|
||||
|
||||
def export_to_json(results: List[Dict[str, Any]], output_file: str):
|
||||
"""Export results to JSON format."""
|
||||
if not results:
|
||||
print("No results to export", file=sys.stderr)
|
||||
return
|
||||
|
||||
# Enhance results with parsed information
|
||||
enhanced_results = []
|
||||
for result in results:
|
||||
parsed = parse_benchmark_name(result['name'])
|
||||
enhanced = {**result, **parsed}
|
||||
enhanced_results.append(enhanced)
|
||||
|
||||
output = {
|
||||
'benchmarks': enhanced_results,
|
||||
'summary': {
|
||||
'total_benchmarks': len(results),
|
||||
'backends': list(set(r.get('backend', '') for r in enhanced_results if r.get('backend')))
|
||||
}
|
||||
}
|
||||
|
||||
with open(output_file, 'w') as f:
|
||||
json.dump(output, f, indent=2)
|
||||
|
||||
print(f"Exported {len(results)} results to {output_file}")
|
||||
|
||||
def print_summary(results: List[Dict[str, Any]]):
|
||||
"""Print a summary of benchmark results."""
|
||||
if not results:
|
||||
print("No results to summarize")
|
||||
return
|
||||
|
||||
print("\n=== Benchmark Summary ===\n")
|
||||
print(f"Total benchmarks: {len(results)}")
|
||||
|
||||
# Group by backend
|
||||
backends = {}
|
||||
for result in results:
|
||||
parsed = parse_benchmark_name(result['name'])
|
||||
backend = parsed['backend']
|
||||
if backend:
|
||||
if backend not in backends:
|
||||
backends[backend] = []
|
||||
backends[backend].append(result)
|
||||
|
||||
for backend, bench_results in backends.items():
|
||||
print(f"\n{backend.upper()}:")
|
||||
print(f" Benchmarks: {len(bench_results)}")
|
||||
|
||||
if bench_results:
|
||||
mean_throughput = sum(r['throughput_ops_sec'] for r in bench_results) / len(bench_results)
|
||||
print(f" Avg throughput: {mean_throughput:.2f} ops/sec")
|
||||
|
||||
fastest = max(bench_results, key=lambda x: x['throughput_ops_sec'])
|
||||
print(f" Fastest: {fastest['name']} ({fastest['throughput_ops_sec']:.2f} ops/sec)")
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python parse_results.py <criterion_dir> [--csv output.csv] [--json output.json]")
|
||||
print("\nExample:")
|
||||
print(" python parse_results.py target/criterion --csv results.csv --json results.json")
|
||||
sys.exit(1)
|
||||
|
||||
criterion_dir = sys.argv[1]
|
||||
|
||||
# Parse command line arguments
|
||||
csv_output = None
|
||||
json_output = None
|
||||
|
||||
i = 2
|
||||
while i < len(sys.argv):
|
||||
if sys.argv[i] == '--csv' and i + 1 < len(sys.argv):
|
||||
csv_output = sys.argv[i + 1]
|
||||
i += 2
|
||||
elif sys.argv[i] == '--json' and i + 1 < len(sys.argv):
|
||||
json_output = sys.argv[i + 1]
|
||||
i += 2
|
||||
else:
|
||||
i += 1
|
||||
|
||||
# Parse results
|
||||
print(f"Parsing benchmark results from {criterion_dir}...")
|
||||
results = parse_criterion_json(criterion_dir)
|
||||
|
||||
if not results:
|
||||
print("No benchmark results found!")
|
||||
sys.exit(1)
|
||||
|
||||
# Export results
|
||||
if csv_output:
|
||||
export_to_csv(results, csv_output)
|
||||
|
||||
if json_output:
|
||||
export_to_json(results, json_output)
|
||||
|
||||
# Print summary
|
||||
print_summary(results)
|
||||
|
||||
# If no output specified, print to stdout
|
||||
if not csv_output and not json_output:
|
||||
print("\n=== CSV Output ===\n")
|
||||
import io
|
||||
output = io.StringIO()
|
||||
fieldnames = ['name', 'mean_ns', 'median_ns', 'throughput_ops_sec']
|
||||
writer = csv.DictWriter(output, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
for result in results:
|
||||
writer.writerow({
|
||||
'name': result['name'],
|
||||
'mean_ns': int(result['mean_ns']),
|
||||
'median_ns': int(result['median_ns']),
|
||||
'throughput_ops_sec': f"{result['throughput_ops_sec']:.2f}"
|
||||
})
|
||||
print(output.getvalue())
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user