#!/usr/bin/env python3 """ Parse Criterion benchmark results and export to CSV/JSON formats. """ import json import csv import sys import os from pathlib import Path from typing import Dict, List, Any def parse_criterion_json(criterion_dir: str) -> List[Dict[str, Any]]: """Parse Criterion benchmark results from the target directory.""" results = [] criterion_path = Path(criterion_dir) if not criterion_path.exists(): print(f"Error: Criterion directory not found: {criterion_dir}", file=sys.stderr) return results # Find all benchmark.json files for benchmark_file in criterion_path.rglob("new/benchmark.json"): try: with open(benchmark_file, 'r') as f: data = json.load(f) # Extract benchmark name from path bench_name = str(benchmark_file.parent.parent.name) # Extract metrics result = { 'name': bench_name, 'mean_ns': data.get('mean', {}).get('point_estimate', 0), 'median_ns': data.get('median', {}).get('point_estimate', 0), 'std_dev_ns': data.get('std_dev', {}).get('point_estimate', 0), } # Calculate throughput if result['mean_ns'] > 0: result['throughput_ops_sec'] = 1_000_000_000 / result['mean_ns'] else: result['throughput_ops_sec'] = 0 results.append(result) except Exception as e: print(f"Warning: Failed to parse {benchmark_file}: {e}", file=sys.stderr) return results def parse_benchmark_name(name: str) -> Dict[str, str]: """Parse benchmark name into components.""" parts = name.split('/') result = { 'suite': parts[0] if len(parts) > 0 else '', 'category': parts[1] if len(parts) > 1 else '', 'operation': parts[2] if len(parts) > 2 else '', 'backend': '', 'parameter': '' } # Try to extract backend name for part in parts: if 'redb' in part.lower(): result['backend'] = 'redb' break elif 'sled' in part.lower(): result['backend'] = 'sled' break # Extract parameter (size, clients, etc.) if len(parts) > 3: result['parameter'] = parts[3] return result def export_to_csv(results: List[Dict[str, Any]], output_file: str): """Export results to CSV format.""" if not results: print("No results to export", file=sys.stderr) return fieldnames = ['name', 'backend', 'operation', 'mean_ns', 'median_ns', 'std_dev_ns', 'throughput_ops_sec'] with open(output_file, 'w', newline='') as f: writer = csv.DictWriter(f, fieldnames=fieldnames) writer.writeheader() for result in results: parsed = parse_benchmark_name(result['name']) row = { 'name': result['name'], 'backend': parsed['backend'], 'operation': parsed['operation'], 'mean_ns': int(result['mean_ns']), 'median_ns': int(result['median_ns']), 'std_dev_ns': int(result['std_dev_ns']), 'throughput_ops_sec': f"{result['throughput_ops_sec']:.2f}" } writer.writerow(row) print(f"Exported {len(results)} results to {output_file}") def export_to_json(results: List[Dict[str, Any]], output_file: str): """Export results to JSON format.""" if not results: print("No results to export", file=sys.stderr) return # Enhance results with parsed information enhanced_results = [] for result in results: parsed = parse_benchmark_name(result['name']) enhanced = {**result, **parsed} enhanced_results.append(enhanced) output = { 'benchmarks': enhanced_results, 'summary': { 'total_benchmarks': len(results), 'backends': list(set(r.get('backend', '') for r in enhanced_results if r.get('backend'))) } } with open(output_file, 'w') as f: json.dump(output, f, indent=2) print(f"Exported {len(results)} results to {output_file}") def print_summary(results: List[Dict[str, Any]]): """Print a summary of benchmark results.""" if not results: print("No results to summarize") return print("\n=== Benchmark Summary ===\n") print(f"Total benchmarks: {len(results)}") # Group by backend backends = {} for result in results: parsed = parse_benchmark_name(result['name']) backend = parsed['backend'] if backend: if backend not in backends: backends[backend] = [] backends[backend].append(result) for backend, bench_results in backends.items(): print(f"\n{backend.upper()}:") print(f" Benchmarks: {len(bench_results)}") if bench_results: mean_throughput = sum(r['throughput_ops_sec'] for r in bench_results) / len(bench_results) print(f" Avg throughput: {mean_throughput:.2f} ops/sec") fastest = max(bench_results, key=lambda x: x['throughput_ops_sec']) print(f" Fastest: {fastest['name']} ({fastest['throughput_ops_sec']:.2f} ops/sec)") def main(): if len(sys.argv) < 2: print("Usage: python parse_results.py [--csv output.csv] [--json output.json]") print("\nExample:") print(" python parse_results.py target/criterion --csv results.csv --json results.json") sys.exit(1) criterion_dir = sys.argv[1] # Parse command line arguments csv_output = None json_output = None i = 2 while i < len(sys.argv): if sys.argv[i] == '--csv' and i + 1 < len(sys.argv): csv_output = sys.argv[i + 1] i += 2 elif sys.argv[i] == '--json' and i + 1 < len(sys.argv): json_output = sys.argv[i + 1] i += 2 else: i += 1 # Parse results print(f"Parsing benchmark results from {criterion_dir}...") results = parse_criterion_json(criterion_dir) if not results: print("No benchmark results found!") sys.exit(1) # Export results if csv_output: export_to_csv(results, csv_output) if json_output: export_to_json(results, json_output) # Print summary print_summary(results) # If no output specified, print to stdout if not csv_output and not json_output: print("\n=== CSV Output ===\n") import io output = io.StringIO() fieldnames = ['name', 'mean_ns', 'median_ns', 'throughput_ops_sec'] writer = csv.DictWriter(output, fieldnames=fieldnames) writer.writeheader() for result in results: writer.writerow({ 'name': result['name'], 'mean_ns': int(result['mean_ns']), 'median_ns': int(result['median_ns']), 'throughput_ops_sec': f"{result['throughput_ops_sec']:.2f}" }) print(output.getvalue()) if __name__ == '__main__': main()