benchmarking
This commit is contained in:
Maxime Van Hees
258
scripts/compare_backends.py
Executable file
258
scripts/compare_backends.py
Executable file
@@ -0,0 +1,258 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Compare performance between redb and sled backends.
|
||||
"""
|
||||
|
||||
import json
|
||||
import csv
|
||||
import sys
|
||||
from typing import Dict, List, Any
|
||||
from pathlib import Path
|
||||
|
||||
def load_results(input_file: str) -> List[Dict[str, Any]]:
|
||||
"""Load benchmark results from CSV or JSON file."""
|
||||
path = Path(input_file)
|
||||
|
||||
if not path.exists():
|
||||
print(f"Error: File not found: {input_file}", file=sys.stderr)
|
||||
return []
|
||||
|
||||
if path.suffix == '.json':
|
||||
with open(input_file, 'r') as f:
|
||||
data = json.load(f)
|
||||
return data.get('benchmarks', [])
|
||||
elif path.suffix == '.csv':
|
||||
results = []
|
||||
with open(input_file, 'r') as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
# Convert numeric fields
|
||||
row['mean_ns'] = float(row.get('mean_ns', 0))
|
||||
row['median_ns'] = float(row.get('median_ns', 0))
|
||||
row['throughput_ops_sec'] = float(row.get('throughput_ops_sec', 0))
|
||||
results.append(row)
|
||||
return results
|
||||
else:
|
||||
print(f"Error: Unsupported file format: {path.suffix}", file=sys.stderr)
|
||||
return []
|
||||
|
||||
def group_by_operation(results: List[Dict[str, Any]]) -> Dict[str, Dict[str, Dict]]:
|
||||
"""Group results by operation and backend."""
|
||||
grouped = {}
|
||||
|
||||
for result in results:
|
||||
operation = result.get('operation', result.get('name', ''))
|
||||
backend = result.get('backend', '')
|
||||
|
||||
if not operation or not backend:
|
||||
continue
|
||||
|
||||
if operation not in grouped:
|
||||
grouped[operation] = {}
|
||||
|
||||
grouped[operation][backend] = result
|
||||
|
||||
return grouped
|
||||
|
||||
def calculate_speedup(redb_value: float, sled_value: float) -> float:
|
||||
"""Calculate speedup factor (positive means redb is faster)."""
|
||||
if sled_value == 0:
|
||||
return 0
|
||||
return sled_value / redb_value
|
||||
|
||||
def format_duration(nanos: float) -> str:
|
||||
"""Format duration in human-readable format."""
|
||||
if nanos < 1_000:
|
||||
return f"{nanos:.0f} ns"
|
||||
elif nanos < 1_000_000:
|
||||
return f"{nanos / 1_000:.2f} µs"
|
||||
elif nanos < 1_000_000_000:
|
||||
return f"{nanos / 1_000_000:.2f} ms"
|
||||
else:
|
||||
return f"{nanos / 1_000_000_000:.2f} s"
|
||||
|
||||
def print_comparison_table(grouped: Dict[str, Dict[str, Dict]]):
|
||||
"""Print a comparison table of backends."""
|
||||
print("\n" + "=" * 100)
|
||||
print("BACKEND PERFORMANCE COMPARISON")
|
||||
print("=" * 100)
|
||||
print()
|
||||
|
||||
# Header
|
||||
print(f"{'Operation':<30} {'redb (mean)':<15} {'sled (mean)':<15} {'Speedup':<12} {'Winner':<10}")
|
||||
print("-" * 100)
|
||||
|
||||
redb_wins = 0
|
||||
sled_wins = 0
|
||||
total_comparisons = 0
|
||||
|
||||
for operation in sorted(grouped.keys()):
|
||||
backends = grouped[operation]
|
||||
|
||||
if 'redb' in backends and 'sled' in backends:
|
||||
redb_mean = backends['redb'].get('mean_ns', 0)
|
||||
sled_mean = backends['sled'].get('mean_ns', 0)
|
||||
|
||||
speedup = calculate_speedup(redb_mean, sled_mean)
|
||||
|
||||
if speedup > 1.0:
|
||||
winner = "redb"
|
||||
redb_wins += 1
|
||||
elif speedup < 1.0:
|
||||
winner = "sled"
|
||||
sled_wins += 1
|
||||
else:
|
||||
winner = "tie"
|
||||
|
||||
total_comparisons += 1
|
||||
|
||||
speedup_str = f"{speedup:.2f}x" if speedup != 0 else "N/A"
|
||||
|
||||
print(f"{operation:<30} {format_duration(redb_mean):<15} {format_duration(sled_mean):<15} "
|
||||
f"{speedup_str:<12} {winner:<10}")
|
||||
|
||||
print("-" * 100)
|
||||
print(f"\nSummary: redb wins: {redb_wins}, sled wins: {sled_wins}, total: {total_comparisons}")
|
||||
|
||||
if total_comparisons > 0:
|
||||
redb_pct = (redb_wins / total_comparisons) * 100
|
||||
sled_pct = (sled_wins / total_comparisons) * 100
|
||||
print(f"Win rate: redb {redb_pct:.1f}%, sled {sled_pct:.1f}%")
|
||||
|
||||
def print_throughput_comparison(grouped: Dict[str, Dict[str, Dict]]):
|
||||
"""Print throughput comparison."""
|
||||
print("\n" + "=" * 100)
|
||||
print("THROUGHPUT COMPARISON (ops/sec)")
|
||||
print("=" * 100)
|
||||
print()
|
||||
|
||||
print(f"{'Operation':<30} {'redb':<20} {'sled':<20} {'Difference':<15}")
|
||||
print("-" * 100)
|
||||
|
||||
for operation in sorted(grouped.keys()):
|
||||
backends = grouped[operation]
|
||||
|
||||
if 'redb' in backends and 'sled' in backends:
|
||||
redb_throughput = backends['redb'].get('throughput_ops_sec', 0)
|
||||
sled_throughput = backends['sled'].get('throughput_ops_sec', 0)
|
||||
|
||||
diff_pct = ((redb_throughput - sled_throughput) / sled_throughput * 100) if sled_throughput > 0 else 0
|
||||
diff_str = f"{diff_pct:+.1f}%"
|
||||
|
||||
print(f"{operation:<30} {redb_throughput:>18,.0f} {sled_throughput:>18,.0f} {diff_str:>13}")
|
||||
|
||||
def generate_recommendations(grouped: Dict[str, Dict[str, Dict]]):
|
||||
"""Generate recommendations based on benchmark results."""
|
||||
print("\n" + "=" * 100)
|
||||
print("RECOMMENDATIONS")
|
||||
print("=" * 100)
|
||||
print()
|
||||
|
||||
redb_strengths = []
|
||||
sled_strengths = []
|
||||
|
||||
for operation, backends in grouped.items():
|
||||
if 'redb' in backends and 'sled' in backends:
|
||||
redb_mean = backends['redb'].get('mean_ns', 0)
|
||||
sled_mean = backends['sled'].get('mean_ns', 0)
|
||||
|
||||
speedup = calculate_speedup(redb_mean, sled_mean)
|
||||
|
||||
if speedup > 1.2: # redb is >20% faster
|
||||
redb_strengths.append((operation, speedup))
|
||||
elif speedup < 0.8: # sled is >20% faster
|
||||
sled_strengths.append((operation, 1/speedup))
|
||||
|
||||
print("Use redb when:")
|
||||
if redb_strengths:
|
||||
for op, speedup in sorted(redb_strengths, key=lambda x: x[1], reverse=True)[:5]:
|
||||
print(f" • {op}: {speedup:.2f}x faster than sled")
|
||||
else:
|
||||
print(" • No significant advantages found")
|
||||
|
||||
print("\nUse sled when:")
|
||||
if sled_strengths:
|
||||
for op, speedup in sorted(sled_strengths, key=lambda x: x[1], reverse=True)[:5]:
|
||||
print(f" • {op}: {speedup:.2f}x faster than redb")
|
||||
else:
|
||||
print(" • No significant advantages found")
|
||||
|
||||
print("\nGeneral guidelines:")
|
||||
print(" • redb: Better for read-heavy workloads, predictable latency")
|
||||
print(" • sled: Better for write-heavy workloads, memory efficiency")
|
||||
|
||||
def export_comparison(grouped: Dict[str, Dict[str, Dict]], output_file: str):
|
||||
"""Export comparison to CSV."""
|
||||
with open(output_file, 'w', newline='') as f:
|
||||
fieldnames = ['operation', 'redb_mean_ns', 'sled_mean_ns', 'speedup',
|
||||
'redb_throughput', 'sled_throughput', 'winner']
|
||||
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
|
||||
for operation, backends in sorted(grouped.items()):
|
||||
if 'redb' in backends and 'sled' in backends:
|
||||
redb_mean = backends['redb'].get('mean_ns', 0)
|
||||
sled_mean = backends['sled'].get('mean_ns', 0)
|
||||
redb_throughput = backends['redb'].get('throughput_ops_sec', 0)
|
||||
sled_throughput = backends['sled'].get('throughput_ops_sec', 0)
|
||||
|
||||
speedup = calculate_speedup(redb_mean, sled_mean)
|
||||
winner = "redb" if speedup > 1.0 else "sled" if speedup < 1.0 else "tie"
|
||||
|
||||
writer.writerow({
|
||||
'operation': operation,
|
||||
'redb_mean_ns': int(redb_mean),
|
||||
'sled_mean_ns': int(sled_mean),
|
||||
'speedup': f"{speedup:.2f}",
|
||||
'redb_throughput': f"{redb_throughput:.0f}",
|
||||
'sled_throughput': f"{sled_throughput:.0f}",
|
||||
'winner': winner
|
||||
})
|
||||
|
||||
print(f"\nComparison exported to {output_file}")
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python compare_backends.py <results_file> [--export comparison.csv]")
|
||||
print("\nExample:")
|
||||
print(" python compare_backends.py results.csv")
|
||||
print(" python compare_backends.py results.json --export comparison.csv")
|
||||
sys.exit(1)
|
||||
|
||||
input_file = sys.argv[1]
|
||||
export_file = None
|
||||
|
||||
# Parse command line arguments
|
||||
if '--export' in sys.argv:
|
||||
idx = sys.argv.index('--export')
|
||||
if idx + 1 < len(sys.argv):
|
||||
export_file = sys.argv[idx + 1]
|
||||
|
||||
# Load results
|
||||
print(f"Loading results from {input_file}...")
|
||||
results = load_results(input_file)
|
||||
|
||||
if not results:
|
||||
print("No results found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Loaded {len(results)} benchmark results")
|
||||
|
||||
# Group by operation
|
||||
grouped = group_by_operation(results)
|
||||
|
||||
if not grouped:
|
||||
print("No comparable results found!")
|
||||
sys.exit(1)
|
||||
|
||||
# Print comparisons
|
||||
print_comparison_table(grouped)
|
||||
print_throughput_comparison(grouped)
|
||||
generate_recommendations(grouped)
|
||||
|
||||
# Export if requested
|
||||
if export_file:
|
||||
export_comparison(grouped, export_file)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
222
scripts/parse_results.py
Executable file
222
scripts/parse_results.py
Executable file
@@ -0,0 +1,222 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Parse Criterion benchmark results and export to CSV/JSON formats.
|
||||
"""
|
||||
|
||||
import json
|
||||
import csv
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Any
|
||||
|
||||
def parse_criterion_json(criterion_dir: str) -> List[Dict[str, Any]]:
|
||||
"""Parse Criterion benchmark results from the target directory."""
|
||||
results = []
|
||||
criterion_path = Path(criterion_dir)
|
||||
|
||||
if not criterion_path.exists():
|
||||
print(f"Error: Criterion directory not found: {criterion_dir}", file=sys.stderr)
|
||||
return results
|
||||
|
||||
# Find all benchmark.json files
|
||||
for benchmark_file in criterion_path.rglob("new/benchmark.json"):
|
||||
try:
|
||||
with open(benchmark_file, 'r') as f:
|
||||
data = json.load(f)
|
||||
|
||||
# Extract benchmark name from path
|
||||
bench_name = str(benchmark_file.parent.parent.name)
|
||||
|
||||
# Extract metrics
|
||||
result = {
|
||||
'name': bench_name,
|
||||
'mean_ns': data.get('mean', {}).get('point_estimate', 0),
|
||||
'median_ns': data.get('median', {}).get('point_estimate', 0),
|
||||
'std_dev_ns': data.get('std_dev', {}).get('point_estimate', 0),
|
||||
}
|
||||
|
||||
# Calculate throughput
|
||||
if result['mean_ns'] > 0:
|
||||
result['throughput_ops_sec'] = 1_000_000_000 / result['mean_ns']
|
||||
else:
|
||||
result['throughput_ops_sec'] = 0
|
||||
|
||||
results.append(result)
|
||||
except Exception as e:
|
||||
print(f"Warning: Failed to parse {benchmark_file}: {e}", file=sys.stderr)
|
||||
|
||||
return results
|
||||
|
||||
def parse_benchmark_name(name: str) -> Dict[str, str]:
|
||||
"""Parse benchmark name into components."""
|
||||
parts = name.split('/')
|
||||
|
||||
result = {
|
||||
'suite': parts[0] if len(parts) > 0 else '',
|
||||
'category': parts[1] if len(parts) > 1 else '',
|
||||
'operation': parts[2] if len(parts) > 2 else '',
|
||||
'backend': '',
|
||||
'parameter': ''
|
||||
}
|
||||
|
||||
# Try to extract backend name
|
||||
for part in parts:
|
||||
if 'redb' in part.lower():
|
||||
result['backend'] = 'redb'
|
||||
break
|
||||
elif 'sled' in part.lower():
|
||||
result['backend'] = 'sled'
|
||||
break
|
||||
|
||||
# Extract parameter (size, clients, etc.)
|
||||
if len(parts) > 3:
|
||||
result['parameter'] = parts[3]
|
||||
|
||||
return result
|
||||
|
||||
def export_to_csv(results: List[Dict[str, Any]], output_file: str):
|
||||
"""Export results to CSV format."""
|
||||
if not results:
|
||||
print("No results to export", file=sys.stderr)
|
||||
return
|
||||
|
||||
fieldnames = ['name', 'backend', 'operation', 'mean_ns', 'median_ns',
|
||||
'std_dev_ns', 'throughput_ops_sec']
|
||||
|
||||
with open(output_file, 'w', newline='') as f:
|
||||
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
|
||||
for result in results:
|
||||
parsed = parse_benchmark_name(result['name'])
|
||||
row = {
|
||||
'name': result['name'],
|
||||
'backend': parsed['backend'],
|
||||
'operation': parsed['operation'],
|
||||
'mean_ns': int(result['mean_ns']),
|
||||
'median_ns': int(result['median_ns']),
|
||||
'std_dev_ns': int(result['std_dev_ns']),
|
||||
'throughput_ops_sec': f"{result['throughput_ops_sec']:.2f}"
|
||||
}
|
||||
writer.writerow(row)
|
||||
|
||||
print(f"Exported {len(results)} results to {output_file}")
|
||||
|
||||
def export_to_json(results: List[Dict[str, Any]], output_file: str):
|
||||
"""Export results to JSON format."""
|
||||
if not results:
|
||||
print("No results to export", file=sys.stderr)
|
||||
return
|
||||
|
||||
# Enhance results with parsed information
|
||||
enhanced_results = []
|
||||
for result in results:
|
||||
parsed = parse_benchmark_name(result['name'])
|
||||
enhanced = {**result, **parsed}
|
||||
enhanced_results.append(enhanced)
|
||||
|
||||
output = {
|
||||
'benchmarks': enhanced_results,
|
||||
'summary': {
|
||||
'total_benchmarks': len(results),
|
||||
'backends': list(set(r.get('backend', '') for r in enhanced_results if r.get('backend')))
|
||||
}
|
||||
}
|
||||
|
||||
with open(output_file, 'w') as f:
|
||||
json.dump(output, f, indent=2)
|
||||
|
||||
print(f"Exported {len(results)} results to {output_file}")
|
||||
|
||||
def print_summary(results: List[Dict[str, Any]]):
|
||||
"""Print a summary of benchmark results."""
|
||||
if not results:
|
||||
print("No results to summarize")
|
||||
return
|
||||
|
||||
print("\n=== Benchmark Summary ===\n")
|
||||
print(f"Total benchmarks: {len(results)}")
|
||||
|
||||
# Group by backend
|
||||
backends = {}
|
||||
for result in results:
|
||||
parsed = parse_benchmark_name(result['name'])
|
||||
backend = parsed['backend']
|
||||
if backend:
|
||||
if backend not in backends:
|
||||
backends[backend] = []
|
||||
backends[backend].append(result)
|
||||
|
||||
for backend, bench_results in backends.items():
|
||||
print(f"\n{backend.upper()}:")
|
||||
print(f" Benchmarks: {len(bench_results)}")
|
||||
|
||||
if bench_results:
|
||||
mean_throughput = sum(r['throughput_ops_sec'] for r in bench_results) / len(bench_results)
|
||||
print(f" Avg throughput: {mean_throughput:.2f} ops/sec")
|
||||
|
||||
fastest = max(bench_results, key=lambda x: x['throughput_ops_sec'])
|
||||
print(f" Fastest: {fastest['name']} ({fastest['throughput_ops_sec']:.2f} ops/sec)")
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python parse_results.py <criterion_dir> [--csv output.csv] [--json output.json]")
|
||||
print("\nExample:")
|
||||
print(" python parse_results.py target/criterion --csv results.csv --json results.json")
|
||||
sys.exit(1)
|
||||
|
||||
criterion_dir = sys.argv[1]
|
||||
|
||||
# Parse command line arguments
|
||||
csv_output = None
|
||||
json_output = None
|
||||
|
||||
i = 2
|
||||
while i < len(sys.argv):
|
||||
if sys.argv[i] == '--csv' and i + 1 < len(sys.argv):
|
||||
csv_output = sys.argv[i + 1]
|
||||
i += 2
|
||||
elif sys.argv[i] == '--json' and i + 1 < len(sys.argv):
|
||||
json_output = sys.argv[i + 1]
|
||||
i += 2
|
||||
else:
|
||||
i += 1
|
||||
|
||||
# Parse results
|
||||
print(f"Parsing benchmark results from {criterion_dir}...")
|
||||
results = parse_criterion_json(criterion_dir)
|
||||
|
||||
if not results:
|
||||
print("No benchmark results found!")
|
||||
sys.exit(1)
|
||||
|
||||
# Export results
|
||||
if csv_output:
|
||||
export_to_csv(results, csv_output)
|
||||
|
||||
if json_output:
|
||||
export_to_json(results, json_output)
|
||||
|
||||
# Print summary
|
||||
print_summary(results)
|
||||
|
||||
# If no output specified, print to stdout
|
||||
if not csv_output and not json_output:
|
||||
print("\n=== CSV Output ===\n")
|
||||
import io
|
||||
output = io.StringIO()
|
||||
fieldnames = ['name', 'mean_ns', 'median_ns', 'throughput_ops_sec']
|
||||
writer = csv.DictWriter(output, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
for result in results:
|
||||
writer.writerow({
|
||||
'name': result['name'],
|
||||
'mean_ns': int(result['mean_ns']),
|
||||
'median_ns': int(result['median_ns']),
|
||||
'throughput_ops_sec': f"{result['throughput_ops_sec']:.2f}"
|
||||
})
|
||||
print(output.getvalue())
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user