goldenChat base source add
This commit is contained in:
aidev
+234
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/benchmark-backtest-throughput.sh [base-ref] [candidate-ref] [output-dir] [-- harness args...]
|
||||
|
||||
Defaults:
|
||||
base-ref: HEAD^
|
||||
candidate-ref: HEAD
|
||||
output-dir: .agents/benchmarks/backtest-throughput/<timestamp>
|
||||
|
||||
The script runs BacktestPerformanceTuningHarness throughput-control mode in
|
||||
temporary worktrees for both refs, then compares matrix_performance.json
|
||||
cells/min and hypotheses/min.
|
||||
|
||||
Both refs must already contain throughput-control support. The intended
|
||||
candidate-flow comparison is HEAD^ vs HEAD after adding the harness, telemetry,
|
||||
and optimization commits.
|
||||
USAGE
|
||||
}
|
||||
|
||||
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
|
||||
usage
|
||||
exit 0
|
||||
fi
|
||||
|
||||
repo_root="$(git rev-parse --show-toplevel)"
|
||||
timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
base_ref="HEAD^"
|
||||
candidate_ref="HEAD"
|
||||
output_dir="$repo_root/.agents/benchmarks/backtest-throughput/$timestamp"
|
||||
if [[ "${1:-}" != "" && "${1:-}" != "--" ]]; then
|
||||
base_ref="$1"
|
||||
shift
|
||||
fi
|
||||
if [[ "${1:-}" != "" && "${1:-}" != "--" ]]; then
|
||||
candidate_ref="$1"
|
||||
shift
|
||||
fi
|
||||
if [[ "${1:-}" != "" && "${1:-}" != "--" ]]; then
|
||||
output_dir="$1"
|
||||
shift
|
||||
fi
|
||||
if [[ "$output_dir" != /* ]]; then
|
||||
output_dir="$repo_root/$output_dir"
|
||||
fi
|
||||
if [[ "${1:-}" == "--" ]]; then
|
||||
shift
|
||||
fi
|
||||
|
||||
default_harness_args=(
|
||||
--throughputControl
|
||||
--matrixStrategyCounts "250,500,1000"
|
||||
--matrixBarCounts "500,1000"
|
||||
--matrixMaxBarCountHints 0
|
||||
--executionMode topK
|
||||
--topK 10
|
||||
--parallelism 1
|
||||
)
|
||||
|
||||
if [[ "$#" -gt 0 ]]; then
|
||||
harness_args=(--throughputControl "$@")
|
||||
else
|
||||
harness_args=("${default_harness_args[@]}")
|
||||
fi
|
||||
|
||||
worktree_parent="$output_dir/worktrees"
|
||||
mkdir -p "$output_dir" "$worktree_parent"
|
||||
worktree_root="$(mktemp -d "$worktree_parent/run-XXXXXX")"
|
||||
base_worktree="$worktree_root/base"
|
||||
candidate_worktree="$worktree_root/candidate"
|
||||
base_output="$output_dir/base"
|
||||
candidate_output="$output_dir/candidate"
|
||||
comparison_json="$output_dir/comparison.json"
|
||||
comparison_md="$output_dir/summary.md"
|
||||
|
||||
cleanup() {
|
||||
git -C "$repo_root" worktree remove --force "$base_worktree" >/dev/null 2>&1 || true
|
||||
git -C "$repo_root" worktree remove --force "$candidate_worktree" >/dev/null 2>&1 || true
|
||||
rm -rf "$worktree_root"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
git -C "$repo_root" worktree add --detach "$base_worktree" "$base_ref" >/dev/null
|
||||
git -C "$repo_root" worktree add --detach "$candidate_worktree" "$candidate_ref" >/dev/null
|
||||
|
||||
quote_exec_arg() {
|
||||
local value="$1"
|
||||
if [[ "$value" =~ ^[A-Za-z0-9_./:=,+%-]+$ ]]; then
|
||||
printf '%s' "$value"
|
||||
return
|
||||
fi
|
||||
value="${value//\\/\\\\}"
|
||||
value="${value//\"/\\\"}"
|
||||
printf '"%s"' "$value"
|
||||
}
|
||||
|
||||
run_ref() {
|
||||
local worktree="$1"
|
||||
local run_output="$2"
|
||||
mkdir -p "$run_output"
|
||||
local exec_args=""
|
||||
local arg
|
||||
for arg in "${harness_args[@]}" --throughputOutputDir "$run_output"; do
|
||||
local quoted_arg
|
||||
quoted_arg="$(quote_exec_arg "$arg")"
|
||||
if [[ -n "$exec_args" ]]; then
|
||||
exec_args+=" "
|
||||
fi
|
||||
exec_args+="$quoted_arg"
|
||||
done
|
||||
(
|
||||
cd "$worktree"
|
||||
mvn -q -pl ta4j-examples -am compile
|
||||
mvn -q -pl ta4j-examples exec:java \
|
||||
-Dexec.mainClass=ta4jexamples.backtesting.BacktestPerformanceTuningHarness \
|
||||
-Dexec.args="$exec_args"
|
||||
)
|
||||
}
|
||||
|
||||
run_ref "$base_worktree" "$base_output"
|
||||
run_ref "$candidate_worktree" "$candidate_output"
|
||||
|
||||
python3 - "$base_output/matrix_performance.json" "$candidate_output/matrix_performance.json" \
|
||||
"$comparison_json" "$comparison_md" "$base_ref" "$candidate_ref" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
base_path = Path(sys.argv[1])
|
||||
candidate_path = Path(sys.argv[2])
|
||||
comparison_path = Path(sys.argv[3])
|
||||
summary_path = Path(sys.argv[4])
|
||||
base_ref = sys.argv[5]
|
||||
candidate_ref = sys.argv[6]
|
||||
|
||||
base = json.loads(base_path.read_text())
|
||||
candidate = json.loads(candidate_path.read_text())
|
||||
|
||||
required_metrics = (
|
||||
"cellsPerMinute",
|
||||
"hypothesesPerMinute",
|
||||
"totalWallTimeMs",
|
||||
"strategyBuildWallTimeMs",
|
||||
"backtestRuntimeMs",
|
||||
)
|
||||
required_meta = (
|
||||
"specFingerprint",
|
||||
"dataset",
|
||||
"cellCount",
|
||||
"hypothesisCount",
|
||||
"parallelism",
|
||||
"resolvedParallelism",
|
||||
"executionMode",
|
||||
"topK",
|
||||
"progress",
|
||||
"gcBetweenRuns",
|
||||
)
|
||||
|
||||
for key in (*required_metrics, *required_meta):
|
||||
if key not in base or key not in candidate:
|
||||
raise SystemExit(f"Missing required key '{key}' in matrix_performance.json")
|
||||
if "host" not in base or "host" not in candidate:
|
||||
raise SystemExit("Missing required key 'host' in matrix_performance.json")
|
||||
if "hostId" not in base["host"] or "hostId" not in candidate["host"]:
|
||||
raise SystemExit("Missing required key 'host.hostId' in matrix_performance.json")
|
||||
|
||||
def pct(before, after):
|
||||
if before == 0:
|
||||
return None
|
||||
return (after - before) * 100.0 / before
|
||||
|
||||
metrics = {}
|
||||
for name in required_metrics:
|
||||
before = float(base[name])
|
||||
after = float(candidate[name])
|
||||
metrics[name] = {
|
||||
"base": before,
|
||||
"candidate": after,
|
||||
"delta": after - before,
|
||||
"deltaPct": pct(before, after),
|
||||
}
|
||||
|
||||
comparison = {
|
||||
"baseRef": base_ref,
|
||||
"candidateRef": candidate_ref,
|
||||
"basePath": str(base_path),
|
||||
"candidatePath": str(candidate_path),
|
||||
"specFingerprintMatch": base.get("specFingerprint") == candidate.get("specFingerprint"),
|
||||
"datasetMatch": base.get("dataset") == candidate.get("dataset"),
|
||||
"cellCountMatch": base.get("cellCount") == candidate.get("cellCount"),
|
||||
"hypothesisCountMatch": base.get("hypothesisCount") == candidate.get("hypothesisCount"),
|
||||
"hostIdMatch": base["host"]["hostId"] == candidate["host"]["hostId"],
|
||||
"metrics": metrics,
|
||||
}
|
||||
comparison_path.write_text(json.dumps(comparison, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
if not all((
|
||||
comparison["specFingerprintMatch"],
|
||||
comparison["datasetMatch"],
|
||||
comparison["cellCountMatch"],
|
||||
comparison["hypothesisCountMatch"],
|
||||
comparison["hostIdMatch"],
|
||||
)):
|
||||
raise SystemExit("Invalid comparison: host/dataset/spec/hypothesis/cell counts differ between refs.")
|
||||
|
||||
def fmt(value):
|
||||
return "n/a" if value is None else f"{value:.2f}"
|
||||
|
||||
lines = [
|
||||
f"# Backtest Throughput Comparison",
|
||||
"",
|
||||
f"- Base: `{base_ref}`",
|
||||
f"- Candidate: `{candidate_ref}`",
|
||||
f"- Dataset match: `{comparison['datasetMatch']}`",
|
||||
f"- Spec fingerprint match: `{comparison['specFingerprintMatch']}`",
|
||||
f"- Cell count match: `{comparison['cellCountMatch']}`",
|
||||
f"- Hypothesis count match: `{comparison['hypothesisCountMatch']}`",
|
||||
f"- Host ID match: `{comparison['hostIdMatch']}`",
|
||||
"",
|
||||
"| Metric | Base | Candidate | Delta % |",
|
||||
"| --- | ---: | ---: | ---: |",
|
||||
]
|
||||
for name in ("cellsPerMinute", "hypothesesPerMinute", "totalWallTimeMs", "strategyBuildWallTimeMs", "backtestRuntimeMs"):
|
||||
metric = metrics[name]
|
||||
lines.append(f"| {name} | {metric['base']:.2f} | {metric['candidate']:.2f} | {fmt(metric['deltaPct'])} |")
|
||||
summary_path.write_text("\n".join(lines) + "\n")
|
||||
print(f"comparison: {comparison_path}")
|
||||
print(f"summary: {summary_path}")
|
||||
PY
|
||||
|
||||
cat "$comparison_md"
|
||||
Reference in New Issue
Block a user