Skip to content

Commit 2c42ac8

Browse files
dergoeggedergoegge
committed
ci(benchmark): change PR comment to be a summary only
1 parent 0546741 commit 2c42ac8

2 files changed

Lines changed: 177 additions & 122 deletions

File tree

‎.github/workflows/benchmark-comment.yml‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,29 @@ jobs:
4949
with:
5050
script: |
5151
const fs = require('fs');
52+
const runId = context.payload.workflow_run.id;
5253
const issue_number = parseInt(fs.readFileSync('pr/pr-number.txt', 'utf8').trim(), 10);
53-
const body = fs.readFileSync('report/evaluation_report.md', 'utf8');
54+
55+
// Post only the summary; the visualizations and interpretation guide live in
56+
// the full-report artifact (their images use relative paths that don't resolve
57+
// in a comment body). Link to that artifact's browser download page.
58+
const summary = fs.readFileSync('report/pr_comment.md', 'utf8');
59+
60+
const { data: { artifacts } } =
61+
await github.rest.actions.listWorkflowRunArtifacts({
62+
...context.repo,
63+
run_id: runId,
64+
});
65+
const report = artifacts.find(a => a.name === 'benchmark-report');
66+
const reportLink = report
67+
? `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${runId}/artifacts/${report.id}`
68+
: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${runId}`;
69+
70+
const body =
71+
summary +
72+
`\n---\n\n📦 **[Download the full report](${reportLink})** ` +
73+
`(visualizations + interpretation guide, as a zip).\n`;
74+
5475
await github.rest.issues.createComment({
5576
...context.repo,
5677
issue_number,

‎ci/benchmark-evaluation.py‎

Lines changed: 155 additions & 121 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@
4747
Output (written to --out, default <root_dir>/results):
4848
- evaluation_report.md detailed Markdown report with summary stats, adjusted
4949
p-values, effect sizes, an interpretation guide and plots
50+
- pr_comment.md summary-only variant posted as the PR comment (the comment
51+
workflow appends a link to the full-report artifact)
5052
- evaluation_metrics.csv full data table (raw p-values, IQRs, medians)
5153
- <target>_boxplot.png box plots of final coverage distributions
5254
- <target>_auc_boxplot.png box plots of AUC (exploration speed) distributions
@@ -336,153 +338,179 @@ def generate_plots(
336338
plt.close()
337339

338340

339-
def write_evaluation_report(report_path, df_results, config_a, config_b, targets):
340-
"""Writes the final comprehensive Markdown evaluation report."""
341-
342-
def fmt(x, nd=3):
343-
"""Format a numeric cell to nd decimals; pass non-numerics through unchanged."""
344-
if isinstance(x, bool):
345-
return str(x)
346-
if isinstance(x, (int, float, np.floating, np.integer)):
347-
return f"{float(x):.{nd}f}"
341+
def _fmt(x, nd=3):
342+
"""Format a numeric cell to nd decimals; pass non-numerics through unchanged."""
343+
if isinstance(x, bool):
348344
return str(x)
345+
if isinstance(x, (int, float, np.floating, np.integer)):
346+
return f"{float(x):.{nd}f}"
347+
return str(x)
349348

350-
with open(report_path, "w") as f:
351-
f.write("# Fuzzing Evaluation Report\n\n")
352-
f.write(f"**Configuration A (Baseline):** `{config_a}`\n")
353-
f.write(f"**Configuration B (Experiment):** `{config_b}`\n\n")
354-
355-
f.write("## 1. Summary Statistics\n\n")
356-
# A benchmark covers a single target, so a one-row-by-many-columns table reads
357-
# poorly. Present each target's metrics transposed, as a Baseline vs. Experiment
358-
# table, with the experiment-vs-baseline statistics in a second small table.
359-
for _, row in df_results.iterrows():
360-
if len(df_results) > 1:
361-
f.write(f"### {row['Target']}\n\n")
362-
f.write(
363-
f"Evaluation window: **{fmt(row['Duration (h)'])} h** · "
364-
f"trials: **{int(row['n (Baseline)'])}** baseline / "
365-
f"**{int(row['n (Exp.)'])}** experiment\n\n"
366-
)
367-
368-
f.write("| Metric | Baseline | Experiment |\n")
369-
f.write("|---|---:|---:|\n")
370-
f.write(
371-
f"| Median final coverage (%) | {fmt(row['Median Cov. (Baseline)'])} "
372-
f"| {fmt(row['Median Cov. (Exp.)'])} |\n"
373-
)
374-
f.write(
375-
f"| Median AUC (coverage·h) | {fmt(row['Median AUC (Baseline)'])} "
376-
f"| {fmt(row['Median AUC (Exp.)'])} |\n"
377-
)
378-
f.write(
379-
f"| Median execs/s | {fmt(row['Execs/s (Baseline)'])} "
380-
f"| {fmt(row['Execs/s (Exp.)'])} |\n"
381-
)
382-
f.write(
383-
f"| Crashes (total) | {int(row['Crashes (Baseline)'])} "
384-
f"| {int(row['Crashes (Exp.)'])} |\n\n"
385-
)
386349

387-
f.write("Experiment vs. baseline comparison:\n\n")
388-
f.write("| Statistic | Coverage | AUC (speed) |\n")
389-
f.write("|---|---:|---:|\n")
390-
f.write(
391-
f"| Adj. p-value | {fmt(row['Adj. p-value (Cov.)'])} "
392-
f"| {fmt(row['Adj. p-value (AUC)'])} |\n"
393-
)
394-
f.write(f"| Â12 | {fmt(row['Â12 (Cov.)'])} | {fmt(row['Â12 (AUC)'])} |\n\n")
350+
def _write_header(f, config_a, config_b):
351+
f.write("# Fuzzing Evaluation Report\n\n")
352+
f.write(f"**Configuration A (Baseline):** `{config_a}`\n")
353+
f.write(f"**Configuration B (Experiment):** `{config_b}`\n\n")
395354

396-
f.write(
397-
"*Raw P-values and Interquartile Ranges (IQRs) are available in "
398-
"`evaluation_metrics.csv`.*\n\n"
399-
)
400355

401-
f.write("## 2. Interpretation Guide\n\n")
356+
def _write_summary(f, df_results):
357+
f.write("## 1. Summary Statistics\n\n")
358+
# A benchmark covers a single target, so a one-row-by-many-columns table reads
359+
# poorly. Present each target's metrics transposed, as a Baseline vs. Experiment
360+
# table, with the experiment-vs-baseline statistics in a second small table.
361+
for _, row in df_results.iterrows():
362+
if len(df_results) > 1:
363+
f.write(f"### {row['Target']}\n\n")
402364
f.write(
403-
"Use the comparison table above to objectively evaluate the experiment "
404-
"configuration.\n\n"
365+
f"Evaluation window: **{_fmt(row['Duration (h)'])} h** · "
366+
f"trials: **{int(row['n (Baseline)'])}** baseline / "
367+
f"**{int(row['n (Exp.)'])}** experiment\n\n"
405368
)
406369

407-
f.write("### Key Metrics\n\n")
408-
f.write(
409-
"- **`Adj. p-value`**: Mann-Whitney U test (Holm-Bonferroni adjusted). "
410-
"A value < 0.05 means the difference is unlikely to be noise.\n"
411-
)
412-
f.write(
413-
"- **`Â12`**: Probability that a random B (experiment) trial outperforms a random "
414-
"A (baseline) trial. `0.5` = no difference; `0.7` = B wins 70% of pairings. "
415-
"Always read alongside the p-value.\n"
416-
)
370+
f.write("| Metric | Baseline | Experiment |\n")
371+
f.write("|---|---:|---:|\n")
417372
f.write(
418-
"- **`IQR`**: Spread of the middle 50% of trials. A much larger IQR in B suggests "
419-
"a few outlier runs may be inflating the median.\n"
373+
f"| Median final coverage (%) | {_fmt(row['Median Cov. (Baseline)'])} "
374+
f"| {_fmt(row['Median Cov. (Exp.)'])} |\n"
420375
)
421376
f.write(
422-
"- **`AUC`**: Coverage *speed* — how much was discovered and how early. "
423-
"Useful when final coverage is similar between configurations.\n"
377+
f"| Median AUC (coverage·h) | {_fmt(row['Median AUC (Baseline)'])} "
378+
f"| {_fmt(row['Median AUC (Exp.)'])} |\n"
424379
)
425380
f.write(
426-
"- **`Execs/s`**: A large drop in B without a coverage gain means the new feature "
427-
"is too expensive.\n"
381+
f"| Median execs/s | {_fmt(row['Execs/s (Baseline)'])} "
382+
f"| {_fmt(row['Execs/s (Exp.)'])} |\n"
428383
)
429384
f.write(
430-
"- **`Crashes`**: Total solutions saved across trials. Non-zero counts warrant "
431-
"manual inspection of the uploaded crash artifacts.\n\n"
385+
f"| Crashes (total) | {int(row['Crashes (Baseline)'])} "
386+
f"| {int(row['Crashes (Exp.)'])} |\n\n"
432387
)
433388

434-
f.write("### Reading the Results\n\n")
435-
f.write("| Adj. p | `Â12` | Conclusion |\n")
436-
f.write("|---|---|---|\n")
389+
f.write("Experiment vs. baseline comparison:\n\n")
390+
f.write("| Statistic | Coverage | AUC (speed) |\n")
391+
f.write("|---|---:|---:|\n")
437392
f.write(
438-
"| < 0.05 | > 0.5 | Meaningful improvement. Check IQRs are comparable, then merge. |\n"
439-
)
440-
f.write(
441-
"| < 0.05 | ~0.5 | Significant but negligible. Check if worth the added complexity. |\n"
442-
)
443-
f.write(
444-
"| > 0.05 | > 0.6 | Promising but underpowered. Re-run with more trials (e.g., 50). |\n"
445-
)
446-
f.write(
447-
"| > 0.05 | ~0.5 | No effect. Try an advanced snapshot or ground-truth evaluation. |\n"
448-
)
449-
f.write(
450-
"| any | < 0.5 | B underperforms A. If significant, reject or redesign the feature. |\n\n"
393+
f"| Adj. p-value | {_fmt(row['Adj. p-value (Cov.)'])} "
394+
f"| {_fmt(row['Adj. p-value (AUC)'])} |\n"
451395
)
396+
f.write(f"| Â12 | {_fmt(row['Â12 (Cov.)'])} | {_fmt(row['Â12 (AUC)'])} |\n\n")
452397

453-
f.write(
454-
"> **Time-series caveat:** If the IQR bands overlap for most of the campaign and "
455-
"only diverge near the end, treat the final-coverage result cautiously — late "
456-
"divergence may reflect noise rather than a sustained advantage.\n\n"
457-
)
458-
f.write(
459-
"> **Note:** Union coverage (the multi-core coverage ceiling) is not reported "
460-
"here, as the bench CSV records a covered-edge count rather than the raw coverage "
461-
"map.\n\n"
462-
)
398+
f.write(
399+
"*Raw P-values and Interquartile Ranges (IQRs) are available in "
400+
"`evaluation_metrics.csv`.*\n\n"
401+
)
402+
403+
404+
def _write_visualizations(f, targets):
405+
f.write("## 2. Visualizations\n\n")
406+
f.write(
407+
"*Note: In the box plots below, the central box represents the Interquartile Range (IQR, "
408+
"the middle 50% of trials), demonstrating the consistency of the fuzzer's performance. "
409+
"The internal line represents the median.*\n\n"
410+
)
463411

464-
f.write("## 3. Visualizations\n\n")
412+
# Embed images directly into the markdown report
413+
for target in targets:
414+
f.write(f"### Target: {target}\n\n")
415+
416+
f.write("#### Median Coverage Over Time\n\n")
417+
f.write(f"![{target} Time Series]({target}_time_series.png)\n\n")
418+
419+
f.write("#### Distribution Comparisons\n\n")
420+
f.write("| Final Coverage | Area Under Curve (Speed) |\n")
421+
f.write("|:---:|:---:|\n")
465422
f.write(
466-
"*Note: In the box plots below, the central box represents the Interquartile Range (IQR, "
467-
"the middle 50% of trials), demonstrating the consistency of the fuzzer's performance. "
468-
"The internal line represents the median.*\n\n"
423+
f"| ![{target} Boxplot]({target}_boxplot.png) | "
424+
f"![{target} AUC]({target}_auc_boxplot.png) |\n\n"
469425
)
426+
f.write("---\n\n")
470427

471-
# Embed images directly into the markdown report
472-
for target in targets:
473-
f.write(f"### Target: {target}\n\n")
474428

475-
f.write("#### Median Coverage Over Time\n\n")
476-
f.write(f"![{target} Time Series]({target}_time_series.png)\n\n")
429+
def _write_guide(f):
430+
f.write("## 3. Interpretation Guide\n\n")
431+
f.write(
432+
"Use the comparison table above to objectively evaluate the experiment "
433+
"configuration.\n\n"
434+
)
477435

478-
f.write("#### Distribution Comparisons\n\n")
479-
f.write("| Final Coverage | Area Under Curve (Speed) |\n")
480-
f.write("|:---:|:---:|\n")
481-
f.write(
482-
f"| ![{target} Boxplot]({target}_boxplot.png) | "
483-
f"![{target} AUC]({target}_auc_boxplot.png) |\n\n"
484-
)
485-
f.write("---\n\n")
436+
f.write("### Key Metrics\n\n")
437+
f.write(
438+
"- **`Adj. p-value`**: Mann-Whitney U test (Holm-Bonferroni adjusted). "
439+
"A value < 0.05 means the difference is unlikely to be noise.\n"
440+
)
441+
f.write(
442+
"- **`Â12`**: Probability that a random B (experiment) trial outperforms a random "
443+
"A (baseline) trial. `0.5` = no difference; `0.7` = B wins 70% of pairings. "
444+
"Always read alongside the p-value.\n"
445+
)
446+
f.write(
447+
"- **`IQR`**: Spread of the middle 50% of trials. A much larger IQR in B suggests "
448+
"a few outlier runs may be inflating the median.\n"
449+
)
450+
f.write(
451+
"- **`AUC`**: Coverage *speed* — how much was discovered and how early. "
452+
"Useful when final coverage is similar between configurations.\n"
453+
)
454+
f.write(
455+
"- **`Execs/s`**: A large drop in B without a coverage gain means the new feature "
456+
"is too expensive.\n"
457+
)
458+
f.write(
459+
"- **`Crashes`**: Total solutions saved across trials. Non-zero counts warrant "
460+
"manual inspection of the uploaded crash artifacts.\n\n"
461+
)
462+
463+
f.write("### Reading the Results\n\n")
464+
f.write("| Adj. p | `Â12` | Conclusion |\n")
465+
f.write("|---|---|---|\n")
466+
f.write(
467+
"| < 0.05 | > 0.5 | Meaningful improvement. Check IQRs are comparable, then merge. |\n"
468+
)
469+
f.write(
470+
"| < 0.05 | ~0.5 | Significant but negligible. Check if worth the added complexity. |\n"
471+
)
472+
f.write(
473+
"| > 0.05 | > 0.6 | Promising but underpowered. Re-run with more trials (e.g., 50). |\n"
474+
)
475+
f.write(
476+
"| > 0.05 | ~0.5 | No effect. Try an advanced snapshot or ground-truth evaluation. |\n"
477+
)
478+
f.write(
479+
"| any | < 0.5 | B underperforms A. If significant, reject or redesign the feature. |\n\n"
480+
)
481+
482+
f.write(
483+
"> **Time-series caveat:** If the IQR bands overlap for most of the campaign and "
484+
"only diverge near the end, treat the final-coverage result cautiously — late "
485+
"divergence may reflect noise rather than a sustained advantage.\n\n"
486+
)
487+
f.write(
488+
"> **Note:** Union coverage (the multi-core coverage ceiling) is not reported "
489+
"here, as the bench CSV records a covered-edge count rather than the raw coverage "
490+
"map.\n\n"
491+
)
492+
493+
494+
def write_evaluation_report(report_path, df_results, config_a, config_b, targets):
495+
"""Writes the full Markdown evaluation report (summary + visualizations + guide)."""
496+
with open(report_path, "w") as f:
497+
_write_header(f, config_a, config_b)
498+
_write_summary(f, df_results)
499+
_write_visualizations(f, targets)
500+
_write_guide(f)
501+
502+
503+
def write_pr_comment(comment_path, df_results, config_a, config_b):
504+
"""Writes the trimmed PR-comment report: summary statistics only.
505+
506+
The visualizations embed images via relative paths that don't resolve in a GitHub
507+
comment body, and the interpretation guide is long; both live in the full report
508+
inside the downloadable artifact instead. The comment workflow appends a link to
509+
that artifact.
510+
"""
511+
with open(comment_path, "w") as f:
512+
_write_header(f, config_a, config_b)
513+
_write_summary(f, df_results)
486514

487515

488516
def process_data(
@@ -649,8 +677,14 @@ def process_data(
649677
report_path = os.path.join(results_dir, "evaluation_report.md")
650678
write_evaluation_report(report_path, df_results, config_a, config_b, targets)
651679

680+
# Summary-only variant for the PR comment (images/guide stay in the full report
681+
# artifact, which the comment workflow links to).
682+
comment_path = os.path.join(results_dir, "pr_comment.md")
683+
write_pr_comment(comment_path, df_results, config_a, config_b)
684+
652685
print(f"\n[*] Evaluation complete. Results saved to {results_dir}")
653686
print(f" - Open {report_path} to interpret the campaign.")
687+
print(f" - PR-comment summary written to {comment_path}.")
654688
print(f" - Metric data exported to {csv_path}.")
655689

656690

0 commit comments

Comments
 (0)