package com.oracle.graal.python.benchmarks.interop;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import org.openjdk.jmh.infra.BenchmarkParams;
import org.openjdk.jmh.infra.IterationParams;
import org.openjdk.jmh.results.BenchmarkResult;
import org.openjdk.jmh.results.IterationResult;
import org.openjdk.jmh.results.RunResult;
import org.openjdk.jmh.runner.format.OutputFormat;
import org.openjdk.jmh.runner.options.VerboseMode;
import org.openjdk.jmh.util.Utils;
public class BenchOutputFormat implements OutputFormat {
final VerboseMode verbose;
final PrintStream out;
final String name;
final HashMap<String, ArrayList<Double>> raws;
public BenchOutputFormat(PrintStream out, VerboseMode verbose, String name) {
this.out = out;
this.verbose = verbose;
this.raws = new HashMap<>();
this.name = name;
}
public BenchOutputFormat(PrintStream out, VerboseMode verbose) {
this(out, verbose, null);
}
private void add(BenchmarkParams params, double value) {
if (!raws.containsKey(params.getBenchmark())) {
raws.put(params.getBenchmark(), new ArrayList<>());
}
raws.get(params.getBenchmark()).add(value);
}
protected String benchName(BenchmarkParams params) {
if (name != null) {
return name;
}
return params.getBenchmark();
}
@Override
public void startBenchmark(BenchmarkParams params) {
String opts = Utils.join(params.getJvmArgs(), " ");
if (opts.trim().isEmpty()) {
opts = "<none>";
}
println("# JMH version: " + params.getJmhVersion());
println("# VM version: JDK " + params.getJdkVersion() + ", " + params.getVmName() + ", " + params.getVmVersion());
println("# VM invoker: " + params.getJvm());
println("# VM options: " + opts);
println("# Benchmark mode: " + params.getMode().longLabel());
hline();
String benchName = benchName(params);
IterationParams warmup = params.getWarmup();
IterationParams measurement = params.getMeasurement();
String info = "### %s, %d warmup iterations, %d bench iterations";
println(String.format(info, benchName, warmup.getCount(), measurement.getCount()));
String s = "";
boolean isFirst = true;
for (String k : params.getParamsKeys()) {
if (isFirst) {
isFirst = false;
} else {
s += ", ";
}
s += k + " = " + params.getParam(k);
}
println(String.format("### args = [%s]", s));
hline();
println("### start benchmark ...");
}
@Override
public void iteration(BenchmarkParams benchmarkParams, IterationParams params, int iteration) {
}
@Override
public void iterationResult(BenchmarkParams benchmParams, IterationParams params, int iteration, IterationResult data) {
double value = data.getPrimaryResult().getScore();
add(benchmParams, value);
String benchName = benchName(benchmParams);
switch (params.getType()) {
case WARMUP:
out.println(String.format("### (pre)warming up for %s iteration=%d, duration=%.3f", benchName, iteration, value));
break;
case MEASUREMENT:
out.println(String.format("### iteration=%d, name=%s, duration=%.3f", iteration, benchName, value));
break;
default:
throw new IllegalStateException("Unknown iteration type: " + params.getType());
}
}
@Override
public void endBenchmark(BenchmarkResult result) {
final ArrayList<Double> raw = raws.get(result.getParams().getBenchmark());
final double[] durations = new double[raw.size()];
for (int i = 0; i < raw.size(); i++) {
durations[i] = raw.get(i);
}
hline();
println("### teardown ...");
println("### benchmark complete");
hline();
double best = min(durations);
println(String.format("### BEST duration: %.3f s", best));
double worst = max(durations);
println(String.format("### WORST duration: %.3f s", worst));
double avg = avg(durations);
println(String.format("### AVG (all runs) duration: %.3f s", avg));
int warmupIndex = detect_warmup(durations);
println(String.format("### WARMUP detected at iteration: %d", warmupIndex));
double avg2 = avg(Arrays.copyOfRange(durations, Math.min(durations.length - 1, warmupIndex + 1), durations.length));
println(String.format("### AVG (no warmup) duration: %.3f s", avg2));
hline();
String s = "";
for (double d : durations) {
s += d + ", ";
}
println(String.format("### RAW DURATIONS: [%s]", s));
hline();
}
private static int[] cusum(double[] values, double threshold) {
double drift = 0.0;
int size = values.length;
double[] csum_pos = new double[size];
double[] csum_neg = new double[size];
int[] change_points = new int[size];
int cp_idx = -1;
for (int i = 1; i < size; i++) {
double diff = values[i] - values[i - 1];
csum_pos[i] = csum_pos[i - 1] + diff - drift;
csum_neg[i] = csum_neg[i - 1] - diff - drift;
if (csum_pos[i] < 0) {
csum_pos[i] = 0;
}
if (csum_neg[i] < 0) {
csum_neg[i] = 0;
}
if (csum_pos[i] > threshold || csum_neg[i] > threshold) {
cp_idx++;
change_points[cp_idx] = i;
csum_pos[i] = 0.;
csum_neg[i] = 0.;
}
}
return Arrays.copyOf(change_points, cp_idx + 1);
}
private static double avg(double[] values) {
return Arrays.stream(values).average().getAsDouble();
}
private static double min(double[] values) {
return Arrays.stream(values).min().getAsDouble();
}
private static double max(double[] values) {
return Arrays.stream(values).max().getAsDouble();
}
private static double[] norm(double[] values) {
double min = min(values);
double max = max(values);
return Arrays.stream(values).map(v -> (v - min) / (max - min) * 100.0).toArray();
}
private static double[] pairwise_slopes(double[] values, int[] cp) {
double[] copy = Arrays.copyOf(values, values.length - 1);
for (int i = 0; i < copy.length; i++) {
copy[i] = Math.abs((values[i + 1] - values[i]) / (cp[i + 1] - cp[i]));
}
return copy;
}
private static double[] last_n_percent_runs(double[] values, double n) {
int size = values.length;
int newSize = size - (int) (size * n);
if (newSize >= size) {
newSize = size - 1;
}
return Arrays.copyOfRange(values, newSize, size);
}
private static int warmup(int idx, int[] cp, int max) {
return cp[idx] < max ? cp[idx] : -1;
}
private static int detect_warmup(double[] durations) {
double cp_threshold = 0.03, stability_slope_grade = 0.01;
stability_slope_grade *= 100.0;
cp_threshold *= 100;
double[] values = norm(durations);
int size = values.length;
int[] cp = cusum(values, cp_threshold);
double[] rolling_avg = new double[cp.length];
for (int i = 0; i < cp.length; i++) {
rolling_avg[i] = avg(Arrays.copyOfRange(values, cp[i], values.length));
}
for (int i = 0; i < rolling_avg.length; i++) {
if (rolling_avg[i] <= cp_threshold) {
return warmup(i, cp, size);
}
}
double n = 0.1;
double[] last_n_vals = last_n_percent_runs(values, n);
int last_n_idx = size - (int) (size * n);
int totalSize = cp.length + last_n_vals.length;
double[] rolling_avg2 = new double[totalSize];
int[] cp2 = new int[totalSize];
for (int i = 0; i < totalSize; i++) {
if (i < cp.length) {
rolling_avg2[i] = rolling_avg[i];
cp2[i] = cp[i];
} else {
int j = i - cp.length;
rolling_avg2[i] = last_n_vals[j];
cp2[i] = last_n_idx++;
}
}
double[] slopes = pairwise_slopes(rolling_avg2, cp2);
for (int i = 0; i < slopes.length; i++) {
if (slopes[i] <= stability_slope_grade) {
return warmup(i, cp, size);
}
}
return -1;
}
private void hline() {
println("-------------------------------------------------------------------------------");
}
@Override
public void print(String s) {
out.print(s);
}
@Override
public void println(String s) {
out.println(s);
}
@Override
public void flush() {
out.flush();
}
@Override
public void verbosePrintln(String s) {
if (verbose == VerboseMode.EXTRA) {
out.println(s);
}
}
@Override
public void write(int b) {
out.write(b);
}
@Override
public void write(byte[] b) throws IOException {
out.write(b);
}
@Override
public void close() {
}
@Override
public void startRun() {
}
@Override
public void endRun(Collection<RunResult> runResults) {
}
}