Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion lib/argument_parser.rb
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ class ArgumentParser
:with_pre_init,
:pvalue,
:interleave,
:zjit_stats,
keyword_init: true
)

Expand Down Expand Up @@ -125,8 +126,9 @@ def parse(argv)
ENV["YJIT_BENCH_STATS"] = str
end

opts.on("--zjit-stats=STATS", "print ZJIT stats at each iteration for the default harness") do |str|
opts.on("--zjit-stats=STAT1,STAT2,...", "print ZJIT stats at each iteration and show them in the summary table") do |str|
ENV["ZJIT_BENCH_STATS"] = str
args.zjit_stats = str.split(",")
end

opts.on("--yjit_opts=OPT_STRING", "string of command-line options to run YJIT with (ignored if you use -e)") do |str|
Expand Down Expand Up @@ -236,6 +238,7 @@ def default_args
rss: false,
pvalue: false,
interleave: false,
zjit_stats: [],
graph: false,
no_pinning: false,
force_pinning: false,
Expand Down
3 changes: 2 additions & 1 deletion lib/benchmark_runner/cli.rb
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,8 @@ def run
executable_names: ruby_descriptions.keys,
bench_data: bench_data,
include_rss: args.rss,
include_pvalue: args.pvalue
include_pvalue: args.pvalue,
zjit_stats: args.zjit_stats
)
table, format = builder.build

Expand Down
31 changes: 26 additions & 5 deletions lib/results_table_builder.rb
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@ class ResultsTableBuilder
SECONDS_TO_MS = 1000.0
BYTES_TO_MIB = 1024.0 * 1024.0

def initialize(executable_names:, bench_data:, include_rss: false, include_pvalue: false)
def initialize(executable_names:, bench_data:, include_rss: false, include_pvalue: false, zjit_stats: [])
@executable_names = executable_names
@bench_data = bench_data
@include_rss = include_rss
@include_pvalue = include_pvalue
@zjit_stats = zjit_stats || []
@include_gc = detect_gc_data(bench_data)
@base_name = executable_names.first
@other_names = executable_names[1..]
Expand Down Expand Up @@ -46,6 +47,7 @@ def build_header
@executable_names.each do |name|
header << "#{name} (ms)"
header << "RSS (MiB)" if @include_rss
@zjit_stats.each { |stat| header << stat }
if @include_gc
header << "#{name} mark (ms)"
header << "#{name} sweep (ms)"
Expand Down Expand Up @@ -85,6 +87,7 @@ def build_format
@executable_names.each do |_name|
format << "%s"
format << "%.1f" if @include_rss
@zjit_stats.each { format << "%s" }
if @include_gc
format << "%s"
format << "%s"
Expand Down Expand Up @@ -127,6 +130,11 @@ def build_row(bench_name)
base_t, *other_ts = times_no_warmup
base_rss, *other_rsss = rsss

# Extract zjit stats: { stat_name => [base_val, other1_val, ...] }
zjit_stat_values = @zjit_stats.map do |stat|
[stat, extract_zjit_stat(bench_name, stat)]
end

if @include_gc
marking_times = extract_gc_times(bench_name, 'gc_marking_time_bench')
sweeping_times = extract_gc_times(bench_name, 'gc_sweeping_time_bench')
Expand All @@ -135,35 +143,42 @@ def build_row(bench_name)
end

row = [bench_name]
build_base_columns(row, base_t, base_rss, base_mark, base_sweep)
build_comparison_columns(row, other_ts, other_rsss, other_marks, other_sweeps)
build_base_columns(row, base_t, base_rss, zjit_stat_values, 0, base_mark, base_sweep)
build_comparison_columns(row, other_ts, other_rsss, zjit_stat_values, other_marks, other_sweeps)
build_ratio_columns(row, base_t0, other_t0s, base_t, other_ts)
build_rss_ratio_columns(row, base_rss, other_rsss)
build_gc_ratio_columns(row, base_mark, other_marks, base_sweep, other_sweeps)

row
end

def build_base_columns(row, base_t, base_rss, base_mark, base_sweep)
def build_base_columns(row, base_t, base_rss, zjit_stat_values, exe_index, base_mark, base_sweep)
row << format_time_with_stddev(base_t)
row << base_rss if @include_rss
zjit_stat_values.each { |_stat, values| row << format_stat(values[exe_index]) }
if @include_gc
row << format_time_with_stddev(base_mark)
row << format_time_with_stddev(base_sweep)
end
end

def build_comparison_columns(row, other_ts, other_rsss, other_marks, other_sweeps)
def build_comparison_columns(row, other_ts, other_rsss, zjit_stat_values, other_marks, other_sweeps)
other_ts.each_with_index do |other_t, i|
row << format_time_with_stddev(other_t)
row << other_rsss[i] if @include_rss
zjit_stat_values.each { |_stat, values| row << format_stat(values[i + 1]) }
if @include_gc
row << format_time_with_stddev(other_marks[i])
row << format_time_with_stddev(other_sweeps[i])
end
end
end

def format_stat(value)
return "N/A" if value.nil?
value.to_s.gsub(/(\d)(?=(\d{3})+(?!\d))/, '\1,')
end

def format_time_with_stddev(values)
return "N/A" if values.nil? || values.empty?
"%.1f ± %.1f%%" % [mean(values), stddev_percent(values)]
Expand Down Expand Up @@ -274,6 +289,12 @@ def extract_rss_values(bench_name)
end
end

def extract_zjit_stat(bench_name, key)
@executable_names.map do |name|
bench_data_for(name, bench_name).dig('zjit_stats', key)
end
end

def extract_gc_times(bench_name, key)
@executable_names.map do |name|
bench_data_for(name, bench_name)[key] || []
Expand Down
Loading