aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--config.yaml12
-rwxr-xr-xplot.py66
-rwxr-xr-xrun.rb519
3 files changed, 374 insertions, 223 deletions
diff --git a/config.yaml b/config.yaml
index 21a57ea..9f2f44d 100644
--- a/config.yaml
+++ b/config.yaml
@@ -14,19 +14,19 @@ hosts:
arch: "armv6l"
pi: true
- - name: "pi3b-peach"
- host: "peach"
+ - name: "pi3b-pecan"
+ host: "pecan"
text: "Raspberry Pi 3 Model B"
arch: "armv7l"
pi: true
- - name: "pi3b+-pecan"
- host: "pecan"
+ - name: "pi3b+-peach"
+ host: "peach"
text: "Raspberry Pi 3 Model B+"
arch: "armv7l"
pi: true
- - name: "pi4b-cherry"
+ - name: "pi4b-cherry"
host: "cherry"
text: "Raspberry Pi 4 Model B"
arch: "armv7l"
@@ -34,7 +34,7 @@ hosts:
- name: "i78650u-flex"
host: "flex"
- text: "Intel i7-8650U"
+ text: "Intel i7-8650U (Lenovo X1 Carbon 6th Gen)"
arch: "amd64"
pi: false
diff --git a/plot.py b/plot.py
index d5f6863..31262c3 100755
--- a/plot.py
+++ b/plot.py
@@ -1,62 +1,44 @@
#!/usr/bin/python3
-# generate chart.svg for data
+#
+# Generate chart from JSON using matplotlib.
+#
+# (See PiBench::Runner#save_svg in run.rb for JSON details).
+#
-import csv
+import json
import sys
import os
-import re
import numpy as np
import matplotlib.pyplot as plt
-def read_csv(path):
- with open(sys.argv[1], newline = '') as fh:
- return list(reversed(list([row for row in csv.DictReader(fh)])))
-
# check arguments
-if len(sys.argv) < 3:
- print("Usage: {} input.csv output.svg title xlimit".format(sys.argv[0]))
+if len(sys.argv) < 2:
+ print("Usage: {} json_data".format(sys.argv[0]))
exit(-1)
-# read csv
-rows = read_csv(sys.argv[1])
-
-# sort by range
-# rows.sort(key = lambda row: int(row['range']))
+# decode json data from first argument
+data = json.loads(sys.argv[1])
# plot values
plt.barh(
- np.arange(len(rows)),
- [float(row['speed']) / 1048576 for row in rows],
+ np.arange(len(data['rows'])),
+ [row[1] for row in data['rows']],
align = 'center',
alpha = 0.5,
- tick_label = ['{} ({} bytes)'.format(
- row['host'],
- row['size']
- ) for row in rows]
+ tick_label = [row[0] for row in data['rows']]
)
-# # build title
-# algo = os.path.splitext(os.path.basename(sys.argv[1]))[0]
-# title = 'OpenSSL Speed Test: {}'.format(algo)
-
-# get title and xlimit
-title = sys.argv[3]
-limit = int(sys.argv[4])
-
-# # build xlimit
-# # if re.match('^aes', algo):
-# limit = 400
-# else:
-# limit = 2000
-
-# add label and title
-plt.yticks(fontsize = 5)
-plt.xlim(0, limit)
-# plt.xscale('log')
-plt.xlabel('Speed (MB/s)')
-plt.title(title, fontsize = 9)
+# set plot parameters
+plt.yticks(fontsize = data['fontsize']['yticks'])
+plt.xlim(0, data['xlimit'])
+plt.xlabel(data['xlabel'])
+plt.title(data['title'], fontsize = data['fontsize']['title'])
plt.tight_layout()
-# save image
-plt.savefig(sys.argv[2])
+# set output size
+fig = plt.gcf()
+fig.set_size_inches(data['size'][0], data['size'][1])
+
+# save plot
+plt.savefig(data['path'], dpi = data['dpi'])
diff --git a/run.rb b/run.rb
index 4591053..ff040cf 100755
--- a/run.rb
+++ b/run.rb
@@ -18,6 +18,7 @@ require 'yaml'
require 'csv'
require 'logger'
require 'json'
+require 'luigi-template'
module PiBench
# block sizes
@@ -94,54 +95,164 @@ module PiBench
}, {
id: 'text',
name: 'Description',
+ }, {
+ id: 'openssl',
+ name: 'OpenSSL Version',
}],
}.freeze
#
- # Architecture titles format strings.
+ # Architecture strings.
#
- ARCH_TITLES = {
- all: 'OpenSSL Speed: All Systems, %s',
- arm: 'OpenSSL Speed: Raspberry Pis, %s',
- x86: 'OpenSSL Speed: x86-64, %s',
- }
+ ARCHS = {
+ all: {
+ name: 'All',
+ text: %{
+ <p>
+ Test results for all systems. Note that the x86-64 systems
+ include AES-NI and SHA2 hardware acceleration.
+ </p>
+ }.strip,
+ },
+
+ arm: {
+ name: 'Pis',
+ text: %{
+ <p>
+ Test results for Raspberry Pi systems only.
+ </p>
+ }.strip,
+ },
+
+ x86: {
+ name: 'x86-64',
+ text: %{
+ <p>
+ Test results for x86-64 systems only.
+ </p>
+ }.strip,
+ },
+ }.freeze
+
+ LINKS = [{
+ href: 'csvs/all-all.csv',
+ title: 'Download all results as a CSV file.',
+ text: 'Download Results (CSV)',
+ }, {
+ href: 'https://github.com/pablotron/p4-bench',
+ title: 'View code on GitHub.',
+ text: 'GitHub Page',
+ }].freeze
+
+ TEMPLATES = Luigi::Cache.new({
+ index: %{
+ <!DOCTYPE html>
+ <html lang='en'>
+ <head>
+ <meta charset='utf-8'/>
+ <meta
+ name='viewport'
+ content='width=device-width, initial-scale=1, shrink-to-fit=no'
+ />
+
+ <link
+ rel='stylesheet'
+ type='text/css'
+ href='https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css'
+ integrity='sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T'
+ crossorigin='anonymous'
+ />
+
+ <title>%{title|h}</title>
+ </head>
+
+ <body>
+ <div class='container'>
+ <h1>%{title|h}</h1>
+
+ <p>
+ This page contains OpenSSL benchmarks across several
+ Raspberry Pis and x86-64 systems generated using the
+ <code>openssl speed</code> command.
+ </p>
+
+ <ul>
+ %{links}
+ </ul>
+
+ <section>
+ <h2>Systems</h2>
+
+ <p>
+ Test system details.
+ </p>
+
+ %{hosts}
+ </section>
+
+ %{sections}
+ </div><!-- container -->
+ </body>
+ </html>
+ }.strip,
- HTML = {
all: %{
- <table class='table table-hover'>
+ <table class='table table-sm table-hover'>
<thead>
- <tr>%s</tr>
+ <tr>%{cols}</tr>
</thead>
<tbody>
- %s
+ %{rows}
</tbody>
</table>
}.strip,
col: %{
- <th>%s</th>
+ <th>%{name|h}</th>
}.strip,
row: %{
- <tr>%s</tr>
+ <tr>%{row}</tr>
}.strip,
cell: %{
- <td>%s</td>
+ <td>%{text|h}</td>
+ }.strip,
+
+ svg_title: %{
+ Speed Test Results (Systems: %{arch|h}, Algorithm: %{algo|h})
}.strip,
svg: %{
- <p><img
- src="%s"
- width="100%%"
- height="auto"
- title="%s"
- alt="%s"
- /></p>
+ <img
+ src='%{path|h}'
+ class='img-fluid'
+ title='%{name|h}'
+ alt='%{name|h}'
+ />
}.strip,
- }.freeze
+ link: %{
+ <li>
+ <a href='%{href|h}' title='%{title|h}'>
+ %{text|h}
+ </a>
+ </li>
+ }.strip,
+
+ section: %{
+ <section>
+ <h2>Results: %{name|h}</h2>
+ %{text}
+ %{svgs}
+ </section>
+ }.strip,
+ })
+
+ #
+ # Background process mixin.
+ #
module BG
#
# Generate SSH command.
@@ -151,8 +262,8 @@ module PiBench
end
#
- # Spawn background task that writes output to given file and return
- # the PID.
+ # Spawn background task that writes standard output to given file
+ # and return the PID.
#
def bg(out, cmd)
@log.debug('bg') do
@@ -171,9 +282,15 @@ module PiBench
end
end
+ #
+ # Process a map of hosts to background command queues in parallel.
+ #
class HostQueue
include BG
+ #
+ # Allow singleton invocation.
+ #
def self.run(log, queues)
new(log, queues).run
end
@@ -184,8 +301,8 @@ module PiBench
end
#
- # Run until all commands have been run successfully on all hosts, or
- # until any command on any host fails.
+ # Block until all commands have been run successfully on all hosts,
+ # or until any command on any host fails.
#
def run
@queues.keys.each do |host|
@@ -310,11 +427,14 @@ module PiBench
spawn_benchmarks
# generate CSVs, SVGs, and HTML fragments, wait for all to
- # complete
- save(parse_data)
-
- # generate hosts.{html,csv}
- save_hosts.each { |t| t.join }
+ # complete, and return HTML fragments by section
+ html = save(parse_data).merge({
+ # generate hosts.csv and hosts html
+ hosts: make_hosts,
+ })
+
+ # save index.html
+ save_index_html(html)
end
private
@@ -334,19 +454,21 @@ module PiBench
end
#
- # Spawn benchmarks in background and return a list of PIDs.
+ # Spawn benchmark tasks in background and block until they are
+ # complete.
#
def spawn_benchmarks
# build map of hosts to commands
- queues = @config['hosts'].reduce(Hash.new do |h, k|
- h[k] = []
- end) do |r, row|
- TESTS.reduce(r) do |r, test|
+ queues = Hash.new { |h, k| h[k] = [] }
+
+ # populate map
+ @config['hosts'].each do |row|
+ TESTS.each do |test|
case test[:type]
when 'algos'
# queue test command for each algorithm
- (@config['algos'] || ALGOS).reduce(r) do |r, algo|
- r[row['host']] << {
+ (@config['algos'] || ALGOS).each do |algo|
+ queues[row['host']] << {
cmd: [*test[:exec], algo],
out: '%s/hosts/%s/%s-%s.txt' % [
out_dir,
@@ -355,12 +477,10 @@ module PiBench
algo,
],
}
-
- r
end
else
# queue command for test
- r[row['host']] << {
+ queues[row['host']] << {
cmd: test[:exec],
out: '%s/hosts/%s/%s.txt' % [
out_dir,
@@ -368,13 +488,11 @@ module PiBench
test[:name],
]
}
-
- r
end
end
end
- # block until all tasks have exited
+ # block until all task queues have completed successfully
HostQueue.run(@log, queues)
end
@@ -408,30 +526,23 @@ module PiBench
# walk block sizes
SIZES.each_with_index do |size, i|
- [{
- algo: 'all',
- arch: 'all',
- }, {
- algo: algo,
- arch: 'all',
- }, {
- algo: 'all',
- arch: arch,
- }, {
- algo: algo,
- arch: arch,
- }].each do |agg|
+ 4.times.map { |j|
+ {
+ algo: ((j & 1) != 0) ? 'all' : algo,
+ arch: ((j & 2) != 0) ? 'all' : arch,
+ }
+ }.each do |agg|
val = vals[i + 3].to_f
max = r[agg[:arch]][agg[:algo]][:max]
r[agg[:arch]][agg[:algo]][:max] = val if val > max
- r[agg[:arch]][agg[:algo]][:rows] << if agg[:algo] == 'all'
+ r[agg[:arch]][agg[:algo]][:rows] << (if agg[:algo] == 'all'
# build row for all-*.csv
[row['name'], algo, size, val]
else
# row for algo-specific CSV
[row['name'], size, val]
- end
+ end)
end
end
end
@@ -442,168 +553,225 @@ module PiBench
end
#
- # Generate CSVs, SVGs, and HTML fragments, then wait for them all
- # to complete.
+ # Generate CSVs, SVGs, and HTML fragments, then return map of arch
+ # to HTML fragments.
#
- def save(all_data, &block)
- # build svg lut
- svgs = Hash.new { |h, k| h[k] = [] }
-
- # generate csvs and svgs, then wait for them to complete
- join('save', all_data.reduce([]) do |r, pair|
- arch, algo_hash = pair
-
- algo_hash.reduce(r) do |r, pair|
- algo, data = pair
-
- # save csv
- csv_path = save_csv(arch, algo, data[:rows])
-
- if algo != 'all'
- # start building svg
- max = get_max_value(all_data, arch, algo)
- row = save_svg(arch, algo, max, csv_path)
- r << row[:pid]
+ def save(all_data)
+ save_csvs(all_data)
+ svgs = save_svgs(all_data)
+ make_html(svgs)
+ end
- # add to svg lut
- svgs[arch] << {
- algo: algo,
- svg: row[:svg],
- title: row[:title],
- }
- end
+ #
+ # Save all CSVs.
+ #
+ def save_csvs(all_data)
+ all_data.each do |arch, algos|
+ algos.each do |algo, data|
+ save_csv(
+ '%s/csvs/%s-%s.csv' % [out_dir, arch, algo],
+ COLS[(algo == 'all') ? :all : :algo].map { |col| col[:id] },
+ data[:rows]
+ )
+ end
+ end
+ end
- # return pids
- r
+ #
+ # Save SVGs and return a lut of arch to svgs.
+ #
+ def save_svgs(all_data)
+ pids = []
+ svgs = Hash.new { |h, k| h[k] = [] }
+ # generate svgs
+ data = all_data.keys.each do |arch|
+ # omit algo=all svgs because they are too large
+ all_data[arch].keys.select { |algo|
+ algo != 'all'
+ }.each do |algo|
+ # get rows
+ rows = all_data[arch][algo][:rows]
+
+ # get maximum value for plot
+ max = get_max_value(all_data, arch, algo)
+
+ # build svg data
+ svg = make_svg(arch, algo, max, rows)
+
+ # add to svg lut
+ svgs[arch] << {
+ algo: algo,
+ path: svg[:path],
+ title: svg[:title],
+ }
+
+ # save in background and add pid list of pids
+ pids << save_svg(svg)
end
- end)
+ end
- # generate html fragments for svgs
- save_html(svgs)
+ # wait for background tasks to complete
+ join('save_svgs', pids)
+
+ # return svg data lut
+ svgs
end
#
# Generate HTML fragments for each architecture.
#
- def save_html(svgs)
- svgs.each do |arch, rows|
- # build path to html fragment
- path = '%s/html/%s.html' % [out_dir, arch]
-
- # write html
- File.write(path, rows.sort { |a, b|
- a[:svg] <=> b[:svg]
+ def make_html(svgs)
+ svgs.keys.reduce({}) do |r, arch|
+ r[arch] = svgs[arch].sort { |a, b|
+ a[:path] <=> b[:path]
}.map { |row|
- svg_path = '../svgs/%s' % [File.basename(row[:svg])]
- HTML[:svg] % [svg_path, row[:title], row[:title]]
- }.join)
+ TEMPLATES[:svg].run({
+ path: 'svgs/%s' % [File.basename(row[:path])],
+ name: row[:title],
+ })
+ }.join
+
+ r
end
end
#
- # Generate CSV and HTML table of hosts and return array of threads.
+ # Generate CSV and HTML table of hosts and return generated HTML.
#
- def save_hosts
- [save_hosts_csv, save_hosts_html]
+ def make_hosts
+ save_hosts_csv
+ make_hosts_html
end
#
# Generate out/csvs/hosts.csv and return thread.
#
def save_hosts_csv
- Thread.new do
- # build csv path
- path = '%s/csvs/hosts.csv' % [out_dir]
-
- # save CSV
- CSV.open(path, 'wb') do |csv|
- # write headers
- csv << COLS[:hosts].map { |col| col[:name] }
-
- # write rows
- @config['hosts'].each do |row|
- csv << COLS[:hosts].map { |col| row[col[:id]] }
- end
- end
- end
+ save_csv(
+ '%s/csvs/hosts.csv' % [out_dir],
+ COLS[:hosts].map { |col| col[:name] },
+ @config['hosts'].map { |row|
+ COLS[:hosts].map { |col| row[col[:id]] }
+ }
+ )
end
#
- # Generate out/html/hosts.html and return thread.
+ # Generate and return hosts HTML.
#
- def save_hosts_html
- Thread.new do
- # build html path
- path = '%s/html/hosts.html' % [out_dir]
+ def make_hosts_html
+ TEMPLATES[:all].run({
+ cols: COLS[:hosts].map { |col|
+ TEMPLATES[:col].run(col)
+ }.join,
- # generate and save html
- File.write(path, HTML[:all] % [
- COLS[:hosts].map { |col|
- HTML[:col] % [col[:name]]
- }.join,
-
- @config['hosts'].map { |row|
- HTML[:row] % [COLS[:hosts].map { |col|
- HTML[:cell] % [row[col[:id]]]
- }.join]
- }.join,
- ])
- end
+ rows: @config['hosts'].map { |row|
+ path = '%s/hosts/%s/version.txt' % [out_dir, row['name']]
+ row.merge({
+ 'openssl' => File.read(path).strip,
+ })
+ }.map { |row|
+ TEMPLATES[:row].run({
+ row: COLS[:hosts].map { |col|
+ TEMPLATES[:cell].run({
+ text: row[col[:id]]
+ })
+ }.join
+ })
+ }.join,
+ })
end
#
- # save CSV of rows.
+ # Generate and write out/index.html.
#
- def save_csv(arch, algo, rows)
- # build path to output csv
- csv_path = '%s/csvs/%s-%s.csv' % [out_dir, arch, algo]
+ def save_index_html(html)
+ File.write('%s/index.html' % [out_dir], TEMPLATES[:index].run({
+ title: 'OpenSSL Benchmark Results',
+ hosts: html[:hosts],
+
+ links: LINKS.map { |row|
+ TEMPLATES[:link].run(row)
+ }.join,
- # write csv
- CSV.open(csv_path, 'wb') do |csv|
- # write column headers
- csv << COLS[(algo == 'all') ? :all : :algo].map { |col| col[:id] }
+ sections: %i{all arm x86}.map { |arch|
+ TEMPLATES[:section].run({
+ svgs: html[arch.to_s],
+ }.merge(ARCHS[arch]))
+ }.join,
+ }))
+ end
+
+ #
+ # Save CSV file.
+ #
+ def save_csv(path, cols, rows)
+ CSV.open(path, 'wb') do |csv|
+ # write headers
+ csv << cols
# write rows
rows.each do |row|
csv << row
end
end
-
- # return csv path
- csv_path
end
#
- # Render CSV as SVG in background and return SVG and PID.
+ # Build data for SVG.
#
- def save_svg(arch, algo, max, csv_path)
- plot_path = '%s/plot.py' % [__dir__]
- svg_path = '%s/svgs/%s-%s.svg' % [out_dir, arch, algo]
+ def make_svg(arch, algo, max, rows)
+ {
+ # build output path
+ path: '%s/svgs/%s-%s.svg' % [out_dir, arch, algo],
+
+ # build title
+ title: TEMPLATES[:svg_title].run({
+ arch: ARCHS[arch.intern][:name],
+ algo: algo,
+ }),
+
+ # output image size (in inches)
+ size: [6.4 * 2, 4.8 * 2],
+
+ # output image DPI
+ dpi: 100,
+
+ # font sizes (in points)
+ fontsize: {
+ yticks: 10,
+ title: 14,
+ },
+
+ # calculate xlimit (round up to nearest 50)
+ xlimit: (max / (1048576 * 50.0)).ceil * 50,
+ xlabel: 'Speed (MB/s)',
+
+ # rows (sorted in reverse order)
+ rows: rows.map { |row|
+ [
+ '%s (%d bytes)' % [row[0], row[1].to_i],
+ row[2].to_f / 1048576,
+ ]
+ }.reverse,
+ }
+ end
- # make chart title
- title = ARCH_TITLES[arch.intern] % [algo]
+ #
+ # Render SVG in background and return SVG path, title, and PID.
+ #
+ def save_svg(svg)
+ # invoke plot in background, return pid
+ bg('/dev/null', [
+ # absolute path to python
+ '/usr/bin/python3',
- # calculate xlimit (round up to nearest 100)
- # xlimit = ((algo =~ /^aes/) ? 400 : 2000).to_s
- xlimit = (max / (1048576 * 50.0)).ceil * 50
+ # build path to plot.py
+ '%s/plot.py' % [__dir__],
- # build plot command
- plot_cmd = [
- '/usr/bin/python3',
- plot_path,
- csv_path,
- svg_path,
- title,
- xlimit.to_s,
- ]
-
- # return svg path and pid
- {
- # create svg in background
- pid: bg('/dev/null', plot_cmd),
- svg: svg_path,
- title: title,
- }
+ # build chart json data
+ JSON.unparse(svg),
+ ])
end
#
@@ -666,6 +834,7 @@ module PiBench
@config['out_dir']
end
end
+
#
# Allow one-shot invocation.
#