erasure-code: rework benchmark suite

Expand the default suite to enumerate all cases that are relevant to the
current code base so that it is easier to consume. Namely it means

 * iterating over object sizes of 4KB (what is used by default) and
   1MB (what was previous benchmarked)
 * grouping results in series that would make sense to plot to get the
   behavior of a given technique for a series of K/M values and all
   possible erasures.

Instead of specifying the iterations to run, set the size of the total
data set to be exercised and compute the iterations by dividing it by
the object size. Since the object size varies, it is impractical to
preset the number of iterations and get meaningful results.

The PARAMETERS environment variable is added to enable the caller to
inject --parameter jerasure-variant=generic, for instance.

The packets size is calculated based on the other parameters. The
options are limited when packets are small (4KB) and it would not make a
real difference to give control over it. The packet size is capped to
a maximum of 3100 bytes which is roughly what has been found to be an
optimal value for large packets (1MB).

Signed-off-by: Loic Dachary <loic@dachary.org>

diff --git a/qa/workunits/erasure-code/bench.sh b/qa/workunits/erasure-code/bench.sh
index db7c4906529422c6ef14f4e0dfc04e3da73d1f5f..71fbade05bcc6ebf5d11239e2b1a2735b32fc68d 100755 (executable)
--- a/qa/workunits/erasure-code/bench.sh
+++ b/qa/workunits/erasure-code/bench.sh
@@ -1,6 +1,6 @@
 #!/bin/bash 
 #
-# Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
+# Copyright (C) 2013,2014 Cloudwatt <libre.licensing@cloudwatt.com>
 #
 # Author: Loic Dachary <loic@dachary.org>
 #
@@ -22,8 +22,9 @@ export PATH=/sbin:$PATH
 : ${CEPH_ERASURE_CODE_BENCHMARK:=ceph_erasure_code_benchmark}
 : ${PLUGIN_DIRECTORY:=/usr/lib/ceph/erasure-code}
 : ${PLUGINS:=example jerasure}
-: ${ITERATIONS:=1024}
-: ${SIZE:=1048576}
+: ${TOTAL_SIZE:=$((10 * 1024 * 1024))}
+: ${SIZES:=4096 $((1024 * 1024))}
+: ${PARAMETERS:=--parameter jerasure-per-chunk-alignment=true}
 
 function bench_header() {
     echo -e "seconds\tKB\tplugin\tk\tm\twork.\titer.\tsize\teras.\tcommand."
@@ -50,53 +51,71 @@ function bench() {
         --iterations $iterations \
         --size $size \
         --erasures $erasures \
-        --parameter erasure-code-k=$k \
-        --parameter erasure-code-m=$m \
-        --parameter erasure-code-directory=$PLUGIN_DIRECTORY)
+        --parameter k=$k \
+        --parameter m=$m \
+        --parameter directory=$PLUGIN_DIRECTORY)
     result=$($command "$@")
     echo -e "$result\t$plugin\t$k\t$m\t$workload\t$iterations\t$size\t$erasures\t$command ""$@"
 }
 
 function example_test() {
     local plugin=example
+    local size
+    for size in $SIZES ; do
+        bench $plugin 2 1 encode $ITERATIONS $size 0
+        bench $plugin 2 1 decode $ITERATIONS $size 1
+    done
+}
+
+function packetsize() {
+    local k=$1
+    local w=$2
+    local vector_wordsize=$3
+    local size=$4
 
-    bench $plugin 2 1 encode $ITERATIONS $SIZE 0
-    bench $plugin 2 1 decode $ITERATIONS $SIZE 1
+    local p=$(( ($size / $k / $w / $vector_wordsize ) * $vector_wordsize))
+    if [ $p -gt 3100 ] ; then
+        p=3100
+    fi
+    echo $p
 }
 
-#
-# The results are expected to be consistent with 
-# https://www.usenix.org/legacy/events/fast09/tech/full_papers/plank/plank_html/
-# 
 function jerasure_test() {
     local plugin=jerasure
+    local w=8
+    local VECTOR_WORDSIZE=16
+    local ks="2 3 4 6 10"
+    declare -A k2ms
+    k2ms[2]="1"
+    k2ms[3]="2"
+    k2ms[4]="2 3"
+    k2ms[6]="2 3 4"
+    k2ms[10]="3 4"
+    for technique in reed_sol_van cauchy_good ; do
+        for size in $SIZES ; do
+            echo "serie encode_${technique}_${size}"
+            for k in $ks ; do
+                for m in ${k2ms[$k]} ; do
+                    bench $plugin $k $m encode $(($TOTAL_SIZE / $size)) $size 0 \
+                        --parameter packetsize=$(packetsize $k $w $VECTOR_WORDSIZE $size) \
+                        ${PARAMETERS} \
+                        --parameter technique=$technique
 
-    for technique in reed_sol_van ; do
-        for k in 4 6 10 ; do
-            for m in $(seq 1 4) ; do
-                bench $plugin $k $m encode $ITERATIONS $SIZE 0 \
-                    --parameter erasure-code-technique=$technique
-
-                for erasures in $(seq 1 $m) ; do
-                    bench $plugin $k $m decode $ITERATIONS $SIZE $erasures \
-                        --parameter erasure-code-technique=$technique
                 done
             done
         done
     done
-
-    for technique in cauchy_orig cauchy_good ; do
-        for packetsize in $(seq 512 512 4096) ; do
-            for k in 4 6 10 ; do
-                for m in $(seq 1 4) ; do
-                    bench $plugin $k $m encode $ITERATIONS $SIZE 0 \
-                        --parameter erasure-code-packetsize=$packetsize \
-                        --parameter erasure-code-technique=$technique
-
+    for technique in reed_sol_van cauchy_good ; do
+        for size in $SIZES ; do
+            echo "serie decode_${technique}_${size}"
+            for k in $ks ; do
+                for m in ${k2ms[$k]} ; do
+                    echo
                     for erasures in $(seq 1 $m) ; do
-                        bench $plugin $k $m decode $ITERATIONS $SIZE $erasures \
-                            --parameter erasure-code-packetsize=$packetsize \
-                            --parameter erasure-code-technique=$technique
+                        bench $plugin $k $m decode $(($TOTAL_SIZE / $size)) $size $erasures \
+                            --parameter packetsize=$(packetsize $k $w $VECTOR_WORDSIZE  $size) \
+                            ${PARAMETERS} \
+                            --parameter technique=$technique
                     done
                 done
             done
@@ -111,13 +130,12 @@ function main() {
     done
 }
 
-if [ "$1" = TEST ]
-then
+if [ "$1" = TEST ] ; then
     set -x
     set -o functrace
     PS4=' ${FUNCNAME[0]}: $LINENO: '
 
-    ITERATIONS=1
+    TOTAL_SIZE=1024
     SIZE=1024
 
     function run_test() {
@@ -126,100 +144,185 @@ then
         mkdir $dir
         expected=$(cat <<EOF
 plugin k       m       work.   iter.   size    eras.
-example        2       1       encode  1       1024    0
-example        2       1       decode  1       1024    1
-jerasure       2       1       encode  1       1024    0
-jerasure       2       1       decode  1       1024    1
-jerasure       2       2       encode  1       1024    0
-jerasure       2       2       decode  1       1024    1
-jerasure       2       2       decode  1       1024    2
-jerasure       2       1       encode  1       1024    0
-jerasure       2       1       decode  1       1024    1
-jerasure       2       2       encode  1       1024    0
-jerasure       2       2       decode  1       1024    1
-jerasure       2       2       decode  1       1024    2
-jerasure       2       1       encode  1       1024    0
-jerasure       2       1       decode  1       1024    1
-jerasure       2       2       encode  1       1024    0
-jerasure       2       2       decode  1       1024    1
-jerasure       2       2       decode  1       1024    2
-jerasure       2       1       encode  1       1024    0
-jerasure       2       1       decode  1       1024    1
-jerasure       2       2       encode  1       1024    0
-jerasure       2       2       decode  1       1024    1
-jerasure       2       2       decode  1       1024    2
-jerasure       2       1       encode  1       1024    0
-jerasure       2       1       decode  1       1024    1
-jerasure       2       2       encode  1       1024    0
-jerasure       2       2       decode  1       1024    1
-jerasure       2       2       decode  1       1024    2
-jerasure       2       1       encode  1       1024    0
-jerasure       2       1       decode  1       1024    1
-jerasure       2       2       encode  1       1024    0
-jerasure       2       2       decode  1       1024    1
-jerasure       2       2       decode  1       1024    2
-jerasure       2       1       encode  1       1024    0
-jerasure       2       1       decode  1       1024    1
-jerasure       2       2       encode  1       1024    0
-jerasure       2       2       decode  1       1024    1
-jerasure       2       2       decode  1       1024    2
-jerasure       6       3       encode  1       1024    0
-jerasure       6       3       decode  1       1024    1
-jerasure       6       3       decode  1       1024    2
-jerasure       6       3       decode  1       1024    3
-jerasure       6       4       encode  1       1024    0
-jerasure       6       4       decode  1       1024    1
-jerasure       6       4       decode  1       1024    2
-jerasure       6       4       decode  1       1024    3
-jerasure       6       4       decode  1       1024    4
-jerasure       10      3       encode  1       1024    0
-jerasure       10      3       decode  1       1024    1
-jerasure       10      3       decode  1       1024    2
-jerasure       10      3       decode  1       1024    3
-jerasure       10      4       encode  1       1024    0
-jerasure       10      4       decode  1       1024    1
-jerasure       10      4       decode  1       1024    2
-jerasure       10      4       decode  1       1024    3
-jerasure       10      4       decode  1       1024    4
-jerasure       6       3       encode  1       1024    0
-jerasure       6       3       decode  1       1024    1
-jerasure       6       3       decode  1       1024    2
-jerasure       6       3       decode  1       1024    3
-jerasure       6       4       encode  1       1024    0
-jerasure       6       4       decode  1       1024    1
-jerasure       6       4       decode  1       1024    2
-jerasure       6       4       decode  1       1024    3
-jerasure       6       4       decode  1       1024    4
-jerasure       10      3       encode  1       1024    0
-jerasure       10      3       decode  1       1024    1
-jerasure       10      3       decode  1       1024    2
-jerasure       10      3       decode  1       1024    3
-jerasure       10      4       encode  1       1024    0
-jerasure       10      4       decode  1       1024    1
-jerasure       10      4       decode  1       1024    2
-jerasure       10      4       decode  1       1024    3
-jerasure       10      4       decode  1       1024    4
-jerasure       6       3       encode  1       1024    0
-jerasure       6       3       decode  1       1024    1
-jerasure       6       3       decode  1       1024    2
-jerasure       6       3       decode  1       1024    3
-jerasure       6       4       encode  1       1024    0
-jerasure       6       4       decode  1       1024    1
-jerasure       6       4       decode  1       1024    2
-jerasure       6       4       decode  1       1024    3
-jerasure       6       4       decode  1       1024    4
-jerasure       10      3       encode  1       1024    0
-jerasure       10      3       decode  1       1024    1
-jerasure       10      3       decode  1       1024    2
-jerasure       10      3       decode  1       1024    3
-jerasure       10      4       encode  1       1024    0
-jerasure       10      4       decode  1       1024    1
-jerasure       10      4       decode  1       1024    2
-jerasure       10      4       decode  1       1024    3
-jerasure       10      4       decode  1       1024    4
+serie encode_reed_sol_van_4096
+jerasure       2       1       encode  0       4096    0
+jerasure       3       2       encode  0       4096    0
+jerasure       4       2       encode  0       4096    0
+jerasure       4       3       encode  0       4096    0
+jerasure       6       2       encode  0       4096    0
+jerasure       6       3       encode  0       4096    0
+jerasure       6       4       encode  0       4096    0
+jerasure       10      3       encode  0       4096    0
+jerasure       10      4       encode  0       4096    0
+serie encode_reed_sol_van_1048576
+jerasure       2       1       encode  0       1048576 0
+jerasure       3       2       encode  0       1048576 0
+jerasure       4       2       encode  0       1048576 0
+jerasure       4       3       encode  0       1048576 0
+jerasure       6       2       encode  0       1048576 0
+jerasure       6       3       encode  0       1048576 0
+jerasure       6       4       encode  0       1048576 0
+jerasure       10      3       encode  0       1048576 0
+jerasure       10      4       encode  0       1048576 0
+serie encode_cauchy_good_4096
+jerasure       2       1       encode  0       4096    0
+jerasure       3       2       encode  0       4096    0
+jerasure       4       2       encode  0       4096    0
+jerasure       4       3       encode  0       4096    0
+jerasure       6       2       encode  0       4096    0
+jerasure       6       3       encode  0       4096    0
+jerasure       6       4       encode  0       4096    0
+jerasure       10      3       encode  0       4096    0
+jerasure       10      4       encode  0       4096    0
+serie encode_cauchy_good_1048576
+jerasure       2       1       encode  0       1048576 0
+jerasure       3       2       encode  0       1048576 0
+jerasure       4       2       encode  0       1048576 0
+jerasure       4       3       encode  0       1048576 0
+jerasure       6       2       encode  0       1048576 0
+jerasure       6       3       encode  0       1048576 0
+jerasure       6       4       encode  0       1048576 0
+jerasure       10      3       encode  0       1048576 0
+jerasure       10      4       encode  0       1048576 0
+serie decode_reed_sol_van_4096
+
+jerasure       2       1       decode  0       4096    1
+
+jerasure       3       2       decode  0       4096    1
+jerasure       3       2       decode  0       4096    2
+
+jerasure       4       2       decode  0       4096    1
+jerasure       4       2       decode  0       4096    2
+
+jerasure       4       3       decode  0       4096    1
+jerasure       4       3       decode  0       4096    2
+jerasure       4       3       decode  0       4096    3
+
+jerasure       6       2       decode  0       4096    1
+jerasure       6       2       decode  0       4096    2
+
+jerasure       6       3       decode  0       4096    1
+jerasure       6       3       decode  0       4096    2
+jerasure       6       3       decode  0       4096    3
+
+jerasure       6       4       decode  0       4096    1
+jerasure       6       4       decode  0       4096    2
+jerasure       6       4       decode  0       4096    3
+jerasure       6       4       decode  0       4096    4
+
+jerasure       10      3       decode  0       4096    1
+jerasure       10      3       decode  0       4096    2
+jerasure       10      3       decode  0       4096    3
+
+jerasure       10      4       decode  0       4096    1
+jerasure       10      4       decode  0       4096    2
+jerasure       10      4       decode  0       4096    3
+jerasure       10      4       decode  0       4096    4
+serie decode_reed_sol_van_1048576
+
+jerasure       2       1       decode  0       1048576 1
+
+jerasure       3       2       decode  0       1048576 1
+jerasure       3       2       decode  0       1048576 2
+
+jerasure       4       2       decode  0       1048576 1
+jerasure       4       2       decode  0       1048576 2
+
+jerasure       4       3       decode  0       1048576 1
+jerasure       4       3       decode  0       1048576 2
+jerasure       4       3       decode  0       1048576 3
+
+jerasure       6       2       decode  0       1048576 1
+jerasure       6       2       decode  0       1048576 2
+
+jerasure       6       3       decode  0       1048576 1
+jerasure       6       3       decode  0       1048576 2
+jerasure       6       3       decode  0       1048576 3
+
+jerasure       6       4       decode  0       1048576 1
+jerasure       6       4       decode  0       1048576 2
+jerasure       6       4       decode  0       1048576 3
+jerasure       6       4       decode  0       1048576 4
+
+jerasure       10      3       decode  0       1048576 1
+jerasure       10      3       decode  0       1048576 2
+jerasure       10      3       decode  0       1048576 3
+
+jerasure       10      4       decode  0       1048576 1
+jerasure       10      4       decode  0       1048576 2
+jerasure       10      4       decode  0       1048576 3
+jerasure       10      4       decode  0       1048576 4
+serie decode_cauchy_good_4096
+
+jerasure       2       1       decode  0       4096    1
+
+jerasure       3       2       decode  0       4096    1
+jerasure       3       2       decode  0       4096    2
+
+jerasure       4       2       decode  0       4096    1
+jerasure       4       2       decode  0       4096    2
+
+jerasure       4       3       decode  0       4096    1
+jerasure       4       3       decode  0       4096    2
+jerasure       4       3       decode  0       4096    3
+
+jerasure       6       2       decode  0       4096    1
+jerasure       6       2       decode  0       4096    2
+
+jerasure       6       3       decode  0       4096    1
+jerasure       6       3       decode  0       4096    2
+jerasure       6       3       decode  0       4096    3
+
+jerasure       6       4       decode  0       4096    1
+jerasure       6       4       decode  0       4096    2
+jerasure       6       4       decode  0       4096    3
+jerasure       6       4       decode  0       4096    4
+
+jerasure       10      3       decode  0       4096    1
+jerasure       10      3       decode  0       4096    2
+jerasure       10      3       decode  0       4096    3
+
+jerasure       10      4       decode  0       4096    1
+jerasure       10      4       decode  0       4096    2
+jerasure       10      4       decode  0       4096    3
+jerasure       10      4       decode  0       4096    4
+serie decode_cauchy_good_1048576
+
+jerasure       2       1       decode  0       1048576 1
+
+jerasure       3       2       decode  0       1048576 1
+jerasure       3       2       decode  0       1048576 2
+
+jerasure       4       2       decode  0       1048576 1
+jerasure       4       2       decode  0       1048576 2
+
+jerasure       4       3       decode  0       1048576 1
+jerasure       4       3       decode  0       1048576 2
+jerasure       4       3       decode  0       1048576 3
+
+jerasure       6       2       decode  0       1048576 1
+jerasure       6       2       decode  0       1048576 2
+
+jerasure       6       3       decode  0       1048576 1
+jerasure       6       3       decode  0       1048576 2
+jerasure       6       3       decode  0       1048576 3
+
+jerasure       6       4       decode  0       1048576 1
+jerasure       6       4       decode  0       1048576 2
+jerasure       6       4       decode  0       1048576 3
+jerasure       6       4       decode  0       1048576 4
+
+jerasure       10      3       decode  0       1048576 1
+jerasure       10      3       decode  0       1048576 2
+jerasure       10      3       decode  0       1048576 3
+
+jerasure       10      4       decode  0       1048576 1
+jerasure       10      4       decode  0       1048576 2
+jerasure       10      4       decode  0       1048576 3
+jerasure       10      4       decode  0       1048576 4
 EOF
 )
-        test "$(main | cut --fields=3-9 )" = "$expected" || return 1
+        test "$(PLUGINS=jerasure main | cut --fields=3-9 )" = "$expected" || return 1
     }
 
     run_test