From: Loic Dachary Date: Tue, 12 May 2015 12:47:32 +0000 (+0200) Subject: erasure-code: bench.sh compares isa & jerasure, vandermonde & cauchy X-Git-Tag: v9.0.2~154^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=28f0230c123d229fd60f0e91730c540412df0bcd;p=ceph.git erasure-code: bench.sh compares isa & jerasure, vandermonde & cauchy ISA and Jerasure can be compared for the default stripe width (4KB) and the two most commonly used Reed Solomon matrices. Comparing the bandwidth for large chunks (1MB) is not relevant because it is not commonly used. Signed-off-by: Loic Dachary --- diff --git a/qa/workunits/erasure-code/bench.html b/qa/workunits/erasure-code/bench.html index d5dd1095ce02..3b4b6c74c002 100644 --- a/qa/workunits/erasure-code/bench.html +++ b/qa/workunits/erasure-code/bench.html @@ -28,27 +28,6 @@

decode: Y = GB/s, X = K/M/erasures

-
-
-
-

encode 4KB: Y = GB/s, X = K/M

- -
-
-
-

decode 4KB: Y = GB/s, X = K/M/erasures

- -
-
-
-

encode 1MB: Y = GB/s, X = K/M

- -
-
-
-

decode 1MB: Y = GB/s, X = K/M/erasures

- - diff --git a/qa/workunits/erasure-code/bench.sh b/qa/workunits/erasure-code/bench.sh index 27689665c4fc..acd70354edcd 100755 --- a/qa/workunits/erasure-code/bench.sh +++ b/qa/workunits/erasure-code/bench.sh @@ -16,7 +16,6 @@ # # Test that it works from sources with: # -# TOTAL_SIZE=$((1024 * 1024)) \ # CEPH_ERASURE_CODE_BENCHMARK=src/ceph_erasure_code_benchmark \ # PLUGIN_DIRECTORY=src/.libs \ # qa/workunits/erasure-code/bench.sh fplot jerasure | @@ -24,7 +23,7 @@ # # This should start immediately and display: # -# var encode_reed_sol_van_4096 = [ +# ... # [ '2/1', .48035538612887358583 ], # [ '3/2', .21648470405675016626 ], # etc. @@ -36,6 +35,7 @@ # Once it is confirmed to work, it can be run with a more significant # volume of data so that the measures are more reliable: # +# TOTAL_SIZE=$((4 * 1024 * 1024 * 1024)) \ # CEPH_ERASURE_CODE_BENCHMARK=src/ceph_erasure_code_benchmark \ # PLUGIN_DIRECTORY=src/.libs \ # qa/workunits/erasure-code/bench.sh fplot jerasure | @@ -49,8 +49,8 @@ export PATH=/sbin:$PATH : ${CEPH_ERASURE_CODE_BENCHMARK:=ceph_erasure_code_benchmark} : ${PLUGIN_DIRECTORY:=/usr/lib/ceph/erasure-code} : ${PLUGINS:=example jerasure isa} -: ${TOTAL_SIZE:=$((10 * 1024 * 1024))} -: ${SIZES:=4096 $((1024 * 1024))} +: ${TOTAL_SIZE:=$((1024 * 1024))} +: ${SIZE:=4096} : ${PARAMETERS:=--parameter jerasure-per-chunk-alignment=true} function bench_header() { @@ -85,15 +85,6 @@ function bench() { echo -e "$result\t$plugin\t$k\t$m\t$workload\t$iterations\t$size\t$erasures\t$command ""$@" } -function example_test() { - local plugin=example - local size - for size in $SIZES ; do - bench $plugin 2 1 encode $ITERATIONS $size 0 - bench $plugin 2 1 decode $ITERATIONS $size 1 - done -} - function packetsize() { local k=$1 local w=$2 @@ -107,7 +98,7 @@ function packetsize() { echo $p } -function jerasure_test() { +function bench_run() { local plugin=jerasure local w=8 local VECTOR_WORDSIZE=16 @@ -118,31 +109,39 @@ function jerasure_test() { k2ms[4]="2 3" k2ms[6]="2 3 4" k2ms[10]="3 4" - for technique in reed_sol_van cauchy_good ; do - for size in $SIZES ; do - echo "serie encode_${technique}_${size}" + local isa2technique_vandermonde='reed_sol_van' + local isa2technique_cauchy='cauchy' + local jerasure_generic2technique_vandermonde='reed_sol_van' + local jerasure_generic2technique_cauchy='cauchy_good' + local jerasure_sse42technique_vandermonde='reed_sol_van' + local jerasure_sse42technique_cauchy='cauchy_good' + for technique in vandermonde cauchy ; do + for plugin in isa jerasure_generic jerasure_sse4 ; do + eval technique_parameter=\$${plugin}2technique_${technique} + echo "serie encode_${technique}_${plugin}" for k in $ks ; do for m in ${k2ms[$k]} ; do - bench $plugin $k $m encode $(($TOTAL_SIZE / $size)) $size 0 \ - --parameter packetsize=$(packetsize $k $w $VECTOR_WORDSIZE $size) \ + bench $plugin $k $m encode $(($TOTAL_SIZE / $SIZE)) $SIZE 0 \ + --parameter packetsize=$(packetsize $k $w $VECTOR_WORDSIZE $SIZE) \ ${PARAMETERS} \ - --parameter technique=$technique + --parameter technique=$technique_parameter done done done done - for technique in reed_sol_van cauchy_good ; do - for size in $SIZES ; do - echo "serie decode_${technique}_${size}" + for technique in vandermonde cauchy ; do + for plugin in isa jerasure_generic jerasure_sse4 ; do + eval technique_parameter=\$${plugin}2technique_${technique} + echo "serie decode_${technique}_${plugin}" for k in $ks ; do for m in ${k2ms[$k]} ; do echo for erasures in $(seq 1 $m) ; do - bench $plugin $k $m decode $(($TOTAL_SIZE / $size)) $size $erasures \ - --parameter packetsize=$(packetsize $k $w $VECTOR_WORDSIZE $size) \ + bench $plugin $k $m decode $(($TOTAL_SIZE / $SIZE)) $SIZE $erasures \ + --parameter packetsize=$(packetsize $k $w $VECTOR_WORDSIZE $SIZE) \ ${PARAMETERS} \ - --parameter technique=$technique + --parameter technique=$technique_parameter done done done @@ -150,46 +149,9 @@ function jerasure_test() { done } -function isa_test() { - local plugin=isa - local ks="2 3 4 6 10" - declare -A k2ms - k2ms[2]="1" - k2ms[3]="2" - k2ms[4]="2 3" - k2ms[6]="2 3 4" - k2ms[10]="3 4" - for technique in reed_sol_van cauchy ; do - for size in $SIZES ; do - echo "serie encode_${technique}_${size}" - for k in $ks ; do - for m in ${k2ms[$k]} ; do - bench $plugin $k $m encode $(($TOTAL_SIZE / $size)) $size 0 \ - --parameter technique=$technique - - done - done - done - done - for technique in reed_sol_van cauchy ; do - for size in $SIZES ; do - echo "serie decode_${technique}_${size}" - for k in $ks ; do - for m in ${k2ms[$k]} ; do - echo - for erasures in $(seq 1 $m) ; do - bench $plugin $k $m decode $(($TOTAL_SIZE / $size)) $size $erasures \ - --parameter technique=$technique - done - done - done - done - done -} function fplot() { - local plugin=$1 local serie - ${plugin}_test | while read seconds total plugin k m workload iteration size erasures rest ; do + bench_run | while read seconds total plugin k m workload iteration size erasures rest ; do if [ -z $seconds ] ; then echo null, elif [ $seconds = serie ] ; then @@ -213,209 +175,10 @@ function fplot() { function main() { bench_header - for plugin in ${PLUGINS} ; do - ${plugin}_test || return 1 - done + bench_run } -if [ "$1" = TEST ] ; then - set -x - set -o functrace - PS4=' ${FUNCNAME[0]}: $LINENO: ' - - TOTAL_SIZE=1024 - SIZE=1024 - - function run_test() { - dir=/tmp/erasure-code - rm -fr $dir - mkdir $dir - expected=$(cat <