Rewrite run.sh script

README.md

@@ -1,4 +1,4 @@
-# Простой бенчмарк HDL симуляторов (преранняя версия)
+# Простой бенчмарк HDL симуляторов (версия альфа)
 
 Для оценки скорости запускается симуляция 1024 софт-процессоров
 [PicoRV32](https://github.com/YosysHQ/picorv32) с программой вычисления хэш-суммы MD5
@@ -14,9 +14,29 @@
 симуляторе. Скрипты называются `__build.sh` (для сборки проекта) и `__run.sh` (для
 запуска симуляции).
 
-Скрипт `run.sh` запускает бенчмарк из выбранной папки или все тесты, если параметром
+Скрипт `run.sh` запускает бенчмарк из выбранной папки или все тесты. В параметрах
-указать `all`. В параметрах можно указать сразу несколько папок с тестами. Результаты
+можно указать количество софт-ядер, размер блока, количество потоков симуляции (пока
-бенчмарка записываются в файл `results.txt`.
+только для верилятора) и список бенчмарков:
+
+```
+  $ ./run.sh -h
+  Usage: ./run.sh [OPTION]... [SIM...]
+  Run simulator benchmark. Calculates MD5 hash from a block data
+  on an array of soft-cores PicoRV32.
+
+  Options:
+    -c [COUNT]    Soft CPU count in simulation. Default: 1024
+    -s [SIZE]     Data block size in bytes. Default: 1024 bytes
+    -t [COUNT]    Simulation threads count. Default: 1
+                  (so far only for Verilator)
+    -l            List of available benchmarks
+    -h            This help
+
+  The SIM parameter is the name of the simulator from the list of
+  option -l. If the parameter is not specified, benchmarks for all
+  simulators will be performed. Be careful, some simulators take
+  a very long time to benchmark.
+```
 
 ## Результаты для 1024 процессоров
 

run.sh

@@ -1,65 +1,181 @@
 #!/usr/bin/env bash
 
-BUILD=__build.sh
+## Default valies
-RUN=__run.sh
+CPU_COUNT=1024
+BLOCK_SIZE=1
+THREADS=1
 
-if [ "$1" == "all" ]
+BLD_SCRIPT="__build.sh"
+RUN_SCRIPT="__run.sh"
+TEST_DIR_PREFIX="test-"
+LOG_PREFIX="####"
+
+function sim_dir_valid()
+{
+    if [ -e "$1/$BLD_SCRIPT" ] && [ -e "$1/$RUN_SCRIPT" ]
+    then
+        return 0
+    else
+        return 1
+    fi
+}
+
+function sim_list()
+{
+    for dir in "$TEST_DIR_PREFIX"*
+    do
+        if sim_dir_valid "$dir"
+        then
+            echo "${dir:5}"
+        fi
+    done
+}
+
+function print_help()
+{
+    echo "Usage: $0 [OPTION]... [SIM...]"
+    echo "Run simulator benchmark. Calculates MD5 hash from a block data"
+    echo "on an array of soft-cores PicoRV32."
+    echo
+    echo "Options:"
+    echo "  -c [COUNT]    Soft CPU count in simulation. Default: 1024"
+    echo "  -s [SIZE]     Data block size in bytes. Default: 1024 bytes"
+    echo "  -t [COUNT]    Simulation threads count. Default: 1"
+    echo "                (so far only for Verilator)"
+    echo "  -l            List of available benchmarks"
+    echo "  -h            This help"
+    echo
+    echo "The SIM parameter is the name of the simulator from the list of"
+    echo "option -l. If the parameter is not specified, benchmarks for all"
+    echo "simulators will be performed. Be careful, some simulators take "
+    echo "a very long time to benchmark."
+    echo
+}
+
+function check_arg(){
+    if [[ $2 == -* ]]
+    then
+	echo "Option $1 requires an argument" >&2
+	exit 1
+    fi
+}
+
+function parse_param()
+{
+    while getopts ":c:s:t:lh" opt
+    do
+	case $opt in
+	    c)
+		check_arg "-c" "$OPTARG"
+                CPU_COUNT=$OPTARG
+		;;
+	    s)
+		check_arg "-s" "$OPTARG"
+                BLOCK_SIZE=$OPTARG
+		;;
+	    t)
+		check_arg "-t" "$OPTARG"
+                THREADS=$OPTARG
+		;;
+	    l)
+		sim_list
+                exit 0
+		;;
+	    h)
+		print_help
+                exit 0
+		;;
+	    \?)
+		echo "Invalid option: -$OPTARG" >&2
+		print_help
+		exit 1
+		;;
+	    :)
+		echo "Option -$OPTARG requires an argument" >&2
+		exit 1
+		;;
+	esac
+    done
+}
+
+function log()
+{
+    echo -n "$LOG_PREFIX "
+    echo "$@"
+}
+
+function run_benchmark()
+{
+    benchmark=$1
+    dir=$TEST_DIR_PREFIX$benchmark
+
+    if sim_dir_valid "$dir"
+    then
+        local t0 t1 ms
+
+        if cd "$dir"
+        then
+            # Build
+            log "Build $benchmark"
+            t0=$(date +%s%N | cut -b1-13)
+
+            if ! ./$BLD_SCRIPT "$CPU_COUNT" "$BLOCK_SIZE" "$THREADS"
+            then
+                cd ..
+                log "Build $benchmark FAILED"
+                return 1
+            fi
+
+            t1=$(date +%s%N | cut -b1-13)
+            ms=$((t1 - t0))
+            log "Build $benchmark time (ms): $ms"
+
+            echo
+
+            # Run
+            log "Run $benchmark"
+            t0=$(date +%s%N | cut -b1-13)
+
+            if ! ./$RUN_SCRIPT "$CPU_COUNT" "$BLOCK_SIZE" "$THREADS"
+            then
+                cd ..
+                log "RUN $benchmark FAILED"
+                return 1
+            fi
+
+            t1=$(date +%s%N | cut -b1-13)
+            ms=$((t1 - t0))
+            log "Run $benchmark time (ms): $ms"
+
+            cd ..
+            return 0
+        else
+            log "Can't change dir to $dir"
+            return 1
+        fi
+    else
+        log "No run scripts found in $dir"
+        return 1
+    fi
+}
+
+parse_param "$@"
+shift $((OPTIND - 1))
+
+if [ $# -gt 0 ]
 then
-    tests=$(ls -1d test-*)
+    benches="$*"
-elif [ -n "$1" ]
-then
-    tests="$@"
 else
-    echo "Usage: $0 <TEST_DIRECTORY | all>"
+    for b in $(sim_list); do benches="$benches$b "; done
-    exit -1
 fi
 
-## Log header
+log "Soft-cores count: $CPU_COUNT"
-echo >> results.txt
+log "Block size: $BLOCK_SIZE"
-echo "---------- Simulator's benchmark -----------" >> results.txt
+log "Threads count: $THREADS"
-echo $(date) >> results.txt
+log "Benchmarks: $benches"
-echo >> results.txt
 
-## Run tests
+for bench in $benches
-for test_dir in $tests
 do
-    if [ ! -d "$test_dir" ]
+    echo
-    then
+    run_benchmark "$bench"
-        echo "Directory $test_dir is not exists"
-        exit -1
-    fi
-
-    if [ -e $test_dir/$BUILD -a -e $test_dir/$RUN ]
-    then
-        echo "#### Run benchmark in $test_dir"
-
-        cd $test_dir
-
-        ./$BUILD
-        if [ $? -eq 0 ]
-        then
-            start_ms=$(date +%s%N | cut -b1-13)
-
-            ./$RUN
-            if [ $? -eq 0 ]
-            then
-                stop_ms=$(date +%s%N | cut -b1-13)
-                ms=$(expr $stop_ms - $start_ms)
-                echo "#### $test_dir: $ms milliseconds"
-            else
-                ms="RUN FAIL"
-                echo "#### $test_dir: run fail"
-            fi
-        else
-            ms="BUILD FAIL"
-            echo "#### $test_dir: build fail"
-        fi
-
-        echo ""
-        cd ..
-
-        echo "$test_dir: $ms" >> results.txt
-    else
-        echo "Skip $test_dir directory"
-    fi
 done

scripts/sim_vars.sh

@@ -0,0 +1,11 @@
+if [ $# -lt 3 ]
+then
+    echo "Usage: $0 <CPU_COUNT> <BLOCK_SIZE> <THREADS_COUNT"
+    exit -1
+fi
+
+CPU_COUNT=$1
+BLOCK_SIZE=$2
+THREADS=$3
+
+FFILE=../source/sources.f

source/testbench.sv

@@ -1,7 +1,8 @@
 `timescale 1ps/1ps
 
-module testbench (input clock);
+module testbench #(parameter CPU_COUNT = 1024)
-    localparam CPU_COUNT = 1024;
+    (input clock);
+
     localparam DATA_ADDR = 32'h00010000;
     localparam DATA_LEN = 1024;
 

test-iverilog/__build.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
 
-FFILE=../source/sources.f
+. ../scripts/sim_vars.sh
 
-iverilog -g2012 -o top -f $FFILE top.sv
+iverilog -g2012 -o top -Ptop.CPU_COUNT=$CPU_COUNT -f $FFILE top.sv

test-iverilog/__run.sh

@@ -1,9 +1,5 @@
 #!/usr/bin/env bash
 
-if [ -n "$1" ]; then
+. ../scripts/sim_vars.sh
-    dlen_arg="+dlen=$1"
-else
-    dlen_arg=""
-fi
 
-vvp -n ./top $dlen_arg
+vvp -n ./top +dlen=$BLOCK_SIZE

test-iverilog/top.sv

@@ -1,7 +1,7 @@
 `timescale 1ps/1ps
 
-module top;
+module top #(parameter CPU_COUNT = 1024);
     logic clock = 1'b0;
     initial forever #(10ns/2) clock = ~clock;
-    testbench testbench (clock);
+    testbench #(CPU_COUNT) testbench (clock);
 endmodule

test-modelsim/__build.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -e
 
-FFILE=../source/sources.f
+. ../scripts/sim_vars.sh
 
 rm -rf testbench
-vlog -sv -work testbench -vopt -f $FFILE top.sv
+vlog -sv -work testbench -vopt $param -f $FFILE top.sv

test-modelsim/__run.sh

@@ -1,9 +1,5 @@
 #!/usr/bin/env bash
 
-if [ -n "$1" ]; then
+. ../scripts/sim_vars.sh
-    dlen_arg="+dlen=$1"
-else
-    dlen_arg=""
-fi
 
-vsim -batch -voptargs=+acc=npr -do "run -all" -quiet $dlen_arg -lib testbench top
+vsim -batch -voptargs=+acc=npr -do "run -all" -quiet +dlen=$BLOCK_SIZE -GCPU_COUNT=$CPU_COUNT -lib testbench top

test-modelsim/top.sv

@@ -1,7 +1,7 @@
 `timescale 1ps/1ps
 
-module top;
+module top #(parameter CPU_COUNT = 1024);
     logic clock = 1'b0;
     initial forever #(10ns/2) clock = ~clock;
-    testbench testbench (clock);
+    testbench #(CPU_COUNT) testbench (clock);
 endmodule

test-verilator/Makefile

@@ -3,10 +3,12 @@ TOP_MODULE = testbench
 SOURCES = top.cpp clock_generator.cpp
 FLAGS_FILE = ../source/sources.f
 INCLUDES =
+PARAMS :=
+THREADS := 1
 
 FLAGS = -Wno-WIDTH -cc --top-module $(TOP_MODULE) +1800-2017ext+sv \
 	--timing --Mdir $(TOP_MODULE) -o $(TOP_MODULE) -f $(FLAGS_FILE) \
-	--timescale "1ps/1ps" --threads 1 -j 0
+	$(PARAMS) --timescale "1ps/1ps" --threads $(THREADS) -j 0
 
 # FLAGS += --trace
 

test-verilator/__build.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 set -e
 
+. ../scripts/sim_vars.sh
+
 make clean
-make OPT_FAST="-Os -march=native" VM_PARALLEL_BUILDS=0
+make OPT_FAST="-Os -march=native" VM_PARALLEL_BUILDS=0 PARAMS="-GCPU_COUNT=$CPU_COUNT" THREADS=$THREADS

test-verilator/__run.sh

@@ -1,9 +1,5 @@
 #!/usr/bin/env bash
 
-if [ -n "$1" ]; then
+. ../scripts/sim_vars.sh
-    dlen_arg="+dlen=$1"
-else
-    dlen_arg=""
-fi
 
-./testbench/testbench $dlen_arg
+./testbench/testbench +dlen=$BLOCK_SIZE

test-xcelium/.dir-locals.el

@@ -0,0 +1 @@
+((verilog-mode . ((flycheck-verilator-include-path . ("../source")))))

test-xcelium/__build.sh

@@ -1,7 +1,11 @@
 #!/usr/bin/env bash
 set -e
 
+. ../scripts/sim_vars.sh
+
 rm -rf xcelium.d
 
-xmvlog -sv -f ../source/sources.f top.sv
+## WARNING: defparam is not tested
-xmelab -timescale 1ps/1ps top
+
+xmvlog -sv -f $FFILE top.sv
+xmelab -timescale 1ps/1ps -defparam top.CPU_COUNT=$CPU_COUNT top

test-xcelium/__run.sh

@@ -1,9 +1,5 @@
 #!/usr/bin/env bash
 
-if [ -n "$1" ]; then
+. ../scripts/sim_vars.sh
-    dlen_arg="+dlen=$1"
-else
-    dlen_arg=""
-fi
 
-xmsim -status top $dlen_arg
+xmsim -status top +dlen=$BLOCK_SIZE

test-xcelium/top.sv

@@ -1,7 +1,7 @@
 `timescale 1ps/1ps
 
-module top;
+module top #(parameter CPU_COUNT = 1024);
     logic clock = 1'b0;
     initial forever #(10ns/2) clock = ~clock;
-    testbench testbench (clock);
+    testbench #(CPU_COUNT) testbench (clock);
 endmodule

test-xsim/__build.sh

@@ -1,7 +1,8 @@
 #!/usr/bin/env bash
 set -e
 
-FFILE=../source/sources.f
+. ../scripts/sim_vars.sh
+
 SOURCES=$(cat $FFILE | sed -ze 's/\n/ /g')
 
 rm -rf xsim.dir
@@ -10,4 +11,4 @@ rm -rf xvlog.* xelab.* xsim.*
 rm -rf top.wdb
 
 xvlog -work work --sv top.sv $SOURCES
-xelab --O3 -L work top
+xelab --O3 --generic_top "CPU_COUNT=$CPU_COUNT" -L work top

test-xsim/__run.sh

@@ -1,9 +1,5 @@
 #!/usr/bin/env bash
 
-if [ -n "$1" ]; then
+. ../scripts/sim_vars.sh
-    dlen_arg="-testplusarg dlen=$1"
-else
-    dlen_arg=""
-fi
 
-xsim top $dlen_arg --runall
+xsim top -testplusarg dlen=$BLOCK_SIZE --runall

test-xsim/top.sv

@@ -1,7 +1,7 @@
 `timescale 1ps/1ps
 
-module top;
+module top #(parameter CPU_COUNT = 1024);
     logic clock = 1'b0;
     initial forever #(10ns/2) clock = ~clock;
-    testbench testbench (clock);
+    testbench #(CPU_COUNT) testbench (clock);
 endmodule