From 88f2bbe99f17a2ec7eb4579f71413ce596d3b34b Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Tue, 23 Feb 2021 13:30:24 -0800 Subject: [PATCH 1/3] Initial checkin of test scripts, copied from wasi-sdk. --- tools/test-harness/exit_status_zero | 1 + tools/test-harness/run.sh | 56 +++++++++++++++ tools/test-harness/testcase.sh | 106 ++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+) create mode 100644 tools/test-harness/exit_status_zero create mode 100755 tools/test-harness/run.sh create mode 100755 tools/test-harness/testcase.sh diff --git a/tools/test-harness/exit_status_zero b/tools/test-harness/exit_status_zero new file mode 100644 index 00000000..573541ac --- /dev/null +++ b/tools/test-harness/exit_status_zero @@ -0,0 +1 @@ +0 diff --git a/tools/test-harness/run.sh b/tools/test-harness/run.sh new file mode 100755 index 00000000..8d5a9933 --- /dev/null +++ b/tools/test-harness/run.sh @@ -0,0 +1,56 @@ +#!/bin/bash +set -ueo pipefail + +# Top-level test runner. Usage is "run.sh" to run tests in compile-only mode, +# or "run.sh " where is a WASI-capable runtime to run the +# tests in full compile and execute mode. +# +# By default this script will look for `clang` and `clang++` in $PATH and +# assume that they are correctly configured with the sysroot in the default +# location. Alternatively, exporting $CC and $CXX allow more flexibility. e.g: +# +# export CXX="/bin/clang++ --sysroot /share/wasi-sysroot" +# export CCC="/bin/clang --sysroot /share/wasi-sysroot" +# + +# Determine the wasm runtime to use, if one is provided. +if [ $# -gt 0 ]; then + runwasi="$1" +else + runwasi="" +fi + +testdir=$(dirname $0) +CC=${CC:=clang} +CXX=${CXX:=clang++} + +echo $CC +echo $CXX + +cd $testdir/compile-only +for options in -O0 -O2 "-O2 -flto"; do + echo "===== Testing compile-only with $options =====" + for file in *.c; do + echo "Testing compile-only $file..." + ../testcase.sh "" "$CC" "$options" "$file" + done + for file in *.cc; do + echo "Testing compile-only $file..." + ../testcase.sh "" "$CXX" "$options" "$file" + done +done +cd - >/dev/null + +cd $testdir/general +for options in -O0 -O2 "-O2 -flto"; do + echo "===== Testing with $options =====" + for file in *.c; do + echo "Testing $file..." + ../testcase.sh "$runwasi" "$CC" "$options" "$file" + done + for file in *.cc; do + echo "Testing $file..." + ../testcase.sh "$runwasi" "$CXX" "$options" "$file" + done +done +cd - >/dev/null diff --git a/tools/test-harness/testcase.sh b/tools/test-harness/testcase.sh new file mode 100755 index 00000000..38db9a37 --- /dev/null +++ b/tools/test-harness/testcase.sh @@ -0,0 +1,106 @@ +#!/bin/bash +set -ueo pipefail + +# A simple testcase runner that runs a command, captures all its command-line +# outputs, and compares them against expected outputs. + +# Command-line parsing; this script is meant to be run from a higher-level +# script, so don't do anything fancy. +runwasi="$1" +compiler="$2" +options="$3" +input="$4" + +# Compile names for generated files. +wasm="$input.$options.wasm" +stdout_observed="$input.$options.stdout.observed" +stderr_observed="$input.$options.stderr.observed" +exit_status_observed="$input.$options.exit_status.observed" + +# Optionally load compiler options from a file. +if [ -e "$input.options" ]; then + file_options=$(cat "$input.options") +else + file_options= +fi + +echo "Testing $input..." + +# Compile the testcase. +$compiler $options $file_options "$input" -o "$wasm" + +# If we don't have a runwasi command, we're just doing compile-only testing. +if [ "$runwasi" == "" ]; then + exit 0 +fi + +# Determine the input file to write to stdin. +if [ -e "$input.stdin" ]; then + stdin="$input.stdin" +else + stdin="/dev/null" +fi + +# Determine any environment variables to set. +if [ -e "$input.env" ]; then + env=$(sed -e 's/^/--env /' < "$input.env") +else + env="" +fi + +# Determine a preopened directory to provide. +if [ -e "$input.dir" ]; then + dir="--dir $input.dir" + dirarg="$input.dir" +else + dir="" + dirarg="" +fi + +# Run the test, capturing stdout, stderr, and the exit status. +exit_status=0 +"$runwasi" $env $dir "$wasm" $dirarg \ + < "$stdin" \ + > "$stdout_observed" \ + 2> "$stderr_observed" \ + || exit_status=$? +echo $exit_status > "$exit_status_observed" + +# Determine the reference files to compare with. +if [ -e "$input.stdout.expected" ]; then + stdout_expected="$input.stdout.expected" + + # Apply output filters. + if [ -e "$input.stdout.expected.filter" ]; then + cat "$stdout_observed" \ + | "$input.stdout.expected.filter" \ + > "${stdout_observed}.filtered" + stdout_observed="${stdout_observed}.filtered" + fi +else + stdout_expected="/dev/null" +fi +if [ -e "$input.stderr.expected" ]; then + stderr_expected="$input.stderr.expected" + + # Apply output filters. + if [ -e "$input.stderr.expected.filter" ]; then + cat "$stderr_observed" \ + | "./$input.stderr.expected.filter" \ + > "${stderr_observed}.filtered" + stderr_observed="${stderr_observed}.filtered" + fi +else + stderr_expected="/dev/null" +fi +if [ -e "$input.exit_status.expected" ]; then + exit_status_expected="$input.exit_status.expected" +else + exit_status_expected=../exit_status_zero +fi + +# If there are any differences, diff will return a non-zero exit status, and +# since this script uses "set -e", it will return a non-zero exit status too. +diff -u "$stderr_expected" "$stderr_observed" +diff -u "$stdout_expected" "$stdout_observed" +diff -u "$exit_status_expected" "$exit_status_observed" From 3da47e74df645880a881c17d40da7c077187f0bf Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Tue, 23 Feb 2021 13:44:34 -0800 Subject: [PATCH 2/3] Add a minimal command test. --- test/minimal-command.wasm | Bin 0 -> 36 bytes test/sources/generate-wasm.sh | 8 ++++++++ test/sources/wat/minimal-command.wat | 3 +++ 3 files changed, 11 insertions(+) create mode 100644 test/minimal-command.wasm create mode 100755 test/sources/generate-wasm.sh create mode 100644 test/sources/wat/minimal-command.wat diff --git a/test/minimal-command.wasm b/test/minimal-command.wasm new file mode 100644 index 0000000000000000000000000000000000000000..ca050857cb7401d40a94e7f18722fd7f75c5d365 GIT binary patch literal 36 rcmZQbEY4+QU|?WmVN76PU}j=uVCQ0Fi!UxoEGl7O;9_B9V&Db Date: Tue, 23 Feb 2021 14:02:01 -0800 Subject: [PATCH 3/3] Adjust the scripts so that they run the minimal-command.wasm test. --- tools/test-harness/run.sh | 51 ++++------------------------------ tools/test-harness/testcase.sh | 32 ++++++--------------- 2 files changed, 14 insertions(+), 69 deletions(-) diff --git a/tools/test-harness/run.sh b/tools/test-harness/run.sh index 8d5a9933..218856a2 100755 --- a/tools/test-harness/run.sh +++ b/tools/test-harness/run.sh @@ -4,53 +4,14 @@ set -ueo pipefail # Top-level test runner. Usage is "run.sh" to run tests in compile-only mode, # or "run.sh " where is a WASI-capable runtime to run the # tests in full compile and execute mode. -# -# By default this script will look for `clang` and `clang++` in $PATH and -# assume that they are correctly configured with the sysroot in the default -# location. Alternatively, exporting $CC and $CXX allow more flexibility. e.g: -# -# export CXX="/bin/clang++ --sysroot /share/wasi-sysroot" -# export CCC="/bin/clang --sysroot /share/wasi-sysroot" -# -# Determine the wasm runtime to use, if one is provided. -if [ $# -gt 0 ]; then - runwasi="$1" -else - runwasi="" -fi +# Determine the wasm runtime to use. +runwasi="$1" -testdir=$(dirname $0) -CC=${CC:=clang} -CXX=${CXX:=clang++} +tooldir=$(dirname $0) -echo $CC -echo $CXX - -cd $testdir/compile-only -for options in -O0 -O2 "-O2 -flto"; do - echo "===== Testing compile-only with $options =====" - for file in *.c; do - echo "Testing compile-only $file..." - ../testcase.sh "" "$CC" "$options" "$file" - done - for file in *.cc; do - echo "Testing compile-only $file..." - ../testcase.sh "" "$CXX" "$options" "$file" - done -done -cd - >/dev/null - -cd $testdir/general -for options in -O0 -O2 "-O2 -flto"; do - echo "===== Testing with $options =====" - for file in *.c; do - echo "Testing $file..." - ../testcase.sh "$runwasi" "$CC" "$options" "$file" - done - for file in *.cc; do - echo "Testing $file..." - ../testcase.sh "$runwasi" "$CXX" "$options" "$file" - done +echo "===== Testing =====" +for file in *.wasm; do + "$tooldir/testcase.sh" "$runwasi" "$file" done cd - >/dev/null diff --git a/tools/test-harness/testcase.sh b/tools/test-harness/testcase.sh index 38db9a37..482fdddc 100755 --- a/tools/test-harness/testcase.sh +++ b/tools/test-harness/testcase.sh @@ -7,33 +7,17 @@ set -ueo pipefail # Command-line parsing; this script is meant to be run from a higher-level # script, so don't do anything fancy. runwasi="$1" -compiler="$2" -options="$3" -input="$4" +input="$2" # Compile names for generated files. -wasm="$input.$options.wasm" -stdout_observed="$input.$options.stdout.observed" -stderr_observed="$input.$options.stderr.observed" -exit_status_observed="$input.$options.exit_status.observed" - -# Optionally load compiler options from a file. -if [ -e "$input.options" ]; then - file_options=$(cat "$input.options") -else - file_options= -fi +wasm="$input" +stdout_observed="$input.stdout.observed" +stderr_observed="$input.stderr.observed" +exit_status_observed="$input.exit_status.observed" +tooldir=$(dirname $0) echo "Testing $input..." -# Compile the testcase. -$compiler $options $file_options "$input" -o "$wasm" - -# If we don't have a runwasi command, we're just doing compile-only testing. -if [ "$runwasi" == "" ]; then - exit 0 -fi - # Determine the input file to write to stdin. if [ -e "$input.stdin" ]; then stdin="$input.stdin" @@ -94,9 +78,9 @@ else stderr_expected="/dev/null" fi if [ -e "$input.exit_status.expected" ]; then - exit_status_expected="$input.exit_status.expected" + exit_status_expected="$tooldir/$input.exit_status.expected" else - exit_status_expected=../exit_status_zero + exit_status_expected="$tooldir/exit_status_zero" fi # If there are any differences, diff will return a non-zero exit status, and