From d9b052891b53b4d60717431e5878cb5974ec40b0 Mon Sep 17 00:00:00 2001 From: bus Date: Fri, 10 Jul 2020 18:31:52 +0200 Subject: [PATCH 01/10] combined code from multiple repositories --- checkBitSize/README.md | 20 ++ checkBitSize/build/MSVC/checkBitSize.sln | 28 +++ checkBitSize/build/MSVC/checkBitSize.vcxproj | 186 ++++++++++++++++++ .../build/MSVC/checkBitSize.vcxproj.filters | 27 +++ checkBitSize/checkBitSize.c | 12 ++ checkBitSize/checkBitSize.h | 60 ++++++ tests/README.md | 35 ++++ tests/buildAndTest.sh | 166 ++++++++++++++++ tests/checkBits.sh | 102 ++++++++++ tests/emulator.sh | 73 +++++++ tests/test_copy.sh | 67 +++++++ tests/test_dega.sh | 53 +++++ tests/test_lzmh.sh | 45 +++++ 13 files changed, 874 insertions(+) create mode 100644 checkBitSize/README.md create mode 100644 checkBitSize/build/MSVC/checkBitSize.sln create mode 100644 checkBitSize/build/MSVC/checkBitSize.vcxproj create mode 100644 checkBitSize/build/MSVC/checkBitSize.vcxproj.filters create mode 100644 checkBitSize/checkBitSize.c create mode 100644 checkBitSize/checkBitSize.h create mode 100644 tests/README.md create mode 100755 tests/buildAndTest.sh create mode 100755 tests/checkBits.sh create mode 100644 tests/emulator.sh create mode 100644 tests/test_copy.sh create mode 100644 tests/test_dega.sh create mode 100644 tests/test_lzmh.sh diff --git a/checkBitSize/README.md b/checkBitSize/README.md new file mode 100644 index 0000000..fbb7f91 --- /dev/null +++ b/checkBitSize/README.md @@ -0,0 +1,20 @@ +# checkBitSize +This little program tells you the size of size_t and void* +# Compilation +## Linux +On Linux, just run gcc on your platform: + +gcc -o checkbitsize checkBitSize.c + +Some examples for cross-compiling: + +arm: arm-linux-gnueabihf-gcc -o checkbitsize checkBitSize.c + +i386: gcc -m32 -o checkbitsize checkBitSize.c + +## Windows +For MSVC, the targets Win32 and x64 are provided in the solution file. Examples: + +/path/to/MSBuild.exe build/MSVC/checkBitSize.sln //t:Rebuild //p:Configuration=Release //p:Platform=Win32 + +/path/to/MSBuild.exe build/MSVC/checkBitSize.sln //t:Rebuild //p:Configuration=Release //p:Platform=x64 diff --git a/checkBitSize/build/MSVC/checkBitSize.sln b/checkBitSize/build/MSVC/checkBitSize.sln new file mode 100644 index 0000000..8208052 --- /dev/null +++ b/checkBitSize/build/MSVC/checkBitSize.sln @@ -0,0 +1,28 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 2013 +VisualStudioVersion = 12.0.31101.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "checkBitSize", "checkBitSize.vcxproj", "{18A05DED-E49F-4BFC-BE57-E27A7AFB0C12}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {18A05DED-E49F-4BFC-BE57-E27A7AFB0C12}.Debug|Win32.ActiveCfg = Debug|Win32 + {18A05DED-E49F-4BFC-BE57-E27A7AFB0C12}.Debug|Win32.Build.0 = Debug|Win32 + {18A05DED-E49F-4BFC-BE57-E27A7AFB0C12}.Debug|x64.ActiveCfg = Debug|x64 + {18A05DED-E49F-4BFC-BE57-E27A7AFB0C12}.Debug|x64.Build.0 = Debug|x64 + {18A05DED-E49F-4BFC-BE57-E27A7AFB0C12}.Release|Win32.ActiveCfg = Release|Win32 + {18A05DED-E49F-4BFC-BE57-E27A7AFB0C12}.Release|Win32.Build.0 = Release|Win32 + {18A05DED-E49F-4BFC-BE57-E27A7AFB0C12}.Release|x64.ActiveCfg = Release|x64 + {18A05DED-E49F-4BFC-BE57-E27A7AFB0C12}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/checkBitSize/build/MSVC/checkBitSize.vcxproj b/checkBitSize/build/MSVC/checkBitSize.vcxproj new file mode 100644 index 0000000..509b687 --- /dev/null +++ b/checkBitSize/build/MSVC/checkBitSize.vcxproj @@ -0,0 +1,186 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {18A05DED-E49F-4BFC-BE57-E27A7AFB0C12} + Win32Proj + checkBitSize + + + + Application + true + v120 + Unicode + + + Application + true + v120 + Unicode + + + Application + false + v120 + true + Unicode + + + Application + false + v120 + true + Unicode + + + + + + + + + + + + + + + + + + + true + + + true + + + false + + + false + + + + + + EnableAllWarnings + Disabled + + + CompileAsC + false + false + true + true + false + false + 4820;4996;%(DisableSpecificWarnings) + + + Console + true + + + + + + + EnableAllWarnings + Disabled + + + CompileAsC + false + false + true + true + false + false + 4820;4996;%(DisableSpecificWarnings) + + + Console + true + + + + + EnableAllWarnings + + + MaxSpeed + true + true + + + CompileAsC + false + false + true + true + false + false + 4820;4996;%(DisableSpecificWarnings) + + + Console + true + true + true + + + + + EnableAllWarnings + + + MaxSpeed + true + true + + + CompileAsC + false + false + true + true + false + false + 4820;4996;%(DisableSpecificWarnings) + + + Console + true + true + true + + + + + + + + + + + + \ No newline at end of file diff --git a/checkBitSize/build/MSVC/checkBitSize.vcxproj.filters b/checkBitSize/build/MSVC/checkBitSize.vcxproj.filters new file mode 100644 index 0000000..dbf7336 --- /dev/null +++ b/checkBitSize/build/MSVC/checkBitSize.vcxproj.filters @@ -0,0 +1,27 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + + + Header Files + + + \ No newline at end of file diff --git a/checkBitSize/checkBitSize.c b/checkBitSize/checkBitSize.c new file mode 100644 index 0000000..73c3a17 --- /dev/null +++ b/checkBitSize/checkBitSize.c @@ -0,0 +1,12 @@ +/* Sebastian Burkhart, 2019 */ +#include "checkBitSize.h" + +int main(void) +{ + const size_t targetSize = TARGET_SIZE; + const char* const result[] = { "error", "success" }; + const int success = !!(sizeof(size_t) == targetSize && sizeof(void*) == targetSize); + fprintf( (success == 1) ? stdout : stderr, "%s! compiled for %s. size_t = " SIZE_T_FORMAT_STRING " bits, void* = " SIZE_T_FORMAT_STRING " bits. " SIZE_T_FORMAT_STRING " bits expected.\n", result[success], PLATFORM, SIZE_T_CAST(8*sizeof(size_t)), SIZE_T_CAST(8*sizeof(void*)), SIZE_T_CAST(8*targetSize)); + MSVC_DEBUG_PAUSE; + return (success == 1) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/checkBitSize/checkBitSize.h b/checkBitSize/checkBitSize.h new file mode 100644 index 0000000..eaab635 --- /dev/null +++ b/checkBitSize/checkBitSize.h @@ -0,0 +1,60 @@ +/* Sebastian Burkhart, 2019 */ +#include +#include +#include +#include + +#define BITS_32 4 +#define BITS_64 8 + +#if defined __GNUC__ + #if defined __i386 + #define TARGET_SIZE BITS_32 + #define PLATFORM "i386 linux" + #elif defined __x86_64 + #define TARGET_SIZE BITS_64 + #define PLATFORM "x86_64 linux" + #elif defined __arm__ + #define TARGET_SIZE BITS_32 + #define PLATFORM "arm linux" + #elif defined __aarch64__ + #define TARGET_SIZE BITS_64 + #define PLATFORM "arm (64 bit) linux" + #else + #define PLATFORM "invalid" + #error "unsupported architecture" + #endif +#elif defined _MSC_VER && _MSC_VER >= 1800 + #if defined _WIN64 && !defined _M_ARM + #define TARGET_SIZE BITS_64 + #define PLATFORM "win64" + #elif defined _WIN32 && !defined _M_ARM + #define TARGET_SIZE BITS_32 + #define PLATFORM "win32" + #else + #define PLATFORM "invalid" + #error "unsupported architecture" + #endif +#else + #define PLATFORM "invalid compiler" + #error "unsupported compiler" +#endif + +#if defined _MSC_VER && _MSC_VER <= 1800 /* MSVC */ + #define SIZE_T_FORMAT "Iu" + #define SIZE_T_CAST(x) ((size_t)(x)) +#elif defined(__GNUC__) /* gcc */ + #define SIZE_T_FORMAT PRIuMAX + #define SIZE_T_CAST(x) ((uintmax_t)(x)) +#else /* fallback */ + #define SIZE_T_FORMAT "zu" + #define SIZE_T_CAST(x) ((size_t)(x)) +#endif + +#define SIZE_T_FORMAT_STRING "%" SIZE_T_FORMAT + +#if defined _DEBUG && defined _MSC_VER + #define MSVC_DEBUG_PAUSE system("pause") +#else + #define MSVC_DEBUG_PAUSE +#endif diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..e1ddb46 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,35 @@ +# Tests for the data-compressor +This folder contains automated tests for the data-compressor: https://github.com/CenterForSecureEnergyInformatics/data-compressor +The tests can be used manually, but are intended for a CI system (buildbot). +The proper environment for building and testing is chosen automatically. + +## prerequisites +On Windows, you have to have installed: +- git (you need gitbash) +- Microsoft C++ Build Tools + +On Linux, you need gcc for your platform. + +The source files for the data-compressor and checkBitSize have to be next to the folder containing ths project. +Other locations can be specified via the variables checkBitSizePath and dataCompressorPath in the scripts. + +## buildAndTest.sh + +buildAndTest.sh + +First, the data-compressor is built for the platform specified via the first argument. +The second argument specifies IO_SIZE_BITS. + +After compilation, testdata (contained in the data-compressor repository) is compressed and decompressed with the following methods: +- dega +- lzmh +- copy + +The resulting data is compared to the input via diff. +If input and output differs for some tests, the script will return an error. + +## checkBits.sh + +checkBits.sh + +This script compiles and runs the program checkBitSize on the specified platform. diff --git a/tests/buildAndTest.sh b/tests/buildAndTest.sh new file mode 100755 index 0000000..97002c4 --- /dev/null +++ b/tests/buildAndTest.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash + +# automated build and test script for the datacompressor. +# the functions containing the actual tests can be found in separate scripts in the same directory +# the emulator function is used by the testfunctions and also contained in another file. + +set -euo pipefail + +workdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source "$workdir/emulator.sh" +source "$workdir/test_dega.sh" +source "$workdir/test_lzmh.sh" +source "$workdir/test_copy.sh" + +dataCompressorPath="../../data-compressor" + +is_numeric_regex='^[0-9]+$' + +linux_dccli_sep="\#" +windows_dccli_sep="\#" + +linux_dccli_build_dir="$dataCompressorPath/DataCompressor/build/gcc/" +linux_dccli_command="$dataCompressorPath/DataCompressor/build/gcc/DCCLI" + +testdata_input="$dataCompressorPath/DataCompressor/DCCLI/testdata/input.txt" +testdata_output="$dataCompressorPath/DataCompressor/DCCLI/testdata/output.txt" + +windows_compiler="/c/Program\ Files\ \(x86\)/MSBuild/12.0/Bin/MSBuild.exe" +windows_project_file="$dataCompressorPath/DataCompressor/build/MSVC/DataCompressor.sln" +win32_dccli_command="$dataCompressorPath/DataCompressor/build/MSVC/Release/DCCLI.exe" +win_x64_dccli_command="$dataCompressorPath/DataCompressor/build/MSVC/x64/Release/DCCLI.exe" + +script_name="$(basename "$0")" + +success="success" + +err_msg() { + echo "Error: $1" + echo "Linux: $script_name " + echo "Raspberry: $script_name " + echo "Windows: $script_name " +} + +tests() { + # arg 1: IO_SIZE_BITS for copy blocksize + if ! [ $# -eq 1 ] || ! [[ $1 =~ $is_numeric_regex ]]; then + echo "tests: exactly one numeric argument expected" + exit 1 + fi + + local result_dega result_lzmh result_copy + echo "==========================================================" + echo "testing DEGA" + echo "==========================================================" + test_dega result_dega + echo "==========================================================" + echo "testing LZMH" + echo "==========================================================" + test_lzmh result_lzmh + echo "==========================================================" +# echo "testing copy" +# echo "==========================================================" +# test_copy "$1" result_copy +# echo "==========================================================" +# echo "test results" +# echo "==========================================================" + echo "dega: $result_dega" + echo "lzmh: $result_lzmh" +# echo "copy: $result_copy" + +# if ! [[ "$result_dega" == "$success" && "$result_lzmh" == "$success" && "$result_copy" == "$success" ]]; then + if ! [[ "$result_dega" == "$success" && "$result_lzmh" == "$success" ]]; then + echo "Error. Some tests did not pass." + exit 1 + fi +} + +# ####################################### +# start of script. +# ####################################### +if ! [ $# -eq 2 ]; then + err_msg "not enough / too much arguments" + exit 1 +fi + +if ! [[ $2 =~ $is_numeric_regex ]] ; then + err_msg "arg 2 has to be numeric" + exit 1 +fi + +platform="$1" +IO_SIZE_BITS="$2" + +kernel="$(uname -s)" +machine="$(uname -m)" + +if [[ $kernel == Linux* ]]; then + sep="$linux_dccli_sep" + dccli_command="$linux_dccli_command" + if [[ $machine == x86_64 ]]; then + + case "$platform" in + "i386") + echo "i386" + make -C "$linux_dccli_build_dir" clean + COMMON_CFLAGS="IO_SIZE_BITS=$IO_SIZE_BITS" CFLAGS="-m32" LDFLAGS="-m32" make -C "$linux_dccli_build_dir" release + tests "$IO_SIZE_BITS" + + ;; + "x86_64") + echo "x86_64" + make -C "$linux_dccli_build_dir" clean + COMMON_CFLAGS="IO_SIZE_BITS=$IO_SIZE_BITS" make -C "$linux_dccli_build_dir" release + tests "$IO_SIZE_BITS" + ;; + "armhf") + echo "armhf" + make -C "$linux_dccli_build_dir" clean + COMMON_CFLAGS="IO_SIZE_BITS=$IO_SIZE_BITS" make CC=arm-linux-gnueabihf-gcc -C "$linux_dccli_build_dir" release + tests "$IO_SIZE_BITS" + ;; + *) + err_msg "no such target" + exit 1 + ;; + esac + elif [[ $machine == armv* ]] || [[ $machine == aarch64 ]]; then + case "$platform" in + "arm") + echo "arch: $machine - building on rpi" + make -C "$linux_dccli_build_dir" clean + COMMON_CFLAGS="IO_SIZE_BITS=$IO_SIZE_BITS" make CC=gcc -C "$linux_dccli_build_dir" release + tests "$IO_SIZE_BITS" + ;; + *) + err_msg "on the raspberry pi, we won't build and test other platforms" + exit 1 + ;; + esac + fi +elif [[ $kernel == MINGW64* && $machine == x86_64 ]]; then + sep="$windows_dccli_sep" + case "$platform" in + "win32") + dccli_command="$win32_dccli_command" + echo "win32" + bash -c "$windows_compiler $windows_project_file //t:Clean" + bash -c "$windows_compiler $windows_project_file //t:Rebuild //p:Configuration=Release //p:Platform=Win32" + tests "$IO_SIZE_BITS" + ;; + "x64") + dccli_command="$win_x64_dccli_command" + echo "win x64" + bash -c "$windows_compiler $windows_project_file //t:Clean" + bash -c "$windows_compiler $windows_project_file //t:Rebuild //p:Configuration=Release //p:Platform=x64" + tests "$IO_SIZE_BITS" + ;; + *) + err_msg "no such target" + exit 1 + ;; + esac +else + echo "Error. This runs on x86_64 machines, either with linux or windows (mingw)" + exit 1 +fi diff --git a/tests/checkBits.sh b/tests/checkBits.sh new file mode 100755 index 0000000..18f857a --- /dev/null +++ b/tests/checkBits.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash + +set -euo pipefail +workdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source "$workdir/emulator.sh" +checkBitSizePath=".." +source_file="$checkBitSizePath/checkBitSize/checkBitSize.c" + +linux_command="$checkBitSizePath/checkBitSize/checkBitSize" + +windows_compiler="/c/Program\ Files\ \(x86\)/MSBuild/12.0/Bin/MSBuild.exe" +windows_project_file="$checkBitSizePath/checkBitSize/build/MSVC/checkBitSize.sln" + +win32_command="$checkBitSizePath/checkBitSize/build/MSVC/Release/checkBitSize.exe" +win_x64_command="$checkBitSizePath/checkBitSize/build/MSVC/x64/Release/checkBitSize.exe" + +script_name="$(basename "$0")" + +err_msg() { + echo "Error: $1" + echo "Linux: $script_name " + echo "Raspberry: $script_name " + echo "Windows: $script_name " +} + + +# ####################################### +# start of script. +# ####################################### +if [ ! $# -eq 1 ]; then + err_msg "not enough / too much arguments" + exit 1 +fi + +platform="$1" +kernel="$(uname -s)" +machine="$(uname -m)" + +if [[ $kernel == Linux* ]]; then + command="$linux_command" + if [[ $machine == x86_64 ]]; then + case "$platform" in + "i386") + echo "i386" + gcc -m32 -o "$command" "$source_file" + emulator "$command" + + ;; + "x86_64") + echo "x86_64" + gcc -o "$command" "$source_file" + emulator "$command" + ;; + "armhf") + echo armhf + arm-linux-gnueabihf-gcc -o "$command" "$source_file" + emulator "$command" + ;; + *) + err_msg "no such target" + exit 1 + ;; + esac + elif [[ $machine == armv* ]] || [[ $machine == aarch64 ]]; then + case "$platform" in + "arm") + echo "arch: $machine - building on rpi" + gcc -o "$command" "$source_file" + emulator "$command" + ;; + *) + err_msg "on the raspberry pi, we won't build and test other platforms" + exit 1 + ;; + esac + fi + +elif [[ $kernel == MINGW64* && $machine == x86_64 ]]; then + case "$platform" in + "win32") + echo "win32" + command="$win32_command" + bash -c "$windows_compiler $windows_project_file //t:Clean" + bash -c "$windows_compiler $windows_project_file //t:Rebuild //p:Configuration=Release //p:Platform=Win32" + emulator "$command" + ;; + "x64") + echo "win x64" + command="$win_x64_command" + bash -c "$windows_compiler $windows_project_file //t:Clean" + bash -c "$windows_compiler $windows_project_file //t:Rebuild //p:Configuration=Release //p:Platform=x64" + emulator "$command" + ;; + *) + err_msg "no such target" + exit 1 + ;; + esac +else + echo "Error. This runs on x86_64 machines, either with linux or windows (mingw)" + exit 1 +fi diff --git a/tests/emulator.sh b/tests/emulator.sh new file mode 100644 index 0000000..3fba741 --- /dev/null +++ b/tests/emulator.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +emulator(){ + # checks the executable's format and the platform we're on. + # on linux, if i386 or arm, it chooses quemu accordingly. + # linux x86_64: native + # windows x64: native + # win32: we let windows figure it out on it's own. + + # args: + # first arg: executable + # 2nd... nth arg: any argument for executable. + if [ ! $# -ge 1 ]; then + echo "Error: emulator requires at least one arg." + echo "Usage: emulator [args]" + exit 1 + fi + echo "choosing emulator for $1" + echo "file type: $(file -b "$1")" + local p + local k + local m + k="$(uname -s)" + m="$(uname -m)" + + if [[ $k == Linux* ]]; then + p=$(file "$1" | cut -d ',' -f 2 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') # get second col of "file" output and trim leading and tailing whitespaces + if [[ $m == x86_64 ]]; then + case "$p" in + "ARM") + echo "command: $*" + echo "using qemu-arm-static:" + bash -c "qemu-arm-static $*" + ;; + "Intel 80386") + echo "command: $*" + echo "using qemu-i386-static:" + bash -c "qemu-i386-static $*" + ;; + "x86-64") + echo "command: $*" + echo "no emulator. x86_64 - native" + bash -c "$*" + ;; + *) + echo "Emulator: Error: executable was compiled for an unsupported variety of Linux. Target=$p, kernel=$k, machine=$m" + exit 1 + ;; + esac + elif [[ $m == arm* ]] || [[ $m == aarch64 ]]; then + echo "native raspberrypi arch=$m" + case "$p" in + "ARM" | "ARM aarch64") + echo "command: $*" + echo "no emulator - native." + bash -c "$*" + ;; + *) + echo "on the raspberry, we do not test other platforms." + exit 1 + ;; + esac + + fi + + elif [[ $k == MINGW64* && $m == x86_64 ]]; then + echo "command: $*" + echo "no emulator, windows." + bash -c "$*" + else + echo "Emulator: Error: this platform is not supported. kernel=$k, machine=$m" + exit 1 + fi +} diff --git a/tests/test_copy.sh b/tests/test_copy.sh new file mode 100644 index 0000000..cc9a7ca --- /dev/null +++ b/tests/test_copy.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +test_copy() { + # function to run the copy algorithm on testdata. + # dccli_command, testdata_input, testdata_output, sep and success are global vars that have to be set in the script sourcing this file. + # first we check if the command itself fails. + # in case of success, we run diff and check if it failed. + # note that we run "set +e" to ignore errors, so you have to use the returnvalue to stop on errors. + + # # usage example: + # source test_copy.sh + # declare returnvalue + # blocksize = 8 # range:1...IO_SIZE_BITS + # test_copy $blocksize $returnvalue + # echo "$returnvalue" + # # end usage example + + # blocksize parameter: https://github.com/CenterForSecureEnergyInformatics/data-compressor/tree/master/DataCompressor/DCLib/doc + + # if successful, returnvalue contains whatever the global success variable contains. + # in case of failure, returnvalue contains a message with details. + + if ! [ $# -eq 2 ]; then + echo "test_copy: exeactly two arguments expected." + echo "test_copy: usage: test_copy returnvar" + exit 1 + fi + if ! [[ $1 =~ ${is_numeric_regex:?} ]]; then + echo "test_copy: arg 1 has to be numeric" + echo "test_copy: usage: test_copy returnvar" + exit 1 + fi + + max_blocksize="$1" + + local __resultvar=$2 + local __result="${success:?}" + local current_blocksize dccli_return diff_return failed_blocksizes + failed_blocksizes=() + + # here we run the copy module with blocksize=1...IO_SIZE_BITS + current_blocksize=1 + while [ "$current_blocksize" -le "$max_blocksize" ]; do + echo "**********************************************************" + echo "copy: blocksize=$current_blocksize" + set +e # start ignoring errors + emulator "${dccli_command:?}" "${testdata_input:?} ${testdata_output:?} encode copy blocksize=$current_blocksize" + dccli_return="$?" + set -e # stop ignoring errors + + if [[ "$dccli_return" -eq 0 ]]; then + set +e # start ignoring errors + diff -q "${testdata_input:?}" "${testdata_output:?}" # line endings (CRLF / LF) should be the same. + diff_return="$?" + set -e # stop ignoring errors + fi + if ! [ "$dccli_return" -eq "0" ] || ! [ "$diff_return" -eq "0" ]; then + failed_blocksizes+=("$current_blocksize") + fi + current_blocksize=$((current_blocksize + 1)) + done + + if ! [ ${#failed_blocksizes[@]} -eq 0 ]; then + __result="Error: Failed blocksizes for copy: ${failed_blocksizes[*]}" + fi + + eval "$__resultvar='$__result'" +} diff --git a/tests/test_dega.sh b/tests/test_dega.sh new file mode 100644 index 0000000..c8a7d71 --- /dev/null +++ b/tests/test_dega.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +test_dega() { + # function to run the dega algorithm on testdata. + # dccli_command, testdata_input, testdata_output, sep and success are global vars that have to be set in the script sourcing this file. + # first we check if the command itself fails. + # in case of success, we run diff and check if it failed. + # note that we run "set +e" to ignore errors, so you have to use the returnvalue to stop on errors. + + # # usage example: + # source test_dega.sh + # declare returnvalue + # test_dega $returnvalue + # echo "$returnvalue" + # # end usage example + + # if successful, returnvalue contains whatever the global success variable contains. + # in case of failure, returnvalue contains a message with details. + + if ! [ $# -eq 1 ]; then + echo "test_dega: exactly one argument (variable for the result) expected" + exit 1 + fi + local __resultvar=$1 + local __result="${success:?}" + local dccli_return diff_return + + set +e # start ignoring errors + emulator "${dccli_command:?}" "${testdata_input:?} ${testdata_output:?} decode csv ${sep:?} encode normalize ${sep:?} encode diff ${sep:?} encode seg ${sep:?} encode bac adaptive ${sep:?} decode bac adaptive ${sep:?} decode seg ${sep:?} decode diff ${sep:?} decode normalize ${sep:?} encode csv" + dccli_return="$?" + set -e # stop ignoring errors + + + if [[ "$dccli_return" -eq 0 ]]; then + set +e # start ignoring errors + # depending on windows / linux line endings might differ. copy and lzmh should preserve line endings. + # because of that we ignore trailing CRs on windows + if [[ $(uname -s) == Linux* ]]; then + diff -q "${testdata_input:?}" "${testdata_output:?}" + else + diff -q --strip-trailing-cr "${testdata_input:?}" "${testdata_output:?}" + fi + diff_return="$?" + set -e # stop ignoring errors + if ! [[ "$diff_return" -eq 0 ]]; then + __result="Error: diff failed with status $diff_return" + fi + else + __result="Error: dccli failed with status $dccli_return" + fi + + eval "$__resultvar='$__result'" +} + diff --git a/tests/test_lzmh.sh b/tests/test_lzmh.sh new file mode 100644 index 0000000..50c2208 --- /dev/null +++ b/tests/test_lzmh.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +test_lzmh() { + # function to run the lzmh algorithm on testdata. + # dccli_command, testdata_input, testdata_output, sep and success are global vars that have to be set in the script sourcing this file. + # first we check if the command itself fails. + # in case of success, we run diff and check if it failed. + # note that we run "set +e" to ignore errors, so you have to use the returnvalue to stop on errors. + + # # usage example: + # source test_lzmh.sh + # declare returnvalue + # test_lzmh $returnvalue + # echo "$returnvalue" + # # end usage example + + # if successful, returnvalue contains whatever the global success variable contains. + # in case of failure, returnvalue contains a message with details. + + if ! [ $# -eq 1 ]; then + echo "test_lzmh: exactly one argument (variable for the result) expected" + exit 1 + fi + local __resultvar=$1 + local __result="${success:?}" + local dccli_return diff_return + + set +e # start ignoring errors + emulator "${dccli_command:?}" "${testdata_input:?} ${testdata_output:?} encode lzmh ${sep:?} decode lzmh" + dccli_return="$?" + set -e # start ignoring errors + + if [[ "$dccli_return" -eq 0 ]]; then + set +e # start ignoring errors + diff -q "${testdata_input:?}" "${testdata_output:?}" # line endings (CRLF / LF) should be the same. + diff_return="$?" + set -e # stop ignoring errors + if ! [[ "$diff_return" -eq 0 ]]; then + __result="Error: diff failed with status $diff_return" + fi + else + __result="Error: dccli failed with status $dccli_return" + fi + + eval "$__resultvar='$__result'" +} From eb2d15c63a33606cb47c878b6e5d5dfe70fcea67 Mon Sep 17 00:00:00 2001 From: bus Date: Fri, 10 Jul 2020 20:32:49 +0200 Subject: [PATCH 02/10] modified paths --- tests/buildAndTest.sh | 2 +- tests/checkBits.sh | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/buildAndTest.sh b/tests/buildAndTest.sh index 97002c4..4bb3750 100755 --- a/tests/buildAndTest.sh +++ b/tests/buildAndTest.sh @@ -12,7 +12,7 @@ source "$workdir/test_dega.sh" source "$workdir/test_lzmh.sh" source "$workdir/test_copy.sh" -dataCompressorPath="../../data-compressor" +dataCompressorPath="../data-compressor" is_numeric_regex='^[0-9]+$' diff --git a/tests/checkBits.sh b/tests/checkBits.sh index 18f857a..152fc94 100755 --- a/tests/checkBits.sh +++ b/tests/checkBits.sh @@ -3,16 +3,16 @@ set -euo pipefail workdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" source "$workdir/emulator.sh" -checkBitSizePath=".." -source_file="$checkBitSizePath/checkBitSize/checkBitSize.c" +checkBitSizePath="../data-compressor-tests/checkBitSize" +source_file="$checkBitSizePath/checkBitSize.c" -linux_command="$checkBitSizePath/checkBitSize/checkBitSize" +linux_command="$checkBitSizePath/checkBitSize" windows_compiler="/c/Program\ Files\ \(x86\)/MSBuild/12.0/Bin/MSBuild.exe" -windows_project_file="$checkBitSizePath/checkBitSize/build/MSVC/checkBitSize.sln" +windows_project_file="$checkBitSizePath/build/MSVC/checkBitSize.sln" -win32_command="$checkBitSizePath/checkBitSize/build/MSVC/Release/checkBitSize.exe" -win_x64_command="$checkBitSizePath/checkBitSize/build/MSVC/x64/Release/checkBitSize.exe" +win32_command="$checkBitSizePath/build/MSVC/Release/checkBitSize.exe" +win_x64_command="$checkBitSizePath/build/MSVC/x64/Release/checkBitSize.exe" script_name="$(basename "$0")" From 848043814e2374e815747e5965f269b41e8d2e11 Mon Sep 17 00:00:00 2001 From: burs Date: Tue, 18 Aug 2020 18:28:18 +0200 Subject: [PATCH 03/10] links, versions, instructions --- tests/README.md | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/tests/README.md b/tests/README.md index e1ddb46..4e0c664 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,35 +1,38 @@ # Tests for the data-compressor -This folder contains automated tests for the data-compressor: https://github.com/CenterForSecureEnergyInformatics/data-compressor +This folder contains automated tests for the [Data Compressor](https://github.com/CenterForSecureEnergyInformatics/data-compressor) The tests can be used manually, but are intended for a CI system (buildbot). The proper environment for building and testing is chosen automatically. ## prerequisites On Windows, you have to have installed: -- git (you need gitbash) -- Microsoft C++ Build Tools +- Git including Git Bash. Download the current version here: https://git-scm.com/download/win +- Microsoft C++ Build Tools 2019. Download: https://visualstudio.microsoft.com/de/downloads/ On Linux, you need gcc for your platform. +Currently, the versions shipped with Raspian / Ubuntu 18.04 / Alpine are used. -The source files for the data-compressor and checkBitSize have to be next to the folder containing ths project. -Other locations can be specified via the variables checkBitSizePath and dataCompressorPath in the scripts. +The source files for the [Data Compressor](https://github.com/CenterForSecureEnergyInformatics/data-compressor) and [checkBitSize](../checkBitSize) have to be next to the folder containing ths project. +Other locations can be specified via the variables `checkBitSizePath` and `dataCompressorPath` in the scripts. ## buildAndTest.sh -buildAndTest.sh +- Linux `buildAndTest.sh ` +- Raspberry Pi `buildAndTest.sh arm ` +- Windows `buildAndTest.sh ` -First, the data-compressor is built for the platform specified via the first argument. -The second argument specifies IO_SIZE_BITS. +First, the [Data Compressor](https://github.com/CenterForSecureEnergyInformatics/data-compressor) is built for the platform specified via the first argument. +The second argument specifies `IO_SIZE_BITS`. After compilation, testdata (contained in the data-compressor repository) is compressed and decompressed with the following methods: - dega - lzmh -- copy The resulting data is compared to the input via diff. If input and output differs for some tests, the script will return an error. ## checkBits.sh +- Linux `checkBits.sh ` +- Raspberry Pi: `checkBits.sh arm` +- Windows: `checkBits.sh ` -checkBits.sh - -This script compiles and runs the program checkBitSize on the specified platform. +This script compiles and runs the program [checkBitSize](../checkBitSize) on the specified platform. From 4bcae93df4d37902db3cc0f6b562d8cf68b03fc7 Mon Sep 17 00:00:00 2001 From: burs Date: Tue, 18 Aug 2020 19:05:56 +0200 Subject: [PATCH 04/10] activated copy test and minor changes --- tests/buildAndTest.sh | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/buildAndTest.sh b/tests/buildAndTest.sh index 4bb3750..e2e2a6f 100755 --- a/tests/buildAndTest.sh +++ b/tests/buildAndTest.sh @@ -37,7 +37,7 @@ success="success" err_msg() { echo "Error: $1" echo "Linux: $script_name " - echo "Raspberry: $script_name " + echo "Raspberry Pi: $script_name arm " echo "Windows: $script_name " } @@ -58,18 +58,17 @@ tests() { echo "==========================================================" test_lzmh result_lzmh echo "==========================================================" -# echo "testing copy" -# echo "==========================================================" -# test_copy "$1" result_copy -# echo "==========================================================" -# echo "test results" -# echo "==========================================================" + echo "testing copy" + echo "==========================================================" + test_copy "$1" result_copy + echo "==========================================================" + echo "test results" + echo "==========================================================" echo "dega: $result_dega" echo "lzmh: $result_lzmh" -# echo "copy: $result_copy" + echo "copy: $result_copy" -# if ! [[ "$result_dega" == "$success" && "$result_lzmh" == "$success" && "$result_copy" == "$success" ]]; then - if ! [[ "$result_dega" == "$success" && "$result_lzmh" == "$success" ]]; then + if ! [[ "$result_dega" == "$success" && "$result_lzmh" == "$success" && "$result_copy" == "$success" ]]; then echo "Error. Some tests did not pass." exit 1 fi @@ -79,12 +78,12 @@ tests() { # start of script. # ####################################### if ! [ $# -eq 2 ]; then - err_msg "not enough / too much arguments" + err_msg "not enough / too many arguments" exit 1 fi if ! [[ $2 =~ $is_numeric_regex ]] ; then - err_msg "arg 2 has to be numeric" + err_msg "arg 2 (IO_SIZE_BITS) has to be numeric" exit 1 fi From 35a21abb95c9e836adbf125ccd6510cebc918a06 Mon Sep 17 00:00:00 2001 From: burs Date: Tue, 18 Aug 2020 19:12:33 +0200 Subject: [PATCH 05/10] added missing word --- tests/checkBits.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/checkBits.sh b/tests/checkBits.sh index 152fc94..3637009 100755 --- a/tests/checkBits.sh +++ b/tests/checkBits.sh @@ -19,7 +19,7 @@ script_name="$(basename "$0")" err_msg() { echo "Error: $1" echo "Linux: $script_name " - echo "Raspberry: $script_name " + echo "Raspberry Pi: $script_name arm" echo "Windows: $script_name " } From 9b7ea8a72994b9c8db52ed37ec32393e29f1c69e Mon Sep 17 00:00:00 2001 From: burs Date: Tue, 18 Aug 2020 19:19:08 +0200 Subject: [PATCH 06/10] better output on windows --- tests/emulator.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/emulator.sh b/tests/emulator.sh index 3fba741..fea2237 100644 --- a/tests/emulator.sh +++ b/tests/emulator.sh @@ -64,7 +64,7 @@ emulator(){ elif [[ $k == MINGW64* && $m == x86_64 ]]; then echo "command: $*" - echo "no emulator, windows." + echo "Windows. No emulator needed, we can run the executable natively." bash -c "$*" else echo "Emulator: Error: this platform is not supported. kernel=$k, machine=$m" From 57bb81db00009f041eb87a6c0a652a26d51bc7ca Mon Sep 17 00:00:00 2001 From: burs Date: Tue, 18 Aug 2020 19:29:00 +0200 Subject: [PATCH 07/10] minor changes --- tests/test_copy.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_copy.sh b/tests/test_copy.sh index cc9a7ca..fd630cf 100644 --- a/tests/test_copy.sh +++ b/tests/test_copy.sh @@ -1,32 +1,32 @@ #!/usr/bin/env bash test_copy() { # function to run the copy algorithm on testdata. - # dccli_command, testdata_input, testdata_output, sep and success are global vars that have to be set in the script sourcing this file. + # is_numeric_regex, xdccli_command, testdata_input, testdata_output, sep and success are global vars that have to be set in the script sourcing this file. # first we check if the command itself fails. # in case of success, we run diff and check if it failed. # note that we run "set +e" to ignore errors, so you have to use the returnvalue to stop on errors. # # usage example: # source test_copy.sh - # declare returnvalue + # declare returnvar # blocksize = 8 # range:1...IO_SIZE_BITS - # test_copy $blocksize $returnvalue - # echo "$returnvalue" + # test_copy $blocksize $returnvar + # echo "$returnvar" # # end usage example # blocksize parameter: https://github.com/CenterForSecureEnergyInformatics/data-compressor/tree/master/DataCompressor/DCLib/doc - # if successful, returnvalue contains whatever the global success variable contains. - # in case of failure, returnvalue contains a message with details. + # if successful, returnvar contains whatever the global success variable contains. + # in case of failure, returnvar contains a message with details. if ! [ $# -eq 2 ]; then echo "test_copy: exeactly two arguments expected." - echo "test_copy: usage: test_copy returnvar" + echo "test_copy: usage: test_copy " exit 1 fi if ! [[ $1 =~ ${is_numeric_regex:?} ]]; then - echo "test_copy: arg 1 has to be numeric" - echo "test_copy: usage: test_copy returnvar" + echo "test_copy: arg 1 (max blocksize) has to be numeric" + echo "test_copy: usage: test_copy " exit 1 fi From b8a52d9582adeb4965b542d3d2770ed6e0caae9a Mon Sep 17 00:00:00 2001 From: burs Date: Tue, 18 Aug 2020 19:32:26 +0200 Subject: [PATCH 08/10] moved the code a bit to make it more readable --- tests/test_copy.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_copy.sh b/tests/test_copy.sh index fd630cf..212c3ba 100644 --- a/tests/test_copy.sh +++ b/tests/test_copy.sh @@ -18,25 +18,25 @@ test_copy() { # if successful, returnvar contains whatever the global success variable contains. # in case of failure, returnvar contains a message with details. - if ! [ $# -eq 2 ]; then echo "test_copy: exeactly two arguments expected." echo "test_copy: usage: test_copy " exit 1 fi - if ! [[ $1 =~ ${is_numeric_regex:?} ]]; then - echo "test_copy: arg 1 (max blocksize) has to be numeric" - echo "test_copy: usage: test_copy " - exit 1 - fi max_blocksize="$1" - local __resultvar=$2 + local __result="${success:?}" local current_blocksize dccli_return diff_return failed_blocksizes failed_blocksizes=() + if ! [[ $max_blocksize =~ ${is_numeric_regex:?} ]]; then + echo "test_copy: arg 1 (max blocksize) has to be numeric" + echo "test_copy: usage: test_copy " + exit 1 + fi + # here we run the copy module with blocksize=1...IO_SIZE_BITS current_blocksize=1 while [ "$current_blocksize" -le "$max_blocksize" ]; do From ed8cfaa5530c153af6d5085e03b86f89b818221d Mon Sep 17 00:00:00 2001 From: burs Date: Tue, 18 Aug 2020 20:25:39 +0200 Subject: [PATCH 09/10] included header from DataCompressor, removed duplicates, inline code in README --- checkBitSize/README.md | 10 +++++----- checkBitSize/checkBitSize.c | 2 +- checkBitSize/checkBitSize.h | 12 +----------- 3 files changed, 7 insertions(+), 17 deletions(-) diff --git a/checkBitSize/README.md b/checkBitSize/README.md index fbb7f91..408fcbf 100644 --- a/checkBitSize/README.md +++ b/checkBitSize/README.md @@ -4,17 +4,17 @@ This little program tells you the size of size_t and void* ## Linux On Linux, just run gcc on your platform: -gcc -o checkbitsize checkBitSize.c +`gcc -o checkbitsize checkBitSize.c` Some examples for cross-compiling: -arm: arm-linux-gnueabihf-gcc -o checkbitsize checkBitSize.c +arm: `arm-linux-gnueabihf-gcc -o checkbitsize checkBitSize.c` -i386: gcc -m32 -o checkbitsize checkBitSize.c +i386: `gcc -m32 -o checkbitsize checkBitSize.c` ## Windows For MSVC, the targets Win32 and x64 are provided in the solution file. Examples: -/path/to/MSBuild.exe build/MSVC/checkBitSize.sln //t:Rebuild //p:Configuration=Release //p:Platform=Win32 +`/path/to/MSBuild.exe build/MSVC/checkBitSize.sln //t:Rebuild //p:Configuration=Release //p:Platform=Win32` -/path/to/MSBuild.exe build/MSVC/checkBitSize.sln //t:Rebuild //p:Configuration=Release //p:Platform=x64 +`/path/to/MSBuild.exe build/MSVC/checkBitSize.sln //t:Rebuild //p:Configuration=Release //p:Platform=x64` diff --git a/checkBitSize/checkBitSize.c b/checkBitSize/checkBitSize.c index 73c3a17..1120bee 100644 --- a/checkBitSize/checkBitSize.c +++ b/checkBitSize/checkBitSize.c @@ -5,7 +5,7 @@ int main(void) { const size_t targetSize = TARGET_SIZE; const char* const result[] = { "error", "success" }; - const int success = !!(sizeof(size_t) == targetSize && sizeof(void*) == targetSize); + const int success = sizeof(size_t) == targetSize && sizeof(void*) == targetSize; fprintf( (success == 1) ? stdout : stderr, "%s! compiled for %s. size_t = " SIZE_T_FORMAT_STRING " bits, void* = " SIZE_T_FORMAT_STRING " bits. " SIZE_T_FORMAT_STRING " bits expected.\n", result[success], PLATFORM, SIZE_T_CAST(8*sizeof(size_t)), SIZE_T_CAST(8*sizeof(void*)), SIZE_T_CAST(8*targetSize)); MSVC_DEBUG_PAUSE; return (success == 1) ? EXIT_SUCCESS : EXIT_FAILURE; diff --git a/checkBitSize/checkBitSize.h b/checkBitSize/checkBitSize.h index eaab635..35d1a94 100644 --- a/checkBitSize/checkBitSize.h +++ b/checkBitSize/checkBitSize.h @@ -3,6 +3,7 @@ #include #include #include +#include "../../data-compressor/DataCompressor/common/inc/io.h" #define BITS_32 4 #define BITS_64 8 @@ -40,17 +41,6 @@ #error "unsupported compiler" #endif -#if defined _MSC_VER && _MSC_VER <= 1800 /* MSVC */ - #define SIZE_T_FORMAT "Iu" - #define SIZE_T_CAST(x) ((size_t)(x)) -#elif defined(__GNUC__) /* gcc */ - #define SIZE_T_FORMAT PRIuMAX - #define SIZE_T_CAST(x) ((uintmax_t)(x)) -#else /* fallback */ - #define SIZE_T_FORMAT "zu" - #define SIZE_T_CAST(x) ((size_t)(x)) -#endif - #define SIZE_T_FORMAT_STRING "%" SIZE_T_FORMAT #if defined _DEBUG && defined _MSC_VER From 4434e413a0b82be66c9b5468b5360d3e9efdf102 Mon Sep 17 00:00:00 2001 From: burs Date: Tue, 18 Aug 2020 21:26:56 +0200 Subject: [PATCH 10/10] better style --- checkBitSize/checkBitSize.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/checkBitSize/checkBitSize.c b/checkBitSize/checkBitSize.c index 1120bee..8d28aeb 100644 --- a/checkBitSize/checkBitSize.c +++ b/checkBitSize/checkBitSize.c @@ -6,7 +6,7 @@ int main(void) const size_t targetSize = TARGET_SIZE; const char* const result[] = { "error", "success" }; const int success = sizeof(size_t) == targetSize && sizeof(void*) == targetSize; - fprintf( (success == 1) ? stdout : stderr, "%s! compiled for %s. size_t = " SIZE_T_FORMAT_STRING " bits, void* = " SIZE_T_FORMAT_STRING " bits. " SIZE_T_FORMAT_STRING " bits expected.\n", result[success], PLATFORM, SIZE_T_CAST(8*sizeof(size_t)), SIZE_T_CAST(8*sizeof(void*)), SIZE_T_CAST(8*targetSize)); + fprintf( success ? stdout : stderr, "%s! compiled for %s. size_t = " SIZE_T_FORMAT_STRING " bits, void* = " SIZE_T_FORMAT_STRING " bits. " SIZE_T_FORMAT_STRING " bits expected.\n", result[success], PLATFORM, SIZE_T_CAST(8*sizeof(size_t)), SIZE_T_CAST(8*sizeof(void*)), SIZE_T_CAST(8*targetSize)); MSVC_DEBUG_PAUSE; - return (success == 1) ? EXIT_SUCCESS : EXIT_FAILURE; + return success ? EXIT_SUCCESS : EXIT_FAILURE; }