From 6c765920f7e1c1715ede4eb72f92c7fb91179f19 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Wed, 8 Oct 2025 12:01:41 +0200 Subject: [PATCH 1/3] docker SEAL HE fixes --- .gitignore | 3 +++ docker/entrypoint.sh | 8 +++----- docker/testsysds.Dockerfile | 5 ++++- src/main/cpp/build_HE.sh | 4 ++-- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index d4b817a1ae5..d2fcdb9a4de 100644 --- a/.gitignore +++ b/.gitignore @@ -154,3 +154,6 @@ scripts/resource/output # docker tests docker/mountFolder/*.bin docker/mountFolder/*.bin.mtd + +SEAL-*/ + diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index c276a707c35..e3859fe6922 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -35,12 +35,10 @@ mvn -ntp -B test -D maven.test.skip=false -D automatedtestbase.outputbuffering=t | grep -v 'WARNING: Using incubator modules' | tee $log # Merge Federated test runs. +# if merged jacoco exist temporarily rename to not overwrite. [ -f target/jacoco.exec ] && mv target/jacoco.exec target/jacoco_main.exec -mvn -ntp -B jacoco:merge - -# Merge Federated test runs. -[ -f target/jacoco.exec ] && mv target/jacoco.exec target/jacoco_main.exec -mvn -ntp -B jacoco:merge +# merge jacoco files. +mvn -ntp -B jacoco:merge 2>&1 | grep -E "BUILD|Total time:|Building SystemDS|jacoco" grep_args="SUCCESS" grepvals="$( tail -n 100 $log | grep $grep_args)" diff --git a/docker/testsysds.Dockerfile b/docker/testsysds.Dockerfile index fc6eb1491e1..7381c993994 100644 --- a/docker/testsysds.Dockerfile +++ b/docker/testsysds.Dockerfile @@ -79,7 +79,10 @@ RUN Rscript installDependencies.R \ && rm -f installDependencies.R # Copy SEAL -COPY --from=seal-build /seal-install /usr/local +COPY --from=seal-build /seal-install/lib/ /usr/local/lib/ +COPY --from=seal-build /seal-install/include/ /usr/local/include/ + +ENV LD_LIBRARY_PATH=/usr/local/lib/ # Finally copy the entrypoint script # This is last to enable quick updates to the script after initial local build. diff --git a/src/main/cpp/build_HE.sh b/src/main/cpp/build_HE.sh index dec3622a423..24be7f7bb74 100755 --- a/src/main/cpp/build_HE.sh +++ b/src/main/cpp/build_HE.sh @@ -20,7 +20,7 @@ # #------------------------------------------------------------- -echo "Build Homomorphic Encryption Liberary: " +echo "Build Homomorphic Encryption Library: " # compile HE cmake he/ -B HE -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=g++ @@ -29,4 +29,4 @@ rm -R HE echo "" -echo "Sucessfull install of Homomorphic Encryption Liberary SEAL" \ No newline at end of file +echo "Successful install of Homomorphic Encryption Library SEAL" \ No newline at end of file From f49998619fb323a0039e4f8fbac9727c80f331a3 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Wed, 8 Oct 2025 13:38:13 +0200 Subject: [PATCH 2/3] test live output --- docker/README.md | 1 + docker/entrypoint.sh | 17 +++++++++-------- .../java/org/apache/sysds/test/TestUtils.java | 10 ++++++++++ 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/docker/README.md b/docker/README.md index d530061019f..42f751618bd 100644 --- a/docker/README.md +++ b/docker/README.md @@ -138,6 +138,7 @@ Test your testing image locally by running the following command: ```bash docker run \ -v $(pwd):/github/workspace \ + -v $HOME/.m2/repository:/root/.m2/repository \ apache/systemds:testing-latest \ org.apache.sysds.test.component.** ``` diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index e3859fe6922..9a77766b3a3 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -29,21 +29,22 @@ cd /github/workspace export MAVEN_OPTS="-Xmx512m" log="/tmp/sysdstest.log" -mvn -ntp -B test-compile 2>&1 | grep -E "BUILD|Total time:|---|Building SystemDS" +mvn -ntp -B test-compile 2>&1 | stdbuf -oL grep -E "BUILD|Total time:|---|Building SystemDS" mvn -ntp -B test -D maven.test.skip=false -D automatedtestbase.outputbuffering=true -D test=$1 2>&1 \ - | grep -v "already exists in destination." \ - | grep -v 'WARNING: Using incubator modules' | tee $log + | stdbuf -oL grep -Ev "already exists in destination.|Using incubator" \ + | tee $log -# Merge Federated test runs. -# if merged jacoco exist temporarily rename to not overwrite. -[ -f target/jacoco.exec ] && mv target/jacoco.exec target/jacoco_main.exec -# merge jacoco files. -mvn -ntp -B jacoco:merge 2>&1 | grep -E "BUILD|Total time:|Building SystemDS|jacoco" grep_args="SUCCESS" grepvals="$( tail -n 100 $log | grep $grep_args)" if [[ $grepvals == *"SUCCESS"* ]]; then + # Merge Federated test runs. + # if merged jacoco exist temporarily rename to not overwrite. + [ -f target/jacoco.exec ] && mv target/jacoco.exec target/jacoco_main.exec + # merge jacoco files. + mvn -ntp -B jacoco:merge 2>&1 | stdbuf -oL grep -E "jacoco" + exit 0 else exit 1 diff --git a/src/test/java/org/apache/sysds/test/TestUtils.java b/src/test/java/org/apache/sysds/test/TestUtils.java index 107d33b1a1c..e470dd82539 100644 --- a/src/test/java/org/apache/sysds/test/TestUtils.java +++ b/src/test/java/org/apache/sysds/test/TestUtils.java @@ -1760,15 +1760,25 @@ public static HashMap convert2DDoubleArrayToHashMap(double[][ } public static double[][] convertHashMapToDoubleArray(HashMap matrix) { + if(matrix.isEmpty()){ + LOG.error("converting empty hashMap"); + return new double[0][0]; + } int max_rows = -1, max_cols= -1; + for(CellIndex ix : matrix.keySet()) { max_rows = Math.max(max_rows, ix.row); max_cols = Math.max(max_cols, ix.column); } + return convertHashMapToDoubleArray(matrix, max_rows, max_cols); } public static double[][] convertHashMapToDoubleArray(HashMap matrix, int rows, int cols) { + if(rows <= -1 || cols <= -1){ + LOG.error("converting negative size hashmap rows: " + rows + " cols: " + cols); + return new double[0][0]; + } double [][] ret_arr = new double[rows][cols]; for(Entry e : matrix.entrySet()) { int i = e.getKey().row-1; From a91810b8c5b5f3c87163dd1a81af1bfdab8f7ea8 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Wed, 8 Oct 2025 16:32:46 +0200 Subject: [PATCH 3/3] fix locale --- docker/testsysds.Dockerfile | 33 ++++++++++++++++--- .../TransformCSVFrameEncodeReadTest.java | 8 ++--- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/docker/testsysds.Dockerfile b/docker/testsysds.Dockerfile index 7381c993994..1b7935428ac 100644 --- a/docker/testsysds.Dockerfile +++ b/docker/testsysds.Dockerfile @@ -19,7 +19,8 @@ # #------------------------------------------------------------- # Stage 1: Build SEAL -FROM debian:bullseye-slim@sha256:b5f9bc44bdfbd9d551dfdd432607cbc6bb5d9d6dea726a1191797d7749166973 AS seal-build +# FROM debian:bullseye-slim@sha256:b5f9bc44bdfbd9d551dfdd432607cbc6bb5d9d6dea726a1191797d7749166973 AS seal-build +FROM ubuntu:noble@sha256:728785b59223d755e3e5c5af178fab1be7031f3522c5ccd7a0b32b80d8248123 AS seal-build RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ @@ -40,16 +41,25 @@ RUN wget -qO- https://github.com/microsoft/SEAL/archive/refs/tags/v3.7.0.tar.gz && cmake --install build --prefix /seal-install # Stage 2: Final image with R, JDK, Maven, SEAL -FROM debian:bullseye-slim@sha256:b5f9bc44bdfbd9d551dfdd432607cbc6bb5d9d6dea726a1191797d7749166973 +# FROM debian:bullseye-slim@sha256:b5f9bc44bdfbd9d551dfdd432607cbc6bb5d9d6dea726a1191797d7749166973 +FROM ubuntu:noble@sha256:728785b59223d755e3e5c5af178fab1be7031f3522c5ccd7a0b32b80d8248123 WORKDIR /usr/src/ ENV MAVEN_VERSION=3.9.9 ENV MAVEN_HOME=/usr/lib/mvn +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8 +ENV LANG=en_US.UTF-8 ENV JAVA_HOME=/usr/lib/jvm/jdk-17.0.15+6 ENV PATH=$JAVA_HOME/bin:$MAVEN_HOME/bin:$PATH -RUN apt-get update && apt-get install -y --no-install-recommends \ +RUN apt-get update && apt-get install -y locales \ + && echo "en_US.UTF-8 UTF-8" > /etc/locale.gen \ + && locale-gen \ + && update-locale LANG=en_US.UTF-8 + +RUN apt-get install -y --no-install-recommends \ r-base \ wget \ cmake \ @@ -78,11 +88,26 @@ COPY ./src/test/scripts/installDependencies.R installDependencies.R RUN Rscript installDependencies.R \ && rm -f installDependencies.R +ENV HADOOP_VERSION=3.3.6 +ENV HADOOP_HOME=/opt/hadoop +ENV LD_LIBRARY_PATH=/opt/hadoop/lib/native +ENV HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib/native" + +RUN mkdir -p $HADOOP_HOME/lib/native \ + && wget -q https://downloads.apache.org/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz && \ + tar --strip-components=2 -xzf hadoop-${HADOOP_VERSION}.tar.gz \ + hadoop-${HADOOP_VERSION}/lib/native && \ + mv native/libhadoop.so.1.0.0 /opt/hadoop/lib/native && \ + mv native/libhadoop.so /opt/hadoop/lib/native && \ + rm hadoop-${HADOOP_VERSION}.tar.gz && \ + rm -rf native + # Copy SEAL COPY --from=seal-build /seal-install/lib/ /usr/local/lib/ COPY --from=seal-build /seal-install/include/ /usr/local/include/ -ENV LD_LIBRARY_PATH=/usr/local/lib/ +ENV LD_LIBRARY_PATH=/opt/hadoop/lib/native;/usr/local/lib/ + # Finally copy the entrypoint script # This is last to enable quick updates to the script after initial local build. diff --git a/src/test/java/org/apache/sysds/test/functions/transform/TransformCSVFrameEncodeReadTest.java b/src/test/java/org/apache/sysds/test/functions/transform/TransformCSVFrameEncodeReadTest.java index 41872c3ba5c..ce2e7ba0620 100644 --- a/src/test/java/org/apache/sysds/test/functions/transform/TransformCSVFrameEncodeReadTest.java +++ b/src/test/java/org/apache/sysds/test/functions/transform/TransformCSVFrameEncodeReadTest.java @@ -148,16 +148,16 @@ private void runTransformTest( ExecMode rt, String ofmt, boolean subset, boolean String[] fromDisk = DataConverter.toString(fb2).split("\n"); String[] printed = stdOut.split("\n"); boolean equal = true; - String err = ""; + StringBuilder err = new StringBuilder(); for(int i = 0; i < fromDisk.length; i++){ - if(! fromDisk[i].equals(printed[i])){ - err += "\n not equal: \n"+ (fromDisk[i] + "\n" + printed[i]); + if(! fromDisk[i].strip().equals(printed[i].strip())){ + err.append("\n not equal: \n'"+ (fromDisk[i] + "'\n'" + printed[i] + "'")); equal = false; } } if(!equal) - fail(err); + fail(err.toString()); } catch(Exception ex) {