[SYSTEMDS-3877] Docker SEAL HE fixes

.gitignore

@@ -154,3 +154,6 @@ scripts/resource/output
 # docker tests
 docker/mountFolder/*.bin
 docker/mountFolder/*.bin.mtd
+
+SEAL-*/
+

docker/README.md

@@ -138,6 +138,7 @@ Test your testing image locally by running the following command:
 ```bash
 docker run \
   -v $(pwd):/github/workspace \
+  -v $HOME/.m2/repository:/root/.m2/repository \
   apache/systemds:testing-latest \
   org.apache.sysds.test.component.**
 ```

docker/entrypoint.sh

@@ -29,23 +29,22 @@ cd /github/workspace
 export MAVEN_OPTS="-Xmx512m"
 
 log="/tmp/sysdstest.log"
-mvn -ntp -B test-compile 2>&1 | grep -E "BUILD|Total time:|---|Building SystemDS"
+mvn -ntp -B test-compile 2>&1 | stdbuf -oL grep -E "BUILD|Total time:|---|Building SystemDS"
 mvn -ntp -B test -D maven.test.skip=false -D automatedtestbase.outputbuffering=true -D test=$1 2>&1 \
-	| grep -v "already exists in destination." \
-	| grep -v 'WARNING: Using incubator modules' | tee $log
+	| stdbuf -oL grep -Ev "already exists in destination.|Using incubator" \
+	| tee $log
 
-# Merge Federated test runs.
-[ -f target/jacoco.exec ] && mv target/jacoco.exec target/jacoco_main.exec
-mvn -ntp -B jacoco:merge
-
-# Merge Federated test runs.
-[ -f target/jacoco.exec ] && mv target/jacoco.exec target/jacoco_main.exec
-mvn -ntp -B jacoco:merge
 
 grep_args="SUCCESS"
 grepvals="$( tail -n 100 $log | grep $grep_args)"
 
 if [[ $grepvals == *"SUCCESS"* ]]; then
+	# Merge Federated test runs.
+	# if merged jacoco exist temporarily rename to not overwrite.
+	[ -f target/jacoco.exec ] && mv target/jacoco.exec target/jacoco_main.exec
+	# merge jacoco files.
+	mvn -ntp -B jacoco:merge 2>&1 | stdbuf -oL grep -E "jacoco"
+
 	exit 0
 else
 	exit 1

docker/testsysds.Dockerfile

@@ -19,7 +19,8 @@
 #
 #-------------------------------------------------------------
 # Stage 1: Build SEAL
-FROM debian:bullseye-slim@sha256:b5f9bc44bdfbd9d551dfdd432607cbc6bb5d9d6dea726a1191797d7749166973 AS seal-build
+# FROM debian:bullseye-slim@sha256:b5f9bc44bdfbd9d551dfdd432607cbc6bb5d9d6dea726a1191797d7749166973 AS seal-build
+FROM ubuntu:noble@sha256:728785b59223d755e3e5c5af178fab1be7031f3522c5ccd7a0b32b80d8248123 AS seal-build
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential \
@@ -40,16 +41,25 @@ RUN wget -qO- https://github.com/microsoft/SEAL/archive/refs/tags/v3.7.0.tar.gz
     && cmake --install build --prefix /seal-install
 
 # Stage 2: Final image with R, JDK, Maven, SEAL
-FROM debian:bullseye-slim@sha256:b5f9bc44bdfbd9d551dfdd432607cbc6bb5d9d6dea726a1191797d7749166973
+# FROM debian:bullseye-slim@sha256:b5f9bc44bdfbd9d551dfdd432607cbc6bb5d9d6dea726a1191797d7749166973
+FROM ubuntu:noble@sha256:728785b59223d755e3e5c5af178fab1be7031f3522c5ccd7a0b32b80d8248123 
 
 WORKDIR /usr/src/
 ENV MAVEN_VERSION=3.9.9
 ENV MAVEN_HOME=/usr/lib/mvn
 
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
+ENV LANG=en_US.UTF-8
 ENV JAVA_HOME=/usr/lib/jvm/jdk-17.0.15+6
 ENV PATH=$JAVA_HOME/bin:$MAVEN_HOME/bin:$PATH
 
-RUN apt-get update && apt-get install -y --no-install-recommends \
+RUN apt-get update && apt-get install -y locales \
+    && echo "en_US.UTF-8 UTF-8" > /etc/locale.gen \
+    && locale-gen \
+    && update-locale LANG=en_US.UTF-8
+
+RUN apt-get install -y --no-install-recommends \
     r-base \
 	wget \
     cmake \
@@ -78,8 +88,26 @@ COPY ./src/test/scripts/installDependencies.R installDependencies.R
 RUN Rscript installDependencies.R \
     && rm -f installDependencies.R
 
+ENV HADOOP_VERSION=3.3.6
+ENV HADOOP_HOME=/opt/hadoop
+ENV LD_LIBRARY_PATH=/opt/hadoop/lib/native
+ENV HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib/native"
+
+RUN mkdir -p $HADOOP_HOME/lib/native \
+	&& wget -q https://downloads.apache.org/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz && \
+    tar --strip-components=2 -xzf hadoop-${HADOOP_VERSION}.tar.gz \
+        hadoop-${HADOOP_VERSION}/lib/native && \
+    mv native/libhadoop.so.1.0.0 /opt/hadoop/lib/native && \
+	mv native/libhadoop.so /opt/hadoop/lib/native && \
+    rm hadoop-${HADOOP_VERSION}.tar.gz && \
+	rm -rf native
+
 # Copy SEAL
-COPY --from=seal-build /seal-install /usr/local
+COPY --from=seal-build /seal-install/lib/ /usr/local/lib/
+COPY --from=seal-build /seal-install/include/ /usr/local/include/
+
+ENV LD_LIBRARY_PATH=/opt/hadoop/lib/native;/usr/local/lib/
+
 
 # Finally copy the entrypoint script
 # This is last to enable quick updates to the script after initial local build.

src/main/cpp/build_HE.sh

@@ -20,7 +20,7 @@
 #
 #-------------------------------------------------------------
 
-echo "Build Homomorphic Encryption Liberary: "
+echo "Build Homomorphic Encryption Library: "
 
 # compile HE
 cmake he/ -B HE -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=g++
@@ -29,4 +29,4 @@ rm -R HE
 
 echo ""
 
-echo "Sucessfull install of Homomorphic Encryption Liberary SEAL"
+echo "Successful install of Homomorphic Encryption Library SEAL"

src/test/java/org/apache/sysds/test/TestUtils.java

@@ -1760,15 +1760,25 @@ public static HashMap<CellIndex, Double> convert2DDoubleArrayToHashMap(double[][
 	}
 
 	public static double[][] convertHashMapToDoubleArray(HashMap <CellIndex, Double> matrix) {
+		if(matrix.isEmpty()){
+			LOG.error("converting empty hashMap");
+			return new double[0][0];
+		}
 		int max_rows = -1, max_cols= -1;
+
 		for(CellIndex ix : matrix.keySet()) {
 			max_rows = Math.max(max_rows, ix.row);
 			max_cols = Math.max(max_cols, ix.column);
 		}
+
 		return convertHashMapToDoubleArray(matrix, max_rows, max_cols);
 	}
 
 	public static double[][] convertHashMapToDoubleArray(HashMap<CellIndex, Double> matrix, int rows, int cols) {
+		if(rows <= -1 || cols <= -1){
+			LOG.error("converting negative size hashmap rows: " + rows + " cols: " + cols);
+			return new double[0][0];
+		}
 		double [][] ret_arr = new double[rows][cols];
 		for(Entry<CellIndex, Double> e : matrix.entrySet()) {
 			int i = e.getKey().row-1;

src/test/java/org/apache/sysds/test/functions/transform/TransformCSVFrameEncodeReadTest.java

@@ -148,16 +148,16 @@ private void runTransformTest( ExecMode rt, String ofmt, boolean subset, boolean
 			String[] fromDisk = DataConverter.toString(fb2).split("\n");
 			String[] printed = stdOut.split("\n");
 			boolean equal = true;
-			String err = "";
+			StringBuilder err = new StringBuilder();
 			for(int i = 0; i < fromDisk.length; i++){
-				if(! fromDisk[i].equals(printed[i])){
-					err += "\n not equal: \n"+ (fromDisk[i] + "\n" + printed[i]);
+				if(! fromDisk[i].strip().equals(printed[i].strip())){
+					err.append("\n not equal: \n'"+ (fromDisk[i] + "'\n'" + printed[i] + "'"));
 					equal = false;
 				}
 
 			}
 			if(!equal)
-				fail(err);
+				fail(err.toString());
 
 		}
 		catch(Exception ex) {