Merge pull request #189 from FloopCZ/v1.15.0

FloopCZ · web-flow · commit 5fefb85684f9 · 2019-10-30T20:31:35.000+01:00
Update TensorFlow to v1.15.0
diff --git a/Dockerfiles/BAZEL_VERSION b/Dockerfiles/BAZEL_VERSION
@@ -1 +1 @@
-0.21.0
+0.26.1
diff --git a/Dockerfiles/archlinux-shared b/Dockerfiles/archlinux-shared
@@ -1,4 +1,4 @@
-FROM archlinux/base
+FROM archlinux:latest
 
 # copy the contents of this repository to the container
 COPY . tensorflow_cc
diff --git a/Dockerfiles/archlinux-shared-cuda b/Dockerfiles/archlinux-shared-cuda
@@ -1,4 +1,4 @@
-FROM archlinux/base
+FROM archlinux:latest
 
 # copy the contents of this repository to the container
 COPY . tensorflow_cc
@@ -10,4 +10,4 @@ RUN ./tensorflow_cc/Dockerfiles/install-archlinux.sh --cuda --shared
 
 ENV NVIDIA_VISIBLE_DEVICES all
 ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
-ENV NVIDIA_REQUIRE_CUDA "cuda>=9.2"
+ENV NVIDIA_REQUIRE_CUDA "cuda>=10.1"
diff --git a/Dockerfiles/install-archlinux.sh b/Dockerfiles/install-archlinux.sh
@@ -30,7 +30,6 @@ pacman -Syu --noconfirm --needed \
 
 if $shared; then
     pacman -S --noconfirm --needed \
-      gcc \
       java-environment=8 \
       libarchive \
       protobuf \
diff --git a/Dockerfiles/install-ubuntu.sh b/Dockerfiles/install-ubuntu.sh
@@ -51,7 +51,7 @@ if $shared; then
 fi
 if $cuda; then
     # install libcupti
-    apt-get -y install cuda-command-line-tools-10-0
+    apt-get -y install cuda-command-line-tools-10-1
 fi
 
 apt-get -y clean
diff --git a/Dockerfiles/ubuntu-shared-cuda b/Dockerfiles/ubuntu-shared-cuda
@@ -1,4 +1,4 @@
-FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04
+FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04
 
 # copy the contents of this repository to the container
 COPY . tensorflow_cc
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # tensorflow_cc
 [![Build Status](http://ash.floop.cz:8080/buildStatus/icon?job=tensorflow_cc)](http://ash.floop.cz:8080/job/tensorflow_cc/)
-[![TF version](https://img.shields.io/badge/TF%20version-1.13.1-brightgreen.svg)]()
+[![TF version](https://img.shields.io/badge/TF%20version-1.15.0-brightgreen.svg)]()
 
 This repository makes possible the usage of the [TensorFlow C++](https://www.tensorflow.org/api_docs/cc/) API from the outside of the TensorFlow source code folders and without the use of the [Bazel](https://bazel.build/) build system.
 
@@ -60,7 +60,7 @@ sudo pacman -S gcc7 bazel cuda cudnn nvidia
 ```
 
 **Warning:** Newer versions of TensorFlow sometimes fail to build with the latest version of Bazel. You may wish
-to install an older version of Bazel (e.g., 0.16.1).
+to install an older version of Bazel (e.g., 0.26.1).
 
 #### 2) Clone this repository
 ```
diff --git a/tensorflow_cc/CMakeLists.txt b/tensorflow_cc/CMakeLists.txt
@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.3 FATAL_ERROR)
 project(
   "tensorflow_cc"
-  VERSION 1.13.1
+  VERSION 1.15.0
 )
 
 # Static library with no GPU support is built by default.
@@ -10,7 +10,7 @@ project(
 option(TENSORFLOW_SHARED "Build shared library (required for GPU support)." OFF)
 option(ALLOW_CUDA "When building the shared library, try to find and use CUDA." ON)
 option(TENSORFLOW_STATIC "Build static library." ON)
-set(TENSORFLOW_TAG "v1.13.1" CACHE STRING "The tensorflow release tag to be checked out (default v1.13.1).")
+set(TENSORFLOW_TAG "v1.15.0" CACHE STRING "The tensorflow release tag to be checked out (default v1.15.0).")
 option(SYSTEM_PROTOBUF "Use system protobuf instead of static protobuf from contrib/makefile." OFF)
 
 # -------------
@@ -79,7 +79,7 @@ if(TENSORFLOW_SHARED)
   )
   target_link_libraries(
     tensorflow_cc_shared INTERFACE
-    "${CMAKE_INSTALL_PREFIX}/lib/tensorflow_cc/libtensorflow_cc.so"
+    "${CMAKE_INSTALL_PREFIX}/lib/libtensorflow_cc.so.${PROJECT_VERSION_MAJOR}"
     dl pthread
   )
   if(SYSTEM_PROTOBUF)
@@ -204,6 +204,12 @@ install(
                  PATTERN "*.tar.*"    EXCLUDE
                  PATTERN "*.zip"      EXCLUDE
 )
+# install all the *.inc files
+install(
+  DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/tensorflow/tensorflow/contrib/makefile/gen/"
+  DESTINATION include/tensorflow/tensorflow/contrib/makefile/gen
+  FILES_MATCHING PATTERN "*.inc"
+)
 # install all files from third_party folder (e.g., Eigen/Tensor)
 install(
   DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/tensorflow/third_party/"
@@ -220,14 +226,23 @@ endif()
 if(TENSORFLOW_SHARED)
   install(
     FILES "${CMAKE_CURRENT_BINARY_DIR}/tensorflow/bazel-bin/tensorflow/libtensorflow_cc.so"
-    DESTINATION lib/tensorflow_cc
+    DESTINATION lib
+    RENAME libtensorflow_cc.so.${PROJECT_VERSION_MAJOR}
     OPTIONAL
   )
   install(
     FILES "${CMAKE_CURRENT_BINARY_DIR}/tensorflow/bazel-bin/bin/tensorflow/libtensorflow_cc.so"
-    DESTINATION lib/tensorflow_cc
+    DESTINATION lib
+    RENAME libtensorflow_cc.so.${PROJECT_VERSION_MAJOR}
     OPTIONAL
   )
+  # Make versioned symlinks to the library
+  install(CODE "execute_process(                          \
+    COMMAND ${CMAKE_COMMAND} -E create_symlink            \
+      libtensorflow_cc.so.${PROJECT_VERSION_MAJOR}        \
+      ${CMAKE_INSTALL_PREFIX}/lib/libtensorflow_cc.so     \
+    )"
+  )
 endif()
 # static library specific
 if(TENSORFLOW_STATIC)
diff --git a/tensorflow_cc/cmake/TensorflowShared.cmake b/tensorflow_cc/cmake/TensorflowShared.cmake
@@ -10,7 +10,12 @@ ExternalProject_Add(
   BUILD_IN_SOURCE 1
   DOWNLOAD_COMMAND ""
   UPDATE_COMMAND ""
-  CONFIGURE_COMMAND tensorflow/contrib/makefile/compile_linux_protobuf.sh
+            # The grpc integrated in v1.15.0 needs patching, see https://github.com/clearlinux/distribution/issues/1151.
+  CONFIGURE_COMMAND cp "${CMAKE_CURRENT_SOURCE_DIR}/patches/v1.15.0-Rename-gettid-functions.patch" .
+            COMMAND git apply v1.15.0-Rename-gettid-functions.patch
+            COMMAND cp "${CMAKE_CURRENT_SOURCE_DIR}/patches/v1.15.0-workspace.bzl.patch" .
+            COMMAND git apply v1.15.0-workspace.bzl.patch
+            COMMAND tensorflow/contrib/makefile/compile_linux_protobuf.sh
             # Do not fail on warnings when building nsync
             COMMAND sed -i "s/ -Werror//g" tensorflow/contrib/makefile/compile_nsync.sh
             COMMAND tensorflow/contrib/makefile/compile_nsync.sh
diff --git a/tensorflow_cc/cmake/build_tensorflow.sh.in b/tensorflow_cc/cmake/build_tensorflow.sh.in
@@ -28,48 +28,25 @@ export TF_NEED_KAFKA=${TF_NEED_KAFKA:-0}
 export TF_DOWNLOAD_CLANG=${TF_DOWNLOAD_CLANG:-0}
 export TF_NEED_IGNITE=${TF_NEED_IGNITE:-0}
 export TF_NEED_ROCM=${TF_NEED_ROCM:-0}
-export TF_NCCL_VERSION=${TF_NCCL_VERSION:-2.3}  # _DEFAULT_NCCL_VERSION from configure.py
 export NCCL_INSTALL_PATH=${NCCL_INSTALL_PATH:-/usr}
 export PYTHON_BIN_PATH=${PYTHON_BIN_PATH:-"$(which python3)"}
 export PYTHON_LIB_PATH="$($PYTHON_BIN_PATH -c 'import site; print(site.getsitepackages()[0])')"
 
-# configure cuda environmental variables
-
-if [ -e /opt/cuda ]; then
-    echo "Using CUDA from /opt/cuda"
-    export CUDA_TOOLKIT_PATH=/opt/cuda
-elif [ -e /usr/local/cuda ]; then
-    echo "Using CUDA from /usr/local/cuda"
-    export CUDA_TOOLKIT_PATH=/usr/local/cuda
-fi
-
-if [ -e /opt/cuda/include/cudnn.h ]; then
-    echo "Using CUDNN from /opt/cuda"
-    export CUDNN_INSTALL_PATH=/opt/cuda
-elif [ -e /usr/local/cuda/include/cudnn.h ]; then
-    echo "Using CUDNN from /usr/local/cuda"
-    export CUDNN_INSTALL_PATH=/usr/local/cuda
-elif [ -e /usr/include/cudnn.h ]; then
-    echo "Using CUDNN from /usr"
-    export CUDNN_INSTALL_PATH=/usr
-fi
-
-if [ "@ALLOW_CUDA@" = "ON" ] && [ -n "${CUDA_TOOLKIT_PATH}" ]; then
-    if [[ -z "${CUDNN_INSTALL_PATH}" ]]; then
-        echo "CUDA found but no cudnn.h found. Please install cuDNN."
-        exit 1
-    fi
+# check if cuda support requested and supported
+if [ "@ALLOW_CUDA@" = "ON" ] && hash nvcc 2>/dev/null; then
     echo "CUDA support enabled"
     cuda_config_opts="--config=cuda"
     export TF_NEED_CUDA=1
     export TF_CUDA_COMPUTE_CAPABILITIES=${TF_CUDA_COMPUTE_CAPABILITIES:-"3.5,7.0"}  # default from configure.py
-    export TF_CUDA_VERSION="$($CUDA_TOOLKIT_PATH/bin/nvcc --version | sed -n 's/^.*release \(.*\),.*/\1/p')"
-    export TF_CUDNN_VERSION="$(sed -n 's/^#define CUDNN_MAJOR\s*\(.*\).*/\1/p' $CUDNN_INSTALL_PATH/include/cudnn.h)"
+    export TF_CUDA_PATHS=${TF_CUDA_PATHS:-"/opt/cuda,/usr/local/cuda,/usr/local,/usr/cuda,/usr"}
+    export TF_CUDA_VERSION="$(nvcc --version | sed -n 's/^.*release \(.*\),.*/\1/p')"
+    export TF_NCCL_VERSION="$(find / -name 'libnccl.so.*' | tail -n1 | sed -r 's/^.*\.so\.//')"
+    export TF_CUDNN_VERSION="$(find / -name 'libcudnn.so.*' | tail -n1 | sed -r 's/^.*\.so\.//')"
 
     # choose the right version of CUDA compiler
     if [ -z "$GCC_HOST_COMPILER_PATH" ]; then
-        if   hash gcc 2>/dev/null && version_gt 8.5 `gcc -dumpversion`; then
-            export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc"}
+        if   hash gcc-8 2>/dev/null && version_gt 8.5 `gcc-8 -dumpversion`; then
+            export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-8"}
         elif hash gcc-7 2>/dev/null && version_gt 7.5 `gcc-7 -dumpversion`; then
             export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-7"}
         elif hash gcc-6 2>/dev/null && version_gt 6.4 `gcc-6 -dumpversion`; then
diff --git a/tensorflow_cc/patches/v1.15.0-Rename-gettid-functions.patch b/tensorflow_cc/patches/v1.15.0-Rename-gettid-functions.patch
@@ -0,0 +1,83 @@
+new file mode 100644
+index 00000000..90bd9115
+--- /dev/null
++++ b/third_party/Rename-gettid-functions.patch
+@@ -0,0 +1,78 @@
++From d1d017390b799c59d6fdf7b8afa6136d218bdd61 Mon Sep 17 00:00:00 2001
++From: Benjamin Peterson <benjamin@dropbox.com>
++Date: Fri, 3 May 2019 08:11:00 -0700
++Subject: [PATCH] Rename gettid() functions.
++
++glibc 2.30 will declare its own gettid; see https://sourceware.org/git/?p=glibc.git;a=commit;h=1d0fc213824eaa2a8f8c4385daaa698ee8fb7c92. Rename the grpc versions to avoid naming conflicts.
++---
++ src/core/lib/gpr/log_linux.cc          | 4 ++--
++ src/core/lib/gpr/log_posix.cc          | 4 ++--
++ src/core/lib/iomgr/ev_epollex_linux.cc | 4 ++--
++ 3 files changed, 6 insertions(+), 6 deletions(-)
++
++diff --git a/src/core/lib/gpr/log_linux.cc b/src/core/lib/gpr/log_linux.cc
++index 561276f0c20..8b597b4cf2f 100644
++--- a/src/core/lib/gpr/log_linux.cc
+++++ b/src/core/lib/gpr/log_linux.cc
++@@ -40,7 +40,7 @@
++ #include <time.h>
++ #include <unistd.h>
++ 
++-static long gettid(void) { return syscall(__NR_gettid); }
+++static long sys_gettid(void) { return syscall(__NR_gettid); }
++ 
++ void gpr_log(const char* file, int line, gpr_log_severity severity,
++              const char* format, ...) {
++@@ -70,7 +70,7 @@ void gpr_default_log(gpr_log_func_args* args) {
++   gpr_timespec now = gpr_now(GPR_CLOCK_REALTIME);
++   struct tm tm;
++   static __thread long tid = 0;
++-  if (tid == 0) tid = gettid();
+++  if (tid == 0) tid = sys_gettid();
++ 
++   timer = static_cast<time_t>(now.tv_sec);
++   final_slash = strrchr(args->file, '/');
++diff --git a/src/core/lib/gpr/log_posix.cc b/src/core/lib/gpr/log_posix.cc
++index b6edc14ab6b..2f7c6ce3760 100644
++--- a/src/core/lib/gpr/log_posix.cc
+++++ b/src/core/lib/gpr/log_posix.cc
++@@ -31,7 +31,7 @@
++ #include <string.h>
++ #include <time.h>
++ 
++-static intptr_t gettid(void) { return (intptr_t)pthread_self(); }
+++static intptr_t sys_gettid(void) { return (intptr_t)pthread_self(); }
++ 
++ void gpr_log(const char* file, int line, gpr_log_severity severity,
++              const char* format, ...) {
++@@ -86,7 +86,7 @@ void gpr_default_log(gpr_log_func_args* args) {
++   char* prefix;
++   gpr_asprintf(&prefix, "%s%s.%09d %7" PRIdPTR " %s:%d]",
++                gpr_log_severity_string(args->severity), time_buffer,
++-               (int)(now.tv_nsec), gettid(), display_file, args->line);
+++               (int)(now.tv_nsec), sys_gettid(), display_file, args->line);
++ 
++   fprintf(stderr, "%-70s %s\n", prefix, args->message);
++   gpr_free(prefix);
++diff --git a/src/core/lib/iomgr/ev_epollex_linux.cc b/src/core/lib/iomgr/ev_epollex_linux.cc
++index 08116b3ab53..76f59844312 100644
++--- a/src/core/lib/iomgr/ev_epollex_linux.cc
+++++ b/src/core/lib/iomgr/ev_epollex_linux.cc
++@@ -1102,7 +1102,7 @@ static void end_worker(grpc_pollset* pollset, grpc_pollset_worker* worker,
++ }
++ 
++ #ifndef NDEBUG
++-static long gettid(void) { return syscall(__NR_gettid); }
+++static long sys_gettid(void) { return syscall(__NR_gettid); }
++ #endif
++ 
++ /* pollset->mu lock must be held by the caller before calling this.
++@@ -1122,7 +1122,7 @@ static grpc_error* pollset_work(grpc_pollset* pollset,
++ #define WORKER_PTR (&worker)
++ #endif
++ #ifndef NDEBUG
++-  WORKER_PTR->originator = gettid();
+++  WORKER_PTR->originator = sys_gettid();
++ #endif
++   if (grpc_polling_trace.enabled()) {
++     gpr_log(GPR_INFO,
diff --git a/tensorflow_cc/patches/v1.15.0-workspace.bzl.patch b/tensorflow_cc/patches/v1.15.0-workspace.bzl.patch
@@ -0,0 +1,11 @@
+index 55d7eb93..33e86087 100755
+--- a/tensorflow/workspace.bzl
++++ b/tensorflow/workspace.bzl
+@@ -486,6 +486,7 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
+     # WARNING: make sure ncteisen@ and vpai@ are cc-ed on any CL to change the below rule
+     tf_http_archive(
+         name = "grpc",
++	patch_file = clean_dep("//third_party:Rename-gettid-functions.patch"),
+         sha256 = "67a6c26db56f345f7cee846e681db2c23f919eba46dd639b09462d1b6203d28c",
+         strip_prefix = "grpc-4566c2a29ebec0835643b972eb99f4306c4234a3",
+         system_build_file = clean_dep("//third_party/systemlibs:grpc.BUILD"),

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-FROM archlinux/base`
	`1`	`+FROM archlinux:latest`
`2`	`2`
`3`	`3`	`# copy the contents of this repository to the container`
`4`	`4`	`COPY . tensorflow_cc`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04`
	`1`	`+FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04`
`2`	`2`
`3`	`3`	`# copy the contents of this repository to the container`
`4`	`4`	`COPY . tensorflow_cc`