diff --git a/WORKSPACE.in b/WORKSPACE.in index 62656440..3b312d3f 100644 --- a/WORKSPACE.in +++ b/WORKSPACE.in @@ -36,20 +36,20 @@ http_archive( http_archive( name = "boringssl", - sha256 = "1188e29000013ed6517168600fc35a010d58c5d321846d6a6dfee74e4c788b45", - strip_prefix = "boringssl-7f634429a04abc48e2eb041c81c5235816c96514", + sha256 = "1c5c5bca468c2bc98d02a035f9e10171cdcbd47b8e517a5e06ec9e0fae8bf983", + strip_prefix = "boringssl-3ccef042c8390706182b81141c5fdd7dfebcfab8", urls = [ - "http://mirror.tensorflow.org/github.com/google/boringssl/archive/7f634429a04abc48e2eb041c81c5235816c96514.tar.gz", - "https://github.com/google/boringssl/archive/7f634429a04abc48e2eb041c81c5235816c96514.tar.gz", + "http://mirror.tensorflow.org/github.com/google/boringssl/archive/3ccef042c8390706182b81141c5fdd7dfebcfab8.tar.gz", + "https://github.com/google/boringssl/archive/3ccef042c8390706182b81141c5fdd7dfebcfab8.tar.gz", ], ) http_archive( name = "bazel_skylib", - sha256 = "1dde365491125a3db70731e25658dfdd3bc5dbdfd11b840b3e987ecf043c7ca0", + sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506", urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/bazel-skylib/releases/download/0.9.0/bazel_skylib-0.9.0.tar.gz", - "https://github.com/bazelbuild/bazel-skylib/releases/download/0.9.0/bazel_skylib-0.9.0.tar.gz", + "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", ], ) diff --git a/configure.sh b/configure.sh index bdba9476..49dd7047 100755 --- a/configure.sh +++ b/configure.sh @@ -42,7 +42,7 @@ function write_action_env_to_bazelrc() { [ -e .bazelrc ] && rm .bazelrc write_to_bazelrc "build -c opt" -write_to_bazelrc 'build --cxxopt="-std=c++11"' +write_to_bazelrc 'build --cxxopt="-std=c++17"' write_to_bazelrc 'build --auto_output_filter=subpackages' write_to_bazelrc 'build --copt="-Wall" --copt="-Wno-sign-compare"' write_to_bazelrc 'build --linkopt="-lrt -lm"' @@ -51,7 +51,7 @@ TF_NEED_CUDA=0 # Check if it's installed TF_CFLAGS="" TF_LFLAGS="" -if ${PIP} list | grep "tensorflow \|tensorflow-gpu\|tensorflow-cpu\|tf-nightly" >/dev/null ; then +if ${PIP} list | grep "tensorflow \|tensorflow-gpu\|tensorflow-cpu\|tf-nightly\|tensorflow-metal" >/dev/null ; then echo 'Using installed tensorflow' TF_CFLAGS=( $(${PYTHON} -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_compile_flags()))') ) TF_LFLAGS="$(${PYTHON} -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_link_flags()))')" @@ -82,16 +82,18 @@ if [[ "$PIP_MANYLINUX2010" == "1" ]]; then write_to_bazelrc "test --config=manylinux2010" fi -export TF_VERSION="${TF_VERSION:-2.5.0}" +export TF_VERSION="${TF_VERSION:-2.11.0}" export TF_VERSION_UNDERSCORE=$(echo $TF_VERSION | sed 's/\./_/g') export TF_VERSION_DASH=$(echo $TF_VERSION | sed 's/\./-/g') cat WORKSPACE.in | sed "s/TF_VERSION/${TF_VERSION_UNDERSCORE}/" > WORKSPACE cat pip_pkg_scripts/setup.py.in | sed "s/TF_VERSION/${TF_VERSION_DASH}/" > pip_pkg_scripts/setup.py -if [[ ${TF_VERSION} == '1.14.0' ]]; then - write_to_bazelrc 'build --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=1"' -else - write_to_bazelrc 'build --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0"' -fi - +case "$TF_VERSION" in + 1.14.0 | 2.9.* | 2.10.* | 2.11.*) + write_to_bazelrc 'build --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=1"' + ;; + *) + write_to_bazelrc 'build --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0"' + ;; +esac diff --git a/pip_pkg_scripts/build.Dockerfile b/pip_pkg_scripts/build.Dockerfile index 0d065973..5e587276 100644 --- a/pip_pkg_scripts/build.Dockerfile +++ b/pip_pkg_scripts/build.Dockerfile @@ -1,16 +1,11 @@ -FROM tensorflow/tensorflow:custom-op-ubuntu16 +FROM tensorflow/tensorflow:2.3.0-custom-op-gpu-ubuntu16 ENV PYTHON_VERSION="3" ENV PYTHON_MINOR_VERSION="" ENV PIP_MANYLINUX2010="1" -ENV TF_VERSION="2.6.0" -RUN wget https://github.com/bazelbuild/bazel/releases/download/4.0.0/bazel-4.0.0-installer-linux-x86_64.sh > /dev/null -RUN bash bazel-4.0.0-installer-linux-x86_64.sh -# There are some problems with the python3 installation from custom-op-ubuntu16. -# Remove it and install new ones. -RUN apt-get remove --purge -y python3.5 python3.6 -RUN rm -f /etc/apt/sources.list.d/jonathonf-ubuntu-python-3_6-xenial.list -RUN apt-key del F06FC659 +ENV TF_VERSION="2.11.0" +RUN wget https://github.com/bazelbuild/bazel/releases/download/5.3.2/bazel-5.3.2-installer-linux-x86_64.sh > /dev/null +RUN bash bazel-5.3.2-installer-linux-x86_64.sh # Deadsnakes PPA no longer supports 16.04 # https://github.com/deadsnakes/issues/issues/195 @@ -20,7 +15,11 @@ RUN apt-get update RUN apt-get install -y apt-utils RUN apt-get install -y build-essential checkinstall libreadline-gplv2-dev libncursesw5-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev libssl-dev zlib1g-dev openssl libffi-dev -RUN for v in 3.6.15 3.7.12 3.8.12 3.9.10; do \ +RUN add-apt-repository ppa:ubuntu-toolchain-r/test +RUN apt-get update +RUN apt-get install + +RUN for v in 3.7.12 3.8.12 3.9.10; do \ wget "https://www.python.org/ftp/python/$v/Python-${v}.tar.xz" && \ tar xvf "Python-${v}.tar.xz" -C /tmp/python && \ cd "/tmp/python/Python-${v}" && \ @@ -31,7 +30,7 @@ RUN for v in 3.6.15 3.7.12 3.8.12 3.9.10; do \ RUN apt-get install -y libopenexr-dev RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py -RUN for python in python3.6 python3.7 python3.8 python3.9; do \ +RUN for python in python3.7 python3.8 python3.9; do \ $python get-pip.py && \ $python -m pip install --upgrade pip setuptools auditwheel && \ $python -m pip install --upgrade grpcio>=1.24.3; \ diff --git a/pip_pkg_scripts/build.sh b/pip_pkg_scripts/build.sh index 9d4c7471..3a462a17 100755 --- a/pip_pkg_scripts/build.sh +++ b/pip_pkg_scripts/build.sh @@ -25,7 +25,7 @@ set -e -x export PYTHON_VERSION="${PYTHON_VERSION:-3}" export PYTHON_MINOR_VERSION="${PYTHON_MINOR_VERSION}" export PIP_MANYLINUX2010="${PIP_MANYLINUX2010:-0}" -export TF_VERSION="${TF_VERSION:-2.5.0}" +export TF_VERSION="${TF_VERSION:-2.11.0}" if [[ -z "${PYTHON_MINOR_VERSION}" ]]; then PYTHON="python${PYTHON_VERSION}" @@ -48,6 +48,15 @@ case "$TF_VERSION" in 2.8.*) export KERAS_VERSION='2.8.0' ;; + 2.9.*) + export KERAS_VERSION='2.9.0' + ;; + 2.10.*) + export KERAS_VERSION='2.10.0' + ;; + 2.11.*) + export KERAS_VERSION='2.11.0' + ;; *) export KERAS_VERSION='nightly' esac @@ -74,4 +83,3 @@ rm -rf "$DST_DIR" || true if [[ "${PIP_MANYLINUX2010}" == "1" ]]; then find "$DST_DIR" -name *.whl | xargs $PYTHON ./third_party/auditwheel_wrapper.py repair --plat manylinux2010_x86_64 -w "$DST_DIR" fi - diff --git a/pip_pkg_scripts/docker_build_all.sh b/pip_pkg_scripts/docker_build_all.sh index ce1b784f..a5d3fa43 100755 --- a/pip_pkg_scripts/docker_build_all.sh +++ b/pip_pkg_scripts/docker_build_all.sh @@ -20,10 +20,9 @@ docker build --tag=open_dataset_pip -f pip_pkg_scripts/build.Dockerfile . mkdir /tmp/artifacts | true mkdir /tmp/artifacts_all | true -for pyv in 7 8 9; do - for tfv in 6; do +for pyv in 7 8 9 10; do + for tfv in 11; do docker run --mount type=bind,source=/tmp/artifacts,target=/tmp/artifacts -e "PYTHON_VERSION=3" -e "PYTHON_MINOR_VERSION=${pyv}" -e "PIP_MANYLINUX2010=1" -e "TF_VERSION=2.${tfv}.0" open_dataset_pip cp -f /tmp/artifacts/* /tmp/artifacts_all done done - diff --git a/tf/workspace_tf2_10_0.bzl b/tf/workspace_tf2_10_0.bzl new file mode 100644 index 00000000..9d406ac9 --- /dev/null +++ b/tf/workspace_tf2_10_0.bzl @@ -0,0 +1,51 @@ +"""Install dependencies.""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def load_tf_version_specific_dependencies(): + """Load TF version specific dependencies.""" + http_archive( + name = "com_google_absl", + sha256 = "94aef187f688665dc299d09286bfa0d22c4ecb86a80b156dff6aabadc5a5c26d", + strip_prefix = "abseil-cpp-273292d1cfc0a94a65082ee350509af1d113344d", + urls = [ + "https://storage.googleapis.com/mirror.tensorflow.org/github.com/abseil/abseil-cpp/archive/273292d1cfc0a94a65082ee350509af1d113344d.tar.gz", + "https://github.com/abseil/abseil-cpp/archive/273292d1cfc0a94a65082ee350509af1d113344d.tar.gz", + ], + ) + + http_archive( + name = "zlib", + build_file = "//third_party:zlib.BUILD", + sha256 = "91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9", + strip_prefix = "zlib-1.2.12", + urls = [ + "http://mirror.tensorflow.org/zlib.net/zlib-1.2.12.tar.gz", + "https://zlib.net/zlib-1.2.12.tar.gz", + ], + ) + + PROTOBUF_URLS = [ + "https://storage.googleapis.com/mirror.tensorflow.org/github.com/protocolbuffers/protobuf/archive/v3.9.2.zip", + "https://github.com/protocolbuffers/protobuf/archive/v3.9.2.zip", + ] + + PROTOBUF_SHA256 = "cfcba2df10feec52a84208693937c17a4b5df7775e1635c1e3baffc487b24c9b" + + PROTOBUF_STRIP_PREFIX = "protobuf-3.9.2" + + # We need to import the protobuf library under the names com_google_protobuf + # and com_google_protobuf_cc to enable proto_library support in bazel. + http_archive( + name = "com_google_protobuf", + sha256 = PROTOBUF_SHA256, + strip_prefix = PROTOBUF_STRIP_PREFIX, + urls = PROTOBUF_URLS, + ) + + http_archive( + name = "com_google_protobuf_cc", + sha256 = PROTOBUF_SHA256, + strip_prefix = PROTOBUF_STRIP_PREFIX, + urls = PROTOBUF_URLS, + ) diff --git a/tf/workspace_tf2_11_0.bzl b/tf/workspace_tf2_11_0.bzl new file mode 100644 index 00000000..9d406ac9 --- /dev/null +++ b/tf/workspace_tf2_11_0.bzl @@ -0,0 +1,51 @@ +"""Install dependencies.""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def load_tf_version_specific_dependencies(): + """Load TF version specific dependencies.""" + http_archive( + name = "com_google_absl", + sha256 = "94aef187f688665dc299d09286bfa0d22c4ecb86a80b156dff6aabadc5a5c26d", + strip_prefix = "abseil-cpp-273292d1cfc0a94a65082ee350509af1d113344d", + urls = [ + "https://storage.googleapis.com/mirror.tensorflow.org/github.com/abseil/abseil-cpp/archive/273292d1cfc0a94a65082ee350509af1d113344d.tar.gz", + "https://github.com/abseil/abseil-cpp/archive/273292d1cfc0a94a65082ee350509af1d113344d.tar.gz", + ], + ) + + http_archive( + name = "zlib", + build_file = "//third_party:zlib.BUILD", + sha256 = "91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9", + strip_prefix = "zlib-1.2.12", + urls = [ + "http://mirror.tensorflow.org/zlib.net/zlib-1.2.12.tar.gz", + "https://zlib.net/zlib-1.2.12.tar.gz", + ], + ) + + PROTOBUF_URLS = [ + "https://storage.googleapis.com/mirror.tensorflow.org/github.com/protocolbuffers/protobuf/archive/v3.9.2.zip", + "https://github.com/protocolbuffers/protobuf/archive/v3.9.2.zip", + ] + + PROTOBUF_SHA256 = "cfcba2df10feec52a84208693937c17a4b5df7775e1635c1e3baffc487b24c9b" + + PROTOBUF_STRIP_PREFIX = "protobuf-3.9.2" + + # We need to import the protobuf library under the names com_google_protobuf + # and com_google_protobuf_cc to enable proto_library support in bazel. + http_archive( + name = "com_google_protobuf", + sha256 = PROTOBUF_SHA256, + strip_prefix = PROTOBUF_STRIP_PREFIX, + urls = PROTOBUF_URLS, + ) + + http_archive( + name = "com_google_protobuf_cc", + sha256 = PROTOBUF_SHA256, + strip_prefix = PROTOBUF_STRIP_PREFIX, + urls = PROTOBUF_URLS, + )