switch tf version to 2.15.0 (#388)

update the CI docker images for TF2.15.x Co-authored-by: rhdong <[email protected]> Co-authored-by: poinwater <[email protected]>
tensorflow · Mar 22, 2024 · ba467f4 · ba467f4
1 parent 095db80
commit ba467f4
Show file tree

Hide file tree

Showing 55 changed files with 625 additions and 3,190 deletions.
diff --git a/.github/workflows/ci_test.yml b/.github/workflows/ci_test.yml
@@ -22,6 +22,9 @@ jobs:
     name: Check that the source code is compliant
     runs-on: ubuntu-latest
     steps:
+      - uses: actions/setup-python@v1
+        with:
+          python-version: 3.9
       - uses: actions/checkout@v2
       - name: Run type check
         run: bash tools/run_build.sh source_code_test
@@ -57,13 +60,19 @@ jobs:
     name: Check that we can build the docs
     runs-on: ubuntu-latest
     steps:
+      - uses: actions/setup-python@v1
+        with:
+          python-version: 3.9
       - uses: actions/checkout@v2
       - name: Building the docs
         run: bash tools/run_build.sh docs_tests
   test_cpu_in_small_docker_image:
     name: Run the cpu tests in a small python docker image
     runs-on: ubuntu-latest
     steps:
+      - uses: actions/setup-python@v1
+        with:
+          python-version: 3.9
       - uses: actions/checkout@v2
       - run: bash tools/run_cpu_tests.sh
   valid-codeowners:
@@ -82,29 +91,33 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/setup-python@v1
+      with:
+        python-version: 3.9
     - uses: actions/checkout@v2
     - name: Install tensorflow-docs
       run: python3 -m pip install -U git+https://github.com/tensorflow/docs
     - name: Check notebook formatting
       run: |
         # Run on all notebooks to prevent upstream change.
         echo "Check formatting with nbfmt:"
-        python3 -m pip install --upgrade protobuf==3.20.0
+        python3 -m pip install --upgrade protobuf==4.23.4
         python3 -m tensorflow_docs.tools.nbfmt --test \
             $(find docs/tutorials/ -type f -name *.ipynb)
   nblint:
     name: Notebook lint
     runs-on: ubuntu-latest
     steps:
     - uses: actions/setup-python@v1
+      with:
+        python-version: 3.9
     - uses: actions/checkout@v2
     - name: Install tensorflow-docs
       run: python3 -m pip install -U git+https://github.com/tensorflow/docs
     - name: Lint notebooks
       run: |
         # Run on all notebooks to prevent upstream change.
         echo "Lint check with nblint:"
-        python3 -m pip install --upgrade protobuf==3.20.0
+        python3 -m pip install --upgrade protobuf==4.23.4
         python3 -m tensorflow_docs.tools.nblint \
             --arg=repo:tensorflow/recommenders-addons \
             --exclude_lint=tensorflow::button_colab \

diff --git a/.github/workflows/make_wheel_Linux_x86.sh b/.github/workflows/make_wheel_Linux_x86.sh
@@ -5,15 +5,27 @@ docker info
 # to get more disk space
 rm -rf /usr/share/dotnet &
 
-if [ $TF_NEED_CUDA -eq "1" ] ; then
-  export TF_NAME='tensorflow-gpu'
-else
-  export TF_NAME='tensorflow'
-fi
+export TF_NAME='tensorflow'
 
-# if tensorflow version >= 2.6.0 and <= 2.11.9
-if [[ "$TF_VERSION" =~ ^2\.([6-9]|10|11)\.[0-9]$ ]] ; then
-  export BUILD_IMAGE="tfra/nosla-cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-python$PY_VERSION"
+# if tensorflow version >= 2.6.0 and <= 2.15.9
+if [[ "$TF_VERSION" =~ ^2\.(16)\.[0-9]+$ ]] ; then
+  export BUILD_IMAGE="tfra/nosla-cuda12.3-cudnn8.9-ubuntu20.04-manylinux2014-python$PY_VERSION"
+  export TF_CUDA_VERSION="12.3"
+  export TF_CUDNN_VERSION="8.9"
+elif [[ "$TF_VERSION" =~ ^2\.(15)\.[0-9]+$ ]] ; then
+  export BUILD_IMAGE="tfra/nosla-cuda12.2-cudnn8.9-ubuntu20.04-manylinux2014-python$PY_VERSION"
+  export TF_CUDA_VERSION="12.2"
+  export TF_CUDNN_VERSION="8.9"
+elif [[ "$TF_VERSION" =~ ^2\.(14)\.[0-9]+$ ]] ; then
+  export BUILD_IMAGE="tfra/nosla-cuda11.8-cudnn8.7-ubuntu20.04-manylinux2014-python$PY_VERSION"
+  export TF_CUDA_VERSION="11.8"
+  export TF_CUDNN_VERSION="8.7"
+elif [[ "$TF_VERSION" =~ ^2\.(12|13)\.[0-9]+$ ]] ; then
+  export BUILD_IMAGE="tfra/nosla-cuda11.8-cudnn8.6-ubuntu20.04-manylinux2014-python$PY_VERSION"
+  export TF_CUDA_VERSION="11.8"
+  export TF_CUDNN_VERSION="8.6"
+elif [[ "$TF_VERSION" =~ ^2\.([6-9]|10|11)\.[0-9]+$ ]] ; then
+  export BUILD_IMAGE="tfra/nosla-cuda11.2-cudnn8-ubuntu20.04-manylinux2014-python$PY_VERSION"
   export TF_CUDA_VERSION="11.2"
   export TF_CUDNN_VERSION="8.1"
 elif [ $TF_VERSION == "2.4.1" ] ; then
@@ -29,8 +41,18 @@ else
   exit 1
 fi
 
+echo "BUILD_IMAGE is $BUILD_IMAGE"
+echo "TF_CUDA_VERSION is $TF_CUDA_VERSION"
+echo "TF_CUDNN_VERSION is $TF_CUDNN_VERSION"
+
 if [ -z $HOROVOD_VERSION ] ; then
-  export HOROVOD_VERSION='0.23.0'
+  export HOROVOD_VERSION='0.28.1'
+fi
+
+# For TensorFlow version 2.13 or later:
+export PROTOBUF_VERSION='3.19.6'
+if [[ "$TF_VERSION" =~ ^2\.1[3-9]\.[0-9]$ ]] ; then
+  export PROTOBUF_VERSION='4.23.4'
 fi
 
 DOCKER_BUILDKIT=1 docker build --no-cache \
@@ -46,4 +68,5 @@ DOCKER_BUILDKIT=1 docker build --no-cache \
     --build-arg BUILD_IMAGE \
     --build-arg NIGHTLY_FLAG \
     --build-arg NIGHTLY_TIME \
+    --build-arg PROTOBUF_VERSION \
     ./
diff --git a/.github/workflows/make_wheel_Windows_x86.sh b/.github/workflows/make_wheel_Windows_x86.sh
@@ -4,12 +4,18 @@ export TF_NEED_CUDA=0
 export BAZEL_VC="C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/"
 
 if [ -z $HOROVOD_VERSION ] ; then
-  export HOROVOD_VERSION='0.23.0'
+  export HOROVOD_VERSION='0.28.1'
 fi
 
 python -m pip install --default-timeout=1000 wheel setuptools tensorflow==$TF_VERSION horovod==$HOROVOD_VERSION
 python -m pip install tensorflow-io
-python -m pip install --upgrade protobuf==3.20.0
+# For TensorFlow version 2.12 or earlier:
+export PROTOBUF_VERSION=3.19.6
+if [[ "$TF_VERSION" =~ ^2\.1[3-9]\.[0-9]$ ]] ; then
+  export PROTOBUF_VERSION=4.23.4
+fi
+
+python -m pip install --upgrade protobuf~=$PROTOBUF_VERSION
 
 bash ./tools/testing/build_and_run_tests.sh
 

diff --git a/.github/workflows/make_wheel_macOS_arm64.sh b/.github/workflows/make_wheel_macOS_arm64.sh
@@ -19,7 +19,7 @@ python --version
 python -m pip install --default-timeout=1000 delocate==0.10.3 wheel setuptools
 # For TensorFlow version 2.13 or later:
 if [[ "$TF_VERSION" =~ ^2\.1[3-9]\.[0-9]$ ]] ; then
-  export PROTOBUF_VERSION=3.20.3
+  export PROTOBUF_VERSION=4.23.4
   export TF_NAME="tensorflow"
 fi
 

diff --git a/.github/workflows/make_wheel_macOS_x86.sh b/.github/workflows/make_wheel_macOS_x86.sh
@@ -6,10 +6,16 @@ if [ -z $HOROVOD_VERSION ] ; then
 fi
 python --version
 
+# For TensorFlow version 2.12 or earlier:
+export PROTOBUF_VERSION=3.19.6
+if [[ "$TF_VERSION" =~ ^2\.1[3-9]\.[0-9]$ ]] ; then
+  export PROTOBUF_VERSION=4.23.4
+fi
+
 brew install open-mpi
 python -m pip install --default-timeout=1000 delocate==0.10.3 wheel==0.36.2 setuptools tensorflow==$TF_VERSION
 python -m pip install tensorflow-io
-python -m pip install --upgrade protobuf==3.20.0
+python -m pip install --upgrade protobuf~=$PROTOBUF_VERSION
 
 bash tools/docker/install/install_horovod.sh $HOROVOD_VERSION --only-cpu
 

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -16,8 +16,8 @@ on:
 
 env:
   MIN_PY_VERSION: '3.7'
-  MAX_PY_VERSION: '3.10'
-  HOROVOD_VERSION: '0.23.0'
+  MAX_PY_VERSION: '3.11'
+  HOROVOD_VERSION: '0.28.1'
 
 jobs:
   test-with-bazel:
@@ -34,7 +34,7 @@ jobs:
           sudo apt install -y redis > /dev/null 2> /dev/null
           bash tools/install_deps/install_bazelisk.sh ./
           python -m pip install tensorflow-io
-          python -m pip install --upgrade protobuf==3.20.0
+          python -m pip install --upgrade protobuf==4.23.4
           python configure.py
           bazel test --local_ram_resources=4096 -c opt -k --test_timeout 300,450,1200,3600 --test_output=errors //tensorflow_recommenders_addons/...
   release-wheel:
@@ -43,16 +43,20 @@ jobs:
       matrix:
         # TODO: add back 'windows-latest' when it can be compiled.
         os: ['macos-12', 'ubuntu-20.04']
-        py-version: ['3.7', '3.8', '3.9', '3.10']
-        tf-version: ['2.8.3', '2.11.0']
+        py-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        tf-version: ['2.11.0', '2.15.1']
         tf-need-cuda: ['0', '1']
-        tf-cuda-version: ['11.2']
-        tf-cudnn-version: ['8.1']
         cpu: ['x86']
         exclude:
           # excludes cuda on macOS
           - os: 'macos-12'
             tf-need-cuda: '1'
+          - tf-version: '2.11.0'
+            py-version: '3.11'
+          - tf-version: '2.15.1'
+            py-version: '3.7'
+          - tf-version: '2.15.1'
+            py-version: '3.8'
       fail-fast: false
     runs-on: ${{ matrix.os }}
     steps:
@@ -65,7 +69,7 @@ jobs:
           script: |
             const commit_details = await github.git.getCommit({owner: context.repo.owner, repo: context.repo.repo, commit_sha: context.sha});
             return commit_details.data.author.date
-      - if: matrix.tf-version != '2.11.0'
+      - if: matrix.tf-version != '2.15.1'
         shell: bash
         run: echo "SKIP_CUSTOM_OP_TESTS=--skip-custom-ops" >> $GITHUB_ENV
       - if: github.event_name == 'push'
@@ -85,13 +89,11 @@ jobs:
           PY_VERSION: ${{ matrix.py-version }}
           TF_VERSION: ${{ matrix.tf-version }}
           TF_NEED_CUDA: ${{ matrix.tf-need-cuda }}
-          TF_CUDA_VERSION: ${{ matrix.tf-cuda-version }}
-          TF_CUDNN_VERSION: ${{ matrix.tf-cudnn-version }}
           NIGHTLY_TIME: ${{ steps.author-date.outputs.result }}
           CPU: ${{ matrix.cpu }}
         shell: bash
         run: |
-          if [[ "$TF_VERSION" =~ ^2\.(9|10|11)\.[0-9]$ ]] ; then
+          if [[ "$TF_VERSION" =~ ^2\.(11|12|13|14|15)\.[0-9]$ ]] ; then
             export HOROVOD_VERSION="0.28.1"
           fi
           bash .github/workflows/make_wheel_${OS}_${CPU}.sh
@@ -113,13 +115,20 @@ jobs:
       matrix:
         # TODO: add back 'Windows' when it can be compiled.
         os: ['macOS', 'Linux']
-        py-version: ['3.7', '3.8', '3.9', '3.10']
+        py-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        tf-version: ['2.15.1']
         tf-need-cuda: ['0', '1']
         cpu: ['x86']
         exclude:
           # excludes cuda on macOS
           - os: 'macOS'
             tf-need-cuda: '1'
+          - tf-version: '2.11.0'
+            py-version: '3.11'
+          - tf-version: '2.15.1'
+            py-version: '3.7'
+          - tf-version: '2.15.1'
+            py-version: '3.8'
       fail-fast: false
     if: (github.event_name == 'push' && github.ref == 'refs/heads/master') || github.event_name == 'release'
     steps:
@@ -147,7 +156,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        py-version: ['3.7', '3.8', '3.9', '3.10']
+        py-version: ['3.10']
     if: (github.event_name == 'push' && github.ref == 'refs/heads/master')
     steps:
       - uses: actions/checkout@v2

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -62,7 +62,7 @@ See our [Style Guide](STYLE_GUIDE.md) for more details.
 ### Additional Requirements
 In addition to the above requirements, contribution also needs to meet the following criteria:
 * The functionality is not otherwise available in TensorFlow.
-* It has to be compatible with TensorFlow 2.6.3 and 2.8.3.
+* It has to be compatible with TensorFlow 2.6.3 and 2.15.1.
 * The change needs to include unit tests and integration tests if any.
 * Each project needs to provide documentation for when and how to use it.
 

diff --git a/README.md b/README.md
@@ -128,8 +128,8 @@ Check [nvidia-support-matrix](https://docs.nvidia.com/deeplearning/cudnn/support
 - Due to the significant changes in the Tensorflow API, we can only ensure version 0.2.0 compatibility with TF1.15.2 on CPU & GPU, 
   but **there are no official releases**, you can only get it through compiling by the following:
 ```sh
-PY_VERSION="3.7" \
-TF_VERSION="1.15.2" \
+PY_VERSION="3.9" \
+TF_VERSION="2.15.1" \
 TF_NEED_CUDA=1 \
 sh .github/workflows/make_wheel_Linux_x86.sh
 
@@ -144,7 +144,7 @@ At the same time, we find some OPs used by TRFA have better performance, so we h
 
 For all developers, we recommend you use the development docker containers which are all GPU enabled:
 ```sh
-docker pull tfra/dev_container:latest-python3.8  # "3.7", "3.9" are all avaliable.
+docker pull tfra/dev_container:latest-python3.9  # "3.9" “3.10” are all avaliable.
 docker run --privileged --gpus all -it --rm -v $(pwd):$(pwd) tfra/dev_container:latest-3.8
 ```
 
@@ -154,8 +154,8 @@ Please install a TensorFlow on your compiling machine, The compiler needs to kno
 its headers according to the installed TensorFlow. 
 
 ```sh
-export TF_VERSION="2.8.3"  # "2.6.3" are well tested.
-pip install tensorflow[-gpu]==$TF_VERSION
+export TF_VERSION="2.15.1"  # "2.11.0" are well tested.
+pip install tensorflow==$TF_VERSION
 
 git clone https://github.com/tensorflow/recommenders-addons.git
 cd recommenders-addons
@@ -171,11 +171,11 @@ pip install artifacts/tensorflow_recommenders_addons-*.whl
 #### GPU Support
 Only `TF_NEED_CUDA=1` is required and other environment variables are optional:
 ```sh
-export TF_VERSION="2.8.3"  # "2.6.3" is well tested.
-export PY_VERSION="3.8" 
+export TF_VERSION="2.15.1"  # "2.11.0" is well tested.
+export PY_VERSION="3.9" 
 export TF_NEED_CUDA=1
-export TF_CUDA_VERSION=11.2
-export TF_CUDNN_VERSION=8.1
+export TF_CUDA_VERSION=12.2
+export TF_CUDNN_VERSION=8.9
 export CUDA_TOOLKIT_PATH="/usr/local/cuda"
 export CUDNN_INSTALL_PATH="/usr/lib/x86_64-linux-gnu"
 

diff --git a/configure.py b/configure.py
@@ -112,12 +112,9 @@ def get_tf_shared_lib_dir():
 # Converts the linkflag namespec to the full shared library name
 def get_shared_lib_name():
   namespec = tf.sysconfig.get_link_flags()
-  if is_macos() and is_arm64():
-    # MacOS arm64
-    return "libtensorflow_framework.2.dylib"
-  elif is_macos():
+  if is_macos():
     # MacOS
-    return "libtensorflow_framework.dylib"
+    return "lib" + namespec[1][2:] + ".dylib"
   elif is_windows():
     # Windows
     return "_pywrap_tensorflow_internal.lib"
@@ -138,6 +135,7 @@ def get_tf_version_integer():
     2.4.1 get 2041
     2.6.3 get 2063
     2.8.3 get 2083
+    2.15.1 get 2151
 
   The 4-digits-string will be passed to C macro to discriminate different
   Tensorflow versions. 
@@ -152,7 +150,8 @@ def get_tf_version_integer():
         '\nPlease install a TensorFlow on your compiling machine, '
         'The compiler needs to know the version of Tensorflow '
         'and get TF c++ headers according to the installed TensorFlow. '
-        '\nNote: Only TensorFlow 2.8.3, 2.6.3, 2.4.1, 1.15.2 are supported.')
+        '\nNote: Only TensorFlow 2.15.1 2.8.3, 2.6.3, 2.4.1, 1.15.2 are supported.'
+    )
   try:
     major, minor, patch = version.split('.')
     assert len(

diff --git a/setup.py b/setup.py
@@ -63,7 +63,6 @@ def get_project_name_version():
 
   if os.getenv("TF_NEED_CUDA", "0") == "1":
     project_name = project_name + "-gpu"
-    version["tf_project_name"] = "tensorflow-gpu"
 
   if "--nightly" in sys.argv:
     project_name = "tfra-nightly"
@@ -118,9 +117,9 @@ def has_ext_modules(self):
         "Intended Audience :: Science/Research",
         "License :: OSI Approved :: Apache Software License",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
         "Topic :: Scientific/Engineering :: Mathematics",
         "Topic :: Software Development :: Libraries :: Python Modules",
         "Topic :: Software Development :: Libraries",

diff --git a/tensorflow_recommenders_addons/__init__.py b/tensorflow_recommenders_addons/__init__.py
@@ -19,13 +19,12 @@
 It's built on Keras and aims to have a gentle learning curve while still giving
 you the flexibility to build complex models.
 """
-__all__ = ['dynamic_embedding', 'embedding_variable']
+__all__ = ['dynamic_embedding']
 
 from tensorflow_recommenders_addons.utils.ensure_tf_install import _check_tf_version
 from tensorflow_recommenders_addons.version import __version__
 
 _check_tf_version()
 
 from tensorflow_recommenders_addons import dynamic_embedding
-from tensorflow_recommenders_addons import embedding_variable
 from tensorflow_recommenders_addons.register import register_all