Upgrade Struct2Tensor to TensorFlow 2.21.0

.bazelrc

@@ -35,15 +35,21 @@ build:linux --copt=-w
 build:macos --copt=-w
 
 # By default, build TF in C++ 17 mode.
-build:linux --cxxopt=-std=c++17
+build --cxxopt=-std=c++17
+build --host_cxxopt=-std=c++17
 build:linux --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=1
-build:linux --host_cxxopt=-std=c++17
-build:macos --cxxopt=-std=c++17
 build:macos --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=1
-build:macos --host_cxxopt=-std=c++17
 
 
 # Suppress all warning messages.
+
 build:short_logs --output_filter=DONT_MATCH_ANYTHING
 
+# Config to disable sframe in assembler to avoid linker errors with gcc 15
+build:sframe_fix --copt=-Wa,--gsframe=no
+
 build:macos --macos_minimum_os=10.12
+
+common --noenable_bzlmod
+
+test --noincompatible_check_sharding_support

.bazelversion

@@ -0,0 +1 @@
+7.7.0

.github/workflows/build.yml

@@ -6,13 +6,17 @@ on:
   pull_request:
     branches: [ master ]
 
+env:
+  USE_BAZEL_VERSION: 7.7.0
+
 jobs:
   build:
-    name: Build (${{ matrix.os }})
+    name: Build (${{ matrix.os }}, Python ${{ matrix.python-version }})
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
 
     steps:
     - uses: actions/checkout@v4
@@ -23,6 +27,8 @@ jobs:
         auto-activate-base: false
         activate-environment: s2t-env
         environment-file: environment.yml
+        python-version: ${{ matrix.python-version }}
+        use-mamba: true
 
     - name: Install Bazel
       shell: bash -l {0}
@@ -55,3 +61,13 @@ jobs:
       if: runner.os == 'Linux'
       shell: bash -l {0}
       run: bazel build //...
+
+  docker-serving-build:
+    name: Build Docker Serving Image
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Build TF-Serving Docker Image
+      run: |
+        cd struct2tensor/tools/tf_serving_docker
+        docker build -t struct2tensor-serving:latest .

.github/workflows/pre-commit.yml

@@ -0,0 +1,21 @@
+name: pre-commit
+
+on:
+  pull_request:
+  push:
+     branches: [master]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4.1.7
+      with:
+        # Ensure the full history is fetched
+        # This is required to run pre-commit on a specific set of commits
+        # TODO: Remove this when all the pre-commit issues are fixed
+        fetch-depth: 0
+    - uses: actions/setup-python@v5.1.1
+      with:
+        python-version: 3.13
+    - uses: pre-commit/action@v3.0.1

.pre-commit-config.yaml

@@ -0,0 +1,9 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+    -   id: trailing-whitespace
+        exclude: \.patch$
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files

LICENSE

@@ -198,4 +198,4 @@
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
-   limitations under the License.
+   limitations under the License.

RELEASE.md

@@ -4,12 +4,28 @@
 
 ## Major Features and Improvements
 
+*   N/A
+
 ## Bug Fixes and Other Changes
 
+*   Depends on `tensorflow 2.21.0`.
+*   Depends on `protobuf==6.31.1`.
+*   Depends on `pyarrow==23.0.1`.
+*   Enforced C++17 in `.bazelrc`.
+*   Disabled Bzlmod in `.bazelrc` to resolve protobuf conflicts.
+*   Added dummy repositories in `WORKSPACE` to bypass circular dependencies with TensorFlow.
+*   Fixed missing `#include <cstdint>` in various files to support compilation with `gcc 15`.
+*   Added support for Python 3.12.
+*   Dropped support for Python 3.9.
+
 ## Breaking Changes
 
+*   N/A
+
 ## Deprecations
 
+*   N/A
+
 # Version 0.48.1
 
 ## Major Features and Improvements

WORKSPACE

@@ -16,6 +16,58 @@
 
 workspace(name = "struct2tensor")
 
+local_repository(
+    name = "python_version_repo",
+    path = "third_party/python_version_repo",
+)
+
+local_repository(
+    name = "python_3_11_host",
+    path = "third_party/python_3_11_host",
+)
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
+
+local_repository(
+    name = "rules_java",
+    path = "third_party/rules_java",
+)
+
+local_repository(
+    name = "local_config_cuda",
+    path = "third_party/local_config_cuda",
+)
+
+local_repository(
+    name = "local_config_tensorrt",
+    path = "third_party/local_config_tensorrt",
+)
+
+local_repository(
+    name = "local_config_rocm",
+    path = "third_party/local_config_rocm",
+)
+
+local_repository(
+    name = "local_config_sycl",
+    path = "third_party/local_config_sycl",
+)
+
+local_repository(
+    name = "tf_wheel_version_suffix",
+    path = "third_party/tf_wheel_version_suffix",
+)
+
+maybe(
+    http_archive,
+    name = "platforms",
+    urls = [
+        "https://github.com/bazelbuild/platforms/releases/download/0.0.11/platforms-0.0.11.tar.gz",
+    ],
+    sha256 = "29742e87275809b5e598dc2f04d86960cc7a55b3067d97221c9abbc9926bff0f",
+)
+
 load("//tf:tf_configure.bzl", "tf_configure")
 
 load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
@@ -26,23 +78,53 @@ tf_configure(name = "local_config_tf")
 
 #####################################################################################
 
+
+
+# ===== Abseil dependency =====
 http_archive(
-    name = "zlib",
-    build_file = "@com_google_protobuf//:third_party/zlib.BUILD",
-    sha256 = "17e88863f3600672ab49182f217281b6fc4d3c762bde361935e436a95214d05c",
-    strip_prefix = "zlib-1.3.1",
-    urls = ["https://github.com/madler/zlib/archive/v1.3.1.tar.gz"],
+    name = "com_google_absl",
+    sha256 = "d1abe9da2003e6cbbd7619b0ced3e52047422f4f4ac6c66a9bef5d2e99fea837",
+    strip_prefix = "abseil-cpp-d38452e1ee03523a208362186fd42248ff2609f6",
+    urls = [
+        "https://github.com/abseil/abseil-cpp/archive/d38452e1ee03523a208362186fd42248ff2609f6.tar.gz",
+    ],
+    patches = [
+        "//third_party:abseil_visibility.patch",
+    ],
+    patch_args = ["-p0"],
+)
+
+http_archive(
+    name = "abseil-cpp",
+    sha256 = "d1abe9da2003e6cbbd7619b0ced3e52047422f4f4ac6c66a9bef5d2e99fea837",
+    strip_prefix = "abseil-cpp-d38452e1ee03523a208362186fd42248ff2609f6",
+    urls = [
+        "https://github.com/abseil/abseil-cpp/archive/d38452e1ee03523a208362186fd42248ff2609f6.tar.gz",
+    ],
+    patches = [
+        "//third_party:abseil_visibility.patch",
+    ],
+    patch_args = ["-p0"],
 )
 
-# ===== Protobuf 4.25.6 dependency =====
+
+
+
+# ===== Protobuf 6.31.1 dependency =====
 # Must be declared BEFORE TensorFlow's workspaces to override the version they pull
 http_archive(
     name = "com_google_protobuf",
-    sha256 = "4e6727bc5d23177edefa3ad86fd2f5a92cd324151636212fd1f7f13aef3fd2b7",
-    strip_prefix = "protobuf-4.25.6",
+    sha256 = "6e09bbc950ba60c3a7b30280210cd285af8d7d8ed5e0a6ed101c72aff22e8d88",
+    strip_prefix = "protobuf-6.31.1",
     urls = [
-        "https://github.com/protocolbuffers/protobuf/archive/v4.25.6.tar.gz",
+        "https://github.com/protocolbuffers/protobuf/archive/refs/tags/v6.31.1.zip",
     ],
+    patches = ["//third_party:protobuf_tensorflow.patch"],
+    patch_args = ["-p1"],
+)
+
+register_toolchains(
+    "@com_google_protobuf//bazel/private/toolchains:cc_source_toolchain_bazel7",
 )
 
 # ===== TensorFlow dependency =====
@@ -60,23 +142,55 @@ http_archive(
 # 3. Request the new archive to be mirrored on mirror.bazel.build for more
 #    reliable downloads.
 
-_TENSORFLOW_GIT_COMMIT = "3c92ac03cab816044f7b18a86eb86aa01a294d95"  # tf 2.17.1
-_TENSORFLOW_ARCHIVE_SHA256 = "317dd95c4830a408b14f3e802698eb68d70d81c7c7cfcd3d28b0ba023fe84a68"
+_TENSORFLOW_GIT_COMMIT = "2.21.0"  # tf 2.21.0
+_TENSORFLOW_ARCHIVE_SHA256 = "ef3568bb4865d6c1b2564fb5689c19b6b9a5311572cd1f2ff9198636a8520921"
 
 http_archive(
     name = "org_tensorflow",
     sha256 = _TENSORFLOW_ARCHIVE_SHA256,
     urls = [
-        "https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
+        "https://github.com/tensorflow/tensorflow/archive/v%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
     ],
     strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT,
     patches = ["//third_party:tensorflow.patch"],
     patch_args = ["-p1"],
+    repo_mapping = {
+        "@abseil-cpp": "@com_google_absl",
+    },
+)
+
+http_archive(
+    name = "llvm-raw",
+    sha256 = "3f986184ee126677dbd77edb16d6b82c057ec869fefd7a9871979941e52e837a",
+    strip_prefix = "llvm-project-909041e4802c4b9a2223ca04099f35bf1dbbd460",
+    urls = [
+        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/909041e4802c4b9a2223ca04099f35bf1dbbd460.tar.gz",
+        "https://github.com/llvm/llvm-project/archive/909041e4802c4b9a2223ca04099f35bf1dbbd460.tar.gz",
+    ],
+    build_file = "@xla//third_party/llvm:llvm.BUILD",
+    patches = [
+        "@xla//third_party/llvm:generated.patch",
+        "@xla//third_party/llvm:build.patch",
+        "@xla//third_party/llvm:mathextras.patch",
+        "@xla//third_party/llvm:toolchains.patch",
+        "@xla//third_party/llvm:zstd.patch",
+        "@xla//third_party/llvm:lit_test.patch",
+        "//third_party:llvm_configure.patch",
+    ],
+    patch_args = ["-p1"],
+)
+
+http_archive(
+    name = "zlib",
+    build_file = "@com_google_protobuf//:third_party/zlib.BUILD",
+    sha256 = "17e88863f3600672ab49182f217281b6fc4d3c762bde361935e436a95214d05c",
+    strip_prefix = "zlib-1.3.1",
+    urls = ["https://github.com/madler/zlib/archive/v1.3.1.tar.gz"],
 )
 
-load("//third_party:python_configure.bzl", "local_python_configure")
-local_python_configure(name = "local_config_python")
-local_python_configure(name = "local_execution_config_python")
+# load("//third_party:python_configure.bzl", "local_python_configure")
+# local_python_configure(name = "local_config_python")
+# local_python_configure(name = "local_execution_config_python")
 
 
 # Please add all new struct2tensor dependencies in workspace.bzl.
@@ -113,4 +227,4 @@ rules_pkg_dependencies()
 
 # Specify the minimum required bazel version.
 load("@bazel_skylib//lib:versions.bzl", "versions")
-versions.check("6.5.0")
+versions.check("7.7.0")

configure.sh

@@ -53,12 +53,7 @@ else
   fi
 fi
 
-# Set Bazel version based on OS
-if [[ "$(uname)" == "Darwin" ]]; then
-  echo "7.4.1" > .bazelversion
-else
-  echo "6.5.0" > .bazelversion
-fi
+# Bazel version is controlled by .bazelversion file.
 
 ensure_tensorflow
 TF_CFLAGS=( $(${PYTHON_BIN_PATH} -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_compile_flags()))') )