GH-49831: [Python] Withhold annotations from Python wheel until they are complete

ci/scripts/python_wheel_macos_build.sh

@@ -148,7 +148,8 @@ popd
 
 echo "=== (${PYTHON_VERSION}) Building wheel ==="
 export PYARROW_BUNDLE_ARROW_CPP=ON
-export PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
+# TODO(GH-49831): Re-enable when pyarrow-stubs are shipped in wheels again.
+# export PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
 export PYARROW_WITH_ACERO=${ARROW_ACERO}
 export PYARROW_WITH_AZURE=${ARROW_AZURE}
 export PYARROW_WITH_DATASET=${ARROW_DATASET}

ci/scripts/python_wheel_validate_contents.py

@@ -37,6 +37,7 @@ def _count_docstrings(source):
     return count
 
 
+# TODO(GH-48970): Check stubs ARE present once annotations are complete
 def validate_wheel(path):
     p = Path(path)
     wheels = list(p.glob('*.whl'))
@@ -54,9 +55,9 @@ def validate_wheel(path):
                 info.filename.split("/")[-1] == filename for info in wheel_zip.filelist
             ), f"{filename} is missing from the wheel."
 
-        assert any(
+        assert not any(
             info.filename == "pyarrow/py.typed" for info in wheel_zip.filelist
-        ), "pyarrow/py.typed is missing from the wheel."
+        ), "pyarrow/py.typed is present in the wheel."
 
         source_root = Path(__file__).resolve().parents[2]
         stubs_dir = source_root / "python" / "pyarrow-stubs" / "pyarrow"
@@ -73,8 +74,8 @@ def validate_wheel(path):
             if info.filename.startswith("pyarrow/") and info.filename.endswith(".pyi")
         }
 
-        assert wheel_stub_files == expected_stub_files, (
-            "Wheel .pyi files differ from python/pyarrow-stubs/pyarrow.\n"
+        assert not (wheel_stub_files == expected_stub_files), (
+            "Wheel .pyi files do not differ from python/pyarrow-stubs/pyarrow.\n"
             f"Missing in wheel: {sorted(expected_stub_files - wheel_stub_files)}\n"
             f"Unexpected in wheel: {sorted(wheel_stub_files - expected_stub_files)}"
         )
@@ -85,7 +86,7 @@ def validate_wheel(path):
         )
 
         print(f"Found {wheel_docstring_count} docstring(s) in wheel stubs.")
-        assert wheel_docstring_count, "No docstrings found in wheel stub files."
+        assert wheel_docstring_count == 0, "Docstrings found in wheel stub files."
 
     print(f"The wheel: {wheels[0]} seems valid.")
 

ci/scripts/python_wheel_windows_build.bat

@@ -116,7 +116,8 @@ popd
 
 echo "=== (%PYTHON%) Building wheel ==="
 set PYARROW_BUNDLE_ARROW_CPP=ON
-set PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
+rem TODO(GH-49831): Re-enable when pyarrow-stubs are shipped in wheels again.
+rem set PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
 set PYARROW_WITH_ACERO=%ARROW_ACERO%
 set PYARROW_WITH_AZURE=%ARROW_AZURE%
 set PYARROW_WITH_DATASET=%ARROW_DATASET%

ci/scripts/python_wheel_xlinux_build.sh

@@ -157,7 +157,8 @@ check_arrow_visibility
 
 echo "=== (${PYTHON_VERSION}) Building wheel ==="
 export PYARROW_BUNDLE_ARROW_CPP=ON
-export PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
+# TODO(GH-49831): Re-enable when pyarrow-stubs are shipped in wheels again.
+# export PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
 export PYARROW_WITH_ACERO=${ARROW_ACERO}
 export PYARROW_WITH_AZURE=${ARROW_AZURE}
 export PYARROW_WITH_DATASET=${ARROW_DATASET}

python/CMakeLists.txt

@@ -1041,32 +1041,39 @@ endif()
 #
 # Type stubs with docstring injection
 #
+# TODO(GH-49831): Temporarily do not install pyarrow-stubs into wheels.
 # Stubs live in pyarrow-stubs/pyarrow/ during development but are installed
 # alongside the package so type checkers can find them (PEP 561).
-set(PYARROW_STUBS_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/pyarrow-stubs/pyarrow")
-if(EXISTS "${PYARROW_STUBS_SOURCE_DIR}")
-  install(DIRECTORY "${PYARROW_STUBS_SOURCE_DIR}/"
-          DESTINATION "."
-          FILES_MATCHING
-          PATTERN "*.pyi")
-
-  if(PYARROW_REQUIRE_STUB_DOCSTRINGS)
-    install(CODE "
-      execute_process(
-        COMMAND \"${Python3_EXECUTABLE}\"
-                \"${CMAKE_CURRENT_SOURCE_DIR}/scripts/update_stub_docstrings.py\"
-                \"${CMAKE_INSTALL_PREFIX}\"
-                \"${CMAKE_CURRENT_SOURCE_DIR}\"
-        RESULT_VARIABLE _pyarrow_stub_docstrings_result
-      )
-      if(NOT _pyarrow_stub_docstrings_result EQUAL 0)
-        message(FATAL_ERROR \"Stub docstring injection failed (exit code: \${_pyarrow_stub_docstrings_result})\")
-      endif()
-    ")
-  endif()
-else()
-  if(PYARROW_REQUIRE_STUB_DOCSTRINGS)
-    message(FATAL_ERROR "PyArrow stub source directory not found at ${PYARROW_STUBS_SOURCE_DIR}; "
-                        "cannot build wheel without .pyi files.")
-  endif()
+# The stubs are currently incomplete, and some type checkers consume .pyi files
+# even without the py.typed marker. Re-enable this when the stubs are complete.
+if(PYARROW_REQUIRE_STUB_DOCSTRINGS)
+  message(FATAL_ERROR "PYARROW_REQUIRE_STUB_DOCSTRINGS cannot be used while "
+                      "pyarrow-stubs are omitted from wheels (GH-49831).")
 endif()
+# set(PYARROW_STUBS_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/pyarrow-stubs/pyarrow")
+# if(EXISTS "${PYARROW_STUBS_SOURCE_DIR}")
+#   install(DIRECTORY "${PYARROW_STUBS_SOURCE_DIR}/"
+#           DESTINATION "."
+#           FILES_MATCHING
+#           PATTERN "*.pyi")
+#
+#   if(PYARROW_REQUIRE_STUB_DOCSTRINGS)
+#     install(CODE "
+#       execute_process(
+#         COMMAND \"${Python3_EXECUTABLE}\"
+#                 \"${CMAKE_CURRENT_SOURCE_DIR}/scripts/update_stub_docstrings.py\"
+#                 \"${CMAKE_INSTALL_PREFIX}\"
+#                 \"${CMAKE_CURRENT_SOURCE_DIR}\"
+#         RESULT_VARIABLE _pyarrow_stub_docstrings_result
+#       )
+#       if(NOT _pyarrow_stub_docstrings_result EQUAL 0)
+#         message(FATAL_ERROR \"Stub docstring injection failed (exit code: \${_pyarrow_stub_docstrings_result})\")
+#       endif()
+#     ")
+#   endif()
+# else()
+#   if(PYARROW_REQUIRE_STUB_DOCSTRINGS)
+#     message(FATAL_ERROR "PyArrow stub source directory not found at ${PYARROW_STUBS_SOURCE_DIR}; "
+#                         "cannot build wheel without .pyi files.")
+#   endif()
+# endif()

python/pyproject.toml

@@ -88,6 +88,12 @@ metadata.version.provider = "scikit_build_core.metadata.setuptools_scm"
 sdist.include = ["pyarrow/_generated_version.py", "cmake_modules/", "pyarrow-stubs/"]
 wheel.packages = ["pyarrow"]
 wheel.install-dir = "pyarrow"
+# TODO(GH-48970): Remove this when stubfiles are complete
+# Withhold the PEP 561 marker until the type stubs are complete. The .pyi files
+# are also temporarily omitted from wheels, so type checkers don't rely on the
+# incomplete stubs and break downstream users (GH-49831). py.typed is kept
+# in-tree for CI type-checking.
+wheel.exclude = ["pyarrow/py.typed"]
 
 [tool.scikit-build.cmake.define]
 PYARROW_BUNDLE_ARROW_CPP = {env = "PYARROW_BUNDLE_ARROW_CPP", default = "OFF"}