diff --git a/CMakeLists.txt b/CMakeLists.txt index 4abc4ab52f0..1a4cfdee527 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -157,6 +157,13 @@ if(NOT PYTHON_EXECUTABLE) endif() announce_configured_options(PYTHON_EXECUTABLE) +# pybind11 >=3.0 uses find_package(Python) which looks for Python_EXECUTABLE, +# not the legacy PYTHON_EXECUTABLE variable. Bridge the two so pybind11 picks up +# the interpreter that setup.py (or the user) specified. +if(PYTHON_EXECUTABLE AND NOT Python_EXECUTABLE) + set(Python_EXECUTABLE "${PYTHON_EXECUTABLE}") +endif() + announce_configured_options(CMAKE_CXX_COMPILER_ID) announce_configured_options(CMAKE_TOOLCHAIN_FILE) announce_configured_options(BUILD_TESTING) @@ -260,6 +267,27 @@ endif() add_subdirectory(third-party) +# On Apple, pybind11 SHARED extension modules must not link libpython directly. +# When python_add_library creates a SHARED library, it links Python::Python (via +# pybind11::embed) which pulls in libpython. This conflicts with pybind11 +# >=3.0's multi-phase module init — the duplicate Python runtime causes a GIL +# state crash at import time. This function replaces pybind11::embed with +# pybind11::module (which links Python::Module instead of Python::Python — +# headers and ABI only, no libpython) and adds -undefined dynamic_lookup for +# symbol resolution. No-op on non-Apple platforms. +function(strip_python_lib target) + if(NOT APPLE) + return() + endif() + get_target_property(_libs ${target} LINK_LIBRARIES) + if(_libs) + list(REMOVE_ITEM _libs Python::Python pybind11::embed) + list(APPEND _libs pybind11::module) + set_target_properties(${target} PROPERTIES LINK_LIBRARIES "${_libs}") + endif() + target_link_options(${target} PRIVATE "LINKER:-undefined,dynamic_lookup") +endfunction() + # Size-optimized builds disable exceptions, RTTI, and unwind tables. # ExecuTorch's runtime uses Result/Error instead of exceptions, so # -fno-exceptions is safe. Suppressing .eh_frame via -fno-unwind-tables is safe @@ -1114,6 +1142,7 @@ if(EXECUTORCH_BUILD_PYBIND) # pybind portable_lib pybind11_add_module(portable_lib SHARED extension/pybindings/pybindings.cpp) + strip_python_lib(portable_lib) # The actual output file needs a leading underscore so it can coexist with # portable_lib.py in the same python package. PyTorch requires C++20, so # pybindings must be compiled with C++20. @@ -1155,6 +1184,7 @@ if(EXECUTORCH_BUILD_PYBIND) pybind11_add_module( data_loader SHARED extension/pybindings/pybindings_data_loader.cpp ) + strip_python_lib(data_loader) target_include_directories(data_loader PRIVATE ${_common_include_directories}) target_compile_options(data_loader PUBLIC ${_pybind_compile_options}) target_link_libraries(data_loader PRIVATE executorch) diff --git a/backends/apple/coreml/CMakeLists.txt b/backends/apple/coreml/CMakeLists.txt index ce41302bb0a..5391a357f1d 100644 --- a/backends/apple/coreml/CMakeLists.txt +++ b/backends/apple/coreml/CMakeLists.txt @@ -175,6 +175,7 @@ if(EXECUTORCH_BUILD_PYBIND) executorchcoreml SHARED runtime/inmemoryfs/inmemory_filesystem_py.cpp runtime/inmemoryfs/inmemory_filesystem_utils.cpp ) + strip_python_lib(executorchcoreml) target_link_libraries( executorchcoreml PRIVATE coreml_util coreml_inmemoryfs nlohmann_json::nlohmann_json diff --git a/codegen/tools/CMakeLists.txt b/codegen/tools/CMakeLists.txt index 2d61a4d68c1..b829e83c340 100644 --- a/codegen/tools/CMakeLists.txt +++ b/codegen/tools/CMakeLists.txt @@ -9,6 +9,7 @@ # Create the selective_build pybind11 module pybind11_add_module(selective_build SHARED selective_build.cpp) +strip_python_lib(selective_build) # Set the output name to match the module name set_target_properties(selective_build PROPERTIES OUTPUT_NAME "selective_build") diff --git a/extension/llm/runner/CMakeLists.txt b/extension/llm/runner/CMakeLists.txt index 9e50513062b..5247a4ba0a6 100644 --- a/extension/llm/runner/CMakeLists.txt +++ b/extension/llm/runner/CMakeLists.txt @@ -112,6 +112,7 @@ if(EXECUTORCH_BUILD_PYBIND) pybind11_add_module( _llm_runner SHARED ${CMAKE_CURRENT_SOURCE_DIR}/pybindings.cpp ) + strip_python_lib(_llm_runner) find_package_torch() find_library( diff --git a/extension/training/CMakeLists.txt b/extension/training/CMakeLists.txt index a4576679909..e835ae0e0a3 100644 --- a/extension/training/CMakeLists.txt +++ b/extension/training/CMakeLists.txt @@ -66,6 +66,7 @@ if(EXECUTORCH_BUILD_PYBIND) _training_lib SHARED ${CMAKE_CURRENT_SOURCE_DIR}/pybindings/_training_lib.cpp ) + strip_python_lib(_training_lib) set_target_properties(_training_lib PROPERTIES CXX_STANDARD 20) target_include_directories(_training_lib PRIVATE ${TORCH_INCLUDE_DIRS}) diff --git a/third-party/pybind11 b/third-party/pybind11 index a2e59f0e706..d03662f0984 160000 --- a/third-party/pybind11 +++ b/third-party/pybind11 @@ -1 +1 @@ -Subproject commit a2e59f0e7065404b44dfe92a28aca47ba1378dc4 +Subproject commit d03662f0984f652b60e7ddce53d3868002275197