pandas: handle non-contiguous memory

scripts/build_pybind_from_subdir.sh

@@ -31,24 +31,93 @@ export CCACHE_TEMPDIR
 
 mkdir -p "${CCACHE_TEMPDIR}"
 
-rm -rf "${BUILD_DIR}"
+# ---------------------------------------------------------------------------
+# Generator selection
+# ---------------------------------------------------------------------------
+# Honour GEN env var (e.g. GEN=Unix\ Makefiles), otherwise default to Ninja.
+if [[ -n "${GEN:-}" ]]; then
+    GENERATOR="${GEN}"
+else
+    GENERATOR="Ninja"
+fi
+echo "[pybind] Generator: ${GENERATOR}"
+
+# If the build directory exists with a different generator, wipe it so
+# CMake doesn't error out with a generator mismatch.
+if [[ -f "${BUILD_DIR}/CMakeCache.txt" ]]; then
+    cached_gen="$(grep -m1 'CMAKE_GENERATOR:INTERNAL=' "${BUILD_DIR}/CMakeCache.txt" 2>/dev/null | cut -d= -f2)"
+    if [[ -n "${cached_gen}" && "${cached_gen}" != "${GENERATOR}" ]]; then
+        echo "[pybind] Generator changed from '${cached_gen}' to '${GENERATOR}' — wiping stale build directory."
+        rm -rf "${BUILD_DIR}"
+    fi
+fi
+
+# ---------------------------------------------------------------------------
+# CMake configuration (always re-runs if CMakeLists.txt changed)
+# ---------------------------------------------------------------------------
+# Only wipe the build tree when BUILD_CLEAN=1 is set explicitly.
+if [[ "${BUILD_CLEAN:-0}" == "1" ]]; then
+  echo "[pybind] BUILD_CLEAN=1 — removing ${BUILD_DIR}"
+  rm -rf "${BUILD_DIR}"
+fi
+
+cmake_args=(
+    -S "${ROOT_DIR}"
+    -B "${BUILD_DIR}"
+    -G "${GENERATOR}"
+    -DCMAKE_BUILD_TYPE=Release
+    -DLBUG_SOURCE_DIR="${LBUG_DIR}"
+    -DPYTHON_EXECUTABLE="${PYTHON_BIN}"
+    -DPython_EXECUTABLE="${PYTHON_BIN}"
+    -DPython3_EXECUTABLE="${PYTHON_BIN}"
+    -DPYBIND11_PYTHON_VERSION="${PYTHON_VERSION}"
+)
+
+# Wire in ccache if it is available (vastly speeds up rebuilds).
+if command -v ccache &>/dev/null; then
+    cmake_args+=(
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+    )
+fi
+
+cmake "${cmake_args[@]}"
+
+# ---------------------------------------------------------------------------
+# Build
+# ---------------------------------------------------------------------------
+# Parallelism: honour CMAKE_BUILD_PARALLEL_LEVEL or PARALLEL first,
+# then fall back to the number of logical CPUs.
+if [[ -n "${CMAKE_BUILD_PARALLEL_LEVEL:-}" ]]; then
+    NPROC="${CMAKE_BUILD_PARALLEL_LEVEL}"
+elif [[ -n "${PARALLEL:-}" ]]; then
+    NPROC="${PARALLEL}"
+else
+    NPROC="$(nproc 2>/dev/null || echo 4)"
+fi
 
-cmake \
-  -S "${ROOT_DIR}" \
-  -B "${BUILD_DIR}" \
-  -DCMAKE_BUILD_TYPE=Release \
-  -DLBUG_SOURCE_DIR="${LBUG_DIR}" \
-  -DPYTHON_EXECUTABLE="${PYTHON_BIN}" \
-  -DPython_EXECUTABLE="${PYTHON_BIN}" \
-  -DPython3_EXECUTABLE="${PYTHON_BIN}" \
-  -DPYBIND11_PYTHON_VERSION="${PYTHON_VERSION}"
+build_args=(
+    --build "${BUILD_DIR}"
+    --config Release
+    --target _lbug
+    --parallel "${NPROC}"
+)
 
-cmake --build "${BUILD_DIR}" --config Release --target _lbug
+echo "[pybind] Starting build with ${NPROC} parallel job(s) ..."
+cmake "${build_args[@]}"
 
-if compgen -G "${ROOT_DIR}/build/ladybug/_lbug*" > /dev/null; then
+# ---------------------------------------------------------------------------
+# Post-build check
+# ---------------------------------------------------------------------------
+if compgen -G "${ROOT_DIR}/build/ladybug/_lbug*" &>/dev/null; then
   echo "[pybind] Built extension into ${ROOT_DIR}/build/ladybug"
+elif compgen -G "${BUILD_DIR}/ladybug/_lbug*" &>/dev/null; then
+  echo "[pybind] Built extension into ${BUILD_DIR}/ladybug"
+  # Copy to expected location so Makefile's test target finds it.
+  mkdir -p "${ROOT_DIR}/build/ladybug"
+  cp "${BUILD_DIR}"/ladybug/_lbug* "${ROOT_DIR}/build/ladybug/"
 else
   echo "[pybind] Build finished, but no _lbug extension artifact was found." >&2
-  echo "Checked: ${ROOT_DIR}/build/ladybug" >&2
+  echo "Checked: ${ROOT_DIR}/build/ladybug and ${BUILD_DIR}/ladybug" >&2
   exit 1
 fi

src_cpp/pandas/pandas_bind.cpp

@@ -7,6 +7,18 @@ namespace lbug {
 
 using namespace lbug::common;
 
+// Ensure a numpy array is C-contiguous so that the scan code can safely use
+// memcpy on the raw data pointer. Non-contiguous arrays can arise from
+// DataFrame columns backed by views into larger 2D arrays (e.g. after
+// transpose, concat, or slicing).
+static py::array ensureContiguous(py::object obj) {
+    auto arr = py::array(obj);
+    if (!arr.attr("flags").attr("c_contiguous").cast<bool>()) {
+        arr = py::module_::import("numpy").attr("ascontiguousarray")(arr);
+    }
+    return arr;
+}
+
 struct PandasBindColumn {
 public:
     PandasBindColumn(py::handle name, py::handle type, py::object column)
@@ -54,25 +66,24 @@ static common::LogicalType bindColumn(PandasBindColumn& bindColumn,
     }
 
     if (bindData->npType.type == NumpyNullableType::FLOAT_16) {
-        auto pandasArray = column.attr("array");
         bindData->pandasCol =
-            std::make_unique<PandasNumpyColumn>(py::array(column.attr("to_numpy")("float32")));
+            std::make_unique<PandasNumpyColumn>(ensureContiguous(column.attr("to_numpy")("float32")));
         bindData->npType.type = NumpyNullableType::FLOAT_32;
         columnType = NumpyTypeUtils::numpyToLogicalType(bindData->npType);
     } else {
         auto pandasArray = column.attr("array");
         if (py::hasattr(pandasArray, "_data")) {
             // This means we can access the numpy array directly.
             bindData->pandasCol =
-                std::make_unique<PandasNumpyColumn>(column.attr("array").attr("_data"));
+                std::make_unique<PandasNumpyColumn>(ensureContiguous(column.attr("array").attr("_data")));
         } else if (py::hasattr(pandasArray, "asi8")) {
             // This is a datetime object, has the option to get the array as int64_t's.
             bindData->pandasCol =
-                std::make_unique<PandasNumpyColumn>(py::array(pandasArray.attr("asi8")));
+                std::make_unique<PandasNumpyColumn>(ensureContiguous(pandasArray.attr("asi8")));
         } else {
             // Otherwise we have to get it through 'to_numpy()'.
             bindData->pandasCol =
-                std::make_unique<PandasNumpyColumn>(py::array(column.attr("to_numpy")()));
+                std::make_unique<PandasNumpyColumn>(ensureContiguous(column.attr("to_numpy")()));
         }
         columnType = NumpyTypeUtils::numpyToLogicalType(bindData->npType);
     }