diff --git a/ci/docker/pixi.dockerfile b/ci/docker/pixi.dockerfile new file mode 100644 index 000000000000..1976253a9676 --- /dev/null +++ b/ci/docker/pixi.dockerfile @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG arch=amd64 +FROM ${arch}/ubuntu:24.04 + +# install build essentials +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update -y -q && \ + apt-get install -y -q \ + curl \ + gdb \ + libc6-dbg \ + tzdata \ + wget && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# install pixi +RUN curl -fsSL https://pixi.sh/install.sh -o /tmp/install-pixi.sh +RUN PIXI_HOME=/opt/pixi PIXI_NO_PATH_UPDATE=1 sh /tmp/install-pixi.sh && rm /tmp/install-pixi.sh +ENV PATH=/opt/pixi/bin:$PATH + +# Verify pixi installation +RUN pixi --version diff --git a/ci/pixi/.gitignore b/ci/pixi/.gitignore new file mode 100644 index 000000000000..8913651c86dc --- /dev/null +++ b/ci/pixi/.gitignore @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +**/.pixi +**/pixi.lock diff --git a/ci/pixi/cpp/pixi.toml b/ci/pixi/cpp/pixi.toml new file mode 100644 index 000000000000..6dfe95204279 --- /dev/null +++ b/ci/pixi/cpp/pixi.toml @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[workspace] +channels = ["https://prefix.dev/conda-forge"] +platforms = ["linux-64", "linux-aarch64", "osx-arm64"] +preview = ["pixi-build"] + +[environments] +default = [] +test = ["test"] + +[feature.test.activation.env] +ARROW_TEST_DATA = "/arrow/testing/data" +PARQUET_TEST_DATA = "/arrow/cpp/submodules/parquet-testing/data" + +[feature.test.dependencies] +binutils = "*" +cmake = "*" +python = "*" + +[feature.test.tasks] +# This path mangling feels extremely finicky. Investigate alternatives to make this more robust. +test = "cd $(ls -d /arrow/ci/pixi/cpp/.pixi/build/work/arrow-cpp-*/work/build | head -n 1) && ctest --output-on-failure" + +[package] +name = "arrow-cpp" +# TODO: Incorporate this to the bump version script and tests +version = "25.0.0.dev" +authors = ["Apache Arrow"] + +[package.build] +source.path = "../../../cpp" + +[package.build.backend] +name = "pixi-build-cmake" +version = "*" + +[package.build.config] +extra-args = [ + "-DCMAKE_BUILD_TYPE=Release", + "-DARROW_BUILD_TESTS=ON", + "-DARROW_COMPUTE=ON", + "-DARROW_CSV=ON", + "-DARROW_DATASET=ON", + "-DARROW_FILESYSTEM=ON", + "-DARROW_JSON=ON", + "-DARROW_PARQUET=ON", + "-DARROW_WITH_BROTLI=ON", + "-DARROW_WITH_BZ2=ON", + "-DARROW_WITH_LZ4=ON", + "-DARROW_WITH_SNAPPY=ON", + "-DARROW_WITH_ZLIB=ON", + "-DARROW_WITH_ZSTD=ON", + # libutf8proc on package-host-dependencies seems to fail. + # More investigation needed. In the meantime use bundled. + "-Dutf8proc_SOURCE=BUNDLED", + "-DOPENSSL_ROOT_DIR=$PREFIX", +] + +[package.host-dependencies] +brotli = "*" +bzip2 = "*" +gflags = "*" +gmock = ">=1.10.0" +gtest = ">=1.10.0" +libboost-devel = "*" +lz4-c = "*" +openssl = "*" +rapidjson = "*" +re2 = "*" +snappy = "*" +thrift-cpp = ">=0.11.0" +xsimd = ">=14.0" +zlib = "*" +zstd = "*" diff --git a/ci/pixi/python/pixi.toml b/ci/pixi/python/pixi.toml new file mode 100644 index 000000000000..6d096882d255 --- /dev/null +++ b/ci/pixi/python/pixi.toml @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[workspace] +channels = ["https://prefix.dev/conda-forge"] +platforms = ["linux-64", "linux-aarch64", "osx-arm64"] +preview = ["pixi-build"] + +[environments] +default = [] +test = ["test"] + +[feature.test] +platforms = ["linux-64"] + +[feature.test.activation.env] +ARROW_TEST_DATA = "/arrow/testing/data" +PARQUET_TEST_DATA = "/arrow/cpp/submodules/parquet-testing/data" + +[feature.test.dependencies] +hypothesis = "*" +pyarrow = { path = "." } +pytest = "*" +pytest-timeout = "*" + +[feature.test.tasks] +test = "pytest -r s --pyargs pyarrow --timeout=60" + +[package] +name = "pyarrow" +# TODO: Incorporate this to the bump version script and tests +version = "25.0.0.dev" +authors = ["Apache Arrow"] + +[package.build] +source.path = "../../../python" + +[package.build.backend] +name = "pixi-build-python" +version = "*" + +[package.build.config] +noarch = false +compilers = ["c", "cxx"] +env = { + PYARROW_BUNDLE_ARROW_CPP = "0", + CMAKE_GENERATOR = "Ninja", + ARROW_HOME = "$PREFIX", +} + +[package.host-dependencies] +arrow-cpp = { path = "../cpp" } # source-dep on the arrow-cpp manifest +cmake = "*" +cython = ">=3.0" +gcc_linux-64 = "*" +git = "*" +gxx_linux-64 = "*" +ninja = "*" +numpy = "*" +pip = "*" +pkg-config = "*" +python = ">=3.10,<3.14" +scikit-build-core = "*" +setuptools-scm = "*" +sysroot_linux-64 = "*" +zlib = "*" + +[package.run-dependencies] +arrow-cpp = { path = "../cpp" } +python = ">=3.10,<3.14" +numpy = "*" diff --git a/compose.yaml b/compose.yaml index be32a95dd945..7a89058529bc 100644 --- a/compose.yaml +++ b/compose.yaml @@ -138,6 +138,8 @@ x-hierarchy: - fedora-cpp: - fedora-python - fedora-r-clang + - pixi-cpp + - pixi-python - python-sdist - ubuntu-cpp: - ubuntu-cpp-static @@ -197,6 +199,10 @@ volumes: name: ${ARCH}-fedora-${FEDORA}-ccache maven-cache: name: maven-cache + pixi-cpp-build: + name: ${ARCH}-pixi-cpp-build + pixi-python-build: + name: ${ARCH}-pixi-python-build python-wheel-manylinux-2-28-ccache: name: python-wheel-manylinux-2-28-ccache python-wheel-musllinux-1-2-ccache: @@ -1158,6 +1164,50 @@ services: volumes: *ubuntu-volumes command: *python-command + ############################ Pixi builds ################################## + + pixi-cpp: + # C++ build with pixi + # + # Usage: + # docker compose build pixi-cpp + # docker compose run --rm pixi-cpp + # Parameters: + # ARCH: amd64, arm32v7 + image: ${REPO}:${ARCH}-pixi + build: + context: . + dockerfile: ci/docker/pixi.dockerfile + args: + arch: ${ARCH} + volumes: + - .:/arrow:delegated + - ${DOCKER_VOLUME_PREFIX}pixi-cpp-build:/arrow/ci/pixi/cpp/.pixi:delegated + command: > + /bin/sh -c "pixi build --path /arrow/ci/pixi/cpp && + pixi run --environment test --manifest-path /arrow/ci/pixi/cpp test" + + pixi-python: + # CPython build with pixi + # + # Usage: + # docker compose build pixi-python + # docker compose run --rm pixi-python + # Parameters: + # ARCH: amd64, arm32v7 + image: ${REPO}:${ARCH}-pixi + build: + context: . + dockerfile: ci/docker/pixi.dockerfile + args: + arch: ${ARCH} + volumes: + - .:/arrow:delegated + - ${DOCKER_VOLUME_PREFIX}pixi-python-build:/arrow/ci/pixi/python/.pixi:delegated + command: > + /bin/sh -c "pixi build --path /arrow/ci/pixi/python && + pixi run --environment test --manifest-path /arrow/ci/pixi/python test" + ############################ Python wheels ################################## # See available versions at: diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 282a8c83f4de..49e80e59cad9 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -364,6 +364,18 @@ tasks: params: image: conda-cpp-valgrind + test-pixi-cpp: + ci: github + template: docker-tests/github.linux.yml + params: + image: pixi-cpp + + test-pixi-python: + ci: github + template: docker-tests/github.linux.yml + params: + image: pixi-python + test-ubuntu-22.04-cpp: ci: github template: docker-tests/github.linux.yml