diff --git a/kedro-docker/kedro_docker/template/Dockerfile.spark b/kedro-docker/kedro_docker/template/Dockerfile.spark index f0911e8c8..e67266e3b 100644 --- a/kedro-docker/kedro_docker/template/Dockerfile.spark +++ b/kedro-docker/kedro_docker/template/Dockerfile.spark @@ -13,6 +13,10 @@ RUN pip install uv COPY requirements.txt /tmp/requirements.txt RUN uv pip install --system --no-cache-dir -r /tmp/requirements.txt && rm -f /tmp/requirements.txt +ARG PYSPARK_VERSION=3.5.1 +ARG PYARROW_VERSION=16.1.0 +RUN uv pip install --system "pyspark==${PYSPARK_VERSION}" "pyarrow==${PYARROW_VERSION}" + # add kedro user ARG KEDRO_UID=999 ARG KEDRO_GID=0