Standardize on Python 3.11 and clarify architecture-specific requirements

montanalow · montanalow · commit 8563bd46de1e · 2025-04-15T13:41:44.000-07:00
- Renames requirements.linux.txt to requirements.amd64.txt
- Renames requirements.macos.txt to requirements.arm64.txt
- Updates build scripts to use Python 3.11 for all Ubuntu versions
- Updates documentation to clarify architecture-specific requirements
- Simplifies Python version configuration in build scripts
diff --git a/.github/workflows/ubuntu-postgresml-python-package.yaml b/.github/workflows/ubuntu-postgresml-python-package.yaml
@@ -36,14 +36,8 @@ jobs:
         sudo add-apt-repository -y ppa:deadsnakes/ppa
         sudo apt update
         
-        # Install specific Python versions based on Ubuntu target
-        if [[ "$UBUNTU_VERSION" == "20.04" ]]; then
-          sudo apt install -y python3.8 python3.8-dev python3.8-venv
-        elif [[ "$UBUNTU_VERSION" == "22.04" ]]; then
-          sudo apt install -y python3.10 python3.10-dev python3.10-venv
-        elif [[ "$UBUNTU_VERSION" == "24.04" ]]; then
-          sudo apt install -y python3.12 python3.12-dev python3.12-venv
-        fi
+        # Install Python 3.11 for all Ubuntu versions for better dependency compatibility
+        sudo apt install -y python3.11 python3.11-dev python3.11-venv
         
         # Ensure pip is updated
         python3 -m pip install --upgrade pip setuptools wheel
diff --git a/packages/postgresml-python/build.sh b/packages/postgresml-python/build.sh
@@ -7,7 +7,7 @@ deb_dir="/tmp/postgresml-python/deb-build"
 # Parse arguments with defaults
 export PACKAGE_VERSION=${1:-"2.10.0"}
 export UBUNTU_VERSION=${2:-"22.04"}
-export PYTHON_VERSION=${3:-"3.10"}
+export PYTHON_VERSION=${3:-"3.11"}
 
 # Handle architecture
 if [[ $(arch) == "x86_64" ]]; then
@@ -16,16 +16,9 @@ else
   export ARCH=arm64
 fi
 
-# Map Ubuntu versions to Python versions if needed
-# For example: Ubuntu 20.04 uses Python 3.8 by default
-declare -A ubuntu_python_versions=(
-  ["20.04"]="3.8"
-  ["22.04"]="3.10"
-  ["24.04"]="3.12"
-)
-
+# We use Python 3.11 for all Ubuntu versions for better dependency compatibility
 if [[ -z "$3" ]]; then
-  PYTHON_VERSION=${ubuntu_python_versions[$UBUNTU_VERSION]:-"3.10"}
+  PYTHON_VERSION="3.11"
 fi
 
 rm -rf "$deb_dir"
@@ -41,18 +34,18 @@ rm "$deb_dir/release.sh"
 (cat ${SCRIPT_DIR}/DEBIAN/postrm | envsubst '${PGVERSION} ${PYTHON_VERSION}') > "$deb_dir/DEBIAN/postrm"
 
 if [[ "$ARCH" == "amd64" ]]; then
-  cp ${SCRIPT_DIR}/../../pgml-extension/requirements.linux.txt "$deb_dir/etc/postgresml-python/requirements.txt"
+  # Use AMD64-specific requirements (x86_64)
+  cp ${SCRIPT_DIR}/../../pgml-extension/requirements.amd64.txt "$deb_dir/etc/postgresml-python/requirements.txt"
 else
-  cp ${SCRIPT_DIR}/../../pgml-extension/requirements.macos.txt "$deb_dir/etc/postgresml-python/requirements.txt"
+  # Use ARM64-specific requirements (aarch64)
+  cp ${SCRIPT_DIR}/../../pgml-extension/requirements.arm64.txt "$deb_dir/etc/postgresml-python/requirements.txt"
 fi
 
 virtualenv --python="python${PYTHON_VERSION}" "$deb_dir/var/lib/postgresml-python/pgml-venv"
 source "$deb_dir/var/lib/postgresml-python/pgml-venv/bin/activate"
 
-# For Python 3.12, ensure PyTorch is installed first
-if [[ "${PYTHON_VERSION}" == "3.12" ]]; then
-  python -m pip install torch
-fi
+# Install PyTorch first to help with dependency resolution
+python -m pip install torch
 
 python -m pip install -r "${deb_dir}/etc/postgresml-python/requirements.txt"
 
diff --git a/pgml-cms/docs/open-source/pgml/developers/installation.md b/pgml-cms/docs/open-source/pgml/developers/installation.md
@@ -71,14 +71,36 @@ virtualenv pgml-venv && \
 source pgml-venv/bin/activate && \
 pip install -r requirements.txt
 ```
+
+PostgresML has architecture-specific requirements files:
+- `requirements.amd64.txt` - For x86_64/AMD64 architectures
+- `requirements.arm64.txt` - For ARM64/aarch64 architectures
+
+When building from source, use the appropriate file for your architecture:
+
+```bash
+# For AMD64/x86_64 systems
+pip install -r requirements.amd64.txt
+
+# For ARM64/aarch64 systems
+pip install -r requirements.arm64.txt
+```
+
+These files contain frozen dependencies that have been tested with PostgresML. We recommend using Python 3.11 for optimal compatibility with all dependencies.
 {% endtab %}
 
 {% tab title="Globally" %}
 Installing Python packages globally can cause issues with your system. If you wish to proceed nonetheless, you can do so:
 
 ```bash
-pip3 install -r requirements.txt
+# For AMD64/x86_64 systems
+pip3 install -r requirements.amd64.txt
+
+# For ARM64/aarch64 systems
+pip3 install -r requirements.arm64.txt
 ```
+
+We recommend using Python 3.11 for optimal compatibility with all dependencies.
 {% endtab %}
 {% endtabs %}
 
diff --git a/pgml-extension/requirements.amd64.txt b/pgml-extension/requirements.amd64.txt
@@ -0,0 +1,198 @@
+accelerate==1.2.1
+aiohappyeyeballs==2.4.4
+aiohttp==3.11.11
+aiohttp-cors==0.7.0
+aiosignal==1.3.2
+airportsdata==20241001
+annotated-types==0.7.0
+anyio==4.8.0
+astor==0.8.1
+attrs==24.3.0
+auto_gptq==0.7.1
+bitsandbytes==0.45.0
+blake3==1.0.2
+cachetools==5.5.0
+catboost==1.2.7
+certifi==2024.12.14
+charset-normalizer==3.4.1
+click==8.1.8
+cloudpickle==3.1.1
+colorama==0.4.6
+coloredlogs==15.0.1
+colorful==0.5.6
+compressed-tensors==0.8.1
+contourpy==1.3.1
+ctransformers==0.2.27
+cycler==0.12.1
+datasets==3.2.0
+deepspeed==0.16.2
+depyf==0.18.0
+dill==0.3.8
+diskcache==5.6.3
+distlib==0.3.9
+distro==1.9.0
+einops==0.8.0
+evaluate==0.4.3
+fastapi==0.115.6
+filelock==3.16.1
+fonttools==4.55.3
+frozenlist==1.5.0
+fsspec==2024.9.0
+gekko==1.2.1
+gguf==0.10.0
+google-api-core==2.24.0
+google-auth==2.37.0
+googleapis-common-protos==1.66.0
+graphviz==0.20.3
+greenlet==3.1.1
+grpcio==1.69.0
+h11==0.14.0
+hjson==3.1.0
+httpcore==1.0.7
+httptools==0.6.4
+httpx==0.28.1
+huggingface-hub==0.27.1
+humanfriendly==10.0
+idna==3.10
+importlib_metadata==8.5.0
+iniconfig==2.0.0
+interegular==0.3.3
+Jinja2==3.1.5
+jiter==0.8.2
+joblib==1.4.2
+jsonpatch==1.33
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+kiwisolver==1.4.8
+langchain==0.3.14
+langchain-core==0.3.29
+langchain-text-splitters==0.3.5
+langsmith==0.2.10
+lark==1.2.2
+lightgbm==4.5.0
+linkify-it-py==2.0.3
+lm-format-enforcer==0.10.9
+lxml==5.3.0
+markdown-it-py==3.0.0
+MarkupSafe==3.0.2
+matplotlib==3.10.0
+mdit-py-plugins==0.4.2
+mdurl==0.1.2
+memray==1.15.0
+mistral_common==1.5.1
+mpmath==1.3.0
+msgpack==1.1.0
+msgspec==0.19.0
+multidict==6.1.0
+multiprocess==0.70.16
+nest-asyncio==1.6.0
+networkx==3.4.2
+ninja==1.11.1.3
+numpy==1.26.4
+nvidia-cublas-cu12==12.4.5.8
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-runtime-cu12==12.4.127
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.2.1.3
+nvidia-curand-cu12==10.3.5.147
+nvidia-cusolver-cu12==11.6.1.9
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-ml-py==12.560.30
+nvidia-nccl-cu12==2.21.5
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nvtx-cu12==12.4.127
+openai==1.59.7
+opencensus==0.11.4
+opencensus-context==0.1.3
+opencv-python-headless==4.10.0.84
+optimum==1.23.3
+orjson==3.10.14
+outlines==0.1.11
+outlines_core==0.1.26
+packaging==24.2
+pandas==2.2.3
+partial-json-parser==0.2.1.1.post5
+peft==0.14.0
+pillow==10.4.0
+platformdirs==4.3.6
+plotly==5.24.1
+pluggy==1.5.0
+portalocker==3.1.1
+prometheus-fastapi-instrumentator==7.0.2
+prometheus_client==0.21.1
+propcache==0.2.1
+proto-plus==1.25.0
+protobuf==5.29.3
+psutil==6.1.1
+py-cpuinfo==9.0.0
+py-spy==0.4.0
+pyarrow==18.1.0
+pyasn1==0.6.1
+pyasn1_modules==0.4.1
+pybind11==2.13.6
+pycountry==24.6.1
+pydantic==2.10.5
+pydantic_core==2.27.2
+Pygments==2.19.1
+pyparsing==3.2.1
+pytest==8.3.4
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+pytz==2024.2
+PyYAML==6.0.2
+pyzmq==26.2.0
+ray==2.40.0
+referencing==0.35.1
+regex==2024.11.6
+requests==2.32.3
+requests-toolbelt==1.0.0
+rich==13.9.4
+rouge==1.0.1
+rpds-py==0.22.3
+rsa==4.9
+sacrebleu==2.5.1
+sacremoses==0.1.1
+safetensors==0.5.2
+scikit-learn==1.6.1
+scipy==1.15.1
+sentence-transformers==3.3.1
+sentencepiece==0.2.0
+six==1.17.0
+smart-open==7.1.0
+sniffio==1.3.1
+SQLAlchemy==2.0.37
+starlette==0.41.3
+sympy==1.13.1
+tabulate==0.9.0
+tenacity==9.0.0
+textual==1.0.0
+threadpoolctl==3.5.0
+tiktoken==0.7.0
+tokenizers==0.21.0
+torch==2.5.1
+torchaudio==2.5.1
+torchvision==0.20.1
+tqdm==4.67.1
+transformers==4.48.0
+transformers-stream-generator==0.0.5
+triton==3.1.0
+trl==0.13.0
+typing_extensions==4.12.2
+tzdata==2024.2
+uc-micro-py==1.0.3
+urllib3==2.3.0
+uvicorn==0.34.0
+uvloop==0.21.0
+virtualenv==20.28.1
+vllm==0.6.6.post1
+watchfiles==1.0.4
+websockets==14.1
+wrapt==1.17.2
+xformers==0.0.28.post3
+xgboost==2.1.3
+xgrammar==0.1.9
+xxhash==3.5.0
+yarl==1.18.3
+zipp==3.21.0
diff --git a/pgml-extension/requirements.arm64.txt b/pgml-extension/requirements.arm64.txt
