Rewrote and simplified requirements.txt to allow gpu usage. Also modi…

…fied specfile so tensorflow is able to find nvidia libraries in service venv.
redBorder · May 21, 2024 · 5734aef · 5734aef
1 parent 47dff04
commit 5734aef
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 74 deletions.
diff --git a/packaging/rpm/rb-aioutliers.spec b/packaging/rpm/rb-aioutliers.spec
@@ -68,6 +68,16 @@ source /opt/rb-aioutliers/aioutliers/bin/activate
 # Install project dependencies
 pip3 install -r /opt/rb-aioutliers/resources/src/requirements.txt
 
+# Add NVIDIA libraries to LD_LIBRARY_PATH (Necessary for using TensorFlow with GPU)
+cp /opt/rb-aioutliers/resources/src/setup_tensorflow_and_cuda.sh /opt/rb-aioutliers/bin/setup_tensorflow_and_cuda.sh
+
+# Append the sourcing of setup script to activate script
+echo "source /opt/rb-aioutliers/bin/setup_tensorflow_and_cuda.sh" | tee -a /opt/rb-aioutliers/aioutliers/bin/activate
+
+# Deactivate and reactivate to apply changes
+deactivate
+source /opt/rb-aioutliers/aioutliers/bin/activate
+
 %changelog
 * Tue Jan 30 2024 Miguel Álvarez <malvarez@redborder.com> - 0.0.3-1
 - Adapt for rhel9 build and user/group creation

diff --git a/resources/src/requirements.txt b/resources/src/requirements.txt
@@ -1,74 +1,15 @@
-absl-py==1.4.0
-astunparse==1.6.3
-cachetools==5.3.1
-certifi==2023.7.22
-charset-normalizer==3.2.0
-click==8.1.7
-Flask==2.2.5
-flatbuffers==23.5.26
-gast==0.4.0
-google-auth==2.23.0
-google-auth-oauthlib>=0.5.0,<1.1
-google-pasta==0.2.0
-grpcio==1.58.0
-h5py==3.8.0
-idna==3.7
-importlib-metadata==6.7.0
-itsdangerous==2.1.2
-Jinja2==3.1.4
-joblib==1.3.2
-keras<2.16,>=2.15.0
-ml-dtypes~=0.2.0
-libclang==16.0.6
-Markdown==3.4.4
-MarkupSafe==2.1.3
-numpy<2.0.0,>=1.23.5
-oauthlib==3.2.2
-opt-einsum==3.3.0
-packaging==23.1
-pandas==1.3.5
-protobuf<4.21.0,>=3.20.3
-pyasn1==0.5.0
-pyasn1-modules==0.3.0
-pylogrus==0.4.0
-python-dateutil==2.8.2
-pytz==2023.3.post1
-requests==2.31.0
-requests-oauthlib>=1.3.0,<1.4.0
-rsa==4.9
-scikit-learn>=1.1.3
-scipy==1.10.0
-six==1.16.0
-tensorboard<2.16,>=2.15
-tensorboard-data-server>=0.7,<0.8.0
-tensorboard-plugin-wit==1.8.1
-tensorflow==2.15.0
-tensorflow-estimator<2.16,>=2.15.0
-tensorflow-io-gcs-filesystem==0.34.0
-termcolor==2.3.0
-threadpoolctl==3.1.0
-typing-extensions<4.6.0,>=3.6.6
-urllib3==1.26.18
-Werkzeug==3.0.3
-wrapt<1.15,>=1.11.0
-zipp==3.15.0
-gunicorn==22.0.0
-schedule==1.2.1
-ntplib==0.4.0
-boto3==1.28.64
-s3transfer>=0.7.0, <0.8.0
-jmespath>=0.7.1, <2.0.0
-botocore>=1.31.64, <1.32.0
-moto==4.2.6
-cryptography>=3.3.1
-xmltodict==0.13.0
-responses>=0.13.0
-cffi>=1.12
-types-PyYAML==6.0.12.12
-pyyaml==6.0.1
-pycparser==2.21
-rq==1.15.1
-redis>=4.0.0
-async-timeout>=4.0.2
-ntp==1.0.0a1
-croniter==2.0.1
+boto3~=1.28.64
+croniter~=2.0.1
+Flask~=2.2.5
+gunicorn~=21.2.0
+moto~=4.2.6
+numpy~=1.26.4
+pandas~=1.3.5
+pylogrus~=0.4.0
+pytz~=2023.4
+redis~=5.0.3
+Requests~=2.32.1
+scikit_learn~=1.4.1.post1
+tensorflow[and-cuda]~=2.15.0
+ntplib~=0.4.0
+rq~=1.16.2
diff --git a/resources/src/setup_tensorflow_and_cuda.sh b/resources/src/setup_tensorflow_and_cuda.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Set the base directory for NVIDIA packages within the virtual environment using wildcard for Python version
+NVIDIA_PACKAGE_DIR="$(echo "$VIRTUAL_ENV")/lib/python*/site-packages/nvidia"
+echo $NVIDIA_PACKAGE_DIR
+# Export LD_LIBRARY_PATH for each lib directory found in the NVIDIA_PACKAGE_DIR
+for dir in $NVIDIA_PACKAGE_DIR/*; do
+    if [ -d "$dir/lib" ]; then
+        export LD_LIBRARY_PATH="$dir/lib:$LD_LIBRARY_PATH"
+    fi
+done
+