From 895b5b27dd4d270e384b6ae76465e661058d305e Mon Sep 17 00:00:00 2001 From: Rafael Padilha Date: Mon, 21 Oct 2024 18:59:14 +0000 Subject: [PATCH 1/5] Adding git lfs as dependency and making git lfs pull when make local --- Makefile | 1 + resources/vm/setup_farmvibes_ai_vm.sh | 3 +++ 2 files changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 3ac73b4b..adf60f4b 100644 --- a/Makefile +++ b/Makefile @@ -123,6 +123,7 @@ revert-data-ops: cluster repo-$(DATA_OPS_REPO) delete-$(DATA_OPS_DEPLOYMENT) ## DEPLOYMENT=$(DATA_OPS_DEPLOYMENT) REPLICAS=$(CURRENT_DATA_OPS_REPLICAS) make scale local-worker: cluster local-worker-repo delete-$(WORKER_DEPLOYMENT) ## Builds and deploys a local WORKER image (enabling debug) + git lfs pull DEPLOYMENT=$(WORKER_DEPLOYMENT) IMAGE_FULL_REFERENCE=$(WORKER_REPO):$(TAG) $(MAKE) -C . set-registry-image DEPLOYMENT=$(WORKER_DEPLOYMENT) $(MAKE) -C . disable-frozen-modules DEPLOYMENT=$(WORKER_DEPLOYMENT) REPLICAS=$(CURRENT_WORKER_REPLICAS) make scale diff --git a/resources/vm/setup_farmvibes_ai_vm.sh b/resources/vm/setup_farmvibes_ai_vm.sh index 762fa3b6..8c5ded9b 100755 --- a/resources/vm/setup_farmvibes_ai_vm.sh +++ b/resources/vm/setup_farmvibes_ai_vm.sh @@ -48,3 +48,6 @@ fi # Run docker without sudo sudo usermod -aG docker $DOCKER_USER + +# Install git lfs +sudo apt install git-lfs \ No newline at end of file From f8305af2df57c54f6fd9b7e4dd711ed4d7a67d92 Mon Sep 17 00:00:00 2001 From: Rafael Padilha Date: Mon, 21 Oct 2024 20:29:33 +0000 Subject: [PATCH 2/5] Updating makefile, setup script and quickstart doc --- Makefile | 8 +++++--- docs/source/docfiles/markdown/QUICKSTART.md | 17 +++++++++++++++++ resources/vm/setup_farmvibes_ai_vm.sh | 8 +++++--- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index adf60f4b..9757147a 100644 --- a/Makefile +++ b/Makefile @@ -43,12 +43,15 @@ help: ## Shows this help message @echo -e This is the farmvibes.ai makefile. Supported targets are:\\n @grep -E -h '\s##\s' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' -local: cluster local-rest-api local-cache local-worker local-orchestrator local-data-ops ## Builds all images locally and deploys them into the local farmvibes.ai cluster +local: cluster restore-git-lfs local-rest-api local-cache local-worker local-orchestrator local-data-ops ## Builds all images locally and deploys them into the local farmvibes.ai cluster [ -z $(WAIT_AT_THE_END) ] || kubectl delete pods -l backend=terravibes && \ kubectl wait --for=condition=Available deployment --timeout=300s -l backend=terravibes revert: cluster revert-rest-api revert-cache revert-worker revert-orchestrator ## Reverts all images to the official version +restore-git-lfs: + git lfs pull + services-base: resources/docker/Dockerfile-services-base @docker manifest inspect `$(subst FILE,$<,$(base_image_name))` || \ az acr login -n `$(subst FILE,$<,$(base_image_name)) | cut -d / -f 1 | sed 's|.azurecr.io||g'` || \ @@ -122,8 +125,7 @@ revert-data-ops: cluster repo-$(DATA_OPS_REPO) delete-$(DATA_OPS_DEPLOYMENT) ## DEPLOYMENT=$(DATA_OPS_DEPLOYMENT) IMAGE_FULL_REFERENCE=$(CONTAINER_REGISTRY_BASE)/$(DATA_OPS_REPO):$(FARMVIBES_AI_IMAGE_TAG) $(MAKE) set-registry-image DEPLOYMENT=$(DATA_OPS_DEPLOYMENT) REPLICAS=$(CURRENT_DATA_OPS_REPLICAS) make scale -local-worker: cluster local-worker-repo delete-$(WORKER_DEPLOYMENT) ## Builds and deploys a local WORKER image (enabling debug) - git lfs pull +local-worker: cluster restore-git-lfs local-worker-repo delete-$(WORKER_DEPLOYMENT) ## Builds and deploys a local WORKER image (enabling debug) DEPLOYMENT=$(WORKER_DEPLOYMENT) IMAGE_FULL_REFERENCE=$(WORKER_REPO):$(TAG) $(MAKE) -C . set-registry-image DEPLOYMENT=$(WORKER_DEPLOYMENT) $(MAKE) -C . disable-frozen-modules DEPLOYMENT=$(WORKER_DEPLOYMENT) REPLICAS=$(CURRENT_WORKER_REPLICAS) make scale diff --git a/docs/source/docfiles/markdown/QUICKSTART.md b/docs/source/docfiles/markdown/QUICKSTART.md index 6963a861..d1172cfc 100644 --- a/docs/source/docfiles/markdown/QUICKSTART.md +++ b/docs/source/docfiles/markdown/QUICKSTART.md @@ -22,6 +22,9 @@ In order to run FarmVibes.AI cluster, you need the following: the repository. If you already have access to the source code, then Git is not required. + * [Git LFS](https://git-lfs.com/) to restore some of the large files in the + repository (e.g., model weights). + * [Docker](https://docs.docker.com/engine/install/ubuntu/). Make sure you can run the docker client without running `sudo` by adding your user account to the `docker` group (which might require a logout/login when adding oneself @@ -60,6 +63,20 @@ bash ./resources/vm/setup_farmvibes_ai_vm.sh You might needed to restart your shell session once the script finishes. +## Optional: Restore files with Git LFS + +In case you did not have Git LFS installed when cloning the repository, you will need to do so +to restore the large files in the repository. Note that the last step +["Installing software dependencies](#optional-installing-software-dependencies) already installs +Git LFS. + +To restore the missing files, you can run the following command in the root of the repository: + +```shell +git lfs install +git lfs pull +``` + ## Install the FarmVibes.AI cluster With python3.8+ and pip installed on your machine, please install diff --git a/resources/vm/setup_farmvibes_ai_vm.sh b/resources/vm/setup_farmvibes_ai_vm.sh index 8c5ded9b..b0b5ee87 100755 --- a/resources/vm/setup_farmvibes_ai_vm.sh +++ b/resources/vm/setup_farmvibes_ai_vm.sh @@ -6,7 +6,7 @@ # Update apt sudo apt update -# Install git +# Install git and git-lfs sudo apt install git -y # Install python @@ -49,5 +49,7 @@ fi # Run docker without sudo sudo usermod -aG docker $DOCKER_USER -# Install git lfs -sudo apt install git-lfs \ No newline at end of file +# Run git-lfs install to restore large files +sudo apt install git-lfs -y +git lfs install +git lfs pull \ No newline at end of file From bc9152585b43a5e730f2a20a1abdd4582592a25c Mon Sep 17 00:00:00 2001 From: Rafael Padilha Date: Mon, 21 Oct 2024 20:36:34 +0000 Subject: [PATCH 3/5] Removing comment --- resources/vm/setup_farmvibes_ai_vm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/vm/setup_farmvibes_ai_vm.sh b/resources/vm/setup_farmvibes_ai_vm.sh index b0b5ee87..21d014ca 100755 --- a/resources/vm/setup_farmvibes_ai_vm.sh +++ b/resources/vm/setup_farmvibes_ai_vm.sh @@ -6,7 +6,7 @@ # Update apt sudo apt update -# Install git and git-lfs +# Install git sudo apt install git -y # Install python From d7bb6705ed317b309b21c8de43f0320ab07074e1 Mon Sep 17 00:00:00 2001 From: Rafael Soares Padilha Date: Mon, 21 Oct 2024 17:42:56 -0300 Subject: [PATCH 4/5] Update Makefile Co-authored-by: Renato L. de F. Cunha --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9757147a..71e3ae9a 100644 --- a/Makefile +++ b/Makefile @@ -50,7 +50,7 @@ local: cluster restore-git-lfs local-rest-api local-cache local-worker local-orc revert: cluster revert-rest-api revert-cache revert-worker revert-orchestrator ## Reverts all images to the official version restore-git-lfs: - git lfs pull +git lfs pull || echo "git lfs was not found. Please see https://git-lfs.com/ to install it." && exit 1 services-base: resources/docker/Dockerfile-services-base @docker manifest inspect `$(subst FILE,$<,$(base_image_name))` || \ From 90d1992fc4b75c0eb82eece1ab44262fc9441459 Mon Sep 17 00:00:00 2001 From: Rafael Padilha Date: Tue, 22 Oct 2024 12:59:41 +0000 Subject: [PATCH 5/5] Addressing PR comments --- docs/source/docfiles/markdown/QUICKSTART.md | 2 +- docs/source/docfiles/markdown/TROUBLESHOOTING.md | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/source/docfiles/markdown/QUICKSTART.md b/docs/source/docfiles/markdown/QUICKSTART.md index d1172cfc..878f3b27 100644 --- a/docs/source/docfiles/markdown/QUICKSTART.md +++ b/docs/source/docfiles/markdown/QUICKSTART.md @@ -63,7 +63,7 @@ bash ./resources/vm/setup_farmvibes_ai_vm.sh You might needed to restart your shell session once the script finishes. -## Optional: Restore files with Git LFS +## Restore files with Git LFS In case you did not have Git LFS installed when cloning the repository, you will need to do so to restore the large files in the repository. Note that the last step diff --git a/docs/source/docfiles/markdown/TROUBLESHOOTING.md b/docs/source/docfiles/markdown/TROUBLESHOOTING.md index d3e54963..ca53bfde 100644 --- a/docs/source/docfiles/markdown/TROUBLESHOOTING.md +++ b/docs/source/docfiles/markdown/TROUBLESHOOTING.md @@ -122,6 +122,15 @@ that are currently being addressed by the development team. +
+ Updating cluster in the `dev` branch after pulling files with Git LFS + + If you did not have Git LFS installed when cloning the repository and checking out to `dev`, + you will be missing some of the large files in the repository (e.g., ONNX models). Make sure + to install and setup Git LFS as described in the [Quickstart guide](QUICKSTART.md#restore-files-with-git-lfs). + You will also need to update your cluster with `make local`. +
+
- **Composing and running workflows:**