-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathstart.sh
132 lines (99 loc) · 3.93 KB
/
start.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
#!/bin/bash
if [ -z "$GCS_BUCKET_NAME" ]; then
echo "Error: GCS_BUCKET_NAME not provided"
exit 1
fi
if [ -z "$BRANCH_NAME" ]; then
echo "Error: BRANCH_NAME not provided"
exit 1
fi
if [ -z "$GITHUB_REPO" ]; then
echo "Error: GITHUB_REPO not provided"
exit 1
fi
mkdir bla
cd bla
# Clone the repository and checkout the specified branch
git clone $GITHUB_REPO .
git checkout ${BRANCH_NAME}
# Install any needed packages specified in requirements.txt
pip install --no-cache-dir -r requirements.txt
gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS
# Check and remove 'gs://' from GCS_BUCKET_NAME if present
if [[ $GCS_BUCKET_NAME == gs://* ]]; then
GCS_BUCKET_NAME=${GCS_BUCKET_NAME#gs://}
echo "Removed 'gs://' prefix from GCS_BUCKET_NAME. New value: $GCS_BUCKET_NAME"
fi
# Create the mount point directory if it doesn't exist
MOUNT_POINT="/mnt/gcs_bucket"
if [ ! -d "$MOUNT_POINT" ]; then
echo "Creating mount point directory: $MOUNT_POINT"
sudo mkdir -p $MOUNT_POINT
sudo chown $USER:$USER $MOUNT_POINT
fi
# Mount the GCS bucket using default credentials
# echo "Mounting GCS bucket: $GCS_BUCKET_NAME to $MOUNT_POINT"
# gcsfuse --implicit-dirs --key-file=$GOOGLE_APPLICATION_CREDENTIALS $GCS_BUCKET_NAME $MOUNT_POINT
# echo "Using publically available bucket"
# sudo mkdir -p image-cache
# gcsfuse --implicit-dirs --anonymous-access $GCS_BUCKET_NAME $MOUNT_POINT
# if [ $? -eq 0 ]; then
# echo "Successfully mounted GCS bucket to $MOUNT_POINT"
# # Sanity check: List contents of the mounted directory
# echo "Performing sanity check - listing contents of $MOUNT_POINT:"
# ls -la $MOUNT_POINT
# # Check if the directory is empty
# if [ -z "$(ls -A $MOUNT_POINT)" ]; then
# echo "Warning: The mounted directory appears to be empty."
# else
# echo "Sanity check passed: Directory contains files/folders."
# # Optional: List contents of a specific subdirectory if you know it exists
# # For example, if you know there's an 'imf' directory:
# if [ -d "$MOUNT_POINT/imf" ]; then
# echo "Contents of $MOUNT_POINT/celebvhq/35666/images:"
# ls -la $MOUNT_POINT/celebvhq/35666/images
# fi
# fi
# else
# echo "Failed to mount GCS bucket. Please check your credentials and bucket name."
# exit 1
# fi
# Create the mount point directory if it doesn't exist
mkdir -p $MOUNT_POINT
# Download and extract zip files from GCS bucket
echo "Downloading and extracting files from GCS bucket..."
gsutil -m cp -r gs://$GCS_BUCKET_NAME/*.zip $MOUNT_POINT/
if [ $? -eq 0 ]; then
echo "Successfully downloaded zip files from GCS bucket to $MOUNT_POINT"
# Extract all zip files
for zip_file in $MOUNT_POINT/*.zip; do
unzip -o $zip_file -d $MOUNT_POINT
rm $zip_file # Remove the zip file after extraction
done
echo "All zip files extracted and removed."
# Sanity check: List contents of the directory
echo "Performing sanity check - listing contents of $MOUNT_POINT:"
ls -la $MOUNT_POINT
# Check if the directory is empty
if [ -z "$(ls -A $MOUNT_POINT)" ]; then
echo "Warning: The directory appears to be empty."
else
echo "Sanity check passed: Directory contains files/folders."
fi
else
echo "Failed to download files from GCS bucket. Please check your credentials and bucket name."
exit 1
fi
echo "Setup complete. $MOUNT_POINT now contains the extracted contents of the GCS bucket."
# echo "Running training.... 🏄"
# # Run the training script
# python train.py
# # Unmount the GCS bucket
# fusermount -u /mnt/gcs_bucket
echo "Environment setup complete. Waiting for interactive web access..."
echo "You can now use the Vertex AI console to access this instance interactively."
echo "To start training manually, run: python train.py"
# Wait indefinitely
while true; do
sleep 3600 # Sleep for 1 hour
done