feat: dataset preparations

.gitignore

@@ -15,3 +15,5 @@ gcs_mount
 datacache
 *.deb
 gcsfuse.yml
+*.csv
+*.tsv

datasets/cc12m downloader.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-img2dataset --url_list ./datacache/cc12m.csv --input_format "csv"\
+img2dataset --url_list ./cc12m.tsv --input_format "tsv"\
          --url_col "image_url" --caption_col "caption" --output_format arrayrecord\
-           --output_folder gs://flaxdiff-datasets-regional/arrayrecord/cc12m --processes_count 64
+           --output_folder gs://flaxdiff-datasets-regional/arrayrecord/cc12m --processes_count 64\
  --thread_count 64 --image_size 256\
              --enable_wandb True --disallowed_header_directives '[]' --compute_hash None --max_shard_retry 3 --timeout 60

datasets/laion_a+coco17.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+img2dataset --url_list $HOME/research/laion-aesthetics-12m+mscoco-2017.parquet --input_format "parquet"\
+         --url_col "url" --caption_col "caption" --output_format arrayrecord\
+           --output_folder gs://flaxdiff-datasets-regional/arrayrecord/laion-aesthetics-12m+mscoco-2017 --processes_count 64\
+          --thread_count 64 --image_size 256 --min_image_size 100 \
+             --enable_wandb True --disallowed_header_directives '[]' --compute_hash None --max_shard_retry 3 --timeout 60

tpu_utils/setup_tpu.sh

@@ -14,23 +14,6 @@ sudo apt update
 sudo apt install -y knot-resolver
 sudo sh -c 'echo `hostname -I` `hostname` >> /etc/hosts'
 sudo sh -c 'echo nameserver 127.0.0.1 > /etc/resolv.conf'
-sudo systemctl stop systemd-resolved
-sudo systemctl start kresd@1.service
-sudo systemctl start kresd@2.service
-sudo systemctl start kresd@3.service
-sudo systemctl start kresd@4.service
-sudo systemctl start kresd@5.service
-sudo systemctl start kresd@6.service
-sudo systemctl start kresd@7.service
-sudo systemctl start kresd@8.service
-sudo systemctl start kresd@9.service
-sudo systemctl start kresd@10.service
-sudo systemctl start kresd@11.service
-sudo systemctl start kresd@12.service
-sudo systemctl start kresd@13.service
-sudo systemctl start kresd@14.service
-sudo systemctl start kresd@15.service
-sudo systemctl start kresd@16.service
 
 # Backup the original resolv.conf
 sudo cp /etc/resolv.conf /etc/resolv.conf.bak
@@ -56,6 +39,24 @@ for ns in "${nameservers[@]}"; do
 done
 echo "Nameservers added to /etc/resolv.conf"
 
+sudo systemctl stop systemd-resolved
+sudo systemctl start kresd@1.service
+sudo systemctl start kresd@2.service
+sudo systemctl start kresd@3.service
+sudo systemctl start kresd@4.service
+sudo systemctl start kresd@5.service
+sudo systemctl start kresd@6.service
+sudo systemctl start kresd@7.service
+sudo systemctl start kresd@8.service
+sudo systemctl start kresd@9.service
+sudo systemctl start kresd@10.service
+sudo systemctl start kresd@11.service
+sudo systemctl start kresd@12.service
+sudo systemctl start kresd@13.service
+sudo systemctl start kresd@14.service
+sudo systemctl start kresd@15.service
+sudo systemctl start kresd@16.service
+
 # Installing and setting up gcsfuse
 export GCSFUSE_REPO=gcsfuse-`lsb_release -c -s`
 echo "deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt $GCSFUSE_REPO main" | sudo tee /etc/apt/sources.list.d/gcsfuse.list
@@ -134,9 +135,9 @@ if [ -n "$GCS_BUCKET" ]; then
 
     # Make the script executable
     chmod +x $LOCAL_FILE
-
+    echo "Mounting GCS bucket: $GCS_BUCKET to $HOME/gcs_mount"
     # Run the script with the specified arguments
-    ./$LOCAL_FILE DATASET_GCS_BUCKET=$GCS_BUCKET MOUNT_PATH=/mnt/gcs_mount
+    ./$LOCAL_FILE DATASET_GCS_BUCKET=$GCS_BUCKET MOUNT_PATH=$HOME/gcs_mount
 fi
 
 if [ "$DEV_MODE" = true ]; then