Added ability to specify a subfolder in the GCS bucket (#264)

Author: abalalaev

Dockerfile

@@ -18,4 +18,4 @@ RUN pip3 --no-cache-dir install .
 
 RUN chown -R 1337:1337 /opt/grafana-backup-tool
 USER 1337
-CMD sh -c 'if [ "$RESTORE" = true ]; then if [ ! -z "$AWS_S3_BUCKET_NAME" ] || [ ! -z "$AZURE_STORAGE_CONTAINER_NAME" ]; then grafana-backup restore $ARCHIVE_FILE; else grafana-backup restore _OUTPUT_/$ARCHIVE_FILE; fi else grafana-backup save; fi'
+CMD sh -c 'if [ "$RESTORE" = true ]; then if [ ! -z "$AWS_S3_BUCKET_NAME" ] || [ ! -z "$AZURE_STORAGE_CONTAINER_NAME" ] || [ ! -z "$GCS_BUCKET_NAME" ]; then grafana-backup restore $ARCHIVE_FILE; else grafana-backup restore _OUTPUT_/$ARCHIVE_FILE; fi else grafana-backup save; fi'

DockerfileSlim

@@ -37,4 +37,4 @@ RUN echo "@edge http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk
     && apk del build-deps
 
 USER ${UID}
-CMD sh -c 'if [ "$RESTORE" = true ]; then if [ ! -z "$AWS_S3_BUCKET_NAME" ] || [ ! -z "$AZURE_STORAGE_CONTAINER_NAME" ]; then grafana-backup restore $ARCHIVE_FILE; else grafana-backup restore _OUTPUT_/$ARCHIVE_FILE; fi else grafana-backup save; fi'
+CMD sh -c 'if [ "$RESTORE" = true ]; then if [ ! -z "$AWS_S3_BUCKET_NAME" ] || [ ! -z "$AZURE_STORAGE_CONTAINER_NAME" ] || [ ! -z "$GCS_BUCKET_NAME" ]; then grafana-backup restore $ARCHIVE_FILE; else grafana-backup restore _OUTPUT_/$ARCHIVE_FILE; fi else grafana-backup save; fi'

README.md

@@ -173,7 +173,8 @@ docker run --user $(id -u):$(id -g) --rm --name grafana-backup-tool \
 
 ***GCS Example:*** Set GCS configurations in `-e` or `grafanaSettings.json`([example](https://github.com/ysde/grafana-backup-tool/blob/master/examples/grafana-backup.example.json))
 ```
-		   -e GCS_BUCKET_NAME="bucket-name" \
+		   -e GCS_BUCKET_NAME="backups-bucket-name" \
+		   -e GCS_BUCKET_PATH="grafana-backup-folder" \
 		   -e GCLOUD_PROJECT="gcp-project-name" \
 		   -e GOOGLE_APPLICATION_CREDENTIALS="credential-file-path"
 ```

examples/grafanaSettings.example.json

@@ -25,6 +25,7 @@
   },
   "gcp": {
     "gcs_bucket_name": "bucket_name",
+    "gcs_bucket_path": "grafana-backup",
     "google_application_credentials": "google_credential_file_path"
   },
   "influxdb": {

grafana_backup/gcs_download.py

@@ -7,11 +7,15 @@ def main(args, settings):
     arg_archive_file = args.get('<archive_file>', None)
 
     bucket_name = settings.get('GCS_BUCKET_NAME')
+    bucket_path = settings.get('GCS_BUCKET_PATH').strip('/')
 
     storage_client = storage.Client()
+
+    gcs_blob_name = arg_archive_file if bucket_path == '' else '{0}/{1}'.format(bucket_path, arg_archive_file)
+
     bucket = storage_client.bucket(bucket_name)
 
-    blob = bucket.blob(arg_archive_file)
+    blob = bucket.blob(gcs_blob_name)
 
     try:
         gcs_data = io.BytesIO(blob.download_as_bytes())
@@ -23,7 +27,7 @@ def main(args, settings):
         print("Permission denied: {0}, please grant `Storage Admin` to service account you used".format(str(e)))
         return False
     except api_core.exceptions.NotFound:
-        print("The file: {0} or gcs bucket: {1} doesn't exist".format(arg_archive_file, bucket_name))
+        print("The file: {0} or gcs bucket: {1} doesn't exist".format(gcs_blob_name, bucket_name))
         return False
     except Exception as e:
         print("Exception: {0}".format(str(e)))

grafana_backup/gcs_upload.py

@@ -4,23 +4,25 @@
 
 def main(args, settings):
     bucket_name = settings.get('GCS_BUCKET_NAME')
+    bucket_path = settings.get('GCS_BUCKET_PATH').strip('/')
     backup_dir = settings.get('BACKUP_DIR')
     timestamp = settings.get('TIMESTAMP')
 
     storage_client = storage.Client()
 
     gcs_file_name = '{0}.tar.gz'.format(timestamp)
     archive_file = '{0}/{1}'.format(backup_dir, gcs_file_name)
+    gcs_blob_name = gcs_file_name if bucket_path == '' else '{0}/{1}'.format(bucket_path, gcs_file_name)
 
     try:
         bucket = storage_client.bucket(bucket_name)
 
-        blob = bucket.blob(gcs_file_name)
+        blob = bucket.blob(gcs_blob_name)
         blob.upload_from_filename(archive_file)
 
         print("Upload to gcs: was successful")
     except FileNotFoundError:  # noqa: F821
-        print("The file: {0} was not found".format(gcs_file_name))
+        print("The file: {0} was not found".format(archive_file))
         return False
     except api_core.exceptions.Forbidden as e:
         print("Permission denied: {0}, please grant `Storage Admin` to service account you used".format(str(e)))

grafana_backup/grafanaSettings.py

@@ -43,6 +43,7 @@ def main(config_path):
     # Cloud storage settings - GCP
     gcp_config = config.get('gcp', {})
     gcs_bucket_name = gcp_config.get('gcs_bucket_name', '')
+    gcs_bucket_path = gcp_config.get('gcs_bucket_path', '')
     google_application_credentials = gcp_config.get('google_application_credentials', '')
 
     influxdb_measurement = config.get('influxdb', {}).get('measurement', 'grafana_backup')
@@ -72,6 +73,7 @@ def main(config_path):
     AZURE_STORAGE_CONNECTION_STRING = os.getenv('AZURE_STORAGE_CONNECTION_STRING', azure_storage_connection_string)
 
     GCS_BUCKET_NAME = os.getenv('GCS_BUCKET_NAME', gcs_bucket_name)
+    GCS_BUCKET_PATH = os.getenv('GCS_BUCKET_PATH', gcs_bucket_path)
     if not os.getenv('GOOGLE_APPLICATION_CREDENTIALS') and google_application_credentials:
         os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = google_application_credentials
 
@@ -177,6 +179,7 @@ def main(config_path):
     config_dict['AZURE_STORAGE_CONTAINER_NAME'] = AZURE_STORAGE_CONTAINER_NAME
     config_dict['AZURE_STORAGE_CONNECTION_STRING'] = AZURE_STORAGE_CONNECTION_STRING
     config_dict['GCS_BUCKET_NAME'] = GCS_BUCKET_NAME
+    config_dict['GCS_BUCKET_PATH'] = GCS_BUCKET_PATH
     config_dict['INFLUXDB_MEASUREMENT'] = INFLUXDB_MEASUREMENT
     config_dict['INFLUXDB_HOST'] = INFLUXDB_HOST
     config_dict['INFLUXDB_PORT'] = INFLUXDB_PORT