Merge pull request #246 from lukaszbudnik/azureblob-support-for-optio…

…nal-prefixes

implemented support for optional prefixes for Azure Blob Container

.gitignore

@@ -1,3 +1,4 @@
+.DS_Store
 migrator
 test/migrator.yaml
 coverage-*.txt

README.md

@@ -552,18 +552,22 @@ If `baseLocation` starts with `s3://` prefix, AWS S3 implementation is used. In
 
 ```
 # S3 bucket
-baseLocation: s3://your-bucket-migrator/application-x/prod
+baseLocation: s3://your-bucket-migrator
+# S3 bucket with optional prefix
+baseLocation: s3://your-bucket-migrator/appcodename/prod/artefacts
 ```
 
 migrator uses official AWS SDK for Go and uses a well known [default credential provider chain](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html). Please setup your env variables accordingly.
 
 ### Azure Blob Containers
 
-If `baseLocation` matches `^https://.*\.blob\.core\.windows\.net/.*` regex, Azure Blob implementation is used. In such case the `baseLocation` property is treated as a container URL:
+If `baseLocation` matches `^https://.*\.blob\.core\.windows\.net/.*` regex, Azure Blob implementation is used. In such case the `baseLocation` property is treated as a container URL. The URL can have optional prefix too:
 
 ```
 # Azure Blob container URL
 baseLocation: https://storageaccountname.blob.core.windows.net/mycontainer
+# Azure Blob container URL with optional prefix
+baseLocation: https://storageaccountname.blob.core.windows.net/mycontainer/appcodename/prod/artefacts
 ```
 
 migrator uses official Azure Blob SDK for Go. Unfortunately as of the time of writing Azure Blob implementation the SDK only supported authentication using Storage Accounts and not for example much more flexible Active Directory (which is supported by the rest of the Azure Go SDK). Issue to watch: [Authorization via Azure AD / RBAC](https://github.com/Azure/azure-storage-blob-go/issues/160). I plan to revisit the authorization once Azure team updates their Azure Blob SDK.

loader/azureblob_loader.go

@@ -28,29 +28,45 @@ func (abl *azureBlobLoader) GetSourceMigrations() []types.Migration {
 	}
 
 	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	if err != nil {
+		panic(err.Error())
+	}
 
 	p := azblob.NewPipeline(credential, azblob.PipelineOptions{})
 
-	u, err := url.Parse(abl.config.BaseLocation)
+	// migrator expects that container as a part of the service url
+	// the URL can contain optional prefixes like prod/artefacts
+	// for example:
+	// https://lukaszbudniktest.blob.core.windows.net/mycontainer/
+	// https://lukaszbudniktest.blob.core.windows.net/mycontainer/prod/artefacts/
+
+	// check if optional prefixes are provided
+	baseLocation := strings.TrimRight(abl.config.BaseLocation, "/")
+	indx := findNthIndex(baseLocation, '/', 4)
+
+	optionalPrefixes := ""
+	if indx > -1 {
+		optionalPrefixes = baseLocation[indx+1:]
+		baseLocation = baseLocation[:indx]
+	}
+
+	u, err := url.Parse(baseLocation)
 	if err != nil {
 		panic(err.Error())
 	}
 
-	// migrator expects that container as a part of the service url
-	// for example: https://lukaszbudniktest.blob.core.windows.net/mycontainer
-	// Azure SDK will correctly parse the account and the container
 	containerURL := azblob.NewContainerURL(*u, p)
 
-	return abl.doGetSourceMigrations(containerURL)
+	return abl.doGetSourceMigrations(containerURL, optionalPrefixes)
 }
 
-func (abl *azureBlobLoader) doGetSourceMigrations(containerURL azblob.ContainerURL) []types.Migration {
+func (abl *azureBlobLoader) doGetSourceMigrations(containerURL azblob.ContainerURL, optionalPrefixes string) []types.Migration {
 	migrations := []types.Migration{}
 
-	singleMigrationsObjects := abl.getObjectList(containerURL, abl.config.SingleMigrations)
-	tenantMigrationsObjects := abl.getObjectList(containerURL, abl.config.TenantMigrations)
-	singleScriptsObjects := abl.getObjectList(containerURL, abl.config.SingleScripts)
-	tenantScriptsObjects := abl.getObjectList(containerURL, abl.config.TenantScripts)
+	singleMigrationsObjects := abl.getObjectList(containerURL, optionalPrefixes, abl.config.SingleMigrations)
+	tenantMigrationsObjects := abl.getObjectList(containerURL, optionalPrefixes, abl.config.TenantMigrations)
+	singleScriptsObjects := abl.getObjectList(containerURL, optionalPrefixes, abl.config.SingleScripts)
+	tenantScriptsObjects := abl.getObjectList(containerURL, optionalPrefixes, abl.config.TenantScripts)
 
 	migrationsMap := make(map[string][]types.Migration)
 	abl.getObjects(containerURL, migrationsMap, singleMigrationsObjects, types.MigrationTypeSingleMigration)
@@ -68,14 +84,21 @@ func (abl *azureBlobLoader) doGetSourceMigrations(containerURL azblob.ContainerU
 	return migrations
 }
 
-func (abl *azureBlobLoader) getObjectList(containerURL azblob.ContainerURL, prefixes []string) []string {
+func (abl *azureBlobLoader) getObjectList(containerURL azblob.ContainerURL, optionalPrefixes string, prefixes []string) []string {
 	objects := []string{}
 
 	for _, prefix := range prefixes {
 
-		for marker := (azblob.Marker{}); marker.NotDone(); { // The parens around Marker{} are required to avoid compiler error.
+		for marker := (azblob.Marker{}); marker.NotDone(); {
 
-			listBlob, err := containerURL.ListBlobsFlatSegment(abl.ctx, marker, azblob.ListBlobsSegmentOptions{Prefix: prefix + "/"})
+			var fullPrefix string
+			if optionalPrefixes != "" {
+				fullPrefix = optionalPrefixes + "/" + prefix + "/"
+			} else {
+				fullPrefix = prefix + "/"
+			}
+
+			listBlob, err := containerURL.ListBlobsFlatSegment(abl.ctx, marker, azblob.ListBlobsSegmentOptions{Prefix: fullPrefix})
 			if err != nil {
 				panic(err.Error())
 			}
@@ -125,3 +148,16 @@ func (abl *azureBlobLoader) getObjects(containerURL azblob.ContainerURL, migrati
 
 	}
 }
+
+func findNthIndex(str string, c byte, n int) int {
+	occur := 0
+	for i := 0; i < len(str); i++ {
+		if str[i] == c {
+			occur += 1
+		}
+		if occur == n {
+			return i
+		}
+	}
+	return -1
+}

loader/azureblob_loader_test.go

@@ -51,3 +51,41 @@ func TestAzureGetSourceMigrations(t *testing.T) {
 	assert.Contains(t, migrations[11].File, "migrations/tenants-scripts/b.sql")
 
 }
+
+func TestAzureGetSourceMigrationsWithOptionalPrefix(t *testing.T) {
+
+	accountName, accountKey := os.Getenv("AZURE_STORAGE_ACCOUNT"), os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+
+	if len(accountName) == 0 || len(accountKey) == 0 {
+		t.Skip("skipping test AZURE_STORAGE_ACCOUNT or AZURE_STORAGE_ACCESS_KEY not set")
+	}
+
+	baseLocation := fmt.Sprintf("https://%v.blob.core.windows.net/myothercontainer/prod/artefacts/", accountName)
+
+	config := &config.Config{
+		BaseLocation:     baseLocation,
+		SingleMigrations: []string{"migrations/config", "migrations/ref"},
+		TenantMigrations: []string{"migrations/tenants"},
+		SingleScripts:    []string{"migrations/config-scripts"},
+		TenantScripts:    []string{"migrations/tenants-scripts"},
+	}
+
+	loader := &azureBlobLoader{baseLoader{context.TODO(), config}}
+	migrations := loader.GetSourceMigrations()
+
+	assert.Len(t, migrations, 12)
+
+	assert.Contains(t, migrations[0].File, "prod/artefacts/migrations/config/201602160001.sql")
+	assert.Contains(t, migrations[1].File, "prod/artefacts/migrations/config/201602160002.sql")
+	assert.Contains(t, migrations[2].File, "prod/artefacts/migrations/tenants/201602160002.sql")
+	assert.Contains(t, migrations[3].File, "prod/artefacts/migrations/ref/201602160003.sql")
+	assert.Contains(t, migrations[4].File, "prod/artefacts/migrations/tenants/201602160003.sql")
+	assert.Contains(t, migrations[5].File, "prod/artefacts/migrations/ref/201602160004.sql")
+	assert.Contains(t, migrations[6].File, "prod/artefacts/migrations/tenants/201602160004.sql")
+	assert.Contains(t, migrations[7].File, "prod/artefacts/migrations/tenants/201602160005.sql")
+	assert.Contains(t, migrations[8].File, "prod/artefacts/migrations/config-scripts/200012181227.sql")
+	assert.Contains(t, migrations[9].File, "prod/artefacts/migrations/tenants-scripts/200001181228.sql")
+	assert.Contains(t, migrations[10].File, "prod/artefacts/migrations/tenants-scripts/a.sql")
+	assert.Contains(t, migrations[11].File, "prod/artefacts/migrations/tenants-scripts/b.sql")
+
+}