Merge pull request #66 from PerfectThymeTech/marvinbuss/datafactory

Add Data Factory Module

.github/workflows/_terraformTestTemplate.yml

@@ -27,6 +27,11 @@ on:
         required: true
         type: string
         description: "Specifies the subscription id of the deployment."
+      run_azure_login:
+        required: false
+        type: boolean
+        description: "Specifies whether the test should be executed."
+        default: false
       run_test:
         required: false
         type: boolean
@@ -135,6 +140,16 @@ jobs:
         id: checkout_repository
         uses: actions/checkout@v4
 
+      # Azure login
+      - name: Azure login
+        id: azure_login
+        uses: azure/login@v2
+        if: inputs.run_azure_login
+        with:
+          client-id: ${{ secrets.CLIENT_ID }}
+          tenant-id: ${{ inputs.tenant_id }}
+          subscription-id: ${{ inputs.subscription_id }}
+
       # Terraform Override
       - name: Terraform Override
         run: |

.github/workflows/terraform-datafactory.yml

@@ -0,0 +1,30 @@
+name: datafactory tests
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "modules/datafactory/**"
+      - ".github/workflows/terraform-datafactory.yml"
+
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "modules/datafactory/**"
+      - ".github/workflows/terraform-datafactory.yml"
+
+jobs:
+  terraform_dev:
+    uses: ./.github/workflows/_terraformTestTemplate.yml
+    name: "Dev"
+    with:
+      environment: "dev"
+      config: "datafactory"
+      terraform_version: "1.9.6"
+      node_version: 20
+      tenant_id: "37963dd4-f4e6-40f8-a7d6-24b97919e452"
+      subscription_id: "1fdab118-1638-419a-8b12-06c9543714a0"
+      run_azure_login: true
+    secrets:
+      CLIENT_ID: ${{ secrets.CLIENT_ID }}

.pre-commit-config.yaml

@@ -9,6 +9,7 @@ exclude: |
     modules/botservice/README.md|
     modules/containerregistry/README.md|
     modules/cosmosdb/README.md|
+    modules/datafactory/README.md|
     modules/fabriccapacity/README.md|
     modules/fabricworkspace/README.md|
     modules/keyvault/README.md|
@@ -48,6 +49,8 @@ repos:
       args: ["-c", "./.terraform-docs.yml", "./modules/containerregistry"]
     - id: terraform-docs-go
       args: ["-c", "./.terraform-docs.yml", "./modules/cosmosdb"]
+    - id: terraform-docs-go
+      args: ["-c", "./.terraform-docs.yml", "./modules/datafactory"]
     - id: terraform-docs-go
       args: ["-c", "./.terraform-docs.yml", "./modules/fabriccapacity"]
     - id: terraform-docs-go

README.md

@@ -12,6 +12,7 @@ This repository contains secure by default Terraform modules for Azure Services.
 - [Azure AI Bot Service](/modules/botservice/)
 - [Azure Container Registry](/modules/containerregistry/)
 - [Azure Cosmos DB](/modules/cosmosdb/)
+- [Azure Data Factory](/modules/datafactory/)
 - [Azure Key Vault](/modules/keyvault/)
 - [Azure Log Analytics Workspace](/modules/loganalytics/)
 - [Azure Storage Account](/modules/storage/)

modules/datafactory/README.md

@@ -0,0 +1,247 @@
+<!-- BEGIN_TF_DOCS -->
+# Azure Data Factory Terraform Module
+
+NOTE: The `data_factory_triggers_start` and `data_factory_pipelines_run` features rely on Azure CLI and therefore require you to be authenticated.
+
+## Documentation
+<!-- markdownlint-disable MD033 -->
+
+## Requirements
+
+The following requirements are needed by this module:
+
+- <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) (>=0.12)
+
+- <a name="requirement_azapi"></a> [azapi](#requirement\_azapi) (>= 1.14.0)
+
+- <a name="requirement_azurerm"></a> [azurerm](#requirement\_azurerm) (>= 4.0.0)
+
+## Modules
+
+No modules.
+
+<!-- markdownlint-disable MD013 -->
+<!-- markdownlint-disable MD034 -->
+## Required Inputs
+
+The following input variables are required:
+
+### <a name="input_data_factory_name"></a> [data\_factory\_name](#input\_data\_factory\_name)
+
+Description: Specifies the name of the data factory.
+
+Type: `string`
+
+### <a name="input_location"></a> [location](#input\_location)
+
+Description: Specifies the location of all resources.
+
+Type: `string`
+
+### <a name="input_resource_group_name"></a> [resource\_group\_name](#input\_resource\_group\_name)
+
+Description: Specifies the resource group name in which all resources will get deployed.
+
+Type: `string`
+
+### <a name="input_subnet_id"></a> [subnet\_id](#input\_subnet\_id)
+
+Description: Specifies the resource id of a subnet in which the private endpoints get created.
+
+Type: `string`
+
+## Optional Inputs
+
+The following input variables are optional (have default values):
+
+### <a name="input_connectivity_delay_in_seconds"></a> [connectivity\_delay\_in\_seconds](#input\_connectivity\_delay\_in\_seconds)
+
+Description: Specifies the delay in seconds after the private endpoint deployment (required for the DNS automation via Policies).
+
+Type: `number`
+
+Default: `120`
+
+### <a name="input_customer_managed_key"></a> [customer\_managed\_key](#input\_customer\_managed\_key)
+
+Description: Specifies the customer managed key configurations.
+
+Type:
+
+```hcl
+object({
+    key_vault_id                     = string,
+    key_vault_key_versionless_id     = string,
+    user_assigned_identity_id        = string,
+    user_assigned_identity_client_id = string,
+  })
+```
+
+Default: `null`
+
+### <a name="input_data_factory_azure_devops_repo"></a> [data\_factory\_azure\_devops\_repo](#input\_data\_factory\_azure\_devops\_repo)
+
+Description: Specifies the Azure Devops repository configuration.
+
+Type:
+
+```hcl
+object(
+    {
+      account_name    = optional(string, "")
+      branch_name     = optional(string, "")
+      project_name    = optional(string, "")
+      repository_name = optional(string, "")
+      root_folder     = optional(string, "")
+      tenant_id       = optional(string, "")
+    }
+  )
+```
+
+Default: `{}`
+
+### <a name="input_data_factory_github_repo"></a> [data\_factory\_github\_repo](#input\_data\_factory\_github\_repo)
+
+Description: Specifies the Github repository configuration.
+
+Type:
+
+```hcl
+object(
+    {
+      account_name    = optional(string, "")
+      branch_name     = optional(string, "")
+      git_url         = optional(string, "")
+      repository_name = optional(string, "")
+      root_folder     = optional(string, "")
+    }
+  )
+```
+
+Default: `{}`
+
+### <a name="input_data_factory_global_parameters"></a> [data\_factory\_global\_parameters](#input\_data\_factory\_global\_parameters)
+
+Description: Specifies the Azure Data Factory global parameters.
+
+Type:
+
+```hcl
+map(object({
+    type  = optional(string, "String")
+    value = optional(any, "")
+  }))
+```
+
+Default: `{}`
+
+### <a name="input_data_factory_managed_private_endpoints"></a> [data\_factory\_managed\_private\_endpoints](#input\_data\_factory\_managed\_private\_endpoints)
+
+Description: Specifies custom template variables to use for the deployment templates from ADF.
+
+Type:
+
+```hcl
+map(object({
+    subresource_name   = string
+    target_resource_id = string
+  }))
+```
+
+Default: `{}`
+
+### <a name="input_data_factory_pipelines_run"></a> [data\_factory\_pipelines\_run](#input\_data\_factory\_pipelines\_run)
+
+Description: Specifies the list of pipeline names that should be started after the deployment.
+
+Type: `list(string)`
+
+Default: `[]`
+
+### <a name="input_data_factory_published_content"></a> [data\_factory\_published\_content](#input\_data\_factory\_published\_content)
+
+Description: Specifies the Azure Devops repository configuration.
+
+Type:
+
+```hcl
+object(
+    {
+      parameters_file = optional(string, "")
+      template_file   = optional(string, "")
+    }
+  )
+```
+
+Default: `{}`
+
+### <a name="input_data_factory_published_content_template_variables"></a> [data\_factory\_published\_content\_template\_variables](#input\_data\_factory\_published\_content\_template\_variables)
+
+Description: Specifies custom template variables to use for the deployment templates from ADF.
+
+Type: `map(string)`
+
+Default: `{}`
+
+### <a name="input_data_factory_purview_id"></a> [data\_factory\_purview\_id](#input\_data\_factory\_purview\_id)
+
+Description: Specifies the resource id of purview that should be connnected to the data factory.
+
+Type: `string`
+
+Default: `null`
+
+### <a name="input_data_factory_triggers_start"></a> [data\_factory\_triggers\_start](#input\_data\_factory\_triggers\_start)
+
+Description: Specifies the list of trigger names that should be started after the deployment.
+
+Type: `list(string)`
+
+Default: `[]`
+
+### <a name="input_diagnostics_configurations"></a> [diagnostics\_configurations](#input\_diagnostics\_configurations)
+
+Description: Specifies the diagnostic configuration for the service.
+
+Type:
+
+```hcl
+list(object({
+    log_analytics_workspace_id = optional(string, ""),
+    storage_account_id         = optional(string, "")
+  }))
+```
+
+Default: `[]`
+
+### <a name="input_private_dns_zone_id_data_factory"></a> [private\_dns\_zone\_id\_data\_factory](#input\_private\_dns\_zone\_id\_data\_factory)
+
+Description: Specifies the resource ID of the private DNS zone for Azure Data Factory. Not required if DNS A-records get created via Azure Policy.
+
+Type: `string`
+
+Default: `""`
+
+### <a name="input_tags"></a> [tags](#input\_tags)
+
+Description: Specifies a key value map of tags to set on every taggable resources.
+
+Type: `map(string)`
+
+Default: `{}`
+
+## Outputs
+
+The following outputs are exported:
+
+### <a name="output_data_factory_id"></a> [data\_factory\_id](#output\_data\_factory\_id)
+
+Description: Specifies the id of the data factory.
+
+### <a name="output_fabric_capacity_name"></a> [fabric\_capacity\_name](#output\_fabric\_capacity\_name)
+
+Description: Specifies the name of the data factory.
+
+<!-- markdownlint-enable -->
+
+<!-- END_TF_DOCS -->

modules/datafactory/README_header.md

@@ -0,0 +1,3 @@
+# Azure Data Factory Terraform Module
+
+NOTE: The `data_factory_triggers_start` and `data_factory_pipelines_run` features rely on Azure CLI and therefore require you to be authenticated.

modules/datafactory/connectivity.tf

@@ -0,0 +1,38 @@
+resource "azurerm_private_endpoint" "data_factory_private_endpoint_data_factory" {
+  name                = "${azurerm_data_factory.data_factory.name}-datafactory-pe"
+  location            = var.location
+  resource_group_name = azurerm_data_factory.data_factory.resource_group_name
+  tags                = var.tags
+
+  custom_network_interface_name = "${azurerm_data_factory.data_factory.name}-datafactory-nic"
+  private_service_connection {
+    name                           = "${azurerm_data_factory.data_factory.name}-datafactory-svc"
+    is_manual_connection           = false
+    private_connection_resource_id = azurerm_data_factory.data_factory.id
+    subresource_names              = ["dataFactory"]
+  }
+  subnet_id = var.subnet_id
+  dynamic "private_dns_zone_group" {
+    for_each = var.private_dns_zone_id_data_factory == "" ? [] : [1]
+    content {
+      name = "${azurerm_data_factory.data_factory.name}-arecord"
+      private_dns_zone_ids = [
+        var.private_dns_zone_id_data_factory
+      ]
+    }
+  }
+
+  lifecycle {
+    ignore_changes = [
+      private_dns_zone_group
+    ]
+  }
+}
+
+resource "time_sleep" "sleep_connectivity" {
+  create_duration = "${var.connectivity_delay_in_seconds}s"
+
+  depends_on = [
+    azurerm_private_endpoint.data_factory_private_endpoint_data_factory,
+  ]
+}