diff --git a/.github/workflows/cd-dev.yml b/.github/workflows/cd-dev.yml index 28457413..3a970f42 100644 --- a/.github/workflows/cd-dev.yml +++ b/.github/workflows/cd-dev.yml @@ -11,8 +11,9 @@ jobs: uses: ./.github/workflows/deployment.yml with: rg: 'MartinkMe-Dev' - uniquelabel: 'MartinkMeDev' + unique-label: 'mkmedev' secrets: azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} - azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} \ No newline at end of file + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + github-pat: ${{ secrets.GH_PAT }} \ No newline at end of file diff --git a/.github/workflows/cd-prod.yml b/.github/workflows/cd-prod.yml index ef26f407..b15140e3 100644 --- a/.github/workflows/cd-prod.yml +++ b/.github/workflows/cd-prod.yml @@ -11,8 +11,9 @@ jobs: uses: ./.github/workflows/deployment.yml with: rg: 'MartinkMe-Prod' - uniquelabel: 'MartinkMeProd' + unique-label: 'mkmeprod' secrets: azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} - azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} \ No newline at end of file + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + github-pat: ${{ secrets.GH_PAT }} \ No newline at end of file diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index d83c9145..fa686348 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -6,7 +6,7 @@ on: rg: type: string required: true - uniquelabel: + unique-label: type: string required: true secrets: @@ -16,6 +16,8 @@ on: required: true azure-subscription-id: required: true + github-pat: + required: true permissions: @@ -54,14 +56,30 @@ jobs: - name: bicep-install run: az bicep upgrade + - name: bicep-parameters + run: | + echo "{ + \"$schema\": \"https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#\", + \"contentVersion\": \"1.0.0.0\", + \"parameters\": { + \"githubPat\": { + \"value\": \"${{ secrets.github-pat }}\" + }, + \"uniqueLabel\": { + \"value\": \"${{ inputs.unique-label }}\" + } + } + }" > parameters.json + - name: bicep-deploy uses: azure/arm-deploy@v2 with: subscriptionId: ${{ secrets.azure-subscription-id }} resourceGroupName: ${{ inputs.rg }} template: ./bicep/main.bicep - failOnStdErr: false - parameters: 'uniqueName=${{ inputs.uniquelabel }}' + failOnStdErr: true + parameters: parameters.json + deploymentMode: Incremental # Web App code - name: bicep-output-webappname @@ -104,3 +122,8 @@ jobs: with: app-name: ${{ env.FunctionAppName }} package: './src/Workflow/output' + + # Capture and log the Bicep outputs + - name: output-unique-name + run: | + echo "uniqueName: ${{ steps.bicep-deploy.outputs.uniqueName }}" diff --git a/.gitignore b/.gitignore index 4d80b01c..aae7f385 100644 --- a/.gitignore +++ b/.gitignore @@ -249,3 +249,4 @@ Src/Workflow/__queuestorage* src/Workflow/local.settings.json src/Web/appsettings.Development.json /src/.idea +src/DataSync/appsettings.development.json diff --git a/bicep/main.bicep b/bicep/main.bicep index 8ffb0604..8e53086c 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -1,11 +1,14 @@ -@description('The name of the Azure Function app.') -param uniqueName string = toLower(uniqueString('${resourceGroup().id}')) +@description('Github PAT.') +param githubPat string + +@description('A unique name for all resources.') +param uniqueLabel string @description('Location for all resources.') -param location string = resourceGroup().location +param location string = resourceGroup().location // Nothing is being passed so this will use the default //STORAGE ACCOUNT -var storageAccountName = toLower('storage${uniqueName}') +var storageAccountName = toLower('storage${uniqueLabel}') resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' = { name: storageAccountName location: location @@ -54,7 +57,7 @@ resource storageAccountTableServiceShortcutsTable 'Microsoft.Storage/storageAcco } //APP INSIGHTS -var appInsightsName = toLower('appinisghts-${uniqueName}') +var appInsightsName = toLower('appinisghts-${uniqueLabel}') resource applicationInsights 'Microsoft.Insights/components@2020-02-02' = { name: appInsightsName location: location @@ -67,7 +70,7 @@ resource applicationInsights 'Microsoft.Insights/components@2020-02-02' = { //APP SERVICE PLAN for FUNCTION APP resource functionAppServicePlan 'Microsoft.Web/serverfarms@2022-03-01' = { - name: toLower('functionapp-service-${uniqueName}') + name: toLower('functionapp-service-${uniqueLabel}') location: location sku: { tier: 'Dynamic', name: 'Y1', family: 'Y', capacity: 1 } properties: { reserved: true } @@ -75,7 +78,7 @@ resource functionAppServicePlan 'Microsoft.Web/serverfarms@2022-03-01' = { //FUNCTION APP resource functionApp 'Microsoft.Web/sites@2022-03-01' = { - name: toLower('functionapp-${uniqueName}') + name: toLower('functionapp-${uniqueLabel}') location: location kind: 'functionapp,linux' properties: { @@ -125,6 +128,10 @@ resource functionApp 'Microsoft.Web/sites@2022-03-01' = { name: 'StorageConfiguration__ConnectionString' value: 'DefaultEndpointsProtocol=https;AccountName=${storageAccountName};EndpointSuffix=${environment().suffixes.storage};AccountKey=${listKeys(storageAccount.id, '2019-06-01').keys[0].value}' } + { + name: 'GithubConfiguration__Pat' + value: githubPat + } ] } } @@ -133,7 +140,7 @@ output functionAppName string = functionApp.name //APP SERVICE PLAN for WEB APP resource webAppServicePlan 'Microsoft.Web/serverfarms@2022-03-01' = { - name: 'webapp-service-${uniqueName}' + name: toLower('webapp-service-${uniqueLabel}') location: location sku: { name: 'B1' @@ -144,7 +151,7 @@ resource webAppServicePlan 'Microsoft.Web/serverfarms@2022-03-01' = { //WEB APP resource webApp 'Microsoft.Web/sites@2022-03-01' = { - name: toLower('webapp-${uniqueName}') + name: toLower('webapp-${uniqueLabel}') location: location kind: 'app,linux' properties: { diff --git a/src/DataSync/DataSync.csproj b/src/DataSync/DataSync.csproj new file mode 100644 index 00000000..01bdd3f5 --- /dev/null +++ b/src/DataSync/DataSync.csproj @@ -0,0 +1,29 @@ + + + + Exe + net8.0 + enable + enable + + + + + + + + + + + + + + + PreserveNewest + + + PreserveNewest + + + + diff --git a/src/DataSync/Program.cs b/src/DataSync/Program.cs new file mode 100644 index 00000000..80d52116 --- /dev/null +++ b/src/DataSync/Program.cs @@ -0,0 +1,141 @@ +using System.Net.Http.Headers; +using System.Runtime.CompilerServices; +using Domain.Models; +using System.Text; +using System.Text.Json; +using AutoFixture; +using AutoFixture.AutoMoq; +using Microsoft.Extensions.Configuration; + +namespace DataSync; + +class Program +{ + static async Task Main(string[] args) + { + // Setup configuration + var builder = new ConfigurationBuilder() + .SetBasePath(Directory.GetCurrentDirectory()) // Set the base path + .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true) // Load default appsettings.json + .AddJsonFile("appsettings.development.json", optional: true, reloadOnChange: true) // Load development-specific settings + .AddEnvironmentVariables(); // Optionally add environment variables + IConfiguration configuration = builder.Build(); + var functionUrl = configuration["FunctionUrl"]!; + var gitHubPat = configuration["GitHubPAT"]!; + + // Get GH blogs + var files = await GetGithubFiles("martinkearn", "Content", "Blogs", gitHubPat); // These values ARE case senitive + Console.WriteLine($"Got {files.Count} files from GitHub"); + foreach (var file in files) + { + Console.WriteLine($"Processing File: {file.Path}"); + + // Get Commit + var commit = await GetGithubLastCommit("martinkearn", "Content", file.Path, gitHubPat); + + // Create Fixture + var fixture = CreateFixture($"Updated {file.Path}", commit.Url, file.Path); + + // Send to Function + if (functionUrl != null) await CallFunction(functionUrl, fixture); + + await Task.Delay(10000); // Pause for 10 seconds + + Console.WriteLine($"Processed File: {file.Path}"); + + //Console.WriteLine("Press any key to continue..."); + //Console.ReadKey(); // Waits for the user to press any key + + Console.WriteLine(""); + + } + + Console.WriteLine("COMPLETED"); + } + + + private static GithubPushWebhookPayload CreateFixture(string message, string commitUrl, string modifiedPath) + { + var fixture = new Fixture().Customize(new AutoMoqCustomization()); + var ghWh = fixture.Create(); + ghWh.Repository.Name = "Content"; + ghWh.HeadCommit.Message = message; + ghWh.HeadCommit.Url = commitUrl; + ghWh.HeadCommit.Timestamp = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:sszzz"); + ghWh.HeadCommit.Author = new Author() + { + Name = "Martin Kearn", + Email = "martin.kearn@microsoft.com", + Username = "martinkearn" + }; + ghWh.HeadCommit.Added = []; + ghWh.HeadCommit.Removed = []; + ghWh.HeadCommit.Modified = [modifiedPath]; + var commit = new Commit() + { + Id = ghWh.HeadCommit.Id, + TreeId = ghWh.HeadCommit.TreeId, + Distinct = ghWh.HeadCommit.Distinct, + Message = ghWh.HeadCommit.Message, + Timestamp = Convert.ToDateTime(ghWh.HeadCommit.Timestamp), + Url = ghWh.HeadCommit.Url, + Author = ghWh.HeadCommit.Author, + Committer = ghWh.HeadCommit.Committer, + Added = [], + Removed = [], + Modified = ghWh.HeadCommit.Modified + }; + ghWh.Commits = + [ + commit + ]; + + return ghWh; + } + + private static async Task> GetGithubFiles(string repoOwner, string repoName, string folderPath, string pat) + { + var url = $"https://api.github.com/repos/{repoOwner}/{repoName}/contents/{folderPath}"; + using var client = new HttpClient(); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", pat); + client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (compatible; dotnet)"); //(GitHub requires this) + var response = await client.GetAsync(url); + response.EnsureSuccessStatusCode(); + var responseBody = await response.Content.ReadAsStringAsync(); + var files = JsonSerializer.Deserialize(responseBody); + + return files!.ToList(); + } + + private static async Task GetGithubLastCommit(string repoOwner, string repoName, string filePath, string pat) + { + var url = $"https://api.github.com/repos/{repoOwner}/{repoName}/commits?path={filePath}&sha=master"; + using var client = new HttpClient(); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", pat); + client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (compatible; dotnet)"); //(GitHub requires this) + var response = await client.GetAsync(url); + response.EnsureSuccessStatusCode(); + var responseBody = await response.Content.ReadAsStringAsync(); + var commits = JsonSerializer.Deserialize(responseBody); + + return commits.FirstOrDefault(); + } + + private static async Task CallFunction(string functionUrl, GithubPushWebhookPayload data) + { + using var client = new HttpClient(); + var jsonData = JsonSerializer.Serialize(data); + var content = new StringContent(jsonData, Encoding.UTF8, "application/json"); + var response = await client.PostAsync(functionUrl, content); + if (response.IsSuccessStatusCode) + { + var result = await response.Content.ReadAsStringAsync(); + Console.WriteLine("Response received successfully:"); + Console.WriteLine(result); + } + else + { + Console.WriteLine($"Failed to send POST request. Status Code: {response.StatusCode}"); + } + } +} \ No newline at end of file diff --git a/src/DataSync/appsettings.json b/src/DataSync/appsettings.json new file mode 100644 index 00000000..09bf5037 --- /dev/null +++ b/src/DataSync/appsettings.json @@ -0,0 +1,4 @@ +{ + "FunctionUrl": "", + "GitHubPAT": "" +} \ No newline at end of file diff --git a/src/Domain/Domain.csproj b/src/Domain/Domain.csproj index b060e28b..ac9eb1ba 100644 --- a/src/Domain/Domain.csproj +++ b/src/Domain/Domain.csproj @@ -12,6 +12,5 @@ - diff --git a/src/Domain/Models/GithubConfiguration.cs b/src/Domain/Models/GithubConfiguration.cs new file mode 100644 index 00000000..62d4dc40 --- /dev/null +++ b/src/Domain/Models/GithubConfiguration.cs @@ -0,0 +1,13 @@ +namespace Domain.Models +{ + /// + /// Used to strongly type the "GithubConfiguration" appsettings section + /// + public class GithubConfiguration + { + /// + /// PAT for acessing API. + /// + public string Pat { get; set; } + } +} \ No newline at end of file diff --git a/src/Domain/Models/GithubFile.cs b/src/Domain/Models/GithubFile.cs new file mode 100644 index 00000000..24d93537 --- /dev/null +++ b/src/Domain/Models/GithubFile.cs @@ -0,0 +1,18 @@ +using System.Text.Json.Serialization; + +namespace Domain.Models; + +public class GithubFile +{ + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("path")] + public string Path { get; set; } + + [JsonPropertyName("type")] + public string Type { get; set; } + + [JsonPropertyName("download_url")] + public string DownloadUrl { get; set; } +} \ No newline at end of file diff --git a/src/Workflow/Models/GithubPushWebhookPayload.cs b/src/Domain/Models/GithubPushWebhookPayload.cs similarity index 99% rename from src/Workflow/Models/GithubPushWebhookPayload.cs rename to src/Domain/Models/GithubPushWebhookPayload.cs index 09813a3a..d793b34a 100644 --- a/src/Workflow/Models/GithubPushWebhookPayload.cs +++ b/src/Domain/Models/GithubPushWebhookPayload.cs @@ -1,6 +1,6 @@ using System.Text.Json.Serialization; -namespace Workflow.Models +namespace Domain.Models { /// /// DTO class for converting GitHub push event payload json. Classes created by https://json2csharp.com/ @@ -479,7 +479,7 @@ public class HeadCommit public string Message { get; set; } [JsonPropertyName("timestamp")] - public DateTime Timestamp { get; set; } + public string Timestamp { get; set; } [JsonPropertyName("url")] public string Url { get; set; } diff --git a/src/MK.sln b/src/MK.sln index 17045d10..13b99d1b 100644 --- a/src/MK.sln +++ b/src/MK.sln @@ -11,6 +11,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Services", "Services\Servic EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Web", "Web\Web.csproj", "{126FC35A-2216-416A-8C3B-B2D7C6CCD69E}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataSync", "DataSync\DataSync.csproj", "{3428DC99-3D45-4F8C-9CDF-BDC857A51360}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -41,6 +43,10 @@ Global {126FC35A-2216-416A-8C3B-B2D7C6CCD69E}.Debug|Any CPU.Build.0 = Debug|Any CPU {126FC35A-2216-416A-8C3B-B2D7C6CCD69E}.Release|Any CPU.ActiveCfg = Release|Any CPU {126FC35A-2216-416A-8C3B-B2D7C6CCD69E}.Release|Any CPU.Build.0 = Release|Any CPU + {3428DC99-3D45-4F8C-9CDF-BDC857A51360}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3428DC99-3D45-4F8C-9CDF-BDC857A51360}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3428DC99-3D45-4F8C-9CDF-BDC857A51360}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3428DC99-3D45-4F8C-9CDF-BDC857A51360}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/Workflow/Program.cs b/src/Workflow/Program.cs index 48ec993b..35e184e3 100644 --- a/src/Workflow/Program.cs +++ b/src/Workflow/Program.cs @@ -25,6 +25,11 @@ { configuration.GetSection(nameof(StorageConfiguration)).Bind(settings); }); + services.AddOptions() + .Configure((settings, configuration) => + { + configuration.GetSection(nameof(GithubConfiguration)).Bind(settings); + }); }) .Build(); diff --git a/src/Workflow/Services/GithubService.cs b/src/Workflow/Services/GithubService.cs index 0dea0e4d..a6cbb7cb 100644 --- a/src/Workflow/Services/GithubService.cs +++ b/src/Workflow/Services/GithubService.cs @@ -1,22 +1,23 @@ -using System.Text.Json; +using System.Net.Http.Headers; +using System.Text.Json; +using Microsoft.Extensions.Options; namespace Workflow.Services { /// - public class GithubService : IGithubService + public class GithubService( + IHttpClientFactory httpClientFactory, + IOptions githubConfigurationOptions) + : IGithubService { - private readonly IHttpClientFactory _clientFactory; - - public GithubService(IHttpClientFactory httpClientFactory) - { - _clientFactory = httpClientFactory; - } + private readonly GithubConfiguration _options = githubConfigurationOptions.Value; public async Task GetGithubContent(string fileApiUrl) { // Make request to Github - var client = _clientFactory.CreateClient(); + var client = httpClientFactory.CreateClient(); client.BaseAddress = new Uri(fileApiUrl); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _options.Pat); client.DefaultRequestHeaders.Add("User-Agent", "Martink.me - GetFileContentsActivity"); var response = await client.GetAsync(fileApiUrl); response.EnsureSuccessStatusCode(); diff --git a/src/Workflow/local.settings.sample.json b/src/Workflow/local.settings.sample.json index 4e0255b2..62cc8a5e 100644 --- a/src/Workflow/local.settings.sample.json +++ b/src/Workflow/local.settings.sample.json @@ -7,6 +7,7 @@ "StorageConfiguration:ConnectionString": "storage connection string here", "StorageConfiguration:ArticlesTable": "articles", "StorageConfiguration:ShortcutsTable": "shortcuts", - "StorageConfiguration:WallpaperBlobsContainer": "wallpaperblobs" + "StorageConfiguration:WallpaperBlobsContainer": "wallpaperblobs", + "GithubConfiguration:Pat": "" } } \ No newline at end of file