Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net9.0</TargetFramework>
<AzureFunctionsVersion>v4</AzureFunctionsVersion>
<OutputType>Exe</OutputType>
<ImplicitUsings>enabled</ImplicitUsings>
Expand All @@ -15,8 +15,8 @@
</None>
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="1.17.4" />
<PackageReference Include="Microsoft.Azure.Functions.Worker" Version="1.23.0" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.ServiceBus" Version="5.22.0" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="2.0.5" />
<PackageReference Include="Microsoft.Azure.Functions.Worker" Version="2.1.0" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.ServiceBus" Version="5.24.0" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,6 @@
"Values": {
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
"ServiceBusConnection__fullyQualifiedNamespace": "{SERVICE_BUS_CONNECTION}"
"ServiceBusConnection__fullyQualifiedNamespace": "{SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}"
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net9.0</TargetFramework>
<AzureFunctionsVersion>v4</AzureFunctionsVersion>
<OutputType>Exe</OutputType>
<ImplicitUsings>enabled</ImplicitUsings>
Expand All @@ -15,8 +15,8 @@
</None>
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="1.17.4" />
<PackageReference Include="Microsoft.Azure.Functions.Worker" Version="1.23.0" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.ServiceBus" Version="5.22.0" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="2.0.5" />
<PackageReference Include="Microsoft.Azure.Functions.Worker" Version="2.1.0" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.ServiceBus" Version="5.24.0" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,6 @@
"Values": {
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
"ServiceBusConnection__fullyQualifiedNamespace": "{SERVICE_BUS_CONNECTION}"
"ServiceBusConnection__fullyQualifiedNamespace": "{SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}"
}
}
11 changes: 7 additions & 4 deletions priority-queue/PriorityQueueSender/PriorityQueueSender.csproj
Original file line number Diff line number Diff line change
@@ -1,15 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net9.0</TargetFramework>
<AzureFunctionsVersion>v4</AzureFunctionsVersion>
<OutputType>Exe</OutputType>
<ImplicitUsings>enabled</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="1.17.4" />
<PackageReference Include="Microsoft.Azure.Functions.Worker" Version="1.23.0" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.ServiceBus" Version="5.22.0" />
<Content Include="local.settings.json" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="2.0.5" />
<PackageReference Include="Microsoft.Azure.Functions.Worker" Version="2.1.0" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.ServiceBus" Version="5.24.0" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Timer" Version="4.3.1" />
</ItemGroup>
<ItemGroup>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,6 @@
"Values": {
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
"ServiceBusConnection__fullyQualifiedNamespace": "{SERVICE_BUS_CONNECTION}"
"ServiceBusConnection__fullyQualifiedNamespace": "{SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}"
}
}
89 changes: 62 additions & 27 deletions priority-queue/Readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,13 @@

This directory contains an example of the [Priority Queue pattern](https://learn.microsoft.com/azure/architecture/patterns/priority-queue).

This example demonstrates how priority queues can be implemented by using Service Bus topics and subscriptions. A time-triggered Azure Function is responsible for sending messages to a topic, each with a priority assigned. The receiving Azure Functions read messages from subscriptions that have the corresponding priority. In this example, the _PriorityQueueConsumerHigh_ Azure function can scale out to 200 instances, while the _PriorityQueueConsumerLow_ function runs only with one instance. This example simulates high priority messages being read from the queue more urgently than low priority messages.
This example demonstrates how priority queues can be implemented by using Service Bus topics and subscriptions. A time-triggered Azure Function is responsible for sending messages to a topic, each with a priority assigned. The receiving Azure Functions read messages from subscriptions that have the corresponding priority.

This example also demonstrates operational aspects of applications running on Azure. Monitoring tools need to be used in order to understand how the sample works. Azure Functions in the solution **must** be configured to use the diagnostics mechanism. If not, you will not see the trace information generated by the example.
For local execution, the sample demonstrates the producer/consumer model, where each consumer processes only one type of message based on its priority.

In the Azure Deployment, the _PriorityQueueConsumerHigh_ Azure function could scale out to 200 instances, while the _PriorityQueueConsumerLow_ Azure function could scale out only to 40 instances. It simulates high priority messages being read from the queue more urgently than low priority messages.

The Azure deployment also demonstrates operational aspects of applications running on Azure. Monitoring tools are essential to understand how the sample operates. Azure Functions in the solution **must** be configured to use the diagnostics mechanism. Otherwise, trace information generated by the example will not be visible.

## :rocket: Deployment guide

Expand All @@ -15,7 +19,7 @@ Install the prerequisites and follow the steps to deploy and run an example of t
- Permission to create a new resource group and resources in an [Azure subscription](https://azure.com/free).
- [Git](https://git-scm.com/downloads)
- [Azure CLI](https://learn.microsoft.com/cli/azure/install-azure-cli)
- [.NET 8 SDK](https://dotnet.microsoft.com/download/dotnet/8.0)
- [.NET 9 SDK](https://dotnet.microsoft.com/download/dotnet/9.0)
- [Azure Functions Core Tools v4.x](https://learn.microsoft.com/azure/azure-functions/functions-run-local#install-the-azure-functions-core-tools)

### Steps
Expand Down Expand Up @@ -49,73 +53,104 @@ Install the prerequisites and follow the steps to deploy and run an example of t
CURRENT_USER_OBJECT_ID=$(az ad signed-in-user show -o tsv --query id)
SERVICE_BUS_NAMESPACE_NAME="sbns-priority-queue-$(LC_ALL=C tr -dc 'a-z0-9' < /dev/urandom | fold -w 7 | head -n 1)"

# This takes about two minute
az deployment group create -n deploy-priority-queue -f bicep/main.bicep -g "${RESOURCE_GROUP_NAME}" -p queueNamespaces=$SERVICE_BUS_NAMESPACE_NAME principalId=$CURRENT_USER_OBJECT_ID
# This takes about two minutes
az deployment group create -n deploy-priority-queue -f bicep/main.bicep -g $RESOURCE_GROUP_NAME -p queueNamespaces=$SERVICE_BUS_NAMESPACE_NAME principalId=$CURRENT_USER_OBJECT_ID
```

1. Configure the samples to use the created Azure resources.

```bash
# Retrieve the primary connection string for the Service Bus namespace.
SERVICE_BUS_CONNECTION="${SERVICE_BUS_NAMESPACE_NAME}.servicebus.windows.net"
SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE="${SERVICE_BUS_NAMESPACE_NAME}.servicebus.windows.net"

sed "s|{SERVICE_BUS_CONNECTION}|${SERVICE_BUS_CONNECTION}|g" ./PriorityQueueSender/local.settings.template.json > ./PriorityQueueSender/local.settings.json
sed "s|{SERVICE_BUS_CONNECTION}|${SERVICE_BUS_CONNECTION}|g" ./PriorityQueueConsumerHigh/local.settings.template.json > ./PriorityQueueConsumerHigh/local.settings.json
sed "s|{SERVICE_BUS_CONNECTION}|${SERVICE_BUS_CONNECTION}|g" ./PriorityQueueConsumerLow/local.settings.template.json > ./PriorityQueueConsumerLow/local.settings.json
sed "s|{SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}|${SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}|g" ./PriorityQueueSender/local.settings.template.json > ./PriorityQueueSender/local.settings.json
sed "s|{SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}|${SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}|g" ./PriorityQueueConsumerHigh/local.settings.template.json > ./PriorityQueueConsumerHigh/local.settings.json
sed "s|{SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}|${SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}|g" ./PriorityQueueConsumerLow/local.settings.template.json > ./PriorityQueueConsumerLow/local.settings.json
```

1. [Run Azurite](https://learn.microsoft.com/azure/storage/common/storage-use-azurite#run-azurite) blob storage emulation service.

> The local storage emulator is required as an Azure Storage account is a required "backing resource" for Azure Functions.
> Azure Functions require an Azure Storage account as a backing resource. When running locally, you can use Azurite, the local storage emulator, to fulfill this requirement.
Alternatively, you may configure the AzureWebJobsStorage setting to use a real Azure Storage account if preferred.

1. Launch the Function PriorityQueueSender to generate Low and High messages.
1. Launch the Azure Function PriorityQueueSender to generate Low and High messages.

```bash
cd ./PriorityQueueSender
func start
```

1. In a new terminal, launch the Function PriorityQueueConsumerLow to consume messages.
1. In a new terminal, launch the Azure Function PriorityQueueConsumerLow to consume messages.

```bash
cd ./PriorityQueueConsumerLow
func start -p 15000
```

> Please note: For demo purposes, the sample application will write content to the the screen.
> Please note: For demo purposes, the sample application will write content to the screen.

1. In a new terminal, launch the Function PriorityQueueConsumerHigh to consume messages.
1. In a new terminal, launch the Azure Function PriorityQueueConsumerHigh to consume messages.

```bash
cd ./PriorityQueueConsumerHigh
func start -p 15001
```

> Please note: For demo purposes, the sample application will write content to the the screen.
> Please note: For demo purposes, the sample application will write content to the screen.

## Deploy the example to Azure (Optional)

To deploy the example to Azure, you need to publish each Azure Functions to Azure. You can do so from Visual Studio by right clicking each function and selecting `Publish` from the menu. Use the same resource group and region from earlier. Be sure to enable Application Insights.

Once each function is published, a new App Setting must be added to store the connection string to the Service Bus namespace. This is the same value that was used in the `SERVICE_BUS_CONNECTION_STRING` variable in the previous steps.
This Bicep template sets up the core infrastructure for a priority-based message processing system using Azure Functions. It creates a secure Storage Account, an Application Insights instance for monitoring, and uses a previously created Service Bus namespace to enable communication between the sender and consumer functions. The deployment includes three Azure Function Apps: one sender and two consumers, each with different scaling limits to simulate message prioritization.
The funcPriorityQueueConsumerHigh function can scale out to 200 instances, allowing it to process high-priority messages quickly. The funcPriorityQueueConsumerLow function is limited to 40 instances, handling lower-priority messages with less urgency. All function apps use the FlexConsumption plan and are connected to Application Insights for diagnostics and monitoring. Role assignments are configured to securely grant access to the Service Bus and Storage resources using managed identities.
All Azure Function Apps share the same Storage Account and Application Insights instance (It is essential to understand how the sample operates), which centralizes observability and logging.

For each function, run the following:
```bash
# This takes about three minutes
az deployment group create -n deploy-priority-queue-sites -f bicep/azure/azure-function-apps.bicep -g $RESOURCE_GROUP_NAME -p serviceBusNamespaceName=$SERVICE_BUS_NAMESPACE_NAME
```
After deploying the infrastructure, you need to publish each Azure Function to its corresponding Function App using Azure Functions Core Tools:

```bash
az webapp config appsettings set -n <function_app_name> -g $RESOURCE_GROUP_NAME --settings ServiceBusConnection__fullyQualifiedNamespace=$SERVICE_BUS_CONNECTION
cd .\PriorityQueueSender\
func azure functionapp publish funcPriorityQueueSender
cd ..
cd .\PriorityQueueConsumerLow\
func azure functionapp publish funcPriorityQueueConsumerLow
cd ..
cd .\PriorityQueueConsumerHigh\
func azure functionapp publish funcPriorityQueueConsumerHigh
```

Once the functions are deployed you need to restrict the maximum number of instances the `PriorityQueueConsumerLow` function can scale out to.
You can view the maximum scaling configuration in the Azure Portal. Go to each Function App, then under Settings, select Scale and concurrency. There, you'll see the Maximum instance count setting.

From the Azure portal:
Once the Azure Functions are deployed, you can use Application Insights to monitor their activity. In the Azure portal, go to the Application Insights resource, then select Logs. Switch to KQL mode and run the following queries to view trace logs for each function:

- Visit the Function App that contains `PriorityQueueConsumerLow`
- Navigate to Scale Out on the left menu
- On the App Scale Out dialog, set the `Enforce Scale Out Limit` to `Yes`
- Set the `Maximum Scale Out Limit` to `1` instance
```
// Traces for the High priority consumer
traces
| where operation_Name contains "High"

// Traces for the Low priority consumer
traces
| where operation_Name contains "Low"

Once the functions are deployed you can visit Application Insights to view the most recent activity for each function.
// Traces for the Sender function
traces
| where operation_Name contains "Sender"
```
In addition to viewing trace logs, you can also use Application Insights to analyze request-level data for each Azure Function. The following queries help you inspect recent requests and understand how frequently each function is being called:

```
// Recent requests with key details
requests
| project timestamp, operation_Name, cloud_RoleName, id, success, resultCode, duration, operation_Id
| order by timestamp desc

// Count of requests by Function
requests
| summarize RequestCount = count() by operation_Name
| order by RequestCount desc
```
## :broom: Clean up resources

Be sure to delete Azure resources when not using them. Since all resources were deployed into a new resource group, you can simply delete the resource group.
Expand Down
82 changes: 82 additions & 0 deletions priority-queue/bicep/azure/azure-function-apps.bicep
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
targetScope = 'resourceGroup'

@minLength(5)
@description('Location of the resources. Defaults to resource group location.')
param location string = resourceGroup().location

@description('The name of the existing Service Bus namespace used for message queuing between the sender and consumer functions.')
param serviceBusNamespaceName string

@description('Defines the name of the Storage Account used by the Function Apps. It uses a unique string based on the resource group ID to ensure global uniqueness.')
param storageAccountName string = 'st${uniqueString(resourceGroup().id)}'

@description('Sets the name of the Application Insights resource for monitoring and diagnostics. Like the storage account, it uses a unique string based on the resource group ID.')
param appInsightsName string = 'ai${uniqueString(resourceGroup().id)}'

var senderRoleId = '69a216fc-b8fb-44d8-bc22-1f3c2cd27a39' //Azure Service Bus Data Sender
var receiverRoleId = '4f6d3b9b-027b-4f4c-9142-0e5a2a2247e0' //Azure Service Bus Data Receiver

resource storageAccount 'Microsoft.Storage/storageAccounts@2025-01-01' = {
name: storageAccountName
location: location
sku: {
name: 'Standard_LRS'
}
kind: 'StorageV2'
properties: {
defaultToOAuthAuthentication: true
publicNetworkAccess: 'Enabled'
allowCrossTenantReplication: false
minimumTlsVersion: 'TLS1_2'
allowBlobPublicAccess: false
allowSharedKeyAccess: false
networkAcls: {
bypass: 'AzureServices'
virtualNetworkRules: []
ipRules: []
defaultAction: 'Allow'
}
supportsHttpsTrafficOnly: true
encryption: {
services: {
file: {
keyType: 'Account'
enabled: true
}
blob: {
keyType: 'Account'
enabled: true
}
}
keySource: 'Microsoft.Storage'
}
}
}

resource appInsights 'Microsoft.Insights/components@2020-02-02' = {
name: appInsightsName
location: location
kind: 'web'
properties: {
Application_Type: 'web'
}
}

module functionApp './sites.bicep' = [
for name in [
'funcPriorityQueueSender'
'funcPriorityQueueConsumerLow'
'funcPriorityQueueConsumerHigh'
]: {
name: name
params: {
location: location
functionAppName: name
storageAccountName: storageAccount.name
serviceBusNamespaceName: serviceBusNamespaceName
roleId: name == 'funcPriorityQueueSender' ? senderRoleId : receiverRoleId
appInsightsName: appInsights.name
scaleUp: name == 'funcPriorityQueueConsumerHigh' ? 200 : 40
}
}
]
Loading