Skip to content

Commit 1c8e814

Browse files
devleaddevlead
authored
feat: migrate to OpenTelemetry and fix MERGE when only key columns (#121)
- Replace Application Insights with OpenTelemetry - Use Azure.Monitor.OpenTelemetry.Exporter, Microsoft.Azure.Functions.Worker.OpenTelemetry, OpenTelemetry.Extensions.Hosting, OpenTelemetry.Exporter.Console - Set host.json telemetryMode to OpenTelemetry and remove applicationInsights config - Configure worker logging: AddSimpleConsole, Logging from config, filter SqlBulkSyncFunction at Information - host.json: set default, Host.Results, Function log level to Information - README: add Logging__LogLevel__Default to example local.settings.json - SqlStatementExtensions: omit WHEN MATCHED THEN UPDATE when there are no updatable columns (only primary/identity), avoid empty SET and format/argument reorder - Bump Microsoft.Data.SqlClient to 7.0.0
1 parent 58d43bb commit 1c8e814

6 files changed

Lines changed: 95 additions & 67 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ To quicker get started testing the function locally example configuration and da
7474
"Values": {
7575
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
7676
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
77+
"Logging__LogLevel__Default": "Information",
7778
"ProcessGlobalChangeTrackingSchedule": "0 23 11 * * *",
7879
"SyncJobsConfig__Jobs__SyncTest__Area": "SyncTest",
7980
"SyncJobsConfig__Jobs__SyncTest__Source__ConnectionString": "Server=localhost;Initial Catalog=SyncTest;Integrated Security=True",

‎src/Directory.Packages.props‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,15 @@
99
<PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.Http" Version="3.3.0" />
1010
<PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.Http.AspNetCore" Version="2.1.0" />
1111
<PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.OpenApi" Version="1.6.0" />
12-
<PackageVersion Include="Microsoft.ApplicationInsights.WorkerService" Version="2.23.0" />
13-
<PackageVersion Include="Microsoft.Azure.Functions.Worker.ApplicationInsights" Version="2.50.0" />
12+
<PackageVersion Include="Azure.Monitor.OpenTelemetry.Exporter" Version="1.6.0" />
13+
<PackageVersion Include="Microsoft.Azure.Functions.Worker.OpenTelemetry" Version="1.1.0" />
14+
<PackageVersion Include="OpenTelemetry.Exporter.Console" Version="1.15.0" />
15+
<PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.15.0" />
1416
<PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.Timer" Version="4.3.1" />
1517
<PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.Storage" Version="6.8.1" />
1618
<PackageVersion Include="Microsoft.Azure.Functions.Worker" Version="2.51.0" />
1719
<PackageVersion Include="Microsoft.Azure.Functions.Worker.Sdk" Version="2.0.7" />
18-
<PackageVersion Include="Microsoft.Data.SqlClient" Version="6.1.4" />
20+
<PackageVersion Include="Microsoft.Data.SqlClient" Version="7.0.0" />
1921
<PackageVersion Include="NuGetizer" Version="1.4.7" />
2022
</ItemGroup>
2123
</Project>

‎src/SqlBulkSyncFunction/Helpers/SqlStatementExtensions.cs‎

Lines changed: 65 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -58,67 +58,85 @@ DROP TABLE {tableName}
5858
public static string GetNewOrUpdatedMergeStatement(this TableSchema tableSchema, bool disableTargetIdentityInsert, bool disableConstraintCheck)
5959
{
6060
var identityInsert = !disableTargetIdentityInsert && tableSchema.Columns.Any(column => column.IsIdentity);
61+
var updatableColumns = tableSchema.Columns
62+
.Where(column => !column.IsPrimary && !column.IsIdentity)
63+
.ToArray();
64+
var whenMatchedBlock = updatableColumns.Length == 0
65+
? string.Empty
66+
: string.Format(
67+
"""
68+
WHEN MATCHED
69+
THEN UPDATE
70+
SET {0}
71+
""",
72+
string.Join(
73+
",\r\n ",
74+
updatableColumns
75+
.Select(
76+
column => string.Concat(
77+
column.QuoteName,
78+
" = source.",
79+
column.QuoteName
80+
)
81+
)
82+
)
83+
);
6184
var statement = string.Format(
6285
"""
63-
{9}
64-
{6};
65-
MERGE {0} AS target
66-
USING {1} AS source
67-
ON {2}
86+
{0}
87+
{1}
88+
89+
MERGE {2} AS target
90+
USING {3} AS source
91+
ON {4}
6892
WHEN NOT MATCHED BY TARGET
6993
THEN INSERT (
70-
{3}
94+
{5}
7195
) VALUES (
72-
{4}
73-
){8}
74-
WHEN MATCHED
75-
THEN UPDATE
76-
SET {5};
77-
SELECT @@ROWCOUNT AS [RowCount];
96+
{6}
97+
)
7898
{7}
99+
{8};
100+
101+
SELECT @@ROWCOUNT AS [RowCount];
102+
103+
{9}
79104
{10}
80105
""",
106+
(
107+
disableConstraintCheck
108+
? $"ALTER TABLE {tableSchema.TargetTableName} NOCHECK CONSTRAINT ALL;"
109+
: string.Empty
110+
),
111+
(
112+
identityInsert
113+
? $"SET IDENTITY_INSERT {tableSchema.TargetTableName} ON;"
114+
: string.Empty
115+
),
81116
tableSchema.TargetTableName,
82117
tableSchema.SyncNewOrUpdatedTableName,
83118
string.Join(
84119
" AND\r\n ",
85-
tableSchema.Columns.Where(column => column.IsPrimary).Select(
86-
column => string.Concat(
87-
"target.",
88-
column.QuoteName,
89-
" = source.",
90-
column.QuoteName
91-
)
92-
)
93-
),
120+
tableSchema
121+
.Columns
122+
.Where(column => column.IsPrimary)
123+
.Select(
124+
column => string.Concat(
125+
"target.",
126+
column.QuoteName,
127+
" = source.",
128+
column.QuoteName
129+
)
130+
)
131+
),
94132
string.Join(
95133
",\r\n ",
96134
tableSchema.Columns.Select(column => column.QuoteName)
97-
),
135+
),
98136
string.Join(
99137
",\r\n ",
100138
tableSchema.Columns.Select(column => string.Concat("source.", column.QuoteName))
101-
),
102-
string.Join(
103-
",\r\n ",
104-
tableSchema.Columns.Where(column => !column.IsPrimary && !column.IsIdentity).Select(
105-
column => string.Concat(
106-
column.QuoteName,
107-
" = source.",
108-
column.QuoteName
109-
)
110-
)
111-
),
112-
(
113-
identityInsert
114-
? $"SET IDENTITY_INSERT {tableSchema.TargetTableName} ON"
115-
: string.Empty
116-
),
117-
(
118-
identityInsert
119-
? $"SET IDENTITY_INSERT {tableSchema.TargetTableName} OFF"
120-
: string.Empty
121-
),
139+
),
122140
(
123141
(tableSchema.TargetVersion.CurrentVersion < 0)
124142
? string.Empty
@@ -128,10 +146,11 @@ THEN UPDATE
128146
// THEN DELETE
129147
//"""
130148
: string.Empty
131-
),
149+
),
150+
whenMatchedBlock,
132151
(
133-
disableConstraintCheck
134-
? $"ALTER TABLE {tableSchema.TargetTableName} NOCHECK CONSTRAINT ALL;"
152+
identityInsert
153+
? $"SET IDENTITY_INSERT {tableSchema.TargetTableName} OFF;"
135154
: string.Empty
136155
),
137156
(

‎src/SqlBulkSyncFunction/Program.cs‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,24 @@
1-
using Microsoft.Azure.Functions.Worker;
1+
using Azure.Monitor.OpenTelemetry.Exporter;
2+
using Microsoft.Azure.Functions.Worker.OpenTelemetry;
23
using Microsoft.Extensions.Azure;
34
using Microsoft.Extensions.Configuration;
45
using Microsoft.Extensions.DependencyInjection;
56
using Microsoft.Extensions.Hosting;
7+
using Microsoft.Extensions.Logging;
68
using SqlBulkSyncFunction.Functions;
79
using SqlBulkSyncFunction.Models.Job;
810
using SqlBulkSyncFunction.Services;
911

1012
await new HostBuilder()
1113
.ConfigureFunctionsWebApplication()
14+
.ConfigureLogging(
15+
static (context, logging) => logging
16+
.AddConfiguration(context.Configuration.GetSection("Logging"))
17+
.SetMinimumLevel(LogLevel.Information)
18+
.AddSimpleConsole(static o => {})
19+
.AddFilter("SqlBulkSyncFunction", LogLevel.Information))
1220
.ConfigureServices(
13-
configure =>
21+
static configure =>
1422
{
1523

1624
_ = configure.AddOptions<SyncJobsConfig>()
@@ -24,19 +32,19 @@
2432
.AddSingleton<ITokenCacheService, TokenCacheService>()
2533
.AddSingleton<SyncProgressService>()
2634
.AddAzureClients(
27-
az => {
35+
static az => {
2836
var connectionString = System.Environment.GetEnvironmentVariable("AzureWebJobsStorage");
2937
_ = az
3038
.AddBlobServiceClient(connectionString).ConfigureOptions(
31-
options => {
39+
static options => {
3240
options.Diagnostics.IsLoggingContentEnabled = false;
3341
options.Diagnostics.IsLoggingEnabled = false;
3442
}
3543
);
3644
_ = az
3745
.AddQueueServiceClient(connectionString)
3846
.ConfigureOptions(
39-
options => {
47+
static options => {
4048
options.MessageEncoding = Azure.Storage.Queues.QueueMessageEncoding.Base64;
4149
options.Diagnostics.IsLoggingContentEnabled = false;
4250
options.Diagnostics.IsLoggingEnabled = false;
@@ -46,8 +54,9 @@
4654
);
4755

4856
_ = configure
49-
.AddApplicationInsightsTelemetryWorkerService()
50-
.ConfigureFunctionsApplicationInsights();
57+
.AddOpenTelemetry()
58+
.UseFunctionsWorkerDefaults()
59+
.UseAzureMonitorExporter();
5160
})
5261
.Build()
5362
.RunAsync();

‎src/SqlBulkSyncFunction/SqlBulkSyncFunction.csproj‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,10 @@
1313
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http" />
1414
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http.AspNetCore" />
1515
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.OpenApi" />
16-
<PackageReference Include="Microsoft.ApplicationInsights.WorkerService" />
17-
<PackageReference Include="Microsoft.Azure.Functions.Worker.ApplicationInsights" />
16+
<PackageReference Include="Azure.Monitor.OpenTelemetry.Exporter" />
17+
<PackageReference Include="Microsoft.Azure.Functions.Worker.OpenTelemetry" />
18+
<PackageReference Include="OpenTelemetry.Exporter.Console" />
19+
<PackageReference Include="OpenTelemetry.Extensions.Hosting" />
1820
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Timer" />
1921
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Storage" />
2022
<PackageReference Include="Microsoft.Azure.Functions.Worker" />

‎src/SqlBulkSyncFunction/host.json‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
{
22
"version": "2.0",
3+
"telemetryMode": "OpenTelemetry",
34
"functionTimeout": "01:00:00",
45
"logging": {
56
"logLevel": {
6-
"default": "Warning",
7-
"Host.Results": "Warning",
8-
"Function": "Warning",
7+
"default": "Information",
8+
"Host.Results": "Information",
9+
"Function": "Information",
910
"Host.Aggregator": "Warning",
1011
"Azure.Core": "Error",
1112
"Azure.Storage": "Warning",
@@ -14,12 +15,6 @@
1415
"Azure": "Error",
1516
"System.Net.Http.HttpClient": "Warning"
1617
},
17-
"applicationInsights": {
18-
"samplingSettings": {
19-
"isEnabled": true,
20-
"excludedTypes": "Request"
21-
}
22-
},
2318
"Console": {
2419
"FormatterOptions": {
2520
"IncludeScopes": true,

0 commit comments

Comments
 (0)