From 1d0b4708e9398f7f168974d5f49c6a4b7e7f8c45 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 12 Oct 2025 17:07:53 +0000 Subject: [PATCH 1/4] Initial plan From 01e0edd72f7dc0ca730bd073340fe42aff523c28 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 12 Oct 2025 17:22:27 +0000 Subject: [PATCH 2/4] Add comprehensive telemetry to all content providers Co-authored-by: csharpfritz <78577+csharpfritz@users.noreply.github.com> --- .../Telemetry/ProviderInstrumentation.cs | 30 ++++++++++++ .../AzureQueueProvider.cs | 33 +++++++++++-- .../BlazotProvider.cs | 49 +++++++++++++++++-- .../BlueskyProvider.cs | 34 +++++++++++-- .../MastodonProvider.cs | 16 +++++- .../TwitchChatProvider.cs | 12 ++++- .../TwitterProvider.cs | 33 ++++++++++++- .../YouTubeChatProvider.cs | 40 ++++++++++++++- .../YoutubeProvider.cs | 37 +++++++++++++- 9 files changed, 263 insertions(+), 21 deletions(-) diff --git a/src/TagzApp.Common/Telemetry/ProviderInstrumentation.cs b/src/TagzApp.Common/Telemetry/ProviderInstrumentation.cs index 568cc9ec..50909246 100644 --- a/src/TagzApp.Common/Telemetry/ProviderInstrumentation.cs +++ b/src/TagzApp.Common/Telemetry/ProviderInstrumentation.cs @@ -5,16 +5,46 @@ namespace TagzApp.Common.Telemetry; public class ProviderInstrumentation { public Counter MessagesReceivedCounter { get; set; } + public Counter ConnectionStatusChangesCounter { get; set; } + public ObservableGauge ConnectionStatusGauge { get; set; } + + private readonly Dictionary _ProviderConnectionStatus = new(); + private readonly object _StatusLock = new(); public ProviderInstrumentation(IMeterFactory meterFactory) { var meter = meterFactory.Create("tagzapp-provider-metrics"); MessagesReceivedCounter = meter.CreateCounter("messages-received", "message", "Counter for messages received"); + ConnectionStatusChangesCounter = meter.CreateCounter("connection-status-changes", "change", "Counter for connection status changes"); + ConnectionStatusGauge = meter.CreateObservableGauge("connection-status", () => GetConnectionStatusMeasurements(), "status", "Current connection status of providers (0=Disabled, 1=Unhealthy, 2=Degraded, 3=Healthy)"); } public void AddMessage(string provider, string author) => MessagesReceivedCounter.Add(1, new KeyValuePair(nameof(provider), provider), new KeyValuePair(nameof(author), author)); + + public void RecordConnectionStatusChange(string provider, SocialMediaStatus status) + { + lock (_StatusLock) + { + _ProviderConnectionStatus[provider] = (int)status; + } + + ConnectionStatusChangesCounter.Add(1, + new KeyValuePair(nameof(provider), provider), + new KeyValuePair(nameof(status), status.ToString())); + } + + private IEnumerable> GetConnectionStatusMeasurements() + { + lock (_StatusLock) + { + foreach (var kvp in _ProviderConnectionStatus) + { + yield return new Measurement(kvp.Value, new KeyValuePair("provider", kvp.Key)); + } + } + } } diff --git a/src/TagzApp.Providers.AzureQueue/AzureQueueProvider.cs b/src/TagzApp.Providers.AzureQueue/AzureQueueProvider.cs index 25d89558..9c309991 100644 --- a/src/TagzApp.Providers.AzureQueue/AzureQueueProvider.cs +++ b/src/TagzApp.Providers.AzureQueue/AzureQueueProvider.cs @@ -1,5 +1,7 @@ using Azure.Storage.Queues; +using Microsoft.Extensions.Logging; using System.Text.Json; +using TagzApp.Common.Telemetry; namespace TagzApp.Providers.AzureQueue; @@ -8,6 +10,8 @@ public class AzureQueueProvider : ISocialMediaProvider private const string QueueName = "tagzapp-content"; private AzureQueueConfiguration _Configuration; private QueueClient _Client; + private readonly ILogger? _Logger; + private readonly ProviderInstrumentation? _Instrumentation; private SocialMediaStatus _Status = SocialMediaStatus.Unhealthy; private string _StatusMessage = "Not started"; private bool _DisposedValue; @@ -20,9 +24,12 @@ public class AzureQueueProvider : ISocialMediaProvider public bool Enabled { get; private set; } - public AzureQueueProvider(AzureQueueConfiguration configuration) + public AzureQueueProvider(AzureQueueConfiguration configuration, ILogger? logger = null, + ProviderInstrumentation? instrumentation = null) { _Configuration = configuration; + _Logger = logger; + _Instrumentation = instrumentation; Enabled = configuration.Enabled; } @@ -49,6 +56,18 @@ public async Task> GetContentForHashtag(Hashtag tag, DateTi } + if (_Instrumentation is not null && outList.Any()) + { + _Logger?.LogInformation("AzureQueue: Retrieved {Count} new messages", outList.Count); + foreach (var content in outList) + { + if (!string.IsNullOrEmpty(content.Author?.UserName)) + { + _Instrumentation.AddMessage(Id.ToLowerInvariant(), content.Author.UserName); + } + } + } + return outList; @@ -67,22 +86,26 @@ public async Task StartAsync() try { await _Client.CreateIfNotExistsAsync(); + _Status = SocialMediaStatus.Healthy; + _StatusMessage = "Connected"; + _Logger?.LogInformation("AzureQueue: Successfully connected to queue"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Healthy); } catch (Exception ex) { _Status = SocialMediaStatus.Unhealthy; _StatusMessage = $"Unable to start a connection to the Azure Queue: {ex.Message}"; + _Logger?.LogError(ex, "AzureQueue: Failed to connect to queue"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Unhealthy); return; } - _Status = SocialMediaStatus.Healthy; - _StatusMessage = "Connected"; - } public Task StopAsync() { - // do nothing + _Logger?.LogInformation("AzureQueue: Provider stopped"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Disabled); return Task.CompletedTask; } diff --git a/src/TagzApp.Providers.Blazot/BlazotProvider.cs b/src/TagzApp.Providers.Blazot/BlazotProvider.cs index 153e5b22..0b9ef70f 100644 --- a/src/TagzApp.Providers.Blazot/BlazotProvider.cs +++ b/src/TagzApp.Providers.Blazot/BlazotProvider.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.Logging; +using TagzApp.Common.Telemetry; using TagzApp.Providers.Blazot.Constants; using TagzApp.Providers.Blazot.Interfaces; using TagzApp.Providers.Blazot.Models; @@ -15,6 +16,7 @@ public sealed class BlazotProvider : ISocialMediaProvider private readonly IContentConverter _ContentConverter; private readonly ITransmissionsService _TransmissionsService; private readonly IAuthService _AuthService; + private readonly ProviderInstrumentation? _Instrumentation; public TimeSpan NewContentRetrievalFrequency => TimeSpan.FromSeconds((double)_WindowSeconds / _WindowRequests); public string Id => BlazotConstants.ProviderId; public string DisplayName => BlazotConstants.DisplayName; @@ -27,13 +29,15 @@ public sealed class BlazotProvider : ISocialMediaProvider public bool Enabled { get; } public BlazotProvider(ILogger logger, BlazotConfiguration settings, - IContentConverter contentConverter, ITransmissionsService transmissionsService, IAuthService authService) + IContentConverter contentConverter, ITransmissionsService transmissionsService, IAuthService authService, + ProviderInstrumentation? instrumentation = null) { _ContentConverter = contentConverter ?? throw new ArgumentNullException(nameof(contentConverter)); _TransmissionsService = transmissionsService ?? throw new ArgumentNullException(nameof(transmissionsService)); _AuthService = authService ?? throw new ArgumentNullException(nameof(authService)); _Logger = logger ?? throw new ArgumentNullException(nameof(logger)); _Settings = settings; + _Instrumentation = instrumentation; _WindowSeconds = settings?.WindowSeconds ?? throw new ArgumentNullException(nameof(settings)); _WindowRequests = settings.WindowRequests; @@ -49,7 +53,11 @@ public async Task> GetContentForHashtag(Hashtag tag, DateTi { var transmissions = new List(); - if (!_Settings.Enabled) return Enumerable.Empty(); + if (!_Settings.Enabled) + { + _Logger.LogDebug("Blazot: Provider is disabled"); + return Enumerable.Empty(); + } try { @@ -59,7 +67,7 @@ public async Task> GetContentForHashtag(Hashtag tag, DateTi // A Blazot hashtags request can currently accept up to 10 hashtags with "?t=sometag&t=anothertag". if (_TransmissionsService.HasMadeTooManyRequests) { - _Logger.LogInformation("Exited Blazot request due to current rate limit exceeded state."); + _Logger.LogInformation("Blazot: Exited request due to current rate limit exceeded state"); return Enumerable.Empty(); } @@ -70,13 +78,17 @@ public async Task> GetContentForHashtag(Hashtag tag, DateTi var (isSuccessStatusCode, _) = await _AuthService.GetAccessTokenAsync(); if (isSuccessStatusCode is null or false) + { + _Logger.LogWarning("Blazot: Failed to obtain access token"); return Enumerable.Empty(); + } } transmissions = await _TransmissionsService.GetHashtagTransmissionsAsync(tag, dateTimeOffset); _Status = SocialMediaStatus.Healthy; _StatusMessage = "OK"; + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Healthy); if (transmissions == null) return Enumerable.Empty(); @@ -84,18 +96,43 @@ public async Task> GetContentForHashtag(Hashtag tag, DateTi } catch (Exception ex) { - _Logger.LogError(ex, "Error fetching Blazot Hashtag Transmissions: {message}", ex.Message); + _Logger.LogError(ex, "Blazot: Error fetching Hashtag Transmissions: {message}", ex.Message); _Status = SocialMediaStatus.Unhealthy; _StatusMessage = $"Error fetching Blazot Hashtag Transmissions: {ex.Message}"; + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Unhealthy); + + } + + var content = _ContentConverter.ConvertToContent(transmissions, tag); + if (_Instrumentation is not null && content.Any()) + { + _Logger.LogInformation("Blazot: Retrieved {Count} new transmissions", content.Count()); + foreach (var msg in content) + { + if (!string.IsNullOrEmpty(msg.Author?.UserName)) + { + _Instrumentation.AddMessage(Id.ToLowerInvariant(), msg.Author.UserName); + } + } } - return _ContentConverter.ConvertToContent(transmissions, tag); + return content; } public Task StartAsync() { + if (Enabled) + { + _Logger.LogInformation("Blazot: Provider started"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Healthy); + } + else + { + _Logger.LogInformation("Blazot: Provider is disabled"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Disabled); + } return Task.CompletedTask; } @@ -103,6 +140,8 @@ public Task StartAsync() public Task StopAsync() { + _Logger.LogInformation("Blazot: Provider stopped"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Disabled); return Task.CompletedTask; } diff --git a/src/TagzApp.Providers.Bluesky/BlueskyProvider.cs b/src/TagzApp.Providers.Bluesky/BlueskyProvider.cs index 8730a8e7..b81841b4 100644 --- a/src/TagzApp.Providers.Bluesky/BlueskyProvider.cs +++ b/src/TagzApp.Providers.Bluesky/BlueskyProvider.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Debug; using System.Collections.Concurrent; +using TagzApp.Common.Telemetry; namespace TagzApp.Providers.Bluesky; @@ -36,16 +37,19 @@ public class BlueskyProvider : ISocialMediaProvider private string? _TheTag; private readonly ILogger _Logger; + private readonly ProviderInstrumentation? _Instrumentation; - public BlueskyProvider(BlueskyConfiguration configuration, ILogger logger) + public BlueskyProvider(BlueskyConfiguration configuration, ILogger logger, ProviderInstrumentation? instrumentation = null) { Enabled = configuration.Enabled; _Config = configuration; _Logger = logger; + _Instrumentation = instrumentation; if (!Enabled) { _status = (SocialMediaStatus.Disabled, "Bluesky is not enabled"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Disabled); } } @@ -76,6 +80,18 @@ public Task> GetContentForHashtag(Hashtag tag, DateTimeOffs _ = _messageQueue.TryDequeue(out _); } + if (_Instrumentation is not null && outMessages.Any()) + { + _Logger.LogInformation("Bluesky: Retrieved {Count} new messages", outMessages.Length); + foreach (var msg in outMessages) + { + if (!string.IsNullOrEmpty(msg.Author?.UserName)) + { + _Instrumentation.AddMessage(Id.ToLowerInvariant(), msg.Author.UserName); + } + } + } + return Task.FromResult(outMessages.AsEnumerable()); } @@ -93,12 +109,15 @@ public async Task SaveConfiguration(IConfigureTagzApp configure, IProviderConfig Enabled = providerConfiguration.Enabled; _Config = (BlueskyConfiguration)providerConfiguration; _status = (SocialMediaStatus.Disabled, "Bluesky is disabled"); + _Logger.LogInformation("Bluesky: Configuration changed - disabling provider"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Disabled); await StopAsync(); } else if (_Config.Enabled != providerConfiguration.Enabled && !_Config.Enabled) { Enabled = providerConfiguration.Enabled; _Config = (BlueskyConfiguration)providerConfiguration; + _Logger.LogInformation("Bluesky: Configuration changed - enabling provider"); await StartAsync(); } @@ -111,12 +130,15 @@ public async Task StartAsync() if (!Enabled) { - _Logger.LogInformation("Bluesky is not starting and is marking as disabled"); + _Logger.LogInformation("Bluesky: Not starting - provider is disabled"); _status.status = SocialMediaStatus.Disabled; _status.message = "Bluesky is disabled"; + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Disabled); return; } + _Logger.LogInformation("Bluesky: Starting connection to AT Protocol"); + var debugLog = new DebugLoggerProvider(); _AtProtocol = new ATProtocolBuilder() @@ -137,7 +159,8 @@ public async Task StartAsync() _status.status = SocialMediaStatus.Healthy; _status.message = "Connected to Bluesky"; - _Logger.LogInformation("Bluesky started successfully"); + _Logger.LogInformation("Bluesky: Successfully connected to AT Protocol"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Healthy); } @@ -208,11 +231,16 @@ public async Task StopAsync() { if (_AtWebSocketProtocol is null) return; + _Logger.LogInformation("Bluesky: Stopping connection to AT Protocol"); + await _AtWebSocketProtocol.StopSubscriptionAsync(); _status.status = SocialMediaStatus.Disabled; _status.message = "Disconnected from Bluesky"; + _Logger.LogInformation("Bluesky: Disconnected from AT Protocol"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Disabled); + } } diff --git a/src/TagzApp.Providers.Mastodon/MastodonProvider.cs b/src/TagzApp.Providers.Mastodon/MastodonProvider.cs index 970a13f0..b14794d0 100644 --- a/src/TagzApp.Providers.Mastodon/MastodonProvider.cs +++ b/src/TagzApp.Providers.Mastodon/MastodonProvider.cs @@ -68,9 +68,10 @@ public async Task> GetContentForHashtag(Hashtag tag, DateTi catch (Exception ex) { - _Logger.LogError(ex, "Error getting content from Mastodon"); + _Logger.LogError(ex, "Mastodon: Error getting content"); _Status = SocialMediaStatus.Unhealthy; _StatusMessage = $"Error getting content from Mastodon: {ex.Message}"; + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Unhealthy); return Enumerable.Empty(); @@ -87,6 +88,7 @@ public async Task> GetContentForHashtag(Hashtag tag, DateTi if (_Instrumentation is not null) { + _Logger.LogInformation("Mastodon: Retrieved {Count} new messages", messages.Length); foreach (var username in messages?.Select(x => x.account?.username)!) { if (!string.IsNullOrEmpty(username)) @@ -137,11 +139,23 @@ public async Task SaveConfiguration(IConfigureTagzApp configure, IProviderConfig public Task StartAsync() { + if (Enabled) + { + _Logger.LogInformation("Mastodon: Provider started"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Healthy); + } + else + { + _Logger.LogInformation("Mastodon: Provider is disabled"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Disabled); + } return Task.CompletedTask; } public Task StopAsync() { + _Logger.LogInformation("Mastodon: Provider stopped"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Disabled); return Task.CompletedTask; } diff --git a/src/TagzApp.Providers.TwitchChat/TwitchChatProvider.cs b/src/TagzApp.Providers.TwitchChat/TwitchChatProvider.cs index 9ee964f1..6b35e4bc 100644 --- a/src/TagzApp.Providers.TwitchChat/TwitchChatProvider.cs +++ b/src/TagzApp.Providers.TwitchChat/TwitchChatProvider.cs @@ -133,14 +133,17 @@ private Task ListenForMessages(IChatClient? chatClient = null) } catch (Exception ex) { - _Logger.LogError(ex, "Failed to initialize TwitchChat client"); + _Logger.LogError(ex, "TwitchChat: Failed to initialize client"); _Status = SocialMediaStatus.Unhealthy; _StatusMessage = $"Failed to initialize TwitchChat client: '{ex.Message}'"; + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Unhealthy); return Task.CompletedTask; } _Status = SocialMediaStatus.Healthy; _StatusMessage = "OK"; + _Logger.LogInformation("TwitchChat: Successfully connected"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Healthy); return Task.CompletedTask; @@ -160,6 +163,8 @@ public Task> GetContentForHashtag(Hashtag tag, DateTimeOffs // mark status as unhealthy and return empty list _Status = SocialMediaStatus.Unhealthy; _StatusMessage = "TwitchChat client is not running"; + _Logger.LogWarning("TwitchChat: Client is not running"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Unhealthy); return Task.FromResult(Enumerable.Empty()); @@ -171,6 +176,8 @@ public Task> GetContentForHashtag(Hashtag tag, DateTimeOffs // mark status as unhealthy and return empty list _Status = SocialMediaStatus.Unhealthy; _StatusMessage = "TwitchChat client is not logged in - check credentials"; + _Logger.LogWarning("TwitchChat: Client is not connected - check credentials"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Unhealthy); return Task.FromResult(Enumerable.Empty()); @@ -195,6 +202,7 @@ public Task> GetContentForHashtag(Hashtag tag, DateTimeOffs if (_Instrumentation is not null) { + _Logger.LogInformation("TwitchChat: Retrieved {Count} new messages", messages.Count); foreach (var username in messages?.Select(x => x.Author?.UserName)!) { if (!string.IsNullOrEmpty(username)) @@ -266,6 +274,8 @@ public Task StopAsync() _Client?.Stop(); _Status = SocialMediaStatus.Disabled; _StatusMessage = "TwitchChat client is stopped"; + _Logger.LogInformation("TwitchChat: Provider stopped"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Disabled); return Task.CompletedTask; } diff --git a/src/TagzApp.Providers.Twitter/TwitterProvider.cs b/src/TagzApp.Providers.Twitter/TwitterProvider.cs index bbfb0eaa..74a7cc92 100644 --- a/src/TagzApp.Providers.Twitter/TwitterProvider.cs +++ b/src/TagzApp.Providers.Twitter/TwitterProvider.cs @@ -6,6 +6,7 @@ using System.Text.Json; using System.Web; +using TagzApp.Common.Telemetry; using TagzApp.Providers.Twitter.Configuration; using TagzApp.Providers.Twitter.Models; @@ -16,6 +17,7 @@ public class TwitterProvider : ISocialMediaProvider, IHasNewestId private readonly HttpClient _HttpClient; private readonly TwitterConfiguration _Configuration; private readonly ILogger _Logger; + private readonly ProviderInstrumentation? _Instrumentation; private const string _SearchFields = "created_at,author_id,entities"; private const int _SearchMaxResults = 100; private const string _SearchExpansions = "author_id,attachments.media_keys"; @@ -35,11 +37,12 @@ public class TwitterProvider : ISocialMediaProvider, IHasNewestId public bool Enabled { get; } public TwitterProvider(IHttpClientFactory httpClientFactory, ILogger logger, - TwitterConfiguration configuration) + TwitterConfiguration configuration, ProviderInstrumentation? instrumentation = null) { _HttpClient = httpClientFactory.CreateClient(nameof(TwitterProvider)); _Configuration = configuration; _Logger = logger; + _Instrumentation = instrumentation; Enabled = configuration.Enabled; if (!string.IsNullOrWhiteSpace(configuration.Description)) @@ -80,6 +83,7 @@ public async Task> GetContentForHashtag(Common.Models.Hasht _Status = SocialMediaStatus.Degraded; _StatusMessage = "Twitter provider is not activated - returning sample tweets"; + _Logger.LogWarning("Twitter: Provider is disabled, returning sample tweets"); var assembly = Assembly.GetExecutingAssembly(); var resourceName = "TagzApp.Providers.Twitter.Models.SampleTweets.json.gz"; @@ -104,12 +108,25 @@ public async Task> GetContentForHashtag(Common.Models.Hasht _Status = SocialMediaStatus.Unhealthy; _StatusMessage = $"Error retrieving tweets: {ex.Message}"; - _Logger.LogError(ex, $"Error retrieving tweets"); + _Logger.LogError(ex, "Twitter: Error retrieving tweets"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Unhealthy); } var outTweets = ConvertToContent(recentTweets, tag); + if (_Instrumentation is not null && outTweets.Any()) + { + _Logger.LogInformation("Twitter: Retrieved {Count} new tweets", outTweets.Count()); + foreach (var tweet in outTweets) + { + if (!string.IsNullOrEmpty(tweet.Author?.UserName)) + { + _Instrumentation.AddMessage(Id.ToLowerInvariant(), tweet.Author.UserName); + } + } + } + return outTweets; } @@ -234,6 +251,16 @@ private static Uri FormatUri(string tweetQuery, string sinceTerm) public Task StartAsync() { + if (Enabled) + { + _Logger.LogInformation("Twitter: Provider started"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Healthy); + } + else + { + _Logger.LogInformation("Twitter: Provider is disabled"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Disabled); + } return Task.CompletedTask; } @@ -246,6 +273,8 @@ public Task StartAsync() public Task StopAsync() { + _Logger.LogInformation("Twitter: Provider stopped"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Disabled); return Task.CompletedTask; } diff --git a/src/TagzApp.Providers.YouTubeChat/YouTubeChatProvider.cs b/src/TagzApp.Providers.YouTubeChat/YouTubeChatProvider.cs index 209f768e..b8229720 100644 --- a/src/TagzApp.Providers.YouTubeChat/YouTubeChatProvider.cs +++ b/src/TagzApp.Providers.YouTubeChat/YouTubeChatProvider.cs @@ -2,12 +2,16 @@ using Google.Apis.YouTube.v3; using Google.Apis.YouTube.v3.Data; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using TagzApp.Common.Telemetry; namespace TagzApp.Providers.YouTubeChat; public class YouTubeChatProvider : ISocialMediaProvider, IDisposable { private readonly YouTubeChatConfiguration _ChatConfig; + private readonly ILogger? _Logger; + private readonly ProviderInstrumentation? _Instrumentation; public const string ProviderName = "YouTubeChat"; public const string ProviderId = "YOUTUBE-CHAT"; @@ -33,9 +37,12 @@ public class YouTubeChatProvider : ISocialMediaProvider, IDisposable private SocialMediaStatus _Status = SocialMediaStatus.Unhealthy; private string _StatusMessage = "Not started"; - public YouTubeChatProvider(YouTubeChatConfiguration config, IConfiguration configuration, HttpClient httpClient) + public YouTubeChatProvider(YouTubeChatConfiguration config, IConfiguration configuration, HttpClient httpClient, + ILogger? logger = null, ProviderInstrumentation? instrumentation = null) { _ChatConfig = config; + _Logger = logger; + _Instrumentation = instrumentation; Enabled = true; // config.Enabled; _HttpClient = httpClient; @@ -44,7 +51,12 @@ public YouTubeChatProvider(YouTubeChatConfiguration config, IConfiguration confi public async Task> GetContentForHashtag(Hashtag tag, DateTimeOffset since) { - if (string.IsNullOrEmpty(_ChatConfig.LiveChatId) || (!string.IsNullOrEmpty(_GoogleException) && _GoogleException.StartsWith(_ChatConfig.LiveChatId))) return Enumerable.Empty(); + if (string.IsNullOrEmpty(_ChatConfig.LiveChatId) || (!string.IsNullOrEmpty(_GoogleException) && _GoogleException.StartsWith(_ChatConfig.LiveChatId))) + { + _Logger?.LogDebug("YouTubeChat: No active live chat or chat is in error state"); + return Enumerable.Empty(); + } + var liveChatListRequest = new LiveChatMessagesResource.ListRequest(_Service, _ChatConfig.LiveChatId, new(new[] { "id", "snippet", "authorDetails" })); liveChatListRequest.MaxResults = 2000; liveChatListRequest.ProfileImageSize = 36; @@ -66,16 +78,20 @@ public async Task> GetContentForHashtag(Hashtag tag, DateTi { _GoogleException = $"{_ChatConfig.LiveChatId}:{ex.Message}"; _ChatConfig.LiveChatId = string.Empty; + _Logger?.LogWarning("YouTubeChat: Live chat is no longer active"); } _Status = SocialMediaStatus.Unhealthy; _StatusMessage = $"Exception while fetching YouTubeChat: {ex.Message}"; + _Logger?.LogError(ex, "YouTubeChat: Error fetching chat messages"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Unhealthy); return Enumerable.Empty(); } _Status = SocialMediaStatus.Healthy; _StatusMessage = $"OK -- adding ({contents.Items.Count}) messages for chatid '{_ChatConfig.LiveChatId}' at {DateTimeOffset.UtcNow}"; + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Healthy); try { @@ -95,6 +111,19 @@ public async Task> GetContentForHashtag(Hashtag tag, DateTi Type = ContentType.Message, HashtagSought = tag?.Text ?? "" }).ToArray(); + + if (_Instrumentation is not null && outItems.Any()) + { + _Logger?.LogInformation("YouTubeChat: Retrieved {Count} new messages", outItems.Length); + foreach (var msg in outItems) + { + if (!string.IsNullOrEmpty(msg.Author?.UserName)) + { + _Instrumentation.AddMessage(Id.ToLowerInvariant(), msg.Author.UserName); + } + } + } + return outItems; } @@ -143,9 +172,14 @@ public async Task StartAsync() { _Status = SocialMediaStatus.Disabled; _StatusMessage = "YouTubeChat client is disabled"; + _Logger?.LogInformation("YouTubeChat: Provider is disabled"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Disabled); return; } + _Logger?.LogInformation("YouTubeChat: Provider started"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Healthy); + } public async Task GetChannelForUserAsync() @@ -303,6 +337,8 @@ public void Dispose() public Task StopAsync() { + _Logger?.LogInformation("YouTubeChat: Provider stopped"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Disabled); return Task.CompletedTask; } diff --git a/src/TagzApp.Providers.Youtube/YoutubeProvider.cs b/src/TagzApp.Providers.Youtube/YoutubeProvider.cs index 4780990f..42d6ec3d 100644 --- a/src/TagzApp.Providers.Youtube/YoutubeProvider.cs +++ b/src/TagzApp.Providers.Youtube/YoutubeProvider.cs @@ -1,5 +1,7 @@ using Google.Apis.Services; using Google.Apis.YouTube.v3; +using Microsoft.Extensions.Logging; +using TagzApp.Common.Telemetry; using TagzApp.Providers.Youtube.Configuration; namespace TagzApp.Providers.Youtube; @@ -7,6 +9,8 @@ namespace TagzApp.Providers.Youtube; internal class YoutubeProvider : ISocialMediaProvider { private readonly YoutubeConfiguration _Configuration; + private readonly ILogger? _Logger; + private readonly ProviderInstrumentation? _Instrumentation; public string Id => "YOUTUBE"; public string DisplayName => "Youtube"; @@ -19,9 +23,11 @@ internal class YoutubeProvider : ISocialMediaProvider public bool Enabled { get; } - public YoutubeProvider(YoutubeConfiguration options) + public YoutubeProvider(YoutubeConfiguration options, ILogger? logger = null, ProviderInstrumentation? instrumentation = null) { _Configuration = options; + _Logger = logger; + _Instrumentation = instrumentation; Enabled = options.Enabled; } @@ -43,13 +49,14 @@ public async Task> GetContentForHashtag(Hashtag tag, DateTi _Status = SocialMediaStatus.Healthy; _StatusMessage = "OK"; + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Healthy); if (searchListResponse.Items == null || (!searchListResponse.Items?.Any() ?? true)) { return Enumerable.Empty(); } - return searchListResponse.Items!.Select(m => new Content + var content = searchListResponse.Items!.Select(m => new Content { Provider = Id, ProviderId = m.Id.VideoId, // TODO: Validate this is what we want here @@ -66,10 +73,34 @@ public async Task> GetContentForHashtag(Hashtag tag, DateTi }, Text = m.Snippet.Title }); + + if (_Instrumentation is not null && content.Any()) + { + _Logger?.LogInformation("YouTube: Retrieved {Count} new videos", content.Count()); + foreach (var video in content) + { + if (!string.IsNullOrEmpty(video.Author?.UserName)) + { + _Instrumentation.AddMessage(Id.ToLowerInvariant(), video.Author.UserName); + } + } + } + + return content; } public Task StartAsync() { + if (Enabled) + { + _Logger?.LogInformation("YouTube: Provider started"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Healthy); + } + else + { + _Logger?.LogInformation("YouTube: Provider is disabled"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Disabled); + } return Task.CompletedTask; } @@ -77,6 +108,8 @@ public Task StartAsync() public Task StopAsync() { + _Logger?.LogInformation("YouTube: Provider stopped"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Disabled); return Task.CompletedTask; } From c37b34ac67f787e0d1713984ebeda5520b2f0b7a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 12 Oct 2025 17:24:46 +0000 Subject: [PATCH 3/4] Add OpenTelemetry metrics configuration for provider metrics Co-authored-by: csharpfritz <78577+csharpfritz@users.noreply.github.com> --- src/TagzApp.ServiceDefaults/Extensions.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/TagzApp.ServiceDefaults/Extensions.cs b/src/TagzApp.ServiceDefaults/Extensions.cs index 238fbf3a..ce6250ff 100644 --- a/src/TagzApp.ServiceDefaults/Extensions.cs +++ b/src/TagzApp.ServiceDefaults/Extensions.cs @@ -53,7 +53,8 @@ public static IHostApplicationBuilder ConfigureOpenTelemetry(this IHostApplicati { metrics.AddAspNetCoreInstrumentation() .AddHttpClientInstrumentation() - .AddRuntimeInstrumentation(); + .AddRuntimeInstrumentation() + .AddMeter("tagzapp-provider-metrics"); }) .WithTracing(tracing => { From c2a8970932a14daf22b45832525511d7eaab633b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Oct 2025 10:18:50 +0000 Subject: [PATCH 4/4] Add telemetry requirements to provider documentation Co-authored-by: csharpfritz <78577+csharpfritz@users.noreply.github.com> --- .github/copilot-instructions.md | 64 +++++++++++++ doc/Provider-Configuration-Pattern.md | 127 ++++++++++++++++++++++++++ 2 files changed, 191 insertions(+) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 770272b9..e39ee17a 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -277,6 +277,7 @@ When adding a new social media provider: 3. **Configuration UI**: Create Blazor component in `TagzApp.Blazor.Client/Components/Admin/[Platform].Config.Ui.razor` 4. **Reference Documentation**: See `doc/Provider-Configuration-Pattern.md` for complete pattern 5. **Add Tests**: Include unit tests in `TagzApp.UnitTest` project +6. **Implement Telemetry**: Add comprehensive logging and metrics (see Telemetry Requirements below) ### Provider Configuration Pattern All providers follow a consistent pattern: @@ -286,6 +287,69 @@ All providers follow a consistent pattern: - Enable/disable toggle for each provider - Graceful error handling with detailed logging +### Telemetry Requirements +**All new providers MUST implement comprehensive telemetry for observability:** + +#### Required Logging +Inject `ILogger` and add structured logging for: +- **Connection lifecycle**: Log when starting, stopping, or changing configuration +- **Message discovery**: Log count of new messages retrieved +- **Error conditions**: Log detailed error messages with context +- **Log prefix**: All logs must start with provider name (e.g., "Twitter:", "Bluesky:") + +Example logging: +```csharp +_Logger.LogInformation("Twitter: Provider started"); +_Logger.LogInformation("Twitter: Retrieved {Count} new tweets", tweets.Count); +_Logger.LogError(ex, "Twitter: Error retrieving tweets"); +_Logger.LogWarning("Twitter: Client is not connected - check credentials"); +``` + +#### Required Metrics +Inject `ProviderInstrumentation?` (optional dependency) and report: +- **Messages received**: Call `_Instrumentation?.AddMessage(providerId, username)` for each message +- **Connection status changes**: Call `_Instrumentation?.RecordConnectionStatusChange(Id, status)` when status changes + +Example metrics: +```csharp +public TwitterProvider(IHttpClientFactory httpClientFactory, ILogger logger, + TwitterConfiguration configuration, ProviderInstrumentation? instrumentation = null) +{ + _Instrumentation = instrumentation; + // ... +} + +// In GetContentForHashtag +if (_Instrumentation is not null && messages.Any()) +{ + _Logger.LogInformation("Twitter: Retrieved {Count} new tweets", messages.Count); + foreach (var tweet in messages) + { + if (!string.IsNullOrEmpty(tweet.Author?.UserName)) + { + _Instrumentation.AddMessage(Id.ToLowerInvariant(), tweet.Author.UserName); + } + } +} + +// In StartAsync +_Logger.LogInformation("Twitter: Provider started"); +_Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Healthy); + +// In StopAsync +_Logger.LogInformation("Twitter: Provider stopped"); +_Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Disabled); +``` + +#### Available Metrics +The `ProviderInstrumentation` class provides: +- `MessagesReceivedCounter`: Tracks messages with provider and author tags +- `ConnectionStatusChangesCounter`: Tracks status transitions with provider and status tags +- `ConnectionStatusGauge`: Observable gauge showing current health (0=Disabled, 1=Unhealthy, 2=Degraded, 3=Healthy) + +#### OpenTelemetry Integration +Metrics are automatically collected via OpenTelemetry. The meter name is `tagzapp-provider-metrics` and is configured in `TagzApp.ServiceDefaults/Extensions.cs`. + ### Supported Providers (7 platforms) - **Blazot**: Developer-focused social platform - **Bluesky**: AT Protocol integration diff --git a/doc/Provider-Configuration-Pattern.md b/doc/Provider-Configuration-Pattern.md index 6e705bfa..b10b0939 100644 --- a/doc/Provider-Configuration-Pattern.md +++ b/doc/Provider-Configuration-Pattern.md @@ -206,3 +206,130 @@ The `ToStaticMonitor()` extension method automatically handles the conversion to - [ ] Update `GetConfiguration` and `SaveConfiguration` methods - [ ] Test reactive configuration updates - [ ] Verify unit tests still work with testing constructor +- [ ] **Implement comprehensive telemetry** (see Telemetry Requirements below) + +## Telemetry Requirements + +**All providers MUST implement comprehensive telemetry for observability and debugging.** + +### Required Dependencies + +1. Inject `ILogger` for structured logging +2. Inject `ProviderInstrumentation?` (optional dependency) for metrics + +```csharp +public class YourProvider : ISocialMediaProvider +{ + private readonly ILogger _Logger; + private readonly ProviderInstrumentation? _Instrumentation; + + public YourProvider(IOptionsMonitor configMonitor, + ILogger logger, + ProviderInstrumentation? instrumentation = null) + { + _Logger = logger; + _Instrumentation = instrumentation; + // ... + } +} +``` + +### Required Logging + +Add structured logging for: + +1. **Connection Lifecycle Events** +```csharp +// In StartAsync +_Logger.LogInformation("YourProvider: Provider started"); +_Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Healthy); + +// In StopAsync +_Logger.LogInformation("YourProvider: Provider stopped"); +_Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Disabled); + +// On configuration changes +_Logger.LogInformation("YourProvider: Configuration changed - enabling provider"); +``` + +2. **Message Discovery** +```csharp +// In GetContentForHashtag +if (_Instrumentation is not null && messages.Any()) +{ + _Logger.LogInformation("YourProvider: Retrieved {Count} new messages", messages.Count); + foreach (var msg in messages) + { + if (!string.IsNullOrEmpty(msg.Author?.UserName)) + { + _Instrumentation.AddMessage(Id.ToLowerInvariant(), msg.Author.UserName); + } + } +} +``` + +3. **Error Conditions** +```csharp +catch (Exception ex) +{ + _Logger.LogError(ex, "YourProvider: Error fetching content"); + _Status = SocialMediaStatus.Unhealthy; + _StatusMessage = $"Error: {ex.Message}"; + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Unhealthy); +} +``` + +4. **Warning States** +```csharp +if (!_Client.IsConnected) +{ + _Logger.LogWarning("YourProvider: Client is not connected - check credentials"); + _Instrumentation?.RecordConnectionStatusChange(Id, SocialMediaStatus.Unhealthy); +} +``` + +### Logging Conventions + +- **Always prefix** logs with the provider name followed by a colon (e.g., "Twitter:", "Bluesky:") +- Use **structured logging** with named parameters: `{Count}`, `{Username}`, etc. +- Use appropriate **log levels**: + - `LogInformation`: Normal operations, message counts, lifecycle events + - `LogWarning`: Degraded states, connection issues + - `LogError`: Exceptions, critical failures + - `LogDebug`: Detailed diagnostic information + +### Available Metrics + +The `ProviderInstrumentation` class provides: + +1. **MessagesReceivedCounter** (`messages-received`) + - Tracks individual messages with provider and author tags + - Call: `_Instrumentation.AddMessage(providerId, username)` + +2. **ConnectionStatusChangesCounter** (`connection-status-changes`) + - Tracks status transitions with provider and status tags + - Call: `_Instrumentation.RecordConnectionStatusChange(providerId, status)` + +3. **ConnectionStatusGauge** (`connection-status`) + - Observable gauge showing current provider health + - Values: 0=Disabled, 1=Unhealthy, 2=Degraded, 3=Healthy + - Updated automatically by `RecordConnectionStatusChange` + +### OpenTelemetry Integration + +Metrics are automatically collected via the `tagzapp-provider-metrics` meter configured in `TagzApp.ServiceDefaults/Extensions.cs`. No additional setup required in the provider. + +### Example Implementation + +See existing providers for reference implementations: +- **TwitchChat**: Most comprehensive with activity tracing +- **Twitter**: Clean logging and metrics pattern +- **Bluesky**: Connection lifecycle tracking +- **Mastodon**: Error handling with telemetry + +### Benefits + +- **Real-time monitoring**: Track provider health and message throughput +- **Faster debugging**: Detailed logs help identify issues quickly +- **Performance insights**: Metrics enable tracking of message volumes +- **Consistent observability**: All providers follow the same pattern