From 9a5d890dbf53393c0889b7017a993e2b11fee05f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Oct 2025 14:50:21 +0000 Subject: [PATCH 1/3] Initial plan From c30f7cbeea8a4f6ee809b5a0ea8b27739c8ed2a6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Oct 2025 15:02:23 +0000 Subject: [PATCH 2/3] Add telemetry and logging to YouTube Chat provider for quota tracking Co-authored-by: csharpfritz <78577+csharpfritz@users.noreply.github.com> --- .../TagzApp.Providers.YouTubeChat.csproj | 1 + .../YouTubeChatProvider.cs | 104 ++++++++++++++++-- 2 files changed, 98 insertions(+), 7 deletions(-) diff --git a/src/TagzApp.Providers.YouTubeChat/TagzApp.Providers.YouTubeChat.csproj b/src/TagzApp.Providers.YouTubeChat/TagzApp.Providers.YouTubeChat.csproj index afc4617f..691c5ef7 100644 --- a/src/TagzApp.Providers.YouTubeChat/TagzApp.Providers.YouTubeChat.csproj +++ b/src/TagzApp.Providers.YouTubeChat/TagzApp.Providers.YouTubeChat.csproj @@ -8,6 +8,7 @@ + diff --git a/src/TagzApp.Providers.YouTubeChat/YouTubeChatProvider.cs b/src/TagzApp.Providers.YouTubeChat/YouTubeChatProvider.cs index 209f768e..87c38eab 100644 --- a/src/TagzApp.Providers.YouTubeChat/YouTubeChatProvider.cs +++ b/src/TagzApp.Providers.YouTubeChat/YouTubeChatProvider.cs @@ -2,6 +2,8 @@ using Google.Apis.YouTube.v3; using Google.Apis.YouTube.v3.Data; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using TagzApp.Common.Telemetry; namespace TagzApp.Providers.YouTubeChat; @@ -23,6 +25,8 @@ public class YouTubeChatProvider : ISocialMediaProvider, IDisposable public bool Enabled { get; } private readonly HttpClient _HttpClient; + private readonly ILogger _Logger; + private readonly ProviderInstrumentation? _Instrumentation; private string _GoogleException = string.Empty; private CancellationTokenSource _TokenSource = new(); @@ -33,11 +37,19 @@ public class YouTubeChatProvider : ISocialMediaProvider, IDisposable private SocialMediaStatus _Status = SocialMediaStatus.Unhealthy; private string _StatusMessage = "Not started"; - public YouTubeChatProvider(YouTubeChatConfiguration config, IConfiguration configuration, HttpClient httpClient) + // YouTube API quota tracking + private long _QuotaUsed = 0; + private const int QUOTA_LIVECHAT_LIST = 5; + private const int QUOTA_SEARCH_LIST = 100; + private const int QUOTA_VIDEO_LIST = 1; + + public YouTubeChatProvider(YouTubeChatConfiguration config, IConfiguration configuration, HttpClient httpClient, ILogger logger, ProviderInstrumentation? instrumentation = null) { _ChatConfig = config; Enabled = true; // config.Enabled; _HttpClient = httpClient; + _Logger = logger; + _Instrumentation = instrumentation; } @@ -58,10 +70,21 @@ public async Task> GetContentForHashtag(Hashtag tag, DateTi _NextPageToken = contents.NextPageToken; NewContentRetrievalFrequency = contents.PollingIntervalMillis.HasValue ? TimeSpan.FromMilliseconds(contents.PollingIntervalMillis.Value * 10) : TimeSpan.FromSeconds(6); + // Track quota usage for LiveChatMessages.list API call + _QuotaUsed += QUOTA_LIVECHAT_LIST; + _Logger.LogInformation("YouTube API call: LiveChatMessages.list - Quota cost: {QuotaCost}, Total quota used: {QuotaUsed}, Messages retrieved: {MessageCount}", + QUOTA_LIVECHAT_LIST, _QuotaUsed, contents.Items.Count); + + // Emit telemetry for API usage + _Instrumentation?.MessagesReceivedCounter.Add(QUOTA_LIVECHAT_LIST, + new KeyValuePair("provider", Id), + new KeyValuePair("api_call", "LiveChatMessages.list"), + new KeyValuePair("quota_used", _QuotaUsed)); + } catch (Exception ex) { - Console.WriteLine($"Exception while fetching YouTubeChat: {ex.Message}"); + _Logger.LogError(ex, "Exception while fetching YouTubeChat: {Message}", ex.Message); if (ex.Message.Contains("live chat is no longer live")) { _GoogleException = $"{_ChatConfig.LiveChatId}:{ex.Message}"; @@ -95,13 +118,26 @@ public async Task> GetContentForHashtag(Hashtag tag, DateTi Type = ContentType.Message, HashtagSought = tag?.Text ?? "" }).ToArray(); + + // Track message authors in instrumentation + if (_Instrumentation is not null) + { + foreach (var item in contents.Items) + { + if (!string.IsNullOrEmpty(item.AuthorDetails.DisplayName)) + { + _Instrumentation.AddMessage("youtubechat", item.AuthorDetails.DisplayName); + } + } + } + return outItems; } catch (Exception ex) { - Console.WriteLine($"Exception while parsing YouTubeChat: {ex.Message}"); + _Logger.LogError(ex, "Exception while parsing YouTubeChat: {Message}", ex.Message); _Status = SocialMediaStatus.Unhealthy; _StatusMessage = $"Exception while parsing YouTubeChat: {ex.Message}"; @@ -158,6 +194,16 @@ public async Task GetChannelForUserAsync() channelRequest.ChannelId = _ChatConfig.ChannelId; var channels = channelRequest.Execute(); + // Track quota usage for Search.list API call + _QuotaUsed += QUOTA_SEARCH_LIST; + _Logger.LogInformation("YouTube API call: Search.list (channel) - Quota cost: {QuotaCost}, Total quota used: {QuotaUsed}", + QUOTA_SEARCH_LIST, _QuotaUsed); + + _Instrumentation?.MessagesReceivedCounter.Add(QUOTA_SEARCH_LIST, + new KeyValuePair("provider", Id), + new KeyValuePair("api_call", "Search.list"), + new KeyValuePair("quota_used", _QuotaUsed)); + // Not sure if this is needed, can't replicate "fisrt" error. (https://github.com/FritzAndFriends/TagzApp/issues/241) return channels.Items?.First().Snippet.Title ?? "Unknown Channel Title"; @@ -179,11 +225,21 @@ public IEnumerable GetBroadcastsForUser() try { broadcasts = listRequest.Execute(); + + // Track quota usage for Search.list API call + _QuotaUsed += QUOTA_SEARCH_LIST; + _Logger.LogInformation("YouTube API call: Search.list (Upcoming) - Quota cost: {QuotaCost}, Total quota used: {QuotaUsed}, Results: {ResultCount}", + QUOTA_SEARCH_LIST, _QuotaUsed, broadcasts.Items?.Count ?? 0); + + _Instrumentation?.MessagesReceivedCounter.Add(QUOTA_SEARCH_LIST, + new KeyValuePair("provider", Id), + new KeyValuePair("api_call", "Search.list"), + new KeyValuePair("quota_used", _QuotaUsed)); } catch (Google.GoogleApiException ex) { // GoogleApiException: The service youtube has thrown an exception. HttpStatusCode is Forbidden. The user is not enabled for live streaming. - Console.WriteLine($"Exception while fetching YouTube broadcasts: {ex.Message}"); + _Logger.LogError(ex, "Exception while fetching YouTube broadcasts: {Message}", ex.Message); _Status = SocialMediaStatus.Unhealthy; _StatusMessage = $"Exception while fetching YouTube broadcasts: {ex.Message}"; @@ -205,11 +261,21 @@ public IEnumerable GetBroadcastsForUser() try { broadcasts = listRequest.Execute(); + + // Track quota usage for Search.list API call + _QuotaUsed += QUOTA_SEARCH_LIST; + _Logger.LogInformation("YouTube API call: Search.list (Live) - Quota cost: {QuotaCost}, Total quota used: {QuotaUsed}, Results: {ResultCount}", + QUOTA_SEARCH_LIST, _QuotaUsed, broadcasts.Items?.Count ?? 0); + + _Instrumentation?.MessagesReceivedCounter.Add(QUOTA_SEARCH_LIST, + new KeyValuePair("provider", Id), + new KeyValuePair("api_call", "Search.list"), + new KeyValuePair("quota_used", _QuotaUsed)); } catch (Google.GoogleApiException ex) { // GoogleApiException: The service youtube has thrown an exception. HttpStatusCode is Forbidden. The user is not enabled for live streaming. - Console.WriteLine($"Exception while fetching YouTube broadcasts: {ex.Message}"); + _Logger.LogError(ex, "Exception while fetching YouTube broadcasts: {Message}", ex.Message); _Status = SocialMediaStatus.Unhealthy; _StatusMessage = $"Exception while fetching YouTube broadcasts: {ex.Message}"; @@ -223,7 +289,7 @@ public IEnumerable GetBroadcastsForUser() } - private static SearchListResponse ConvertToYouTubeBroadcasts(YouTubeService service, SearchResource.ListRequest listRequest, SearchListResponse broadcasts, List outBroadcasts) + private SearchListResponse ConvertToYouTubeBroadcasts(YouTubeService service, SearchResource.ListRequest listRequest, SearchListResponse broadcasts, List outBroadcasts) { var first = true; while (first || !string.IsNullOrEmpty(broadcasts.NextPageToken))// && outBroadcasts.Count < 20) @@ -237,6 +303,16 @@ private static SearchListResponse ConvertToYouTubeBroadcasts(YouTubeService serv { listRequest.PageToken = broadcasts.NextPageToken; broadcasts = listRequest.Execute(); + + // Track quota for paginated Search.list calls + _QuotaUsed += QUOTA_SEARCH_LIST; + _Logger.LogInformation("YouTube API call: Search.list (pagination) - Quota cost: {QuotaCost}, Total quota used: {QuotaUsed}", + QUOTA_SEARCH_LIST, _QuotaUsed); + + _Instrumentation?.MessagesReceivedCounter.Add(QUOTA_SEARCH_LIST, + new KeyValuePair("provider", Id), + new KeyValuePair("api_call", "Search.list"), + new KeyValuePair("quota_used", _QuotaUsed)); } foreach (var broadcast in broadcasts.Items) @@ -246,6 +322,16 @@ private static SearchListResponse ConvertToYouTubeBroadcasts(YouTubeService serv videoRequest.Id = broadcast.Id.VideoId; var videoResponse = videoRequest.Execute(); + // Track quota for Videos.list calls + _QuotaUsed += QUOTA_VIDEO_LIST; + _Logger.LogDebug("YouTube API call: Videos.list - Quota cost: {QuotaCost}, Total quota used: {QuotaUsed}, VideoId: {VideoId}", + QUOTA_VIDEO_LIST, _QuotaUsed, broadcast.Id.VideoId); + + _Instrumentation?.MessagesReceivedCounter.Add(QUOTA_VIDEO_LIST, + new KeyValuePair("provider", Id), + new KeyValuePair("api_call", "Videos.list"), + new KeyValuePair("quota_used", _QuotaUsed)); + if (videoResponse.Items.First().LiveStreamingDetails is null) continue; var liveChatId = videoResponse.Items.First().LiveStreamingDetails.ActiveLiveChatId; @@ -299,7 +385,11 @@ public void Dispose() GC.SuppressFinalize(this); } - public Task<(SocialMediaStatus Status, string Message)> GetHealth() => Task.FromResult((_Status, _StatusMessage)); + public Task<(SocialMediaStatus Status, string Message)> GetHealth() + { + var message = $"{_StatusMessage} | API Quota Used: {_QuotaUsed} units"; + return Task.FromResult((_Status, message)); + } public Task StopAsync() { From b8e230f393ab75630a2ef3f866cd3cdf2f9b1f49 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Oct 2025 15:04:42 +0000 Subject: [PATCH 3/3] Add comprehensive documentation for YouTube Chat provider telemetry Co-authored-by: csharpfritz <78577+csharpfritz@users.noreply.github.com> --- src/TagzApp.Providers.YouTubeChat/README.md | 80 +++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 src/TagzApp.Providers.YouTubeChat/README.md diff --git a/src/TagzApp.Providers.YouTubeChat/README.md b/src/TagzApp.Providers.YouTubeChat/README.md new file mode 100644 index 00000000..25918149 --- /dev/null +++ b/src/TagzApp.Providers.YouTubeChat/README.md @@ -0,0 +1,80 @@ +# TagzApp YouTube Chat Provider + +This provider integrates with YouTube Data API v3 to monitor live chat messages during YouTube live streams. + +## YouTube API Quota Management + +YouTube Data API v3 has daily quota limits (default: 10,000 units per day). This provider tracks API usage to help monitor quota consumption. + +### API Quota Costs + +Different API operations have different quota costs: + +| API Call | Quota Cost | Usage | +|----------|-----------|--------| +| `LiveChatMessages.list` | 5 units | Fetching live chat messages (main polling operation) | +| `Search.list` | 100 units | Searching for broadcasts and channels | +| `Videos.list` | 1 unit | Getting video details and live chat IDs | + +### Telemetry and Logging + +The provider includes comprehensive telemetry to track API usage: + +#### Structured Logging + +Every API call logs: +- API operation name +- Quota cost for the call +- Cumulative quota used +- Additional context (e.g., message count, video ID) + +Example log entry: +``` +YouTube API call: LiveChatMessages.list - Quota cost: 5, Total quota used: 125, Messages retrieved: 42 +``` + +#### Metrics (via ProviderInstrumentation) + +The provider emits counter metrics with the following dimensions: +- `provider` - Set to "YOUTUBE-CHAT" +- `api_call` - The API operation (e.g., "LiveChatMessages.list", "Search.list", "Videos.list") +- `quota_used` - Cumulative quota consumed + +These metrics can be monitored using OpenTelemetry exporters (e.g., Prometheus, Application Insights). + +#### Health Check + +The provider's health status includes current quota usage: +``` +Status: Healthy +Message: OK -- adding (42) messages for chatid 'xyz' at 2025-01-15T10:30:00Z | API Quota Used: 125 units +``` + +### Monitoring Recommendations + +1. **Set up alerts** when quota usage approaches the daily limit (e.g., 8,000 units) +2. **Monitor the quota_used metric** to track consumption trends +3. **Review logs** to identify high-cost operations (especially Search.list calls) +4. **Consider quota reset timing** - YouTube quotas reset at midnight Pacific Time + +### Reducing Quota Usage + +If you're hitting quota limits: +1. Increase polling interval (`NewContentRetrievalFrequency`) +2. Reduce broadcast search frequency +3. Request a quota increase from Google Cloud Console +4. Cache broadcast and channel information + +## Configuration + +See `YouTubeChatConfiguration.cs` for available configuration options including: +- `YouTubeApiKey` - Your YouTube Data API key +- `ChannelId` - The YouTube channel to monitor +- `BroadcastId` - Specific broadcast to monitor +- `LiveChatId` - Live chat ID for the stream +- `Enabled` - Enable/disable the provider + +## Resources + +- [YouTube Data API v3 Quota Calculator](https://developers.google.com/youtube/v3/determine_quota_cost) +- [YouTube Data API v3 Documentation](https://developers.google.com/youtube/v3)