Skip to content

Commit ee9bfc1

Browse files
rlhagermrlhagerm
committed
Update LargeQueryWorkflow.cs
1 parent 249a535 commit ee9bfc1

File tree

1 file changed

+76
-55
lines changed

1 file changed

+76
-55
lines changed

‎dotnetv4/CloudWatchLogs/LargeQuery/Scenarios/LargeQueryWorkflow.cs‎

Lines changed: 76 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -393,83 +393,104 @@ private static async Task<List<List<ResultField>>> PerformLargeQuery(
393393

394394
var startDate = DateTimeOffset.FromUnixTimeSeconds(startTime).ToString("yyyy-MM-ddTHH:mm:ss.fffZ");
395395
var endDate = DateTimeOffset.FromUnixTimeSeconds(endTime).ToString("yyyy-MM-ddTHH:mm:ss.fffZ");
396-
Console.WriteLine($"Query date range: {startDate} to {endDate}. Found {results.Count} logs.");
396+
Console.WriteLine($"Query date range: {startDate} ({startTime}s) to {endDate} ({endTime}s). Found {results.Count} logs.");
397397

398398
if (results.Count < limit)
399399
{
400+
Console.WriteLine($" -> Returning {results.Count} logs (less than limit of {limit})");
400401
return results;
401402
}
402403

403-
var lastTimestamp = results[results.Count - 1].Find(f => f.Field == "@timestamp")?.Value;
404-
if (lastTimestamp == null)
404+
Console.WriteLine($" -> Hit limit of {limit}. Need to split and recurse.");
405+
406+
// Get the timestamp of the last log (sorted to find the actual last one)
407+
var lastLogTimestamp = GetLastLogTimestamp(results);
408+
if (lastLogTimestamp == null)
405409
{
410+
Console.WriteLine($" -> No timestamp found in results. Returning {results.Count} logs.");
406411
return results;
407412
}
408413

409-
// Parse the timestamp - CloudWatch returns ISO 8601 format with milliseconds
410-
var lastTime = DateTimeOffset.Parse(lastTimestamp).ToUnixTimeSeconds();
414+
Console.WriteLine($" -> Last log timestamp: {lastLogTimestamp}");
415+
416+
// Parse the timestamp and add 1 millisecond to avoid querying the same log again
417+
var lastLogDate = DateTimeOffset.Parse(lastLogTimestamp + " +0000");
418+
Console.WriteLine($" -> Last log as DateTimeOffset: {lastLogDate:yyyy-MM-ddTHH:mm:ss.fffZ} ({lastLogDate.ToUnixTimeSeconds()}s)");
419+
420+
var offsetLastLogDate = lastLogDate.AddMilliseconds(1);
421+
Console.WriteLine($" -> Offset timestamp (last + 1ms): {offsetLastLogDate:yyyy-MM-ddTHH:mm:ss.fffZ} ({offsetLastLogDate.ToUnixTimeSeconds()}s)");
411422

423+
// Convert back to seconds for the API
424+
var offsetLastLogTime = offsetLastLogDate.ToUnixTimeSeconds();
425+
426+
Console.WriteLine($" -> Comparing: offsetLastLogTime={offsetLastLogTime}s vs endTime={endTime}s");
427+
Console.WriteLine($" -> End time as date: {DateTimeOffset.FromUnixTimeSeconds(endTime):yyyy-MM-ddTHH:mm:ss.fffZ}");
428+
412429
// Check if there's any time range left to query
413-
if (lastTime >= endTime)
430+
if (offsetLastLogTime >= endTime)
414431
{
432+
Console.WriteLine($" -> No time range left to query. Offset time ({offsetLastLogTime}s) >= end time ({endTime}s)");
415433
return results;
416434
}
417435

418-
// Calculate midpoint between last result and end time
419-
var midpoint = (lastTime + endTime) / 2;
420-
421-
// Ensure we have enough range to split
422-
if (midpoint <= lastTime || midpoint >= endTime)
423-
{
424-
// Range too small to split, just query the remaining range
425-
var remainingResults = await PerformLargeQuery(logGroupName, queryString, lastTime, endTime, limit);
426-
427-
var allResults = new List<List<ResultField>>(results);
428-
// Skip the first result if it's a duplicate of the last result from previous query
429-
if (remainingResults.Count > 0)
430-
{
431-
var firstTimestamp = remainingResults[0].Find(f => f.Field == "@timestamp")?.Value;
432-
if (firstTimestamp == lastTimestamp)
433-
{
434-
remainingResults.RemoveAt(0);
435-
}
436-
}
437-
allResults.AddRange(remainingResults);
438-
return allResults;
439-
}
440-
441-
// Split the remaining range in half
442-
var results1 = await PerformLargeQuery(logGroupName, queryString, lastTime, midpoint, limit);
443-
var results2 = await PerformLargeQuery(logGroupName, queryString, midpoint, endTime, limit);
444-
445-
var combinedResults = new List<List<ResultField>>(results);
436+
// Split the remaining date range in half
437+
var (range1Start, range1End, range2Start, range2End) = SplitDateRange(offsetLastLogTime, endTime);
446438

447-
// Remove duplicate from results1 if it matches the last result
448-
if (results1.Count > 0)
449-
{
450-
var firstTimestamp1 = results1[0].Find(f => f.Field == "@timestamp")?.Value;
451-
if (firstTimestamp1 == lastTimestamp)
452-
{
453-
results1.RemoveAt(0);
454-
}
455-
}
439+
var range1StartDate = DateTimeOffset.FromUnixTimeSeconds(range1Start).ToString("yyyy-MM-ddTHH:mm:ss.fffZ");
440+
var range1EndDate = DateTimeOffset.FromUnixTimeSeconds(range1End).ToString("yyyy-MM-ddTHH:mm:ss.fffZ");
441+
var range2StartDate = DateTimeOffset.FromUnixTimeSeconds(range2Start).ToString("yyyy-MM-ddTHH:mm:ss.fffZ");
442+
var range2EndDate = DateTimeOffset.FromUnixTimeSeconds(range2End).ToString("yyyy-MM-ddTHH:mm:ss.fffZ");
456443

457-
combinedResults.AddRange(results1);
444+
Console.WriteLine($" -> Splitting remaining range:");
445+
Console.WriteLine($" Range 1: {range1StartDate} ({range1Start}s) to {range1EndDate} ({range1End}s)");
446+
Console.WriteLine($" Range 2: {range2StartDate} ({range2Start}s) to {range2EndDate} ({range2End}s)");
447+
448+
// Query both halves recursively
449+
Console.WriteLine($" -> Querying range 1...");
450+
var results1 = await PerformLargeQuery(logGroupName, queryString, range1Start, range1End, limit);
451+
Console.WriteLine($" -> Range 1 returned {results1.Count} logs");
458452

459-
// Remove duplicate from results2 if it matches the last result from results1
460-
if (results2.Count > 0 && results1.Count > 0)
453+
Console.WriteLine($" -> Querying range 2...");
454+
var results2 = await PerformLargeQuery(logGroupName, queryString, range2Start, range2End, limit);
455+
Console.WriteLine($" -> Range 2 returned {results2.Count} logs");
456+
457+
// Combine all results
458+
var allResults = new List<List<ResultField>>(results);
459+
allResults.AddRange(results1);
460+
allResults.AddRange(results2);
461+
462+
Console.WriteLine($" -> Combined total: {allResults.Count} logs ({results.Count} + {results1.Count} + {results2.Count})");
463+
464+
return allResults;
465+
}
466+
467+
/// <summary>
468+
/// Gets the timestamp string of the most recent log from a list of logs.
469+
/// Sorts timestamps to find the actual last one.
470+
/// </summary>
471+
private static string? GetLastLogTimestamp(List<List<ResultField>> logs)
472+
{
473+
var timestamps = logs
474+
.Select(log => log.Find(f => f.Field == "@timestamp")?.Value)
475+
.Where(t => !string.IsNullOrEmpty(t))
476+
.OrderBy(t => t)
477+
.ToList();
478+
479+
if (timestamps.Count == 0)
461480
{
462-
var lastTimestamp1 = results1[results1.Count - 1].Find(f => f.Field == "@timestamp")?.Value;
463-
var firstTimestamp2 = results2[0].Find(f => f.Field == "@timestamp")?.Value;
464-
if (firstTimestamp2 == lastTimestamp1)
465-
{
466-
results2.RemoveAt(0);
467-
}
481+
return null;
468482
}
469-
470-
combinedResults.AddRange(results2);
471483

472-
return combinedResults;
484+
return timestamps[timestamps.Count - 1];
485+
}
486+
487+
/// <summary>
488+
/// Splits a date range in half.
489+
/// </summary>
490+
private static (long range1Start, long range1End, long range2Start, long range2End) SplitDateRange(long startTime, long endTime)
491+
{
492+
var midpoint = startTime + (endTime - startTime) / 2;
493+
return (startTime, midpoint, midpoint, endTime);
473494
}
474495

475496
/// <summary>

0 commit comments

Comments
 (0)