diff --git a/.gitignore b/.gitignore index 94d7f7b5..2d3878b6 100644 --- a/.gitignore +++ b/.gitignore @@ -174,5 +174,8 @@ ctre_sim #Claude .claude +# Robot log files fetched by fetch-logs.ps1 +robot-logs/ + # Prevent accidentally committing the entire repo back in SeriouslyCommonLib/ \ No newline at end of file diff --git a/fetch-logs.ps1 b/fetch-logs.ps1 new file mode 100644 index 00000000..754b5585 --- /dev/null +++ b/fetch-logs.ps1 @@ -0,0 +1,171 @@ +# fetch-logs.ps1 +# Downloads .wpilog files from roboRIO-488-frc.local to a local directory. +# Skips files that already exist locally with the same size (rsync-like behavior). +# Transfers run in parallel to overlap SSH handshake overhead across files. +# +# Usage: .\fetch-logs.ps1 [-Destination ] [-Force] [-Throttle ] +# -Destination Where to save logs (default: .\robot-logs) +# -Force Re-download everything, ignoring local copies +# -Throttle Max simultaneous transfers (default: 4) + +param( + [string]$Destination = "robot-logs", + [switch]$Force, + [int]$Throttle = 4 +) + +$Robot = "roboRIO-488-frc.local" +$User = "admin" +$RemoteLogDir = "/u/logs" # roboRIO mounts USB at /u + +$SshOpts = @( + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-o", "ConnectTimeout=5", + # chacha20 is faster than AES on ARM CPUs that lack hardware AES acceleration (like the roboRIO) + "-o", "Ciphers=chacha20-poly1305@openssh.com,aes128-ctr" +) + +New-Item -ItemType Directory -Force -Path $Destination | Out-Null +$Destination = (Resolve-Path $Destination).Path + +Write-Host "Connecting to $User@$Robot ..." + +# Fetch remote file list as "path size mtime" triples using stat +$RemoteListing = & ssh @SshOpts "$User@$Robot" ` + "find $RemoteLogDir -maxdepth 1 -name '*.wpilog' -exec stat -c '%n %s %Y' {} +" 2>&1 + +if ($LASTEXITCODE -ne 0 -or -not $RemoteListing) { + Write-Error "Could not list log files on robot. Is it connected and is a USB key inserted? (checked $RemoteLogDir)" + exit 1 +} + +# Parse into objects: { Path, Name, RemoteSize, Mtime }, sorted newest first +$RemoteFiles = $RemoteListing -split "`n" | + Where-Object { $_ -match "\.wpilog" } | + ForEach-Object { + $parts = $_.Trim() -split " " + [PSCustomObject]@{ + Path = $parts[0] + Name = Split-Path $parts[0] -Leaf + RemoteSize = [long]$parts[1] + Mtime = [long]$parts[2] + } + } | + Sort-Object Mtime -Descending + +Write-Host "Found $($RemoteFiles.Count) log file(s) on robot." +Write-Host "" + +# Decide what needs downloading +$ToDownload = [System.Collections.Generic.List[PSCustomObject]]::new() +$Skipped = 0 + +foreach ($f in $RemoteFiles) { + $LocalPath = Join-Path $Destination $f.Name + if (-not $Force -and (Test-Path $LocalPath)) { + $LocalSize = (Get-Item $LocalPath).Length + if ($LocalSize -eq $f.RemoteSize) { + Write-Host " [skip] $($f.Name) ($($f.RemoteSize) bytes, already synced)" + $Skipped++ + continue + } + Write-Host " [queue/update] $($f.Name) (local $LocalSize B -> remote $($f.RemoteSize) B)" + } else { + Write-Host " [queue] $($f.Name) ($($f.RemoteSize) bytes)" + } + $f | Add-Member -NotePropertyName LocalPath -NotePropertyValue $LocalPath + $ToDownload.Add($f) +} + +if ($ToDownload.Count -eq 0) { + Write-Host "" + Write-Host "Nothing to download. Skipped: $Skipped" + exit 0 +} + +Write-Host "" +Write-Host "Downloading $($ToDownload.Count) file(s) with up to $Throttle parallel transfer(s) ..." +Write-Host "" + +$TotalBytes = ($ToDownload | Measure-Object -Property RemoteSize -Sum).Sum +$DoneBytes = 0 +$Copied = 0 +$Failed = 0 +$Queue = [System.Collections.Generic.Queue[PSCustomObject]]($ToDownload) +$Active = [System.Collections.Generic.List[object]]::new() + +$Stopwatch = [System.Diagnostics.Stopwatch]::StartNew() + +# Single unified loop: launch new jobs when slots are free, collect finished ones, update progress. +while ($Queue.Count -gt 0 -or $Active.Count -gt 0) { + + # Launch as many jobs as throttle allows + while ($Queue.Count -gt 0 -and $Active.Count -lt $Throttle) { + $f = $Queue.Dequeue() + $job = Start-Job -ScriptBlock { + param($User, $Robot, $RemotePath, $LocalPath) + $opts = @( + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-o", "ConnectTimeout=5", + "-o", "Ciphers=chacha20-poly1305@openssh.com,aes128-ctr" + ) + & scp @opts "${User}@${Robot}:${RemotePath}" "$LocalPath" 2>&1 + return $LASTEXITCODE + } -ArgumentList $User, $Robot, $f.Path, $f.LocalPath + + $job | Add-Member -NotePropertyName FileName -NotePropertyValue $f.Name + $job | Add-Member -NotePropertyName LocalPath -NotePropertyValue $f.LocalPath + $job | Add-Member -NotePropertyName RemoteSize -NotePropertyValue $f.RemoteSize + $Active.Add($job) + } + + # Collect any finished jobs + $Done = @($Active | Where-Object { $_.State -ne 'Running' }) + foreach ($job in $Done) { + $ExitCode = Receive-Job $job + Remove-Job $job + $Active.Remove($job) | Out-Null + if ($ExitCode -eq 0) { + Write-Host " [done] $($job.FileName)" + $DoneBytes += $job.RemoteSize + $Copied++ + } else { + Write-Warning " [fail] $($job.FileName)" + $Failed++ + } + } + + # Update progress bar: count bytes written so far for in-flight files + $InFlightBytes = 0 + $ActiveNames = @() + foreach ($job in $Active) { + if (Test-Path $job.LocalPath) { $InFlightBytes += (Get-Item $job.LocalPath).Length } + $ActiveNames += $job.FileName + } + + $ReceivedBytes = $DoneBytes + $InFlightBytes + $Pct = if ($TotalBytes -gt 0) { [int]($ReceivedBytes / $TotalBytes * 100) } else { 0 } + $ElapsedSec = $Stopwatch.Elapsed.TotalSeconds + $MBps = if ($ElapsedSec -gt 0) { [math]::Round($ReceivedBytes / 1MB / $ElapsedSec, 1) } else { 0 } + $Queued = $Queue.Count + $StatusMsg = "{0:F1} / {1:F1} MB | {2} MB/s | {3} queued | {4}" -f ($ReceivedBytes/1MB), ($TotalBytes/1MB), $MBps, $Queued, ($ActiveNames -join ', ') + Write-Progress -Id 1 -Activity "Fetching logs" -Status $StatusMsg -PercentComplete $Pct + + Start-Sleep -Milliseconds 300 +} + +$Stopwatch.Stop() +Write-Progress -Id 1 -Activity "Fetching logs" -Completed + +$Elapsed = $Stopwatch.Elapsed +$TotalBytes = ($ToDownload | Measure-Object -Property RemoteSize -Sum).Sum +$MBps = if ($Elapsed.TotalSeconds -gt 0) { [math]::Round($TotalBytes / 1MB / $Elapsed.TotalSeconds, 2) } else { "?" } + +Write-Host "" +Write-Host "Done in $($Elapsed.ToString('mm\:ss\.ff')). Copied: $Copied Skipped: $Skipped Failed: $Failed" +Write-Host "Transfer rate: $MBps MB/s ($([math]::Round($TotalBytes / 1MB, 1)) MB total)" +Write-Host "Logs saved to: $Destination"