From 86177a66a0fd56023d2a31f7737ea008f6c5740c Mon Sep 17 00:00:00 2001 From: Alex Schokking Date: Sat, 4 Apr 2026 13:31:34 -0700 Subject: [PATCH 1/2] initial script written by claude --- .gitignore | 3 ++ fetch-logs.ps1 | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 fetch-logs.ps1 diff --git a/.gitignore b/.gitignore index 94d7f7b5..2d3878b6 100644 --- a/.gitignore +++ b/.gitignore @@ -174,5 +174,8 @@ ctre_sim #Claude .claude +# Robot log files fetched by fetch-logs.ps1 +robot-logs/ + # Prevent accidentally committing the entire repo back in SeriouslyCommonLib/ \ No newline at end of file diff --git a/fetch-logs.ps1 b/fetch-logs.ps1 new file mode 100644 index 00000000..a3d4d52d --- /dev/null +++ b/fetch-logs.ps1 @@ -0,0 +1,86 @@ +# fetch-logs.ps1 +# Downloads .wpilog files from roboRIO-488-frc.local to a local directory. +# Skips files that already exist locally with the same size (rsync-like behavior). +# +# Usage: .\fetch-logs.ps1 [-Destination ] [-Force] +# -Destination Where to save logs (default: .\robot-logs) +# -Force Re-download everything, ignoring local copies + +param( + [string]$Destination = "robot-logs", + [switch]$Force +) + +$Robot = "roboRIO-488-frc.local" +$User = "admin" +$RemoteLogDir = "/u/logs" # roboRIO mounts USB at /u + +$SshOpts = @( + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-o", "ConnectTimeout=5" +) + +New-Item -ItemType Directory -Force -Path $Destination | Out-Null +$Destination = (Resolve-Path $Destination).Path + +Write-Host "Connecting to $User@$Robot ..." + +# Fetch remote file list as "path size mtime" triples using stat +$RemoteListing = & ssh @SshOpts "$User@$Robot" ` + "find $RemoteLogDir -maxdepth 1 -name '*.wpilog' -exec stat -c '%n %s %Y' {} +" 2>&1 + +if ($LASTEXITCODE -ne 0 -or -not $RemoteListing) { + Write-Error "Could not list log files on robot. Is it connected and is a USB key inserted? (checked $RemoteLogDir)" + exit 1 +} + +# Parse into objects: { Path, Name, RemoteSize, Mtime }, sorted newest first +$RemoteFiles = $RemoteListing -split "`n" | + Where-Object { $_ -match "\.wpilog" } | + ForEach-Object { + $parts = $_.Trim() -split " " + [PSCustomObject]@{ + Path = $parts[0] + Name = Split-Path $parts[0] -Leaf + RemoteSize = [long]$parts[1] + Mtime = [long]$parts[2] + } + } | + Sort-Object Mtime -Descending + +Write-Host "Found $($RemoteFiles.Count) log file(s) on robot." +Write-Host "" + +$Copied = 0 +$Skipped = 0 +$Failed = 0 + +foreach ($f in $RemoteFiles) { + $LocalPath = Join-Path $Destination $f.Name + + if (-not $Force -and (Test-Path $LocalPath)) { + $LocalSize = (Get-Item $LocalPath).Length + if ($LocalSize -eq $f.RemoteSize) { + Write-Host " [skip] $($f.Name) ($($f.RemoteSize) bytes, already synced)" + $Skipped++ + continue + } + Write-Host " [update] $($f.Name) (local $LocalSize B → remote $($f.RemoteSize) B)" + } else { + Write-Host " [copy] $($f.Name) ($($f.RemoteSize) bytes)" + } + + & scp @SshOpts "${User}@${Robot}:$($f.Path)" "$LocalPath" + if ($LASTEXITCODE -eq 0) { + $Copied++ + } else { + Write-Warning " Failed to copy $($f.Name)" + $Failed++ + } +} + +Write-Host "" +Write-Host "Done. Copied: $Copied Updated: (see above) Skipped: $Skipped Failed: $Failed" +Write-Host "Logs saved to: $Destination" From 00ed6f02927b94d633d1d6eaed62cb02eeb555fc Mon Sep 17 00:00:00 2001 From: Alex Schokking Date: Sat, 4 Apr 2026 13:51:36 -0700 Subject: [PATCH 2/2] Improve fetch-logs.ps1: parallel transfers, live progress, faster cipher - Parallel scp jobs with configurable throttle (default 4) - Unified launch+collect loop so [done] messages appear throughout - Write-Progress bar tracking bytes written to disk by in-flight transfers - chacha20/aes128-ctr cipher preference for faster transfers on roboRIO ARM CPU - robot-logs/ output directory added to .gitignore Co-Authored-By: Claude Sonnet 4.6 --- fetch-logs.ps1 | 115 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 100 insertions(+), 15 deletions(-) diff --git a/fetch-logs.ps1 b/fetch-logs.ps1 index a3d4d52d..754b5585 100644 --- a/fetch-logs.ps1 +++ b/fetch-logs.ps1 @@ -1,14 +1,17 @@ # fetch-logs.ps1 # Downloads .wpilog files from roboRIO-488-frc.local to a local directory. # Skips files that already exist locally with the same size (rsync-like behavior). +# Transfers run in parallel to overlap SSH handshake overhead across files. # -# Usage: .\fetch-logs.ps1 [-Destination ] [-Force] +# Usage: .\fetch-logs.ps1 [-Destination ] [-Force] [-Throttle ] # -Destination Where to save logs (default: .\robot-logs) # -Force Re-download everything, ignoring local copies +# -Throttle Max simultaneous transfers (default: 4) param( [string]$Destination = "robot-logs", - [switch]$Force + [switch]$Force, + [int]$Throttle = 4 ) $Robot = "roboRIO-488-frc.local" @@ -19,7 +22,9 @@ $SshOpts = @( "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", "-o", "LogLevel=ERROR", - "-o", "ConnectTimeout=5" + "-o", "ConnectTimeout=5", + # chacha20 is faster than AES on ARM CPUs that lack hardware AES acceleration (like the roboRIO) + "-o", "Ciphers=chacha20-poly1305@openssh.com,aes128-ctr" ) New-Item -ItemType Directory -Force -Path $Destination | Out-Null @@ -53,13 +58,12 @@ $RemoteFiles = $RemoteListing -split "`n" | Write-Host "Found $($RemoteFiles.Count) log file(s) on robot." Write-Host "" -$Copied = 0 +# Decide what needs downloading +$ToDownload = [System.Collections.Generic.List[PSCustomObject]]::new() $Skipped = 0 -$Failed = 0 foreach ($f in $RemoteFiles) { $LocalPath = Join-Path $Destination $f.Name - if (-not $Force -and (Test-Path $LocalPath)) { $LocalSize = (Get-Item $LocalPath).Length if ($LocalSize -eq $f.RemoteSize) { @@ -67,20 +71,101 @@ foreach ($f in $RemoteFiles) { $Skipped++ continue } - Write-Host " [update] $($f.Name) (local $LocalSize B → remote $($f.RemoteSize) B)" + Write-Host " [queue/update] $($f.Name) (local $LocalSize B -> remote $($f.RemoteSize) B)" } else { - Write-Host " [copy] $($f.Name) ($($f.RemoteSize) bytes)" + Write-Host " [queue] $($f.Name) ($($f.RemoteSize) bytes)" } + $f | Add-Member -NotePropertyName LocalPath -NotePropertyValue $LocalPath + $ToDownload.Add($f) +} - & scp @SshOpts "${User}@${Robot}:$($f.Path)" "$LocalPath" - if ($LASTEXITCODE -eq 0) { - $Copied++ - } else { - Write-Warning " Failed to copy $($f.Name)" - $Failed++ +if ($ToDownload.Count -eq 0) { + Write-Host "" + Write-Host "Nothing to download. Skipped: $Skipped" + exit 0 +} + +Write-Host "" +Write-Host "Downloading $($ToDownload.Count) file(s) with up to $Throttle parallel transfer(s) ..." +Write-Host "" + +$TotalBytes = ($ToDownload | Measure-Object -Property RemoteSize -Sum).Sum +$DoneBytes = 0 +$Copied = 0 +$Failed = 0 +$Queue = [System.Collections.Generic.Queue[PSCustomObject]]($ToDownload) +$Active = [System.Collections.Generic.List[object]]::new() + +$Stopwatch = [System.Diagnostics.Stopwatch]::StartNew() + +# Single unified loop: launch new jobs when slots are free, collect finished ones, update progress. +while ($Queue.Count -gt 0 -or $Active.Count -gt 0) { + + # Launch as many jobs as throttle allows + while ($Queue.Count -gt 0 -and $Active.Count -lt $Throttle) { + $f = $Queue.Dequeue() + $job = Start-Job -ScriptBlock { + param($User, $Robot, $RemotePath, $LocalPath) + $opts = @( + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-o", "ConnectTimeout=5", + "-o", "Ciphers=chacha20-poly1305@openssh.com,aes128-ctr" + ) + & scp @opts "${User}@${Robot}:${RemotePath}" "$LocalPath" 2>&1 + return $LASTEXITCODE + } -ArgumentList $User, $Robot, $f.Path, $f.LocalPath + + $job | Add-Member -NotePropertyName FileName -NotePropertyValue $f.Name + $job | Add-Member -NotePropertyName LocalPath -NotePropertyValue $f.LocalPath + $job | Add-Member -NotePropertyName RemoteSize -NotePropertyValue $f.RemoteSize + $Active.Add($job) + } + + # Collect any finished jobs + $Done = @($Active | Where-Object { $_.State -ne 'Running' }) + foreach ($job in $Done) { + $ExitCode = Receive-Job $job + Remove-Job $job + $Active.Remove($job) | Out-Null + if ($ExitCode -eq 0) { + Write-Host " [done] $($job.FileName)" + $DoneBytes += $job.RemoteSize + $Copied++ + } else { + Write-Warning " [fail] $($job.FileName)" + $Failed++ + } + } + + # Update progress bar: count bytes written so far for in-flight files + $InFlightBytes = 0 + $ActiveNames = @() + foreach ($job in $Active) { + if (Test-Path $job.LocalPath) { $InFlightBytes += (Get-Item $job.LocalPath).Length } + $ActiveNames += $job.FileName } + + $ReceivedBytes = $DoneBytes + $InFlightBytes + $Pct = if ($TotalBytes -gt 0) { [int]($ReceivedBytes / $TotalBytes * 100) } else { 0 } + $ElapsedSec = $Stopwatch.Elapsed.TotalSeconds + $MBps = if ($ElapsedSec -gt 0) { [math]::Round($ReceivedBytes / 1MB / $ElapsedSec, 1) } else { 0 } + $Queued = $Queue.Count + $StatusMsg = "{0:F1} / {1:F1} MB | {2} MB/s | {3} queued | {4}" -f ($ReceivedBytes/1MB), ($TotalBytes/1MB), $MBps, $Queued, ($ActiveNames -join ', ') + Write-Progress -Id 1 -Activity "Fetching logs" -Status $StatusMsg -PercentComplete $Pct + + Start-Sleep -Milliseconds 300 } +$Stopwatch.Stop() +Write-Progress -Id 1 -Activity "Fetching logs" -Completed + +$Elapsed = $Stopwatch.Elapsed +$TotalBytes = ($ToDownload | Measure-Object -Property RemoteSize -Sum).Sum +$MBps = if ($Elapsed.TotalSeconds -gt 0) { [math]::Round($TotalBytes / 1MB / $Elapsed.TotalSeconds, 2) } else { "?" } + Write-Host "" -Write-Host "Done. Copied: $Copied Updated: (see above) Skipped: $Skipped Failed: $Failed" +Write-Host "Done in $($Elapsed.ToString('mm\:ss\.ff')). Copied: $Copied Skipped: $Skipped Failed: $Failed" +Write-Host "Transfer rate: $MBps MB/s ($([math]::Round($TotalBytes / 1MB, 1)) MB total)" Write-Host "Logs saved to: $Destination"