From 33450a926cac27924bd8a54db2a9d36e8d5ea9fc Mon Sep 17 00:00:00 2001 From: Scott Frazer Date: Mon, 6 Oct 2025 16:19:14 -0400 Subject: [PATCH] latest from master --- note/dfu.go | 55 +++++ note/errors.go | 12 + note/event.go | 96 ++++++-- note/notefile.go | 6 + note/session.go | 25 +- notecard-driver-windows7.inf | 74 ------ notecard/cobs_test.go | 8 +- notecard/notecard.go | 211 +++++++++++------ notecard/request.go | 3 + notecard/test.go | 49 +++- notehub/api/devices.go | 109 ++++----- notehub/api/environment_variables.go | 7 + notehub/api/errors.go | 10 - notehub/api/fleet.go | 11 +- notehub/api/job.go | 45 ++++ notehub/api/job_reconciliation.go | 94 ++++++++ notehub/auth.go | 327 +++++++++++++++++++++++++++ notehub/request.go | 49 ++++ 18 files changed, 932 insertions(+), 259 deletions(-) create mode 100644 note/dfu.go delete mode 100644 notecard-driver-windows7.inf create mode 100644 notehub/api/job.go create mode 100644 notehub/api/job_reconciliation.go create mode 100644 notehub/auth.go diff --git a/note/dfu.go b/note/dfu.go new file mode 100644 index 0000000..0e57a2e --- /dev/null +++ b/note/dfu.go @@ -0,0 +1,55 @@ +// Copyright 2025 Blues Inc. All rights reserved. +// Use of this source code is governed by licenses granted by the +// copyright holder including that found in the LICENSE file. + +// Package note dfu.go contains DFU-related structures generated/parsed by the notecard +package note + +// DFUState is the state of the DFU in progress +type DFUState struct { + Type string `json:"type,omitempty"` + File string `json:"file,omitempty"` + Length uint32 `json:"length,omitempty"` + CRC32 uint32 `json:"crc32,omitempty"` + MD5 string `json:"md5,omitempty"` + Phase string `json:"mode,omitempty"` + Status string `json:"status,omitempty"` + BeganSecs uint32 `json:"began,omitempty"` + RetryCount uint32 `json:"retry,omitempty"` + ConsecutiveErrors uint32 `json:"errors,omitempty"` + BinaryRetries uint32 `json:"binretry,omitempty"` + DFUStartCount uint32 `json:"dfu_started,omitempty"` + DFUCompletedCount uint32 `json:"dfu_completed,omitempty"` + ODFUStartedCount uint32 `json:"odfu_started,omitempty"` + ODFUTarget string `json:"odfu_target,omitempty"` + ReadFromService uint32 `json:"read,omitempty"` + UpdatedSecs uint32 `json:"updated,omitempty"` + DownloadComplete bool `json:"dl_complete,omitempty"` + DisabledReason string `json:"disabled,omitempty"` + MinNotecardVersion string `json:"min_card_version,omitempty"` + + // This will always point to the current running version + Version string `json:"version,omitempty"` +} + +// DFUEnv is the data structure passed to Notehub when DFU info changes +type DFUEnv struct { + Card *DFUState `json:"card,omitempty"` + User *DFUState `json:"user,omitempty"` + Modem *DFUState `json:"modem,omitempty"` + Star *DFUState `json:"star,omitempty"` +} + +type DfuPhase string + +const ( + DfuPhaseUnknown DfuPhase = "" + DfuPhaseIdle DfuPhase = "idle" + DfuPhaseError DfuPhase = "error" + DfuPhaseDownloading DfuPhase = "downloading" + DfuPhaseSideloading DfuPhase = "sideloading" + DfuPhaseReady DfuPhase = "ready" + DfuPhaseReadyRetry DfuPhase = "ready-retry" + DfuPhaseUpdating DfuPhase = "updating" + DfuPhaseCompleted DfuPhase = "completed" +) diff --git a/note/errors.go b/note/errors.go index 1777602..4c3d2d7 100644 --- a/note/errors.go +++ b/note/errors.go @@ -13,6 +13,12 @@ import ( // ErrTimeout (golint) const ErrTimeout = "{timeout}" +// ErrInternalTimeout of a notehub-to-notehub transaction (golint) +const ErrInternalTimeout = "{internal-timeout}" + +// ErrRouteTimeout of a notehub-to-customer-service transaction (golint) +const ErrRouteTimeout = "{route-timeout}" + // ErrClosed (golint) const ErrClosed = "{closed}" @@ -115,6 +121,9 @@ const ErrDeviceNotFound = "{device-noexist}" // ErrDeviceNotSpecified (golint) const ErrDeviceNotSpecified = "{device-none}" +// ErrDeviceId (golint) +const ErrDeviceId = "{device-id-invalid}" + // ErrDeviceDisabled (golint) const ErrDeviceDisabled = "{device-disabled}" @@ -142,6 +151,9 @@ const ErrFleetNotFound = "{fleet-noexist}" // ErrCardIo (golint) const ErrCardIo = "{io}" +// ErrCardHeartbeat (golint) +const ErrCardHeartbeat = "{heartbeat}" + // ErrAccessDenied (golint) const ErrAccessDenied = "{access-denied}" diff --git a/note/event.go b/note/event.go index 5d94cf0..46a115f 100644 --- a/note/event.go +++ b/note/event.go @@ -41,6 +41,9 @@ const EventSessionEnd = "session.end" // EventGeolocation (golint) const EventGeolocation = "device.geolocation" +// EventSocket (golint) +const EventSocket = "web.socket" + // EventWebhook (golint) const EventWebhook = "webhook" @@ -65,23 +68,28 @@ type Event struct { Payload []byte `json:"payload,omitempty"` Details *map[string]interface{} `json:"details,omitempty"` // Metadata - SessionUID string `json:"session,omitempty"` - SessionBegan int64 `json:"session_began,omitempty"` - TLS bool `json:"tls,omitempty"` - Transport string `json:"transport,omitempty"` - Continuous bool `json:"continuous,omitempty"` - BestID string `json:"best_id,omitempty"` - DeviceUID string `json:"device,omitempty"` - DeviceSN string `json:"sn,omitempty"` - ProductUID string `json:"product,omitempty"` - AppUID string `json:"app,omitempty"` - Received float64 `json:"received,omitempty"` - Req string `json:"req,omitempty"` - Error string `json:"err,omitempty"` - Updates int32 `json:"updates,omitempty"` - Deleted bool `json:"deleted,omitempty"` - Sent bool `json:"queued,omitempty"` - Bulk bool `json:"bulk,omitempty"` + SessionUID string `json:"session,omitempty"` + SessionBegan int64 `json:"session_began,omitempty"` + TLS bool `json:"tls,omitempty"` + Transport string `json:"transport,omitempty"` + Continuous bool `json:"continuous,omitempty"` + BestID string `json:"best_id,omitempty"` + DeviceUID string `json:"device,omitempty"` + DeviceSN string `json:"sn,omitempty"` + ProductUID string `json:"product,omitempty"` + AppUID string `json:"app,omitempty"` + Received float64 `json:"received,omitempty"` + Req string `json:"req,omitempty"` + Error string `json:"err,omitempty"` + Updates int32 `json:"updates,omitempty"` + Deleted bool `json:"deleted,omitempty"` + Sent bool `json:"queued,omitempty"` + Bulk bool `json:"bulk,omitempty"` + BulkReceived float64 `json:"batch_received,omitempty"` + BulkNumber uint32 `json:"batch_number,omitempty"` + BulkTotal uint32 `json:"batch_total,omitempty"` + FirmwareHost string `json:"firmware_host,omitempty"` + FirmwareNotecard string `json:"firmware_notecard,omitempty"` // This field is ONLY used when we remove the payload for storage reasons, to show the app how large it was MissingPayloadLength int64 `json:"payload_length,omitempty"` // Location @@ -179,13 +187,53 @@ const ( // RouteLogEntry is the log entry used by notification processing type RouteLogEntry struct { - EventSerial int64 `json:"event,omitempty"` - RouteSerial int64 `json:"route,omitempty"` - Date time.Time `json:"date,omitempty"` - Attn bool `json:"attn,omitempty"` - Status string `json:"status,omitempty"` - Text string `json:"text,omitempty"` - URL string `json:"url,omitempty"` + EventSerial int64 `json:"event,omitempty"` + RouteSerial int64 `json:"route,omitempty"` + Date time.Time `json:"date,omitempty"` + Attn bool `json:"attn,omitempty"` + Status string `json:"status,omitempty"` + Text string `json:"text,omitempty"` + URL string `json:"url,omitempty"` + Source RoutingSource `json:"source,omitempty"` + + // Time in milliseconds that the route took to process + // We're making a simplifying assumption that the route will always + // take at least 1ms. So 0 means we didn't record the duration. + Duration int64 `json:"duration,omitempty"` +} + +type RoutingSource uint8 + +const ( + RoutingSourceUnknown RoutingSource = iota + RoutingSourceNormal + RoutingSourceProxy + RoutingSourceRetry + RoutingSourceManual + RoutingSourceDirect + RoutingSourceTest +) + +// String returns a string representation of the routing source +func (s RoutingSource) String() string { + switch s { + case RoutingSourceUnknown: + return "" // display nothing if no entry/default + case RoutingSourceNormal: + return "Normal Routing" + case RoutingSourceProxy: + return "Web Proxy Request" + case RoutingSourceRetry: + return "Auto-Retry" + case RoutingSourceManual: + return "Manual Reroute" + case RoutingSourceDirect: + return "Direct Routing" //only used for test events, should never show in route logs + case RoutingSourceTest: + return "Test" // only used for tests + default: + return "invalid" + } } // GetAggregateEventStatus returns the status of the event given all diff --git a/note/notefile.go b/note/notefile.go index f86ed76..f161f9c 100644 --- a/note/notefile.go +++ b/note/notefile.go @@ -31,9 +31,15 @@ const HealthHostNotefile = "_health_host.qo" // GeolocationNotefile is the hard-wired notefile that the notehub uses when performing a geolocation const GeolocationNotefile = "_geolocate.qo" +// SocketNotefile is the hard-wired notefile that the notehub uses when doing websocket I/O +const SocketNotefile = "_socket.qo" + // WebNotefile is the hard-wired notefile that the notehub uses when performing web requests const WebNotefile = "_web.qo" +// WatchdogNotefile is the hard-wired notefile that the notehub uses when adding watchdog messages +const WatchdogNotefile = "_watchdog.qo" + // SyncPriorityLowest (golint) const SyncPriorityLowest = -3 diff --git a/note/session.go b/note/session.go index aa99981..a9d3998 100644 --- a/note/session.go +++ b/note/session.go @@ -92,6 +92,14 @@ type DeviceSession struct { PowerPrimary bool `json:"power_primary,omitempty"` // Mojo power usage PowerMahUsed float64 `json:"power_mah,omitempty"` + // Information about failed connections PRIOR to this one + PenaltySecs uint32 `json:"penalty_secs,omitempty"` + FailedConnects uint32 `json:"failed_connects,omitempty"` + // Socket-relate + SocketAlias string `json:"socket_alias,omitempty"` + SocketConnectError string `json:"socket_connect_error,omitempty"` + SocketBytesSent int64 `json:"socket_bytes_sent,omitempty"` + SocketBytesRcvd int64 `json:"socket_bytes_rcvd,omitempty"` } func (s *DeviceSession) This() *DeviceUsage { @@ -115,20 +123,21 @@ func (s *DeviceSession) Period() *DeviceUsage { return s.PeriodPtr } -// TowerLocation is the cell tower location structure generated by the tower utility +// TowerLocation is a location structure generated by a lookup type TowerLocation struct { - When int64 `json:"time,omitempty"` // time when this location was ascertained - Name string `json:"n,omitempty"` // name of the location - CountryCode string `json:"c,omitempty"` // country code - Lat float64 `json:"lat,omitempty"` // latitude - Lon float64 `json:"lon,omitempty"` // longitude - TimeZone string `json:"zone,omitempty"` // timezone name + Source string `json:"source,omitempty"` // source of this location + When int64 `json:"time,omitempty"` // time when this location was ascertained + Name string `json:"n,omitempty"` // name of the location + CountryCode string `json:"c,omitempty"` // country code + Lat float64 `json:"lat,omitempty"` // latitude + Lon float64 `json:"lon,omitempty"` // longitude + TimeZone string `json:"zone,omitempty"` // timezone name MCC int `json:"mcc,omitempty"` MNC int `json:"mnc,omitempty"` LAC int `json:"lac,omitempty"` CID int `json:"cid,omitempty"` OLC string `json:"l,omitempty"` // open location code TimeZoneID int `json:"z,omitempty"` // timezone id (see tz.go) - Count int64 `json:"count,omitempty"` // number of times this location was recently used + Deprecated int64 `json:"count,omitempty"` // (no longer used or supported) Towers int `json:"towers,omitempty"` // number of triangulation points } diff --git a/notecard-driver-windows7.inf b/notecard-driver-windows7.inf deleted file mode 100644 index 2a1c6cf..0000000 --- a/notecard-driver-windows7.inf +++ /dev/null @@ -1,74 +0,0 @@ -;--------------------------------------------------------------------------------- -;Generic serial port driver for Windows versions prior to Windows 10 -;--------------------------------------------------------------------------------- - -[Version] -Signature="$Windows NT$" -Class=Ports -;Standard Windows serial/parallel port class -ClassGuid={4D36E978-E325-11CE-BFC1-08002BE10318} -Provider=%MFGNAME% -CatalogFile=%MFGFILENAME%.cat -DriverVer=12/06/2012,5.1.2600.7 - -[Manufacturer] -%MFGNAME%=DeviceList,NTamd64 - -[DeviceList] -%DESCRIPTION%=DriverInstall,USB\VID_30A4&PID_0001 - -[DeviceList.NTamd64] -%DESCRIPTION%=DriverInstall,USB\VID_30A4&PID_0001 - -[DriverInstall.nt] -include=mdmcpq.inf -CopyFiles=FakeModemCopyFileSection -AddReg=DriverInstall.nt.AddReg - -[DriverInstall.nt.AddReg] -HKR,,DevLoader,,*ntkern -HKR,,NTMPDriver,,%DRIVERFILENAME%.sys -HKR,,EnumPropPages32,,"MsPorts.dll,SerialPortPropPageProvider" - -[DriverInstall.NT.Services] -include=mdmcpq.inf -AddService=usbser, 0x00000002, LowerFilter_Service_Inst - -[DriverInstall.NTamd64] -include=mdmcpq.inf -CopyFiles=FakeModemCopyFileSection -AddReg=DriverInstall.NTamd64.AddReg - -[DriverInstall.NTamd64.AddReg] -HKR,,DevLoader,,*ntkern -HKR,,NTMPDriver,,%DRIVERFILENAME%.sys -HKR,,EnumPropPages32,,"MsPorts.dll,SerialPortPropPageProvider" - -[DriverInstall.NTamd64.Services] -include=mdmcpq.inf -AddService=usbser, 0x00000002, LowerFilter_Service_Inst - -[DestinationDirs] -DefaultDestDir=12 - -[SourceDisksNames] - -[SourceDisksFiles] - -[FakeModemCopyFileSection] - -[LowerFilter_Service_Inst] -DisplayName= %SERVICE% -ServiceType= 1 -StartType = 3 -ErrorControl = 0 -ServiceBinary = %12%\usbser.sys - -[Strings] -MFGFILENAME="notecard" -DRIVERFILENAME ="usbser" -MFGNAME="Blues Wireless" -DESCRIPTION="Notecard" -SERVICE="USB Driver" - - diff --git a/notecard/cobs_test.go b/notecard/cobs_test.go index a7cfbd7..8d7fbcc 100644 --- a/notecard/cobs_test.go +++ b/notecard/cobs_test.go @@ -9,14 +9,14 @@ import ( ) func TestCob(t *testing.T) { - rand.Seed(time.Now().UnixNano()) + rng := rand.New(rand.NewSource(time.Now().UnixNano())) min := 100 max := 1000 - len := rand.Intn(max-min+1) + min + len := rng.Intn(max-min+1) + min buf := make([]byte, len) - xor := byte(rand.Int()) + xor := byte(rng.Int()) - _, err := rand.Read(buf) + _, err := rng.Read(buf) require.NoError(t, err) encoded, err := CobsEncode(buf, xor) diff --git a/notecard/notecard.go b/notecard/notecard.go index 820e908..3c6da96 100644 --- a/notecard/notecard.go +++ b/notecard/notecard.go @@ -6,6 +6,7 @@ package notecard import ( "bytes" + "encoding/json" "fmt" "hash/crc32" "io" @@ -44,9 +45,8 @@ var ( multiportTransLock [128]sync.RWMutex ) -// SerialTimeoutMs is the response timeout for Notecard serial communications. (This is public -// in case someone wants to alter it.) -var SerialTimeoutMs = 30000 +// Default transaction timeout (before receiving anything from the notecard) +const transactionTimeoutMsDefault = 30000 // IgnoreWindowsHWErrSecs is the amount of time to ignore a Windows serial communiction error. var IgnoreWindowsHWErrSecs = 2 @@ -127,6 +127,13 @@ type Context struct { ResetFn func(context *Context, portConfig int) (err error) TransactionFn func(context *Context, portConfig int, noResponse bool, reqJSON []byte) (rspJSON []byte, err error) + // Transaction timeout (0 for default) + transactionTimeoutMs int + + // User-specified heartbeat function + HeartbeatCtx interface{} + HeartbeatFn func(context *Context, userCtx interface{}, response []byte) bool + // Trace functions traceOpenFn func(context *Context) (err error) traceReadFn func(context *Context) (data []byte, err error) @@ -176,6 +183,25 @@ func cardReportError(context *Context, err error) { } } +// Set the transaction function +func (context *Context) GetTransactionTimeoutMs() int { + if context.transactionTimeoutMs == 0 { + return transactionTimeoutMsDefault + } + return context.transactionTimeoutMs +} + +// Set the request timeout (0 to restore for default) +func (context *Context) SetTransactionTimeoutMs(msec int) { + context.transactionTimeoutMs = msec +} + +// Set or clear the heartbeat function +func (context *Context) SetTransactionHeartbeatFn(userFn func(context *Context, userCtx interface{}, rsp []byte) bool, userCtx interface{}) { + context.HeartbeatFn = userFn + context.HeartbeatCtx = userCtx +} + // DebugOutput enables/disables debug output func (context *Context) DebugOutput(enabled bool, pretty bool) { context.Debug = enabled @@ -264,7 +290,7 @@ func cardResetSerial(context *Context, portConfig int) (err error) { if debugSerialIO { fmt.Printf("cardResetSerial: about to write newline\n") } - serialIOBegin(context, SerialTimeoutMs) + serialIOBegin(context, context.GetTransactionTimeoutMs()) _, err = context.serialPort.Write([]byte("\n")) err = serialIOEnd(context, err) if debugSerialIO { @@ -293,7 +319,7 @@ func cardResetSerial(context *Context, portConfig int) (err error) { } if err != nil { // Ignore errors after reset, as the only purpose of reset is to drain the input buffer - err = cardReopenSerial(context, portConfig) + err = CardReopenSerial(context, portConfig) return err } somethingFound := false @@ -381,7 +407,7 @@ func OpenSerial(port string, portConfig int) (context *Context, err error) { context.PortEnumFn = serialPortEnum context.PortDefaultsFn = serialDefault context.CloseFn = cardCloseSerial - context.ReopenFn = cardReopenSerial + context.ReopenFn = CardReopenSerial context.ResetFn = cardResetSerial context.TransactionFn = cardTransactionSerial context.traceOpenFn = serialTraceOpen @@ -390,11 +416,14 @@ func OpenSerial(port string, portConfig int) (context *Context, err error) { // Record serial configuration, and whether or not we are using the default context.isSerial = true + context.serialName, context.serialConfig.BaudRate = serialDefault() if port == "" { context.serialUseDefault = true - context.serialName, context.serialConfig.BaudRate = serialDefault() } else { context.serialName = port + + } + if portConfig != 0 { context.serialConfig.BaudRate = portConfig } @@ -519,7 +548,7 @@ func cardCloseSerial(context *Context) { // Close I2C func cardCloseI2C(context *Context) { - i2cClose() + _ = i2cClose() context.portIsOpen = false } @@ -539,7 +568,7 @@ func (context *Context) Reopen(portConfig int) (err error) { } // Reopen serial -func cardReopenSerial(context *Context, portConfig int) (err error) { +func CardReopenSerial(context *Context, portConfig int) (err error) { // Close if open cardCloseSerial(context) @@ -552,6 +581,9 @@ func cardReopenSerial(context *Context, portConfig int) (err error) { return fmt.Errorf("error opening serial port: serial device not available %s", note.ErrCardIo) } + if portConfig != 0 { + context.serialConfig.BaudRate = portConfig + } // Set default speed if not set if context.serialConfig.BaudRate == 0 { _, context.serialConfig.BaudRate = serialDefault() @@ -559,7 +591,7 @@ func cardReopenSerial(context *Context, portConfig int) (err error) { // Open the serial port if debugSerialIO { - fmt.Printf("cardReopenSerial: about to open '%s'\n", context.serialName) + fmt.Printf("CardReopenSerial: about to open '%s'\n", context.serialName) } context.serialPort, err = serial.Open(context.serialName, &context.serialConfig) if debugSerialIO { @@ -746,7 +778,7 @@ func (context *Context) SendBytes(reqBytes []byte) (err error) { // Do a reset if one was pending if context.resetRequired { - context.Reset(portConfig) + _ = context.Reset(portConfig) } // Do the send, with no response requested @@ -775,7 +807,7 @@ func (context *Context) receiveBytes(portConfig int) (rspBytes []byte, err error // Do a reset if one was pending if context.resetRequired { - context.Reset(portConfig) + _ = context.Reset(portConfig) } // Request is empty @@ -878,10 +910,7 @@ func (context *Context) transactionJSON(reqJSON []byte, multiport bool, portConf // Transaction retry loop. Note that "err" must be set before breaking out of loop err = nil - for { - if lastRequestRetries > requestRetriesAllowed { - break - } + for lastRequestRetries <= requestRetriesAllowed { // Only do reopen/reset in the single-port case, because we may not be talking to the port in error if !multiport { @@ -898,7 +927,7 @@ func (context *Context) transactionJSON(reqJSON []byte, multiport bool, portConf // Do a reset if one was pending if context.resetRequired { - context.Reset(portConfig) + _ = context.Reset(portConfig) } } @@ -909,7 +938,7 @@ func (context *Context) transactionJSON(reqJSON []byte, multiport bool, portConf // We can defer the error if a single port, but we need to reset it NOW if multiport if multiport { if context.ResetFn != nil { - context.ResetFn(context, portConfig) + _ = context.ResetFn(context, portConfig) } } else { context.resetRequired = true @@ -950,7 +979,7 @@ func (context *Context) transactionJSON(reqJSON []byte, multiport bool, portConf // We can defer the error if a single port, but we need to reset it NOW if multiport if multiport { if context.ResetFn != nil { - context.ResetFn(context, portConfig) + _ = context.ResetFn(context, portConfig) } } else { context.resetRequired = true @@ -1055,7 +1084,7 @@ func cardTransactionSerial(context *Context, portConfig int, noResponse bool, re } // Set the serial read timeout to 30 seconds, preventing reads under windows from stalling indefinitely on a serial error. - context.serialPort.SetReadTimeout(30 * time.Second) + _ = context.serialPort.SetReadTimeout(30 * time.Second) // Handle the special case where we are looking only for a reply if len(reqJSON) > 0 { @@ -1071,7 +1100,7 @@ func cardTransactionSerial(context *Context, portConfig int, noResponse bool, re if debugSerialIO { fmt.Printf("cardTransactionSerial: about to write %d bytes\n", segLen) } - serialIOBegin(context, SerialTimeoutMs) + serialIOBegin(context, context.GetTransactionTimeoutMs()) _, err = context.serialPort.Write(reqJSON[segOff : segOff+segLen]) err = serialIOEnd(context, err) if debugSerialIO { @@ -1098,7 +1127,8 @@ func cardTransactionSerial(context *Context, portConfig int, noResponse bool, re } // Read the reply until we get '\n' at the end - waitBeganSecs := time.Now().Unix() + waitBegan := time.Now() + waitExpires := waitBegan.Add(time.Duration(context.GetTransactionTimeoutMs()) * time.Millisecond) for { var length int buf := make([]byte, 2048) @@ -1106,7 +1136,8 @@ func cardTransactionSerial(context *Context, portConfig int, noResponse bool, re fmt.Printf("cardTransactionSerial: about to read up to %d bytes\n", len(buf)) } readBeganMs := int(time.Now().UnixNano() / 1000000) - serialIOBegin(context, SerialTimeoutMs) + waitRemainingMs := int(time.Until(waitExpires).Milliseconds()) + serialIOBegin(context, waitRemainingMs) length, err = context.serialPort.Read(buf) err = serialIOEnd(context, err) readElapsedMs := int(time.Now().UnixNano()/1000000) - readBeganMs @@ -1130,7 +1161,7 @@ func cardTransactionSerial(context *Context, portConfig int, noResponse bool, re continue } // Ignore [flaky, rare, Windows] hardware errors for up to several seconds - if (time.Now().Unix() - waitBeganSecs) > int64(IgnoreWindowsHWErrSecs) { + if (time.Now().Unix() - waitBegan.Unix()) > int64(IgnoreWindowsHWErrSecs) { err = fmt.Errorf("error reading from module: %s %s", err, note.ErrCardIo) cardReportError(context, err) return @@ -1139,36 +1170,64 @@ func cardTransactionSerial(context *Context, portConfig int, noResponse bool, re continue } rspJSON = append(rspJSON, buf[:length]...) - if strings.Contains(string(rspJSON), "\n") { + if !strings.Contains(string(rspJSON), "\n") { + continue + } - // At this point, if we split the string at \n its len must be >= 2 - lines := strings.Split(string(rspJSON), "\n") - lastLine := lines[len(lines)-1] - secondToLastLine := lines[len(lines)-2] + // We now have at least one whole line. If we're just gathering a reply, we're done. + if len(reqJSON) == 0 { + break + } + // At this point, if we split the string at \n its len must be >= 2 + // If the json didn't END in \n, we are still collecting a partial line + lines := strings.Split(string(rspJSON), "\n") + lastLine := lines[len(lines)-1] + secondToLastLine := lines[len(lines)-2] + if lastLine != "" { // The reply should be only a single line. However, if the user had been // in trace mode (likely on USB) we may be receiving trace lines that // were sent to us and inserted into the serial buffer prior to the JSON reply. - if lastLine != "" { - // If the json didn't END in \n, we are still collecting a partial line - rspJSON = []byte(lastLine) - } else if len(reqJSON) == 0 { - // If we're just gathering a reply, we accept binary because it may be COBS - break - } else { + rspJSON = []byte(lastLine) + continue + } - // We're done if and only if the response looks like JSON - if len(secondToLastLine) > 0 && secondToLastLine[0] == '{' { - break - } + // Skip the line if it's empty or doesn't look like JSON + if len(secondToLastLine) == 0 || secondToLastLine[0] != '{' { + rspJSON = []byte{} + continue + } - // Drop it, because the line doesn't look like JSON - rspJSON = []byte{} + // ** We now have a clean response in rspJSON ** - } + // We're done if it's not a heartbeat + fn := context.HeartbeatFn + if fn == nil { + break + } + m := make(map[string]string) + if json.Unmarshal(rspJSON, &m) != nil { + break + } + v, errPresent := m["err"] + if !errPresent { + break + } + if !strings.Contains(v, note.ErrCardHeartbeat) { + break + } + // Call the heartbeat function, and abort if it requests that we do so + if fn(context, context.HeartbeatCtx, rspJSON) { + err = fmt.Errorf("aborted by heartbeat function") + cardReportError(context, err) + return } + // Reset the JSON and timeout and start again + rspJSON = []byte{} + waitBegan = time.Now() + waitExpires = waitBegan.Add(time.Duration(context.GetTransactionTimeoutMs()) * time.Millisecond) } // Done @@ -1219,10 +1278,8 @@ func cardTransactionI2C(context *Context, portConfig int, noResponse bool, reqJS jsonbufLen = 0 receivedNewline := false chunklen := 0 - expireSecs := 60 - expires := time.Now().Add(time.Duration(expireSecs) * time.Second) - longExpireSecs := 240 - longexpires := time.Now().Add(time.Duration(longExpireSecs) * time.Second) + waitBegan := time.Now() + waitExpires := waitBegan.Add(time.Duration(context.GetTransactionTimeoutMs()) * time.Millisecond) for { // Read the next chunk @@ -1237,9 +1294,10 @@ func cardTransactionI2C(context *Context, portConfig int, noResponse bool, reqJS readlen := len(readbuf) jsonbufLen += readlen - // If we received something, reset the expiration + // If we received something, reset the expiration to what we'd expect for just the + // I/O portion of a transaction. if readlen > 0 { - expires = time.Now().Add(time.Duration(90) * time.Second) + waitExpires = time.Now().Add(time.Duration(5) * time.Second) } // If the last byte of the chunk is \n, chances are that we're done. However, just so @@ -1260,26 +1318,53 @@ func cardTransactionI2C(context *Context, portConfig int, noResponse bool, reqJS continue } - // If there's nothing available and we received a newline, we're done - if receivedNewline { - break + // See if we're done + if !receivedNewline { + + // If we've timed out and nothing's available, exit + expired := time.Now().After(waitExpires) + if context.i2cMultiport { + expired = time.Now().After(waitBegan.Add(time.Duration(90) * time.Second)) + } + if expired { + err = fmt.Errorf("transaction timeout (%d bytes received before timeout) %s", jsonbufLen, note.ErrCardIo+note.ErrTimeout) + return + } + + // Continue receiving + continue } - // If we've timed out and nothing's available, exit - expired := false - timeoutSecs := 0 - if !context.i2cMultiport || jsonbufLen == 0 { - expired = time.Now().After(expires) - timeoutSecs = expireSecs - } else { - expired = time.Now().After(longexpires) - timeoutSecs = longExpireSecs + // ** We now have a clean response in rspJSON ** + + // We're done if it's not a heartbeat + fn := context.HeartbeatFn + if fn == nil { + break + } + m := make(map[string]string) + if json.Unmarshal(rspJSON, &m) != nil { + break + } + v, errPresent := m["err"] + if !errPresent { + break } - if expired { - err = fmt.Errorf("transaction timeout (received %d bytes in %d secs) %s", jsonbufLen, timeoutSecs, note.ErrCardIo+note.ErrTimeout) + if !strings.Contains(v, note.ErrCardHeartbeat) { + break + } + + // Call the heartbeat function, and abort if it requests that we do so + if fn(context, context.HeartbeatCtx, rspJSON) { + err = fmt.Errorf("aborted by heartbeat function") + cardReportError(context, err) return } + // Reset the JSON and timeout and start again + rspJSON = []byte{} + waitBegan = time.Now() + waitExpires = waitBegan.Add(time.Duration(context.GetTransactionTimeoutMs()) * time.Millisecond) } // Done diff --git a/notecard/request.go b/notecard/request.go index 1c32237..4c92238 100644 --- a/notecard/request.go +++ b/notecard/request.go @@ -19,6 +19,9 @@ const ReqFileSet = "file.set" // ReqFileDelete (golint) const ReqFileDelete = "file.delete" +// ReqFileClear (golint) +const ReqFileClear = "file.clear" + // ReqFileGetL (golint) const ReqFileGetL = "file.get" diff --git a/notecard/test.go b/notecard/test.go index 1fe4bb4..f11df30 100644 --- a/notecard/test.go +++ b/notecard/test.go @@ -7,7 +7,6 @@ package notecard // CardTest is a structure that is returned by the notecard after completing its self-test type CardTest struct { DeviceUID string `json:"device,omitempty"` - DefaultProductUID string `json:"default_product,omitempty"` Error string `json:"err,omitempty"` Status string `json:"status,omitempty"` Tests string `json:"tests,omitempty"` @@ -22,16 +21,20 @@ type CardTest struct { IMSI string `json:"imsi,omitempty"` IMSI2 string `json:"imsi2,omitempty"` IMEI string `json:"imei,omitempty"` + Apn string `json:"apn,omitempty"` + Band string `json:"band,omitempty"` + Channel string `json:"channel,omitempty"` When uint32 `json:"when,omitempty"` SKU string `json:"sku,omitempty"` OrderingCode string `json:"ordering_code,omitempty"` + DefaultProductUID string `json:"default_product,omitempty"` SIMActivationKey string `json:"key,omitempty"` + SIMless bool `json:"simless,omitempty"` Station string `json:"station,omitempty"` Operator string `json:"operator,omitempty"` Check uint32 `json:"check,omitempty"` CellUsageBytes uint32 `json:"cell_used,omitempty"` CellProvisionedTime uint32 `json:"cell_provisioned,omitempty"` - LSEStability string `json:"lse,omitempty"` // Firmware info FirmwareOrg string `json:"org,omitempty"` FirmwareProduct string `json:"product,omitempty"` @@ -41,6 +44,13 @@ type CardTest struct { FirmwarePatch uint32 `json:"ver_patch,omitempty"` FirmwareBuild uint32 `json:"ver_build,omitempty"` FirmwareBuilt string `json:"built,omitempty"` + // Certificate and cert info + CertSN string `json:"certsn,omitempty"` + Cert string `json:"cert,omitempty"` + // Card initialization requests + SetupRequests string `json:"setup,omitempty"` + // Detailed information about LSE stability + LSEStability string `json:"lse,omitempty"` // LoRa notecard provisioning info DevEui string `json:"deveui,omitempty"` AppEui string `json:"appeui,omitempty"` @@ -48,9 +58,34 @@ type CardTest struct { FreqPlan string `json:"freqplan,omitempty"` LWVersion string `json:"lorawan,omitempty"` PHVersion string `json:"regional,omitempty"` - // Certificate and cert info - CertSN string `json:"certsn,omitempty"` - Cert string `json:"cert,omitempty"` - // Card initialization requests - SetupRequests string `json:"setup,omitempty"` + // For manufacturing + CPN string `json:"cpn,omitempty"` + // For Iridium + IriSku string `json:"iri_sku,omitempty"` + IriSn string `json:"iri_sn,omitempty"` + IriImei string `json:"iri_imei,omitempty"` + IriIccid string `json:"iri_iccid,omitempty"` +} + +// Remove fields that are not useful or are sensitive when externalizing for public consumption +func CardTestExternalized(ct CardTest) CardTest { + ct.BoardVersion = 0 // distracting + ct.BoardType = 0 // distracting + ct.SIMActivationKey = "" // security + ct.Station = "" // privacy + ct.Operator = "" // privacy + ct.Check = 0 // invalid after externalizing + ct.FirmwareOrg = "" // distracting + ct.FirmwareProduct = "" // distracting + ct.FirmwareMajor = 0 // distracting + ct.FirmwareMinor = 0 // distracting + ct.FirmwarePatch = 0 // distracting + ct.FirmwareBuild = 0 // distracting + ct.FirmwareBuilt = "" // distracting + ct.CertSN = "" // security + ct.Cert = "" // security + ct.LSEStability = "" // distracting + ct.SetupRequests = "" // security + ct.LSEStability = "" // distracting + return ct } diff --git a/notehub/api/devices.go b/notehub/api/devices.go index d78214b..1d9b081 100644 --- a/notehub/api/devices.go +++ b/notehub/api/devices.go @@ -10,14 +10,21 @@ import ( // // The response object for getting devices. type GetDevicesResponse struct { - Devices []DeviceResponse `json:"devices"` - HasMore bool `json:"has_more"` + Devices []GetDeviceResponse `json:"devices"` + HasMore bool `json:"has_more"` +} + +// Part of the response object for a device. +type DeviceHealthLogEntry struct { + When string `json:"when"` + Text string `json:"text"` + Alert bool `json:"alert"` } // DeviceResponse v1 // // The response object for a device. -type DeviceResponse struct { +type GetDeviceResponse struct { UID string `json:"uid"` SerialNumber string `json:"serial_number,omitempty"` SKU string `json:"sku,omitempty"` @@ -26,6 +33,9 @@ type DeviceResponse struct { Provisioned string `json:"provisioned"` LastActivity *string `json:"last_activity"` + FirmwareHost string `json:"firmware_host,omitempty"` + FirmwareNotecard string `json:"firmware_notecard,omitempty"` + Contact *ContactResponse `json:"contact,omitempty"` ProductUID string `json:"product_uid"` @@ -36,11 +46,20 @@ type DeviceResponse struct { GPSLocation *Location `json:"gps_location,omitempty"` TriangulatedLocation *Location `json:"triangulated_location,omitempty"` - Voltage float64 `json:"voltage"` - Temperature float64 `json:"temperature"` - DFUEnv *DFUEnv `json:"dfu,omitempty"` - Disabled bool `json:"disabled,omitempty"` - Tags string `json:"tags,omitempty"` + Voltage float64 `json:"voltage"` + Temperature float64 `json:"temperature"` + DFUEnv *note.DFUEnv `json:"dfu,omitempty"` + Disabled bool `json:"disabled,omitempty"` + Tags string `json:"tags,omitempty"` + + // Activity + RecentActivityBase string `json:"recent_event_base,omitempty"` + RecentEventCount []int `json:"recent_event_count,omitempty"` + RecentSessionCount []int `json:"recent_session_count,omitempty"` + RecentSessionSeconds []int `json:"recent_session_seconds,omitempty"` + + // Health + HealthLog []DeviceHealthLogEntry `json:"health_log,omitempty"` } // GetDevicesPublicKeysResponse v1 @@ -116,72 +135,30 @@ type HealthLogEntry struct { Text string `json:"text"` } -// DFUState is the state of the DFU in progress -type DFUState struct { - Type string `json:"type,omitempty"` - File string `json:"file,omitempty"` - Length uint32 `json:"length,omitempty"` - CRC32 uint32 `json:"crc32,omitempty"` - MD5 string `json:"md5,omitempty"` - Phase string `json:"mode,omitempty"` - Status string `json:"status,omitempty"` - BeganSecs uint32 `json:"began,omitempty"` - RetryCount uint32 `json:"retry,omitempty"` - ConsecutiveErrors uint32 `json:"errors,omitempty"` - ReadFromService uint32 `json:"read,omitempty"` - UpdatedSecs uint32 `json:"updated,omitempty"` - DownloadComplete bool `json:"dl_complete,omitempty"` - DisabledReason string `json:"disabled,omitempty"` - MinNotecardVersion string `json:"min_card_version,omitempty"` - - // This will always point to the current running version - Version string `json:"version,omitempty"` -} - -// DFUEnv is the data structure passed to Notehub when DFU info changes -type DFUEnv struct { - Card *DFUState `json:"card,omitempty"` - User *DFUState `json:"user,omitempty"` -} - -type DfuPhase string - -const ( - DfuPhaseUnknown DfuPhase = "" - DfuPhaseIdle DfuPhase = "idle" - DfuPhaseError DfuPhase = "error" - DfuPhaseDownloading DfuPhase = "downloading" - DfuPhaseSideloading DfuPhase = "sideloading" - DfuPhaseReady DfuPhase = "ready" - DfuPhaseReadyRetry DfuPhase = "ready-retry" - DfuPhaseUpdating DfuPhase = "updating" - DfuPhaseCompleted DfuPhase = "completed" -) - -var allDfuPhases = []DfuPhase{ - DfuPhaseUnknown, - DfuPhaseIdle, - DfuPhaseError, - DfuPhaseDownloading, - DfuPhaseSideloading, - DfuPhaseReady, - DfuPhaseReadyRetry, - DfuPhaseUpdating, - DfuPhaseCompleted, +var allDfuPhases = []note.DfuPhase{ + note.DfuPhaseUnknown, + note.DfuPhaseIdle, + note.DfuPhaseError, + note.DfuPhaseDownloading, + note.DfuPhaseSideloading, + note.DfuPhaseReady, + note.DfuPhaseReadyRetry, + note.DfuPhaseUpdating, + note.DfuPhaseCompleted, } -func ParseDfuPhase(phase string) DfuPhase { +func ParseDfuPhase(phase string) note.DfuPhase { phase = strings.ToLower(phase) for _, validPhase := range allDfuPhases { if phase == string(validPhase) { return validPhase } } - return DfuPhaseUnknown + return note.DfuPhaseUnknown } -func (phase DfuPhase) IsTerminal() bool { - return phase == DfuPhaseError || - phase == DfuPhaseCompleted || - phase == DfuPhaseIdle +func IsDfuTerminal(phase note.DfuPhase) bool { + return phase == note.DfuPhaseError || + phase == note.DfuPhaseCompleted || + phase == note.DfuPhaseIdle } diff --git a/notehub/api/environment_variables.go b/notehub/api/environment_variables.go index 8427e56..484bd09 100644 --- a/notehub/api/environment_variables.go +++ b/notehub/api/environment_variables.go @@ -117,6 +117,13 @@ type GetDeviceEnvironmentVariablesResponse struct { // // required: true EnvironmentVariablesEnvDefault map[string]string `json:"environment_variables_env_default"` + + // EnvironmentVariablesEffective + // + // The environment variables for the device as though they were fully resolved by resolution rules + // + // required: true + EnvironmentVariablesEffective map[string]string `json:"environment_variables_effective"` } // PutDeviceEnvironmentVariablesRequest v1 diff --git a/notehub/api/errors.go b/notehub/api/errors.go index 509a319..7c36908 100644 --- a/notehub/api/errors.go +++ b/notehub/api/errors.go @@ -5,7 +5,6 @@ package api import ( - "io" "net/http" ) @@ -62,18 +61,9 @@ var SuspendedBillingAccountResponse = ErrorResponse{ // and adds the request Body (if it exists) into the response.Details["body"] as a string func (e ErrorResponse) WithRequest(r *http.Request) ErrorResponse { e.Request = r.RequestURI - var bodyBytes []byte - if r.Body != nil { - var err error - bodyBytes, err = io.ReadAll(r.Body) - if err != nil { - return e - } - } if len(e.Details) == 0 { e.Details = make(map[string]interface{}) } - e.Details["body"] = string(bodyBytes) return e } diff --git a/notehub/api/fleet.go b/notehub/api/fleet.go index d00f7b2..17cd61d 100644 --- a/notehub/api/fleet.go +++ b/notehub/api/fleet.go @@ -16,7 +16,10 @@ type FleetResponse struct { // RFC3339 timestamp, in UTC. Created string `json:"created"` - SmartRule string `json:"smart_rule,omitempty"` + EnvironmentVariables map[string]string `json:"environment_variables"` + + SmartRule string `json:"smart_rule,omitempty"` + WatchdogMins int64 `json:"watchdog_mins,omitempty"` } // PutDeviceFleetsRequest v1 @@ -49,7 +52,8 @@ type DeleteDeviceFleetsRequest struct { type PostFleetRequest struct { Label string `json:"label"` - SmartRule string `json:"smart_rule,omitempty"` + SmartRule string `json:"smart_rule,omitempty"` + WatchdogMins int64 `json:"watchdog_mins,omitempty"` } // PutFleetRequest v1 @@ -60,5 +64,6 @@ type PutFleetRequest struct { AddDevices []string `json:"addDevices,omitempty"` RemoveDevices []string `json:"removeDevices,omitempty"` - SmartRule string `json:"smart_rule,omitempty"` + SmartRule string `json:"smart_rule,omitempty"` + WatchdogMins int64 `json:"watchdog_mins,omitempty"` } diff --git a/notehub/api/job.go b/notehub/api/job.go new file mode 100644 index 0000000..f52aefa --- /dev/null +++ b/notehub/api/job.go @@ -0,0 +1,45 @@ +// Copyright 2025 Blues Inc. All rights reserved. +// Use of this source code is governed by licenses granted by the +// copyright holder including that found in the LICENSE file. + +package api + +// This header is present for every type of job +type HubJob struct { + Type HubJobType `json:"type,omitempty"` + Version string `json:"version,omitempty"` + Name string `json:"name,omitempty"` + Comment string `json:"comment,omitempty"` + Created int64 `json:"created,omitempty"` + CreatedBy string `json:"created_by,omitempty"` +} + +// This header is present for every type of report +type HubJobReport struct { + Type HubJobType `json:"type,omitempty"` + Version string `json:"version,omitempty"` + Comment string `json:"comment,omitempty"` + JobId string `json:"job_id"` + JobName string `json:"job_name"` + Status string `json:"status,omitempty"` + DryRun bool `json:"dry_run,omitempty"` + Cancel bool `json:"cancel,omitempty"` + SubmittedBy string `json:"who_submitted,omitempty"` + Submitted int64 `json:"when_submitted,omitempty"` + Started int64 `json:"when_started,omitempty"` + Updated int64 `json:"when_updated,omitempty"` + Completed int64 `json:"when_completed,omitempty"` +} + +// Types of jobs +type HubJobType string + +const ( + HubJobTypeUnspecified HubJobType = "" + HubJobTypeReconciliation HubJobType = "reconciliation" +) + +const ( + HubJobStatusCancelled = "cancelled" + HubJobStatusSubmitted = "submitted" +) diff --git a/notehub/api/job_reconciliation.go b/notehub/api/job_reconciliation.go new file mode 100644 index 0000000..b14c7f8 --- /dev/null +++ b/notehub/api/job_reconciliation.go @@ -0,0 +1,94 @@ +// Copyright 2025 Blues Inc. All rights reserved. +// Use of this source code is governed by licenses granted by the +// copyright holder including that found in the LICENSE file. + +package api + +// Current data format of the reconciliation job type. Note that major types +// require conversion, while minor types can be handled by the same code. +const HubJobReconciliationMajorVersion = 1 +const HubJobReconciliationMinorVersion = 1 + +// HubJobReconciliation is the format of a batch request file +type HubJobReconciliation struct { + Header HubJob `json:"job,omitempty"` + Comment string `json:"comment,omitempty"` + Select struct { + Comment string `json:"comment,omitempty"` + AllDevices bool `json:"all_devices,omitempty"` + DevicesInFleets []string `json:"devices_in_fleets,omitempty"` + Devices []string `json:"devices,omitempty"` + DevicesBySn []string `json:"devices_by_sn,omitempty"` + } `json:"select,omitempty"` + DefaultRequests HubJobReconciliationRequests `json:"default_requests,omitempty"` + DeviceRequests map[string]HubJobReconciliationRequests `json:"device_requests,omitempty"` + ReportOptions HubJobReconciliationReportOptions `json:"report_options,omitempty"` +} + +// HubReportReconciliation is the format of the report generated by a reconciliation job. +type HubReportReconciliation struct { + Comment string `json:"comment,omitempty"` + Header HubJobReport `json:"job,omitempty"` + Status HubJobReconciliationReportStatus `json:"status,omitempty"` + Report *HubJobReconciliationReport `json:"output,omitempty"` +} + +// HubJobReconciliationRequests is a structure defining requests to apply to a set of selected devices. +// Note that if ProvisionProductUID is specified, the device will be provisioned if it isn't already provisioned, +// else it will fail if not provisioned. Also note that sn_to_set and vars_to_set use the same syntax +// as our standard API calls in that the value '-' means to clear the value. +type HubJobReconciliationRequests struct { + Comment string `json:"comment,omitempty"` + ProvisionProductUID string `json:"provision_product,omitempty"` + Disable bool `json:"disable,omitempty"` + Enable bool `json:"enable,omitempty"` + CaDisable bool `json:"connectivity_assurance_disable,omitempty"` + CaEnable bool `json:"connectivity_assurance_enable,omitempty"` + SnToDefault string `json:"sn_to_default,omitempty"` + SnToSet string `json:"sn_to_set,omitempty"` + VarsToDefault map[string]string `json:"vars_to_default,omitempty"` + VarsToSet map[string]string `json:"vars_to_set,omitempty"` + FleetsToDefault []string `json:"fleets_to_default,omitempty"` + FleetsToJoin []string `json:"fleets_to_join,omitempty"` + FleetsToLeave []string `json:"fleets_to_leave,omitempty"` +} + +// HubJobReconciliationReportStatus is the status portion of a batch report file +type HubJobReconciliationReportStatus struct { + Error string `json:"error,omitempty"` + Errors map[string]string `json:"errors,omitempty"` + Actions map[string]string `json:"actions,omitempty"` + DeviceCount int `json:"device_count,omitempty"` + Provisioned []string `json:"provisioned,omitempty"` +} + +// HubJobReconciliationReport is the format of a batch report file +type HubJobReconciliationReport struct { + App *HubJobReconciliationAppReport `json:"project,omitempty"` + Devices map[string]HubJobReconciliationDeviceReport `json:"devices,omitempty"` +} + +// HubJobReconciliationReportOptions is a structure defining options for the report +type HubJobReconciliationReportOptions struct { + Comment string `json:"comment,omitempty"` + AppInfo bool `json:"app_info,omitempty"` + AppVars bool `json:"app_vars,omitempty"` + AppFleets bool `json:"app_fleets,omitempty"` + DeviceInfo bool `json:"device_info,omitempty"` + DeviceActivity bool `json:"device_activity,omitempty"` + DeviceHealth bool `json:"device_health,omitempty"` + DeviceVars bool `json:"device_vars,omitempty"` +} + +// HubJobReconciliationAppReport is a structure defining the app report +type HubJobReconciliationAppReport struct { + Info *GetAppResponse `json:"project_info,omitempty"` + Vars *GetAppEnvironmentVariablesResponse `json:"project_vars,omitempty"` + Fleets *GetFleetsResponse `json:"project_fleets,omitempty"` +} + +// HubJobReconciliationDeviceReport is a structure defining the device report +type HubJobReconciliationDeviceReport struct { + Info *GetDeviceResponse `json:"device_info,omitempty"` + Vars *GetDeviceEnvironmentVariablesResponse `json:"device_vars,omitempty"` +} diff --git a/notehub/auth.go b/notehub/auth.go new file mode 100644 index 0000000..2d3723a --- /dev/null +++ b/notehub/auth.go @@ -0,0 +1,327 @@ +package notehub + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "math/rand" + "net" + "net/http" + "net/url" + "os" + "os/exec" + "os/signal" + "runtime" + "strings" + "time" +) + +type AccessToken struct { + Host string + Email string + AccessToken string + ExpiresAt time.Time +} + +// open opens the specified URL in the default browser of the user. +func open(url string) error { + var cmd string + var args []string + + switch runtime.GOOS { + case "windows": + cmd = "cmd" + args = []string{"/c", "start"} + case "darwin": + cmd = "open" + default: // "linux", "freebsd", "openbsd", "netbsd" + cmd = "xdg-open" + } + args = append(args, url) + return exec.Command(cmd, args...).Start() +} + +// listenOnAny tries each port in order and returns a bound net.Listener for the first available one. +func listenOnAny(ports []int) (net.Listener, int, error) { + for _, p := range ports { + ln, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", p)) + if err == nil { + return ln, p, nil + } + } + return nil, 0, errors.New("no ports available") +} + +func randString(n int) string { + letterRunes := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + b := make([]rune, n) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] + } + return string(b) +} + +func RevokeAccessToken(hub, token string) error { + form := url.Values{ + "token": {token}, + "token_type_hint": {"access_token"}, + "client_id": {"notehub_cli"}, + } + + req, err := http.NewRequestWithContext( + context.Background(), + http.MethodPost, + fmt.Sprintf("https://%s/oauth2/revoke", hub), + strings.NewReader(form.Encode()), + ) + + if err != nil { + return fmt.Errorf("creating request: %w", err) + } + + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("making request: %w", err) + } + defer resp.Body.Close() + + // Per RFC 7009: 200 OK is returned even if the token is already revoked + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + return nil +} + +// InitiateBrowserBasedLogin starts the OAuth2 login flow by opening the user's browser. +// the `hub` parameter is the hostname of Notehub where it is assumed that an OAuth2 client +// with client ID `notehub_cli` is configured for authorization code flow. +func InitiateBrowserBasedLogin(hub string) (*AccessToken, error) { + // this is the hard-coded OAuth client ID that's persisted in Hydra + clientId := "notehub_cli" + + // Try these ports in order until one is available: + // + // these ports are randomly chosen and hard-coded into + // the OAuth client in Hydra within Notehub (in the redirect_uris field) + ports := []int{58766, 58767, 58768, 58769, 42100, 42101, 42102, 42103} + + // Return values + var accessToken *AccessToken + var accessTokenErr error + + state := randString(16) + codeVerifier := randString(50) // must be at least 43 characters + hash := sha256.Sum256([]byte(codeVerifier)) + codeChallenge := base64.RawURLEncoding.EncodeToString(hash[:]) + + done := make(chan bool, 1) + quit := make(chan os.Signal, 1) + signal.Notify(quit, os.Interrupt) + defer signal.Reset(os.Interrupt) + + router := http.NewServeMux() + + // We'll fill this after we pick a port but declare it now so the handler can close over it. + chosenPort := 0 + + // The browser will be redirected to this endpoint with an authorization code + // and then this endpoint will exchange that authorization code for an access token + router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + authorizationCode := r.URL.Query().Get("code") + callbackState := r.URL.Query().Get("state") + + errHandler := func(msg string) { + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprintf(w, "error: %s", msg) + fmt.Printf("error: %s\n", msg) + accessTokenErr = errors.New(msg) + } + + if callbackState != state { + errHandler("state mismatch") + return + } + + /////////////////////////////////////////// + // Exchange code for access token + /////////////////////////////////////////// + + tokenResp, err := http.Post( + (&url.URL{ + Scheme: "https", + Host: hub, + Path: "/oauth2/token", + }).String(), + "application/x-www-form-urlencoded", + strings.NewReader(url.Values{ + "client_id": {clientId}, + "code": {authorizationCode}, + "code_verifier": {codeVerifier}, + "grant_type": {"authorization_code"}, + "redirect_uri": {fmt.Sprintf("http://localhost:%d", chosenPort)}, + }.Encode()), + ) + if err != nil { + errHandler("error on /oauth2/token: " + err.Error()) + return + } + defer tokenResp.Body.Close() + + body, err := io.ReadAll(tokenResp.Body) + if err != nil { + errHandler("could not read body from /oauth2/token: " + err.Error()) + return + } + + var tokenData map[string]interface{} + if err := json.Unmarshal(body, &tokenData); err != nil { + errHandler("could not unmarshal body from /oauth2/token: " + err.Error()) + return + } + + if errCode, ok := tokenData["error"].(string); ok { + if errDescription, ok2 := tokenData["error_description"].(string); ok2 { + errHandler(fmt.Sprintf("%s: %s", errCode, errDescription)) + } else { + errHandler(errCode) + } + return + } + + accessTokenString, ok := tokenData["access_token"].(string) + if !ok { + errHandler("unexpected error: no access token returned") + return + } + + // be defensive about type + var expiresIn time.Duration + switch v := tokenData["expires_in"].(type) { + case float64: + expiresIn = time.Duration(v) * time.Second + case int: + expiresIn = time.Duration(v) * time.Second + default: + expiresIn = 0 + } + + /////////////////////////////////////////// + // Get user's information (specifically email) + /////////////////////////////////////////// + + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://%s/userinfo", hub), nil) + if err != nil { + errHandler("could not create request for /userinfo: " + err.Error()) + return + } + req.Header.Set("Authorization", "Bearer "+accessTokenString) + userinfoResp, err := http.DefaultClient.Do(req) + if err != nil { + errHandler("could not get userinfo: " + err.Error()) + return + } + defer userinfoResp.Body.Close() + + userinfoBody, err := io.ReadAll(userinfoResp.Body) + if err != nil { + errHandler("could not read body from /userinfo: " + err.Error()) + return + } + + var userinfoData map[string]interface{} + if err := json.Unmarshal(userinfoBody, &userinfoData); err != nil { + errHandler("could not unmarshal body from /userinfo: " + err.Error()) + return + } + + email, ok := userinfoData["email"].(string) + if !ok { + errHandler("could not retrieve email") + return + } + + /////////////////////////////////////////// + // Build the access token response + /////////////////////////////////////////// + + accessToken = &AccessToken{ + Host: hub, + Email: email, + AccessToken: accessTokenString, + ExpiresAt: time.Now().Add(expiresIn), + } + + /////////////////////////////////////////// + // respond to the browser and quit + /////////////////////////////////////////// + + w.Header().Set("Content-Type", "text/html") + w.WriteHeader(http.StatusOK) + fmt.Fprintf(w, "

Token exchange completed successfully

You may now close this window and return to the CLI application

") + + quit <- os.Interrupt + }) + + // Pick first available port and get a listener + listener, port, err := listenOnAny(ports) + if err != nil { + return nil, fmt.Errorf("could not bind any callback port: %w", err) + } + chosenPort = port + + server := &http.Server{ + Addr: fmt.Sprintf(":%d", chosenPort), + Handler: router, + } + + // Wait for OAuth callback to be hit, then shutdown HTTP server + go func() { + <-quit + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + server.SetKeepAlivesEnabled(false) + if err := server.Shutdown(ctx); err != nil { + log.Printf("error: %v", err) + } + close(done) + }() + + // Start HTTP server waiting for OAuth callback + go func() { + if err := server.Serve(listener); err != nil && err != http.ErrServerClosed { + log.Printf("error: %v", err) + } + }() + + // Build the authorize URL using the chosen port + authorizeUrl := url.URL{ + Scheme: "https", + Host: hub, + Path: "/oauth2/auth", + RawQuery: url.Values{ + "client_id": {clientId}, + "code_challenge": {codeChallenge}, + "code_challenge_method": {"S256"}, + "redirect_uri": {fmt.Sprintf("http://localhost:%d", chosenPort)}, + "response_type": {"code"}, + "scope": {"openid email"}, + "state": {state}, + }.Encode(), + } + + // Open web browser to authorize + fmt.Printf("Opening web browser to initiate authentication (redirect port %d)...\n", chosenPort) + open(authorizeUrl.String()) + + // Wait for exchange to finish + <-done + return accessToken, accessTokenErr +} diff --git a/notehub/request.go b/notehub/request.go index 09417de..2d7cd20 100644 --- a/notehub/request.go +++ b/notehub/request.go @@ -6,6 +6,7 @@ package notehub import ( "fmt" + "strings" "github.com/blues/note-go/note" "github.com/blues/note-go/notecard" @@ -37,6 +38,33 @@ const HubUpload = "hub.upload.add" // HubAppUploads (golint) const HubAppUploads = "hub.app.upload.query" +// HubAppJobSubmit (golint) +const HubAppJobSubmit = "hub.app.job.submit" + +// HubAppJobGet (golint) +const HubAppJobGet = "hub.app.job.get" + +// HubAppJobPut (golint) +const HubAppJobPut = "hub.app.job.put" + +// HubAppJobDelete (golint) +const HubAppJobDelete = "hub.app.job.delete" + +// HubAppJobsGet (golint) +const HubAppJobsGet = "hub.app.jobs.get" + +// HubAppReportGet (golint) +const HubAppReportGet = "hub.app.report.get" + +// HubAppReportDelete (golint) +const HubAppReportDelete = "hub.app.report.delete" + +// HubAppReportCancel (golint) +const HubAppReportCancel = "hub.app.report.cancel" + +// HubAppReportsGet (golint) +const HubAppReportsGet = "hub.app.reports.get" + // HubUploads (golint) const HubUploads = "hub.upload.query" @@ -113,6 +141,7 @@ type HubRequest struct { Compress string `json:"compress,omitempty"` MD5 string `json:"md5,omitempty"` DeviceEndpoint bool `json:"device_endpoint,omitempty"` + DryRun bool `json:"dry_run,omitempty"` } type UploadType string @@ -122,7 +151,9 @@ const ( UploadTypeHostFirmware UploadType = "firmware" UploadTypeNotecardFirmware UploadType = "notecard" UploadTypeModemFirmware UploadType = "modem" + UploadTypeStarnoteFirmware UploadType = "starnote" UploadTypeUserData UploadType = "data" + UploadTypeJob UploadType = "job" ) var allFileTypes = []UploadType{ @@ -130,7 +161,9 @@ var allFileTypes = []UploadType{ UploadTypeHostFirmware, UploadTypeNotecardFirmware, UploadTypeModemFirmware, + UploadTypeStarnoteFirmware, UploadTypeUserData, + UploadTypeJob, } func ParseUploadType(fileType string) UploadType { @@ -191,9 +224,25 @@ type UploadMetadata struct { Tags string `json:"tags,omitempty"` // comma-separated, no spaces, case-insensitive Notes string `json:"notes,omitempty"` // Should be simple text Firmware *HubRequestFileFirmware `json:"firmware,omitempty"` // This value is pulled out of the firmware binary itself + Version string `json:"version,omitempty"` // User-specified version string provided at time of upload // Arbitrary metadata that the user may define - we don't interpret the schema at all Info map[string]interface{} `json:"info,omitempty"` } +func (upload UploadMetadata) IsArchSpecificNotecardFirmware() bool { + return upload.FileType == UploadTypeNotecardFirmware && (strings.Contains(upload.Name, "-s3-") || + strings.Contains(upload.Name, "-u5-") || + strings.Contains(upload.Name, "-wl-")) +} + +func (upload UploadMetadata) IsPublished() bool { + for _, tag := range strings.Split(upload.Tags, ",") { + if strings.TrimSpace(strings.ToLower(tag)) == "publish" { + return true + } + } + return false +} + // HubRequestFileTagPublish indicates that this should be published in the UI const HubRequestFileTagPublish = "publish"