From 4885699feae06a18208f11c2e95d0e242255a0bd Mon Sep 17 00:00:00 2001 From: Michael Borde Date: Fri, 29 Aug 2025 17:01:37 +0200 Subject: [PATCH 1/6] fix(parsing): more reliable read data parsing with 2-bytes characters --- CHANGELOG.md | 4 +++ cli.ts | 6 ++-- lib/createCommandRunner.spec.ts | 8 ++--- lib/createCommandRunner.ts | 8 ++--- lib/createTransport.ts | 10 +++--- lib/makeParseConsoleOutput.spec.ts | 53 ++++++++++++++++++++---------- lib/makeParseConsoleOutput.ts | 6 ++-- lib/types.ts | 6 ++-- 8 files changed, 61 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 263cbdb..45c5b07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 2.1.0 - UNRELEASED + +- fix: more reliable read data parsing with 2-bytes characters ([#20](https://github.com/eove/serial-console-com/issues/20)) + ## 2.1.1 - 2025-07-24 - fix: actually write to transport diff --git a/cli.ts b/cli.ts index 04e3b85..b681e9d 100755 --- a/cli.ts +++ b/cli.ts @@ -108,8 +108,8 @@ program debugEnabled, baudRate: Number(baudrate), }); - const received: string[] = []; - serial.data$.subscribe((d: string) => received.push(d)); + const received: Buffer[] = []; + serial.data$.subscribe((d) => received.push(d)); debug(`connecting to ${portName} at ${baudrate}`); await serial.connect(portName); @@ -127,7 +127,7 @@ program await serial.write('\n'); } await delayMS(1000); - console.log('received:', received.join()); + console.log('received:', Buffer.concat(received).toString()); process.exit(0); }); diff --git a/lib/createCommandRunner.spec.ts b/lib/createCommandRunner.spec.ts index e6d98eb..460e457 100644 --- a/lib/createCommandRunner.spec.ts +++ b/lib/createCommandRunner.spec.ts @@ -1,11 +1,11 @@ -import { createCommandRunner, CommandRunner } from './createCommandRunner'; +import { CommandRunner, createCommandRunner } from './createCommandRunner'; import { makeParseConsoleOutput } from './makeParseConsoleOutput'; import { createTransportMock } from './test'; import { from, Subject } from 'rxjs'; describe('command runner', () => { let runner: CommandRunner; - let subject: Subject; + let subject: Subject; beforeEach(() => { subject = new Subject(); @@ -63,8 +63,8 @@ describe('command runner', () => { }); function emitReceivedData(data: string) { - for (const c of data.split('')) { - subject.next(c); + for (let i = 0; i < Buffer.from(data).length; i++) { + subject.next(Buffer.from(data[i])); } } }); diff --git a/lib/createCommandRunner.ts b/lib/createCommandRunner.ts index d1a16ba..b435b9a 100644 --- a/lib/createCommandRunner.ts +++ b/lib/createCommandRunner.ts @@ -34,7 +34,7 @@ export interface CommandRunner { interface CommandRunnerDependencies { parseData: ParseConsoleOutputFunction; transport: Transport; - data$: Observable; + data$: Observable; debugEnabled?: boolean; } @@ -55,14 +55,14 @@ export function createCommandRunner( const answer$ = data$.pipe( scan( - (acc: ParseConsoleOutputResult, byte: any) => { + (acc: ParseConsoleOutputResult, current: any) => { const { remaining: remainingBytes } = acc; - const received = remainingBytes.concat(...byte); + const received = Buffer.concat([remainingBytes, current]); const { remaining, lines } = parseData(received); return { remaining, lines }; }, { - remaining: '', + remaining: Buffer.alloc(0), lines: [], } ), diff --git a/lib/createTransport.ts b/lib/createTransport.ts index fbb64c2..22595dc 100644 --- a/lib/createTransport.ts +++ b/lib/createTransport.ts @@ -3,7 +3,7 @@ import * as _ from 'lodash'; import { Subject } from 'rxjs'; import { SerialPort } from 'serialport'; -import { Device, Transport, IOCTLOptions } from './types'; +import { Device, IOCTLOptions, Transport } from './types'; type UninstallHandler = () => void; @@ -15,7 +15,7 @@ interface TransportCreationOptions { export function createTransport(options?: TransportCreationOptions): Transport { const { debugEnabled = false } = options || {}; const debug = Object.assign(debugLib('transport'), { enabled: debugEnabled }); - const dataSource = new Subject(); + const dataSource = new Subject(); const eventSource = new Subject(); let port: SerialPort; let uninstallPortListeners: UninstallHandler; @@ -66,10 +66,10 @@ export function createTransport(options?: TransportCreationOptions): Transport { _sendEvent({ type: 'TRANSPORT_CONNECTED', payload: undefined }); }; - const onDataHandler = (data: any) => { + const onDataHandler = (data: Buffer) => { const received = data.toString(); debug('received:', received.replace('\r', '\\r')); - _sendData(received); + _sendData(data); }; const onCloseHandler = () => { @@ -173,7 +173,7 @@ export function createTransport(options?: TransportCreationOptions): Transport { eventSource.next(event); } - function _sendData(data: any) { + function _sendData(data: Buffer) { dataSource.next(data); } } diff --git a/lib/makeParseConsoleOutput.spec.ts b/lib/makeParseConsoleOutput.spec.ts index 5b95a24..2e0e0bf 100644 --- a/lib/makeParseConsoleOutput.spec.ts +++ b/lib/makeParseConsoleOutput.spec.ts @@ -2,7 +2,7 @@ import { makeParseConsoleOutput } from './makeParseConsoleOutput'; import { ParseConsoleOutputResult } from './types'; describe('parse console output', () => { - let parseConsoleOutput: (data: string) => ParseConsoleOutputResult; + let parseConsoleOutput: (data: Buffer) => ParseConsoleOutputResult; beforeEach(() => { parseConsoleOutput = makeParseConsoleOutput({ @@ -13,91 +13,108 @@ describe('parse console output', () => { it('should return a single line when output contains a line separator and prompt', () => { expect( - parseConsoleOutput('drwxr-xr-x 18 root root 0 Sep 11 15:48 . \n/ #') + parseConsoleOutput( + Buffer.from('drwxr-xr-x 18 root root 0 Sep 11 15:48 . \n/ #') + ) ).toEqual({ lines: ['drwxr-xr-x 18 root root 0 Sep 11 15:48 .'], - remaining: '', + remaining: Buffer.alloc(0), }); }); it('should return many lines when output contains line separator and prompt', () => { expect( parseConsoleOutput( - 'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \ndrwxr-xr-x 18 root root 0 Sep 11 15:48 .. \n/ #' + Buffer.from( + 'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \ndrwxr-xr-x 18 root root 0 Sep 11 15:48 .. \n/ #' + ) ) ).toEqual({ lines: [ 'drwxr-xr-x 18 root root 0 Sep 11 15:48 .', 'drwxr-xr-x 18 root root 0 Sep 11 15:48 ..', ], - remaining: '', + remaining: Buffer.alloc(0), }); }); it('should return lines when output contains expected prompt with pre escape chars', () => { expect( parseConsoleOutput( - 'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \ndrwxr-xr-x 18 root root 0 Sep 11 15:48 .. \n\u001b[1;32m/ #' + Buffer.from( + 'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \ndrwxr-xr-x 18 root root 0 Sep 11 15:48 .. \n\u001b[1;32m/ #' + ) ) ).toEqual({ lines: [ 'drwxr-xr-x 18 root root 0 Sep 11 15:48 .', 'drwxr-xr-x 18 root root 0 Sep 11 15:48 ..', ], - remaining: '', + remaining: Buffer.alloc(0), }); }); it('should return lines when output contains expected prompt with post escape chars', () => { expect( parseConsoleOutput( - 'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \ndrwxr-xr-x 18 root root 0 Sep 11 15:48 .. \n/ #\u001b[1;32m' + Buffer.from( + 'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \ndrwxr-xr-x 18 root root 0 Sep 11 15:48 .. \n/ #\u001b[1;32m' + ) ) ).toEqual({ lines: [ 'drwxr-xr-x 18 root root 0 Sep 11 15:48 .', 'drwxr-xr-x 18 root root 0 Sep 11 15:48 ..', ], - remaining: '', + remaining: Buffer.alloc(0), }); }); it('should return remaining data when no prompt in output', () => { expect( parseConsoleOutput( - 'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \ndrwxr-xr-x 18 root root 0 Sep 11 15:48 .. \n' + Buffer.from( + 'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \ndrwxr-xr-x 18 root root 0 Sep 11 15:48 .. \n' + ) ) ).toEqual({ lines: [], - remaining: - 'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \ndrwxr-xr-x 18 root root 0 Sep 11 15:48 .. \n', + remaining: Buffer.from( + 'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \ndrwxr-xr-x 18 root root 0 Sep 11 15:48 .. \n' + ), }); }); it('should return remaining data when no line separator in output', () => { expect( - parseConsoleOutput('drwxr-xr-x 18 root root 0 Sep 11 15:48 . / #') + parseConsoleOutput( + Buffer.from('drwxr-xr-x 18 root root 0 Sep 11 15:48 . / #') + ) ).toEqual({ lines: [], - remaining: 'drwxr-xr-x 18 root root 0 Sep 11 15:48 . / #', + remaining: Buffer.from('drwxr-xr-x 18 root root 0 Sep 11 15:48 . / #'), }); }); it('should return remaining data when prompt is not the expected one', () => { expect( - parseConsoleOutput('drwxr-xr-x 18 root root 0 Sep 11 15:48 . \n/ $') + parseConsoleOutput( + Buffer.from('drwxr-xr-x 18 root root 0 Sep 11 15:48 . \n/ $') + ) ).toEqual({ lines: [], - remaining: 'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \n/ $', + remaining: Buffer.from('drwxr-xr-x 18 root root 0 Sep 11 15:48 . \n/ $'), }); }); it('should return remaining data when line separator is not the expected one', () => { expect( - parseConsoleOutput('drwxr-xr-x 18 root root 0 Sep 11 15:48 . \r/ #') + parseConsoleOutput( + Buffer.from('drwxr-xr-x 18 root root 0 Sep 11 15:48 . \r/ #') + ) ).toEqual({ lines: [], - remaining: 'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \r/ #', + remaining: Buffer.from('drwxr-xr-x 18 root root 0 Sep 11 15:48 . \r/ #'), }); }); }); diff --git a/lib/makeParseConsoleOutput.ts b/lib/makeParseConsoleOutput.ts index 47df025..5d34234 100644 --- a/lib/makeParseConsoleOutput.ts +++ b/lib/makeParseConsoleOutput.ts @@ -15,16 +15,16 @@ export function makeParseConsoleOutput( lineSeparator: '\n', }); - return (data: string) => { + return (data: Buffer) => { const regex = new RegExp(`(.*)${lineSeparator}(.*${prompt}.*)`, 'sm'); - const found = data.match(regex); + const found = data.toString().match(regex); if (found && found.length) { return { lines: found[1] .split(/\r\n|\r|\n/) .map((l) => l.trim()) .filter((x) => x), - remaining: '', + remaining: Buffer.alloc(0), }; } return { diff --git a/lib/types.ts b/lib/types.ts index 81f0843..03d0e66 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -2,11 +2,11 @@ import { Observable } from 'rxjs'; export interface ParseConsoleOutputResult { lines: string[]; - remaining: string; + remaining: Buffer; } export type ParseConsoleOutputFunction = ( - data: string + data: Buffer ) => ParseConsoleOutputResult; export interface Device { @@ -26,7 +26,7 @@ export interface Transport { write: (bytes: string) => Promise; discover: () => Promise; ioctl: (options: IOCTLOptions) => Promise; - data$: Observable; + data$: Observable; event$: Observable; connected: boolean; } From 4964bbb67b0825f4ed16ec947cd52c91e1024934 Mon Sep 17 00:00:00 2001 From: Michael Borde Date: Fri, 29 Aug 2025 17:05:56 +0200 Subject: [PATCH 2/6] fix(typing): node typing was not 18.x --- package-lock.json | 21 ++++++++++++++++----- package.json | 2 +- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 89d8a82..44f3f21 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@types/debug": "4.1.5", "@types/jest": "25.1.3", "@types/lodash": "4.14.149", - "@types/node": "13.7.4", + "@types/node": "18.19.123", "@types/rx": "4.1.1", "@types/serialport": "8.0.2", "jest": "25.1.0", @@ -1224,10 +1224,14 @@ "dev": true }, "node_modules/@types/node": { - "version": "13.7.4", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.4.tgz", - "integrity": "sha512-oVeL12C6gQS/GAExndigSaLxTrKpQPxewx9bOcwfvJiJge4rr7wNaph4J+ns5hrmIV2as5qxqN8YKthn9qh0jw==", - "dev": true + "version": "18.19.123", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.123.tgz", + "integrity": "sha512-K7DIaHnh0mzVxreCR9qwgNxp3MH9dltPNIEddW9MYUlcKAzm+3grKNSTe2vCJHI1FaLpvpL5JGJrz1UZDKYvDg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } }, "node_modules/@types/rx": { "version": "4.1.1", @@ -7746,6 +7750,13 @@ "node": ">=4.2.0" } }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true, + "license": "MIT" + }, "node_modules/union-value": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", diff --git a/package.json b/package.json index 8b017e5..f4ec091 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ "@types/debug": "4.1.5", "@types/jest": "25.1.3", "@types/lodash": "4.14.149", - "@types/node": "13.7.4", + "@types/node": "18.19.123", "@types/rx": "4.1.1", "@types/serialport": "8.0.2", "jest": "25.1.0", From 11bd14f22f128f93d9dcd18424d95cf0e564439a Mon Sep 17 00:00:00 2001 From: Michael Borde Date: Fri, 29 Aug 2025 17:07:31 +0200 Subject: [PATCH 3/6] docs: using the right next version in changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45c5b07..fc8b702 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## 2.1.0 - UNRELEASED +## 2.1.2 - UNRELEASED - fix: more reliable read data parsing with 2-bytes characters ([#20](https://github.com/eove/serial-console-com/issues/20)) From 5abdaee074b69091999ca2c761b9ae1b9966b974 Mon Sep 17 00:00:00 2001 From: Michael Borde Date: Fri, 29 Aug 2025 17:11:04 +0200 Subject: [PATCH 4/6] Revert "fix(typing): node typing was not 18.x" This reverts commit 4964bbb67b0825f4ed16ec947cd52c91e1024934. --- package-lock.json | 21 +++++---------------- package.json | 2 +- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/package-lock.json b/package-lock.json index 44f3f21..89d8a82 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@types/debug": "4.1.5", "@types/jest": "25.1.3", "@types/lodash": "4.14.149", - "@types/node": "18.19.123", + "@types/node": "13.7.4", "@types/rx": "4.1.1", "@types/serialport": "8.0.2", "jest": "25.1.0", @@ -1224,14 +1224,10 @@ "dev": true }, "node_modules/@types/node": { - "version": "18.19.123", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.123.tgz", - "integrity": "sha512-K7DIaHnh0mzVxreCR9qwgNxp3MH9dltPNIEddW9MYUlcKAzm+3grKNSTe2vCJHI1FaLpvpL5JGJrz1UZDKYvDg==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~5.26.4" - } + "version": "13.7.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.4.tgz", + "integrity": "sha512-oVeL12C6gQS/GAExndigSaLxTrKpQPxewx9bOcwfvJiJge4rr7wNaph4J+ns5hrmIV2as5qxqN8YKthn9qh0jw==", + "dev": true }, "node_modules/@types/rx": { "version": "4.1.1", @@ -7750,13 +7746,6 @@ "node": ">=4.2.0" } }, - "node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "dev": true, - "license": "MIT" - }, "node_modules/union-value": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", diff --git a/package.json b/package.json index f4ec091..8b017e5 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ "@types/debug": "4.1.5", "@types/jest": "25.1.3", "@types/lodash": "4.14.149", - "@types/node": "18.19.123", + "@types/node": "13.7.4", "@types/rx": "4.1.1", "@types/serialport": "8.0.2", "jest": "25.1.0", From b436bd6f1db95830fb61eb321a2cc75d8db11b81 Mon Sep 17 00:00:00 2001 From: Michael Borde Date: Fri, 29 Aug 2025 17:14:28 +0200 Subject: [PATCH 5/6] fix: typescript compilation --- tsconfig.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tsconfig.json b/tsconfig.json index cbc6726..5cfd28f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -8,7 +8,8 @@ "forceConsistentCasingInFileNames": true, "noImplicitReturns": true, "noUnusedLocals": true, - "noUnusedParameters": true + "noUnusedParameters": true, + "skipLibCheck": true }, "include": ["./lib/**/*"], "exclude": ["node_modules", "build", "**/*.spec.ts", "**/*.spec.js"] From 9c1c1588ea69694ae0842ad6255d889fe6ba0dd5 Mon Sep 17 00:00:00 2001 From: Michael Borde Date: Fri, 29 Aug 2025 17:14:39 +0200 Subject: [PATCH 6/6] 2.1.2-i20.1 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 89d8a82..222f237 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@eove/serial-console-com", - "version": "2.1.1", + "version": "2.1.2-i20.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@eove/serial-console-com", - "version": "2.1.1", + "version": "2.1.2-i20.1", "license": "MIT", "dependencies": { "@arpinum/promising": "3.1.0", diff --git a/package.json b/package.json index 8b017e5..7c6b9f2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@eove/serial-console-com", - "version": "2.1.1", + "version": "2.1.2-i20.1", "description": "library to communicate with a (unix) console over a serial line", "bin": "./cli.js", "main": "build/index.js",