fix(parsing): more reliable read data parsing with 2-bytes characters (#20)

CHANGELOG.md

@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 2.1.2 - UNRELEASED
+
+- fix: more reliable read data parsing with 2-bytes characters ([#20](https://github.com/eove/serial-console-com/issues/20))
+
 ## 2.1.1 - 2025-07-24
 
 - fix: actually write to transport

cli.ts

@@ -108,8 +108,8 @@ program
       debugEnabled,
       baudRate: Number(baudrate),
     });
-    const received: string[] = [];
-    serial.data$.subscribe((d: string) => received.push(d));
+    const received: Buffer[] = [];
+    serial.data$.subscribe((d) => received.push(d));
     debug(`connecting to ${portName} at ${baudrate}`);
     await serial.connect(portName);
 
@@ -127,7 +127,7 @@ program
       await serial.write('\n');
     }
     await delayMS(1000);
-    console.log('received:', received.join());
+    console.log('received:', Buffer.concat(received).toString());
     process.exit(0);
   });
 

lib/createCommandRunner.spec.ts

@@ -1,11 +1,11 @@
-import { createCommandRunner, CommandRunner } from './createCommandRunner';
+import { CommandRunner, createCommandRunner } from './createCommandRunner';
 import { makeParseConsoleOutput } from './makeParseConsoleOutput';
 import { createTransportMock } from './test';
 import { from, Subject } from 'rxjs';
 
 describe('command runner', () => {
   let runner: CommandRunner;
-  let subject: Subject<string>;
+  let subject: Subject<Buffer>;
 
   beforeEach(() => {
     subject = new Subject();
@@ -63,8 +63,8 @@ describe('command runner', () => {
   });
 
   function emitReceivedData(data: string) {
-    for (const c of data.split('')) {
-      subject.next(c);
+    for (let i = 0; i < Buffer.from(data).length; i++) {
+      subject.next(Buffer.from(data[i]));
     }
   }
 });

lib/createCommandRunner.ts

@@ -34,7 +34,7 @@ export interface CommandRunner {
 interface CommandRunnerDependencies {
   parseData: ParseConsoleOutputFunction;
   transport: Transport;
-  data$: Observable<string>;
+  data$: Observable<Buffer>;
   debugEnabled?: boolean;
 }
 
@@ -55,14 +55,14 @@ export function createCommandRunner(
 
   const answer$ = data$.pipe(
     scan(
-      (acc: ParseConsoleOutputResult, byte: any) => {
+      (acc: ParseConsoleOutputResult, current: any) => {
         const { remaining: remainingBytes } = acc;
-        const received = remainingBytes.concat(...byte);
+        const received = Buffer.concat([remainingBytes, current]);
         const { remaining, lines } = parseData(received);
         return { remaining, lines };
       },
       {
-        remaining: '',
+        remaining: Buffer.alloc(0),
         lines: [],
       }
     ),

lib/createTransport.ts

@@ -3,7 +3,7 @@ import * as _ from 'lodash';
 import { Subject } from 'rxjs';
 import { SerialPort } from 'serialport';
 
-import { Device, Transport, IOCTLOptions } from './types';
+import { Device, IOCTLOptions, Transport } from './types';
 
 type UninstallHandler = () => void;
 
@@ -15,7 +15,7 @@ interface TransportCreationOptions {
 export function createTransport(options?: TransportCreationOptions): Transport {
   const { debugEnabled = false } = options || {};
   const debug = Object.assign(debugLib('transport'), { enabled: debugEnabled });
-  const dataSource = new Subject<string>();
+  const dataSource = new Subject<Buffer>();
   const eventSource = new Subject();
   let port: SerialPort;
   let uninstallPortListeners: UninstallHandler;
@@ -66,10 +66,10 @@ export function createTransport(options?: TransportCreationOptions): Transport {
         _sendEvent({ type: 'TRANSPORT_CONNECTED', payload: undefined });
       };
 
-      const onDataHandler = (data: any) => {
+      const onDataHandler = (data: Buffer) => {
         const received = data.toString();
         debug('received:', received.replace('\r', '\\r'));
-        _sendData(received);
+        _sendData(data);
       };
 
       const onCloseHandler = () => {
@@ -173,7 +173,7 @@ export function createTransport(options?: TransportCreationOptions): Transport {
     eventSource.next(event);
   }
 
-  function _sendData(data: any) {
+  function _sendData(data: Buffer) {
     dataSource.next(data);
   }
 }

lib/makeParseConsoleOutput.spec.ts

@@ -2,7 +2,7 @@ import { makeParseConsoleOutput } from './makeParseConsoleOutput';
 import { ParseConsoleOutputResult } from './types';
 
 describe('parse console output', () => {
-  let parseConsoleOutput: (data: string) => ParseConsoleOutputResult;
+  let parseConsoleOutput: (data: Buffer) => ParseConsoleOutputResult;
 
   beforeEach(() => {
     parseConsoleOutput = makeParseConsoleOutput({
@@ -13,91 +13,108 @@ describe('parse console output', () => {
 
   it('should return a single line when output contains a line separator and prompt', () => {
     expect(
-      parseConsoleOutput('drwxr-xr-x 18 root root 0 Sep 11 15:48 . \n/ #')
+      parseConsoleOutput(
+        Buffer.from('drwxr-xr-x 18 root root 0 Sep 11 15:48 . \n/ #')
+      )
     ).toEqual({
       lines: ['drwxr-xr-x 18 root root 0 Sep 11 15:48 .'],
-      remaining: '',
+      remaining: Buffer.alloc(0),
     });
   });
 
   it('should return many lines when output contains line separator and prompt', () => {
     expect(
       parseConsoleOutput(
-        'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \ndrwxr-xr-x 18 root root 0 Sep 11 15:48 .. \n/ #'
+        Buffer.from(
+          'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \ndrwxr-xr-x 18 root root 0 Sep 11 15:48 .. \n/ #'
+        )
       )
     ).toEqual({
       lines: [
         'drwxr-xr-x 18 root root 0 Sep 11 15:48 .',
         'drwxr-xr-x 18 root root 0 Sep 11 15:48 ..',
       ],
-      remaining: '',
+      remaining: Buffer.alloc(0),
     });
   });
 
   it('should return lines when output contains expected prompt with pre escape chars', () => {
     expect(
       parseConsoleOutput(
-        'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \ndrwxr-xr-x 18 root root 0 Sep 11 15:48 .. \n\u001b[1;32m/ #'
+        Buffer.from(
+          'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \ndrwxr-xr-x 18 root root 0 Sep 11 15:48 .. \n\u001b[1;32m/ #'
+        )
       )
     ).toEqual({
       lines: [
         'drwxr-xr-x 18 root root 0 Sep 11 15:48 .',
         'drwxr-xr-x 18 root root 0 Sep 11 15:48 ..',
       ],
-      remaining: '',
+      remaining: Buffer.alloc(0),
     });
   });
 
   it('should return lines when output contains expected prompt with post escape chars', () => {
     expect(
       parseConsoleOutput(
-        'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \ndrwxr-xr-x 18 root root 0 Sep 11 15:48 .. \n/ #\u001b[1;32m'
+        Buffer.from(
+          'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \ndrwxr-xr-x 18 root root 0 Sep 11 15:48 .. \n/ #\u001b[1;32m'
+        )
       )
     ).toEqual({
       lines: [
         'drwxr-xr-x 18 root root 0 Sep 11 15:48 .',
         'drwxr-xr-x 18 root root 0 Sep 11 15:48 ..',
       ],
-      remaining: '',
+      remaining: Buffer.alloc(0),
     });
   });
 
   it('should return remaining data when no prompt in output', () => {
     expect(
       parseConsoleOutput(
-        'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \ndrwxr-xr-x 18 root root 0 Sep 11 15:48 .. \n'
+        Buffer.from(
+          'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \ndrwxr-xr-x 18 root root 0 Sep 11 15:48 .. \n'
+        )
       )
     ).toEqual({
       lines: [],
-      remaining:
-        'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \ndrwxr-xr-x 18 root root 0 Sep 11 15:48 .. \n',
+      remaining: Buffer.from(
+        'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \ndrwxr-xr-x 18 root root 0 Sep 11 15:48 .. \n'
+      ),
     });
   });
 
   it('should return remaining data when no line separator in output', () => {
     expect(
-      parseConsoleOutput('drwxr-xr-x 18 root root 0 Sep 11 15:48 . / #')
+      parseConsoleOutput(
+        Buffer.from('drwxr-xr-x 18 root root 0 Sep 11 15:48 . / #')
+      )
     ).toEqual({
       lines: [],
-      remaining: 'drwxr-xr-x 18 root root 0 Sep 11 15:48 . / #',
+      remaining: Buffer.from('drwxr-xr-x 18 root root 0 Sep 11 15:48 . / #'),
     });
   });
 
   it('should return remaining data when prompt is not the expected one', () => {
     expect(
-      parseConsoleOutput('drwxr-xr-x 18 root root 0 Sep 11 15:48 . \n/ $')
+      parseConsoleOutput(
+        Buffer.from('drwxr-xr-x 18 root root 0 Sep 11 15:48 . \n/ $')
+      )
     ).toEqual({
       lines: [],
-      remaining: 'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \n/ $',
+      remaining: Buffer.from('drwxr-xr-x 18 root root 0 Sep 11 15:48 . \n/ $'),
     });
   });
 
   it('should return remaining data when line separator is not the expected one', () => {
     expect(
-      parseConsoleOutput('drwxr-xr-x 18 root root 0 Sep 11 15:48 . \r/ #')
+      parseConsoleOutput(
+        Buffer.from('drwxr-xr-x 18 root root 0 Sep 11 15:48 . \r/ #')
+      )
     ).toEqual({
       lines: [],
-      remaining: 'drwxr-xr-x 18 root root 0 Sep 11 15:48 . \r/ #',
+      remaining: Buffer.from('drwxr-xr-x 18 root root 0 Sep 11 15:48 . \r/ #'),
     });
   });
 });

lib/makeParseConsoleOutput.ts

@@ -15,16 +15,16 @@ export function makeParseConsoleOutput(
     lineSeparator: '\n',
   });
 
-  return (data: string) => {
+  return (data: Buffer) => {
     const regex = new RegExp(`(.*)${lineSeparator}(.*${prompt}.*)`, 'sm');
-    const found = data.match(regex);
+    const found = data.toString().match(regex);
     if (found && found.length) {
       return {
         lines: found[1]
           .split(/\r\n|\r|\n/)
           .map((l) => l.trim())
           .filter((x) => x),
-        remaining: '',
+        remaining: Buffer.alloc(0),
       };
     }
     return {

lib/types.ts

@@ -2,11 +2,11 @@ import { Observable } from 'rxjs';
 
 export interface ParseConsoleOutputResult {
   lines: string[];
-  remaining: string;
+  remaining: Buffer;
 }
 
 export type ParseConsoleOutputFunction = (
-  data: string
+  data: Buffer
 ) => ParseConsoleOutputResult;
 
 export interface Device {
@@ -26,7 +26,7 @@ export interface Transport {
   write: (bytes: string) => Promise<any>;
   discover: () => Promise<Device[]>;
   ioctl: (options: IOCTLOptions) => Promise<void>;
-  data$: Observable<string>;
+  data$: Observable<Buffer>;
   event$: Observable<unknown>;
   connected: boolean;
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@eove/serial-console-com",
-  "version": "2.1.1",
+  "version": "2.1.2-i20.1",
   "description": "library to communicate with a (unix) console over a serial line",
   "bin": "./cli.js",
   "main": "build/index.js",

tsconfig.json

@@ -8,7 +8,8 @@
     "forceConsistentCasingInFileNames": true,
     "noImplicitReturns": true,
     "noUnusedLocals": true,
-    "noUnusedParameters": true
+    "noUnusedParameters": true,
+    "skipLibCheck": true
   },
   "include": ["./lib/**/*"],
   "exclude": ["node_modules", "build", "**/*.spec.ts", "**/*.spec.js"]