fix: ensure connection encodes utf8 correctly

Signed-off-by: Gordon Smith <GordonJSmith@gmail.com>

Author: GordonSmith

packages/comms/.vscode/launch.json

@@ -33,7 +33,7 @@
             "name": "index.html",
             "request": "launch",
             "type": "msedge",
-            "url": "http://localhost:5521/index.html",
+            "url": "http://localhost:5173/index.html",
             "runtimeArgs": [
                 "--disable-web-security"
             ],

packages/comms/index.html

@@ -35,14 +35,19 @@ <h1>ESM Quick Test</h1>
 
         Workunit.submit({ baseUrl: "http://localhost:8010" }, "hthor", "'Hello and Welcome!';")
             .then((wu) => {
+                console.log("Submitted WU: " + wu.Wuid);
                 return wu.watchUntilComplete();
             }).then((wu) => {
+                console.log("WU: " + wu.Wuid, wu.StateID, wu.State, wu.isComplete());
                 return wu.fetchResults().then((results) => {
+                    console.log("Results: " + results.length);
                     return results[0].fetchRows();
                 }).then((rows) => {
+                    console.log("Rows: " + rows.length);
                     return wu;
                 });
             }).then((wu) => {
+                console.log("Deleting WU: " + wu.Wuid);
                 return wu.delete().then(() => wu);
             }).then(wu => {
                 console.log("Deleted WU: " + wu.Wuid, wu.isDeleted());

packages/comms/src/connection.ts

@@ -1,4 +1,4 @@
-import { join, promiseTimeout, scopedLogger } from "@hpcc-js/util";
+import { join, promiseTimeout, scopedLogger, utf8ToBase64 } from "@hpcc-js/util";
 
 const logger = scopedLogger("comms/connection.ts");
 
@@ -137,7 +137,7 @@ export function jsonp(opts: IOptions, action: string, request: any = {}, respons
 }
 
 function authHeader(opts: IOptions): object {
-    return opts.userID ? { Authorization: `Basic ${btoa(`${opts.userID}:${opts.password}`)}` } : {};
+    return opts.userID ? { Authorization: `Basic ${utf8ToBase64(`${opts.userID}:${opts.password}`)}` } : {};
 }
 
 //  _omitMap is a workaround for older HPCC-Platform instances without credentials ---

packages/comms/src/index.common.ts

@@ -1,5 +1,4 @@
 export * from "./__package__.ts";
-
 export * from "./services/fileSpray.ts";
 export * from "./services/wsAccess.ts";
 export * from "./services/wsAccount.ts";

packages/comms/src/index.node.ts

@@ -7,6 +7,7 @@ root.DOMParser = DOMParser;
 import fetch, { Headers, Request, Response, } from "node-fetch";
 
 import * as https from "node:https";
+import { Buffer } from "node:buffer";
 import { Agent, setGlobalDispatcher } from "undici";
 
 if (typeof root.fetch === "undefined") {
@@ -57,14 +58,6 @@ root.fetch.__trustwaveAgent = new https.Agent({
     ca: globalCA + trustwave
 });
 
-//  btoa polyfill  ---
-import { Buffer } from "node:buffer";
-if (typeof root.btoa === "undefined") {
-    root.btoa = function (str: string) {
-        return Buffer.from(str || "", "utf8").toString("base64");
-    };
-}
-
 export * from "./index.common.ts";
 
 //  Client Tools  ---

packages/util/src/index.ts

@@ -19,3 +19,4 @@ export * from "./stack.ts";
 export * from "./stateful.ts";
 export * from "./string.ts";
 export * from "./url.ts";
+export * from "./utf8ToBase64.ts";

packages/util/src/utf8ToBase64.ts

@@ -0,0 +1,47 @@
+const BASE64_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+function toUTF8Bytes(value: string): Uint8Array {
+    if (typeof TextEncoder !== "undefined") {
+        return new TextEncoder().encode(value);
+    }
+
+    const encoded = encodeURIComponent(value);
+    const bytes: number[] = [];
+    for (let i = 0; i < encoded.length; ++i) {
+        if (encoded[i] === "%") {
+            bytes.push(parseInt(encoded.substring(i + 1, i + 3), 16));
+            i += 2;
+        } else {
+            bytes.push(encoded.charCodeAt(i));
+        }
+    }
+    return Uint8Array.from(bytes);
+}
+
+function bytesToBase64(bytes: Uint8Array): string {
+    let output = "";
+    for (let i = 0; i < bytes.length; i += 3) {
+        const byte1 = bytes[i];
+        const hasByte2 = i + 1 < bytes.length;
+        const hasByte3 = i + 2 < bytes.length;
+        const byte2 = hasByte2 ? bytes[i + 1] : 0;
+        const byte3 = hasByte3 ? bytes[i + 2] : 0;
+
+        output += BASE64_ALPHABET[byte1 >> 2];
+        output += BASE64_ALPHABET[((byte1 & 0x03) << 4) | (byte2 >> 4)];
+        output += hasByte2 ? BASE64_ALPHABET[((byte2 & 0x0f) << 2) | (byte3 >> 6)] : "=";
+        output += hasByte3 ? BASE64_ALPHABET[byte3 & 0x3f] : "=";
+    }
+    return output;
+}
+
+export function utf8ToBase64(value: string = ""): string {
+    const normalized = value == null ? "" : String(value);
+
+    const maybeBuffer = (globalThis as any)?.Buffer;
+    if (maybeBuffer?.from) {
+        return maybeBuffer.from(normalized, "utf8").toString("base64");
+    }
+
+    return bytesToBase64(toUTF8Bytes(normalized));
+}

packages/util/tests/utf8ToBase64.node.spec.ts

@@ -0,0 +1,22 @@
+import { describe, it, expect } from "vitest";
+import { utf8ToBase64 } from "@hpcc-js/util";
+
+describe("utf8ToBase64", () => {
+    it("encodes multi-byte characters", () => {
+        const sample = "Привет, 世界 👋";
+        const expected = Buffer.from(sample, "utf8").toString("base64");
+        expect(utf8ToBase64(sample)).toEqual(expected);
+    });
+
+    it("falls back when Buffer is unavailable", () => {
+        const sample = "mañana ☀️";
+        const expected = Buffer.from(sample, "utf8").toString("base64");
+        const original = (globalThis as any).Buffer;
+        (globalThis as any).Buffer = undefined;
+        try {
+            expect(utf8ToBase64(sample)).toEqual(expected);
+        } finally {
+            (globalThis as any).Buffer = original;
+        }
+    });
+});

packages/util/tests/utf8ToBase64.spec.ts

@@ -0,0 +1,32 @@
+import { describe, it, expect } from "vitest";
+import { utf8ToBase64 } from "@hpcc-js/util";
+
+describe("utf8ToBase64", () => {
+    it("encodes ASCII input", () => {
+        expect(utf8ToBase64("hello world"))
+            .toEqual("aGVsbG8gd29ybGQ=");
+    });
+
+    it("treats nullish values as empty string", () => {
+        expect(utf8ToBase64(undefined as unknown as string)).toEqual("");
+        expect(utf8ToBase64(null as unknown as string)).toEqual("");
+    });
+
+    it("check frequest characters", () => {
+        expect(utf8ToBase64("hello world")).toEqual(btoa("hello world"));
+        expect(utf8ToBase64("g@s*!")).toEqual(btoa("g@s*!"));
+        expect(utf8ToBase64("~@:!$%^&*()_-;'#***")).toEqual(btoa("~@:!$%^&*()_-;'#***"));
+        expect(utf8ToBase64("!$%^&*()_-;'#:@~")).toEqual(btoa("!$%^&*()_-;'#:@~"));
+        // expect(utf8ToBase64("¬!£$%^&*()_-;'#:@~")).toEqual(btoa("¬!£$%^&*()_-;'#:@~"));
+    });
+
+    describe("incremental ASCII coverage", () => {
+        const printableAscii = Array.from({ length: 95 }, (_, i) => String.fromCharCode(32 + i)).join("");
+        const checkpoints = [1, 5, 10, 20, 32, 48, 64, 80, 95];
+        const cases = checkpoints.map((len) => [len, printableAscii.slice(0, len)] as const);
+
+        it.each(cases)("encodes first %i printable ASCII chars", (_length, sample) => {
+            expect(utf8ToBase64(sample)).toEqual(btoa(sample));
+        });
+    });
+});