From e47d09df2d9d4720603fb573c4d6aea86c89da50 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 10 Jan 2023 10:08:50 +0000 Subject: [PATCH 001/231] Start for bpmux replacement --- packages/verser/package.json | 2 +- packages/verser/src/index.ts | 1 + .../src/lib/tc-verser/codecs/frame-decoder.ts | 53 +++++++++++ .../src/lib/tc-verser/codecs/frame-encoder.ts | 68 +++++++++++++ .../verser/src/lib/tc-verser/codecs/index.ts | 2 + .../verser/src/lib/tc-verser/playground.ts | 95 +++++++++++++++++++ .../verser/src/lib/tc-verser/utils/index.ts | 10 ++ yarn.lock | 5 + 8 files changed, 235 insertions(+), 1 deletion(-) create mode 100644 packages/verser/src/lib/tc-verser/codecs/frame-decoder.ts create mode 100644 packages/verser/src/lib/tc-verser/codecs/frame-encoder.ts create mode 100644 packages/verser/src/lib/tc-verser/codecs/index.ts create mode 100644 packages/verser/src/lib/tc-verser/playground.ts create mode 100644 packages/verser/src/lib/tc-verser/utils/index.ts diff --git a/packages/verser/package.json b/packages/verser/package.json index 5f7875cf8..203fff311 100644 --- a/packages/verser/package.json +++ b/packages/verser/package.json @@ -26,7 +26,7 @@ "ts-node": "^10.9.1", "typedoc": "^0.23.17", "typedoc-plugin-markdown": "^3.13.6", - "typescript": "~4.7.4" + "typescript": "~4.9.4" }, "ava": { "extensions": [ diff --git a/packages/verser/src/index.ts b/packages/verser/src/index.ts index 19bca6084..b962fa556 100644 --- a/packages/verser/src/index.ts +++ b/packages/verser/src/index.ts @@ -1,3 +1,4 @@ export * from "./lib/verser"; export * from "./lib/verser-client"; export * from "./lib/verser-connection"; +export * from "./lib/tc-verser/playground"; diff --git a/packages/verser/src/lib/tc-verser/codecs/frame-decoder.ts b/packages/verser/src/lib/tc-verser/codecs/frame-decoder.ts new file mode 100644 index 000000000..2795da985 --- /dev/null +++ b/packages/verser/src/lib/tc-verser/codecs/frame-decoder.ts @@ -0,0 +1,53 @@ +import { ObjLogger } from "@scramjet/obj-logger"; +import { Duplex, Transform, TransformCallback, TransformOptions } from "stream"; +import { HEADER_LENGTH, toHex } from "../utils"; + +export class FrameDecoder extends Transform { + buff: Buffer; + size = 0; + logger: ObjLogger; + + _streams = new Map(); + + constructor(opts?: TransformOptions, params: { name: string } = { name: "FrameDecoder"}) { + super(opts); + this.buff = Buffer.alloc(64 * 1024, 0, undefined); //@TODO: optimize + this.logger = new ObjLogger(params.name); + + this.on("pipe", () => { + this.logger.debug("onPipe"); + }); + } + + _transform(chunk: Buffer, encoding: BufferEncoding, callback: TransformCallback): void { + this.logger.trace("Decoding frame...", toHex(chunk), "Size:", chunk.length); + + if (!Buffer.isBuffer(chunk)) { + this.push(JSON.stringify({ error: "not a buffer"}), undefined); + callback(); + + return; + } + + chunk.copy(this.buff, this.size, 0, chunk.length); + + this.size += chunk.length; + + if (this.size >= 10 && this.buff.readInt32LE(10) === this.size) { + this.push(JSON.stringify({ + sourceAddress: `${this.buff.readInt8(0).toString()}.${this.buff.readInt8(1).toString()}.${this.buff.readInt8(2).toString()}.${this.buff.readInt8(3).toString()}`, + destinationAddress: `${this.buff.readInt8(4).toString()}.${this.buff.readInt8(5).toString()}.${this.buff.readInt8(6).toString()}.${this.buff.readInt8(7).toString()}`, + chunk: this.buff.subarray(32, this.buff.readInt32LE(10) - HEADER_LENGTH + 32), + dataLength: this.buff.readInt32LE(10) - HEADER_LENGTH, + stringified: this.buff.subarray(32, this.buff.readInt32LE(10) - HEADER_LENGTH + 32).toString() + }, null, 0) + "\n"); + + this.size = 0; + this.buff.fill(0); + } else { + this.logger.error("too few data", this.size, this.buff.readInt32LE(10)); + } + + callback(); + } +} diff --git a/packages/verser/src/lib/tc-verser/codecs/frame-encoder.ts b/packages/verser/src/lib/tc-verser/codecs/frame-encoder.ts new file mode 100644 index 000000000..a353e0d4b --- /dev/null +++ b/packages/verser/src/lib/tc-verser/codecs/frame-encoder.ts @@ -0,0 +1,68 @@ +import { Transform, TransformCallback, TransformOptions } from "stream"; +import { FrameTarget, HEADER_LENGTH, toHex as getHexString } from "../utils"; +import { ObjLogger } from "@scramjet/obj-logger"; + +export class FrameEncoder extends Transform { + sequenceNumber = 0; + logger = new ObjLogger("FrameEncoder",); + + constructor(private frameTarget: FrameTarget, opts?: TransformOptions, params: { name: string } = { name: "FrameEncoder"}) { + super(opts); + this.logger = new ObjLogger(params.name, { id: this.frameTarget.toString() }); + + this.on("pipe", () => { + this.logger.debug("onPipe"); + }); + } + + createChunkFrame(chunk: any) { + const checksum = 255; + + const buffer = Buffer.concat([ + // 0: source address 0 - 3 + new Uint8Array([10, 0, 0, 1]), + // 32: destination address 4 - 7 + new Uint8Array([10, 0, 0, 2]), + + // 64: zeroes (8bit), protocol (8bit), 8 - 9 + new Uint8Array([0, 1]), + // tcp length (16bit) 10 - 11 + new Uint8Array(new Uint16Array([chunk.length + HEADER_LENGTH]).buffer), + + // 96: Source port, destination port 12 - 15 + new Uint8Array(new Uint16Array([0, this.frameTarget]).buffer), + + // 128: sequenceNumber(32 bit, acnowledge number) 16 - 19 + new Uint8Array(new Uint32Array([this.sequenceNumber++]).buffer), + + // 160: Acknowledgement number 20-23 + new Uint8Array([0,0,0,0]), + + // 192: data offset (4bit), reserved (4bit), 24 + new Uint8Array([0b00000000]), + // 224: flags (8bit), 25 + new Uint8Array([0b00000000]), + // window(16bit) 26 - 27 + new Uint8Array(new Uint16Array([0]).buffer), + + // checksum(16bit) 28 - 29 + new Uint8Array(new Uint16Array([checksum]).buffer), + // pointer (16bit) 30 - 31 + new Uint8Array(new Uint16Array([checksum]).buffer), + + // 256: data 32 - + new Uint8Array(chunk) + ]); + + return buffer; + } + + _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void { + const buffer = this.createChunkFrame(chunk); + + this.logger.trace("Encoded frame", getHexString(buffer), "Size: ", buffer.length, "Pushing"); + + this.push(buffer, undefined); + callback(null); + }; +} diff --git a/packages/verser/src/lib/tc-verser/codecs/index.ts b/packages/verser/src/lib/tc-verser/codecs/index.ts new file mode 100644 index 000000000..2a0fcd378 --- /dev/null +++ b/packages/verser/src/lib/tc-verser/codecs/index.ts @@ -0,0 +1,2 @@ +export * from "./frame-decoder"; +export * from "./frame-encoder"; diff --git a/packages/verser/src/lib/tc-verser/playground.ts b/packages/verser/src/lib/tc-verser/playground.ts new file mode 100644 index 000000000..8ba3ea03a --- /dev/null +++ b/packages/verser/src/lib/tc-verser/playground.ts @@ -0,0 +1,95 @@ +/*eslint no-unused-vars: ["error", { "args": "none" }]*/ + +import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; +import { IncomingMessage, ServerResponse, createServer, request } from "http"; +import { DataStream } from "scramjet"; +import { FrameTarget } from "./utils"; +import { FrameDecoder, FrameEncoder } from "./codecs"; + +(async () => { + const logger = new ObjLogger("Sandbox"); + + logger.pipe(new DataStream().map(prettyPrint({ colors: true }))).pipe(process.stdout); + + const PORT = 6660; + const server = createServer(); + + server.setTimeout(0); + server.requestTimeout = 0; + + server.on("request", (req: IncomingMessage, res: ServerResponse) => { + res.socket!.setNoDelay(true); + + res.writeHead(200, { "Content-Type": "application/octet-stream", "Transfer-Encoding" : "chunked" }); + res.flushHeaders(); + + logger.debug("on request"); + + const serverFrameEncoder = new FrameEncoder(FrameTarget.API, { encoding: undefined }, { name: "ServerEncoder"}); + serverFrameEncoder.logger.pipe(logger); + + serverFrameEncoder.pipe(res); + let i = 0; + + req.on("pause", () => { logger.warn("Request paused") }); + + const serverFrameDecoder = new FrameDecoder({ highWaterMark: 60 * 1024, encoding: undefined, emitClose: false }, { name: "ServerDecoder"}); + serverFrameDecoder.logger.pipe(logger); + + req.pipe(serverFrameDecoder).pipe(process.stdout); + + serverFrameDecoder.on("data", (d: any) => { + logger.debug(`Server on request data [${i++}]`, JSON.parse(d).chunk, d.length); + serverFrameEncoder.write(Buffer.from(new Uint8Array([255, 255, 255, 255]))); + }) + }); + + server.listen(PORT); + + const req = request({ + hostname: "0.0.0.0", + port: PORT, + headers: { "Content-Type": "application/octet-stream", "Transfer-Encoding": "chunked" } + }); + + + req.on("response", (response) => { + req.socket!.setNoDelay(true); + + logger.debug("Response"); + + let i = 0; + let m = 0; + + const frameEncoder = new FrameEncoder(FrameTarget.API, { encoding: undefined, emitClose: false }, { name: "RequestEncoder"}); + frameEncoder.logger.pipe(logger); + + frameEncoder.pipe(req); + + const responseFrameDecoder = new FrameDecoder({ highWaterMark: 60 * 1024, encoding: undefined, emitClose: false }, { name: "RequestDecoder"}); + + responseFrameDecoder.logger.pipe(logger); + + response.pipe( + responseFrameDecoder + ).on("data", (d) => { + logger.debug(`Echo from server [${i++}]`, JSON.parse(d).chunk, d.length); + }); + + response.on("data", (d) => console.log("plain response", d)); + + req.on("error", (err) => { + console.error(err); + }); + + setInterval(async () => { + logger.debug(`Writing [${++m}]..`); + + frameEncoder.write(Buffer.from(new Uint8Array([0x61, 0x62, 0x63, 0x64]))); + }, 500); + }); + + req.flushHeaders(); + + await new Promise((_res, _rej) => {}); +})(); diff --git a/packages/verser/src/lib/tc-verser/utils/index.ts b/packages/verser/src/lib/tc-verser/utils/index.ts new file mode 100644 index 000000000..6dacf2285 --- /dev/null +++ b/packages/verser/src/lib/tc-verser/utils/index.ts @@ -0,0 +1,10 @@ +export function toHex(chunk: Buffer) { + return chunk.toString('hex').match(/../g)?.join(' '); +} + +export const HEADER_LENGTH = 32; + +export enum FrameTarget { + API, + INPUT = 1001 +} diff --git a/yarn.lock b/yarn.lock index eb667583a..6c8a47843 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7366,6 +7366,11 @@ typescript@~4.7.4: resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.7.4.tgz#1a88596d1cf47d59507a1bcdfb5b9dfe4d488235" integrity sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ== +typescript@~4.9.4: + version "4.9.4" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.4.tgz#a2a3d2756c079abda241d75f149df9d561091e78" + integrity sha512-Uz+dTXYzxXXbsFpM86Wh3dKCxrQqUcVMxwU54orwlJjOpO3ao8L7j5lH+dWfTwgCwIuM9GQ2kvVotzYJMXTBZg== + uglify-js@^3.1.4: version "3.17.4" resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.17.4.tgz#61678cf5fa3f5b7eb789bb345df29afb8257c22c" From 3a8104ea36fdf7efab4975a3bb548b43f2560404 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 10 Jan 2023 12:14:59 +0000 Subject: [PATCH 002/231] tc-verser Switch to connect method --- .../src/lib/tc-verser/codecs/frame-decoder.ts | 4 +- .../src/lib/tc-verser/codecs/frame-encoder.ts | 8 ++-- .../verser/src/lib/tc-verser/playground.ts | 44 ++++++++++--------- .../verser/src/lib/tc-verser/utils/index.ts | 2 +- packages/verser/src/lib/verser-client.ts | 4 +- 5 files changed, 33 insertions(+), 29 deletions(-) diff --git a/packages/verser/src/lib/tc-verser/codecs/frame-decoder.ts b/packages/verser/src/lib/tc-verser/codecs/frame-decoder.ts index 2795da985..f1fcb1c45 100644 --- a/packages/verser/src/lib/tc-verser/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tc-verser/codecs/frame-decoder.ts @@ -9,7 +9,7 @@ export class FrameDecoder extends Transform { _streams = new Map(); - constructor(opts?: TransformOptions, params: { name: string } = { name: "FrameDecoder"}) { + constructor(opts?: TransformOptions, params: { name: string } = { name: "FrameDecoder" }) { super(opts); this.buff = Buffer.alloc(64 * 1024, 0, undefined); //@TODO: optimize this.logger = new ObjLogger(params.name); @@ -23,7 +23,7 @@ export class FrameDecoder extends Transform { this.logger.trace("Decoding frame...", toHex(chunk), "Size:", chunk.length); if (!Buffer.isBuffer(chunk)) { - this.push(JSON.stringify({ error: "not a buffer"}), undefined); + this.push(JSON.stringify({ error: "not a buffer" }), undefined); callback(); return; diff --git a/packages/verser/src/lib/tc-verser/codecs/frame-encoder.ts b/packages/verser/src/lib/tc-verser/codecs/frame-encoder.ts index a353e0d4b..b1fe54f36 100644 --- a/packages/verser/src/lib/tc-verser/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tc-verser/codecs/frame-encoder.ts @@ -6,9 +6,9 @@ export class FrameEncoder extends Transform { sequenceNumber = 0; logger = new ObjLogger("FrameEncoder",); - constructor(private frameTarget: FrameTarget, opts?: TransformOptions, params: { name: string } = { name: "FrameEncoder"}) { + constructor(private frameTarget: FrameTarget, opts?: TransformOptions, params: { name: string } = { name: "FrameEncoder" }) { super(opts); - this.logger = new ObjLogger(params.name, { id: this.frameTarget.toString() }); + this.logger = new ObjLogger(params.name, { id: this.frameTarget.toString() }); this.on("pipe", () => { this.logger.debug("onPipe"); @@ -36,7 +36,7 @@ export class FrameEncoder extends Transform { new Uint8Array(new Uint32Array([this.sequenceNumber++]).buffer), // 160: Acknowledgement number 20-23 - new Uint8Array([0,0,0,0]), + new Uint8Array([0, 0, 0, 0]), // 192: data offset (4bit), reserved (4bit), 24 new Uint8Array([0b00000000]), @@ -64,5 +64,5 @@ export class FrameEncoder extends Transform { this.push(buffer, undefined); callback(null); - }; + } } diff --git a/packages/verser/src/lib/tc-verser/playground.ts b/packages/verser/src/lib/tc-verser/playground.ts index 8ba3ea03a..7c11e815c 100644 --- a/packages/verser/src/lib/tc-verser/playground.ts +++ b/packages/verser/src/lib/tc-verser/playground.ts @@ -1,10 +1,13 @@ /*eslint no-unused-vars: ["error", { "args": "none" }]*/ +/* eslint-disable @typescript-eslint/no-floating-promises */ +/* eslint-disable no-console */ import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; -import { IncomingMessage, ServerResponse, createServer, request } from "http"; +import { IncomingMessage, createServer, request } from "http"; import { DataStream } from "scramjet"; import { FrameTarget } from "./utils"; import { FrameDecoder, FrameEncoder } from "./codecs"; +import { Socket } from "net"; (async () => { const logger = new ObjLogger("Sandbox"); @@ -17,60 +20,61 @@ import { FrameDecoder, FrameEncoder } from "./codecs"; server.setTimeout(0); server.requestTimeout = 0; - server.on("request", (req: IncomingMessage, res: ServerResponse) => { - res.socket!.setNoDelay(true); + server.on("connect", (req: IncomingMessage, socket: Socket) => { + socket.setNoDelay(true); + socket.write(`HTTP/1.1 ${200} \r\n\r\n`); - res.writeHead(200, { "Content-Type": "application/octet-stream", "Transfer-Encoding" : "chunked" }); - res.flushHeaders(); + logger.debug("on connect"); - logger.debug("on request"); + const serverFrameEncoder = new FrameEncoder(FrameTarget.API, { encoding: undefined }, { name: "ServerEncoder" }); - const serverFrameEncoder = new FrameEncoder(FrameTarget.API, { encoding: undefined }, { name: "ServerEncoder"}); serverFrameEncoder.logger.pipe(logger); - serverFrameEncoder.pipe(res); + serverFrameEncoder.pipe(socket); let i = 0; - req.on("pause", () => { logger.warn("Request paused") }); + req.on("pause", () => { logger.warn("Request paused"); }); + + const serverFrameDecoder = new FrameDecoder({ highWaterMark: 60 * 1024, encoding: undefined, emitClose: false }, { name: "ServerDecoder" }); - const serverFrameDecoder = new FrameDecoder({ highWaterMark: 60 * 1024, encoding: undefined, emitClose: false }, { name: "ServerDecoder"}); serverFrameDecoder.logger.pipe(logger); - req.pipe(serverFrameDecoder).pipe(process.stdout); + socket.pipe(serverFrameDecoder).pipe(process.stdout); serverFrameDecoder.on("data", (d: any) => { logger.debug(`Server on request data [${i++}]`, JSON.parse(d).chunk, d.length); serverFrameEncoder.write(Buffer.from(new Uint8Array([255, 255, 255, 255]))); - }) + }); }); server.listen(PORT); const req = request({ hostname: "0.0.0.0", + method: "connect", port: PORT, headers: { "Content-Type": "application/octet-stream", "Transfer-Encoding": "chunked" } }); + req.on("connect", (response, socket, head) => { + socket.setNoDelay(true); - req.on("response", (response) => { - req.socket!.setNoDelay(true); - - logger.debug("Response"); + logger.debug("Response. Head:", head.toString()); let i = 0; let m = 0; - const frameEncoder = new FrameEncoder(FrameTarget.API, { encoding: undefined, emitClose: false }, { name: "RequestEncoder"}); + const frameEncoder = new FrameEncoder(FrameTarget.API, { encoding: undefined, emitClose: false }, { name: "RequestEncoder" }); + frameEncoder.logger.pipe(logger); - frameEncoder.pipe(req); + frameEncoder.pipe(socket); - const responseFrameDecoder = new FrameDecoder({ highWaterMark: 60 * 1024, encoding: undefined, emitClose: false }, { name: "RequestDecoder"}); + const responseFrameDecoder = new FrameDecoder({ highWaterMark: 60 * 1024, encoding: undefined, emitClose: false }, { name: "RequestDecoder" }); responseFrameDecoder.logger.pipe(logger); - response.pipe( + socket.pipe( responseFrameDecoder ).on("data", (d) => { logger.debug(`Echo from server [${i++}]`, JSON.parse(d).chunk, d.length); diff --git a/packages/verser/src/lib/tc-verser/utils/index.ts b/packages/verser/src/lib/tc-verser/utils/index.ts index 6dacf2285..229a29b8d 100644 --- a/packages/verser/src/lib/tc-verser/utils/index.ts +++ b/packages/verser/src/lib/tc-verser/utils/index.ts @@ -1,5 +1,5 @@ export function toHex(chunk: Buffer) { - return chunk.toString('hex').match(/../g)?.join(' '); + return chunk.toString("hex").match(/../g)?.join(" "); } export const HEADER_LENGTH = 32; diff --git a/packages/verser/src/lib/verser-client.ts b/packages/verser/src/lib/verser-client.ts index f70471f30..a33fad356 100644 --- a/packages/verser/src/lib/verser-client.ts +++ b/packages/verser/src/lib/verser-client.ts @@ -108,11 +108,11 @@ export class VerserClient extends TypedEmitter { reject(err); }); - connectRequest.on("connect", (req, socket) => { + connectRequest.on("connect", (response, socket) => { this.socket = socket; this.mux(); - resolve({ req, socket }); + resolve({ req: response, socket }); }); connectRequest.flushHeaders(); From 6a839fdeca608970a431cc36b86622f7c0b3c22e Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 10 Jan 2023 17:27:04 +0000 Subject: [PATCH 003/231] Rename to tecemux --- packages/verser/src/index.ts | 1 - .../src/lib/{tc-verser => tecemux}/codecs/frame-decoder.ts | 0 .../src/lib/{tc-verser => tecemux}/codecs/frame-encoder.ts | 0 packages/verser/src/lib/{tc-verser => tecemux}/codecs/index.ts | 0 packages/verser/src/lib/{tc-verser => tecemux}/playground.ts | 0 packages/verser/src/lib/{tc-verser => tecemux}/utils/index.ts | 0 6 files changed, 1 deletion(-) rename packages/verser/src/lib/{tc-verser => tecemux}/codecs/frame-decoder.ts (100%) rename packages/verser/src/lib/{tc-verser => tecemux}/codecs/frame-encoder.ts (100%) rename packages/verser/src/lib/{tc-verser => tecemux}/codecs/index.ts (100%) rename packages/verser/src/lib/{tc-verser => tecemux}/playground.ts (100%) rename packages/verser/src/lib/{tc-verser => tecemux}/utils/index.ts (100%) diff --git a/packages/verser/src/index.ts b/packages/verser/src/index.ts index b962fa556..19bca6084 100644 --- a/packages/verser/src/index.ts +++ b/packages/verser/src/index.ts @@ -1,4 +1,3 @@ export * from "./lib/verser"; export * from "./lib/verser-client"; export * from "./lib/verser-connection"; -export * from "./lib/tc-verser/playground"; diff --git a/packages/verser/src/lib/tc-verser/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts similarity index 100% rename from packages/verser/src/lib/tc-verser/codecs/frame-decoder.ts rename to packages/verser/src/lib/tecemux/codecs/frame-decoder.ts diff --git a/packages/verser/src/lib/tc-verser/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts similarity index 100% rename from packages/verser/src/lib/tc-verser/codecs/frame-encoder.ts rename to packages/verser/src/lib/tecemux/codecs/frame-encoder.ts diff --git a/packages/verser/src/lib/tc-verser/codecs/index.ts b/packages/verser/src/lib/tecemux/codecs/index.ts similarity index 100% rename from packages/verser/src/lib/tc-verser/codecs/index.ts rename to packages/verser/src/lib/tecemux/codecs/index.ts diff --git a/packages/verser/src/lib/tc-verser/playground.ts b/packages/verser/src/lib/tecemux/playground.ts similarity index 100% rename from packages/verser/src/lib/tc-verser/playground.ts rename to packages/verser/src/lib/tecemux/playground.ts diff --git a/packages/verser/src/lib/tc-verser/utils/index.ts b/packages/verser/src/lib/tecemux/utils/index.ts similarity index 100% rename from packages/verser/src/lib/tc-verser/utils/index.ts rename to packages/verser/src/lib/tecemux/utils/index.ts From de371b88065d524c53d937725b87c5f701b4e24c Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 11 Jan 2023 11:10:31 +0000 Subject: [PATCH 004/231] DecodedFrame type --- .../src/lib/tecemux/codecs/frame-decoder.ts | 26 ++++++---- .../verser/src/lib/tecemux/codecs/index.ts | 48 +++++++++++++++++++ packages/verser/src/lib/tecemux/playground.ts | 10 ++-- .../verser/src/lib/tecemux/utils/index.ts | 13 +++++ 4 files changed, 85 insertions(+), 12 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index f1fcb1c45..2af72925d 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -9,8 +9,9 @@ export class FrameDecoder extends Transform { _streams = new Map(); - constructor(opts?: TransformOptions, params: { name: string } = { name: "FrameDecoder" }) { - super(opts); + constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { + super(Object.assign({}, { readableObjectMode: true }, opts)); + this.buff = Buffer.alloc(64 * 1024, 0, undefined); //@TODO: optimize this.logger = new ObjLogger(params.name); @@ -34,13 +35,20 @@ export class FrameDecoder extends Transform { this.size += chunk.length; if (this.size >= 10 && this.buff.readInt32LE(10) === this.size) { - this.push(JSON.stringify({ - sourceAddress: `${this.buff.readInt8(0).toString()}.${this.buff.readInt8(1).toString()}.${this.buff.readInt8(2).toString()}.${this.buff.readInt8(3).toString()}`, - destinationAddress: `${this.buff.readInt8(4).toString()}.${this.buff.readInt8(5).toString()}.${this.buff.readInt8(6).toString()}.${this.buff.readInt8(7).toString()}`, - chunk: this.buff.subarray(32, this.buff.readInt32LE(10) - HEADER_LENGTH + 32), - dataLength: this.buff.readInt32LE(10) - HEADER_LENGTH, - stringified: this.buff.subarray(32, this.buff.readInt32LE(10) - HEADER_LENGTH + 32).toString() - }, null, 0) + "\n"); + const frameSize = this.buff.readInt32LE(10); + + const payload = { + sourceAddress: [this.buff.readInt8(0), this.buff.readInt8(1), this.buff.readInt8(2), this.buff.readInt8(3)], + destinationAddress: [this.buff.readInt8(4), this.buff.readInt8(5), this.buff.readInt8(6), this.buff.readInt8(7)], + destinationPort: this.buff.readInt8(9), + chunk: this.buff.subarray(32, this.buff.readInt32LE(10)), + dataLength: frameSize - HEADER_LENGTH, + chunkLength: frameSize, + stringified: this.buff.subarray(32, frameSize).toString() + }; + + this.push(JSON.stringify(payload) + "\n"); + //this.push(payload); this.size = 0; this.buff.fill(0); diff --git a/packages/verser/src/lib/tecemux/codecs/index.ts b/packages/verser/src/lib/tecemux/codecs/index.ts index 2a0fcd378..2e0d50d40 100644 --- a/packages/verser/src/lib/tecemux/codecs/index.ts +++ b/packages/verser/src/lib/tecemux/codecs/index.ts @@ -1,2 +1,50 @@ +import { Socket } from "net"; +import { Duplex } from "stream"; +import { FrameEncoder } from "./frame-encoder"; +import { FrameTarget } from "../utils"; +import { TypedEmitter } from "@scramjet/utility"; +import { FrameDecoder } from "./frame-decoder"; + export * from "./frame-decoder"; export * from "./frame-encoder"; + +type TeceMuxEvents = { + channel(socket: Duplex): void; +} + +export class TeceMux extends TypedEmitter{ + socket: Duplex; + socketCount = 0; + decoder = new FrameDecoder(); + + channels: Duplex[] = []; + + constructor(socket: Socket) { + super(); + this.socket = socket; + + this.decoder.on("data", (chunk) => { + const frame = JSON.parse(chunk); + + const channel = frame.destinationPort; + + if (!this.channels[channel]) { + this.channels[channel] = new Duplex({ }); + this.channels[channel].pipe(new FrameEncoder(channel).pipe(this.socket)); // ? lot of encoders / encoder.writeToChannel ?; + this.emit("channel", this.channels[channel]); + } + + this.channels[channel].write(frame.chunk.data); + }); + + socket.pipe(this.decoder); + } + + multiplex() { + const multiplex = new Socket(); + + multiplex.pipe(new FrameEncoder(FrameTarget.API)).pipe(this.socket); + + return multiplex; + } +} diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/src/lib/tecemux/playground.ts index 7c11e815c..41d7894cc 100644 --- a/packages/verser/src/lib/tecemux/playground.ts +++ b/packages/verser/src/lib/tecemux/playground.ts @@ -5,7 +5,7 @@ import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; import { IncomingMessage, createServer, request } from "http"; import { DataStream } from "scramjet"; -import { FrameTarget } from "./utils"; +import { DecodedFrame, FrameTarget } from "./utils"; import { FrameDecoder, FrameEncoder } from "./codecs"; import { Socket } from "net"; @@ -42,7 +42,9 @@ import { Socket } from "net"; socket.pipe(serverFrameDecoder).pipe(process.stdout); serverFrameDecoder.on("data", (d: any) => { - logger.debug(`Server on request data [${i++}]`, JSON.parse(d).chunk, d.length); + const parsed = JSON.parse(d) as DecodedFrame; + + logger.debug(`Server on request data [${i++}]`, parsed.chunk, parsed.dataLength); serverFrameEncoder.write(Buffer.from(new Uint8Array([255, 255, 255, 255]))); }); }); @@ -77,7 +79,9 @@ import { Socket } from "net"; socket.pipe( responseFrameDecoder ).on("data", (d) => { - logger.debug(`Echo from server [${i++}]`, JSON.parse(d).chunk, d.length); + const parsed = JSON.parse(d) as DecodedFrame; + + logger.debug(`Echo from server [${i++}]`, parsed.chunk, parsed.chunkLength); }); response.on("data", (d) => console.log("plain response", d)); diff --git a/packages/verser/src/lib/tecemux/utils/index.ts b/packages/verser/src/lib/tecemux/utils/index.ts index 229a29b8d..631dba5d0 100644 --- a/packages/verser/src/lib/tecemux/utils/index.ts +++ b/packages/verser/src/lib/tecemux/utils/index.ts @@ -8,3 +8,16 @@ export enum FrameTarget { API, INPUT = 1001 } + +export type DecodedFrame = { + sourceAddress: [number, number, number, number]; + destinationAddress: [number, number, number, number]; + destinationPort: number; + chunk: { + type: string; + data: any; + }; + dataLength: number; + chunkLength: number; + stringified: string; +}; From f2ab88367b4134d28c01cfbbeb0071b694ae54ac Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 12 Jan 2023 20:49:34 +0000 Subject: [PATCH 005/231] Flags parsers --- .../src/lib/tecemux/codecs/frame-decoder.ts | 14 +++- .../src/lib/tecemux/codecs/frame-encoder.ts | 64 +++++++++++++++-- .../verser/src/lib/tecemux/codecs/index.ts | 53 ++++---------- packages/verser/src/lib/tecemux/playground.ts | 2 +- packages/verser/src/lib/tecemux/tecemux.ts | 70 +++++++++++++++++++ 5 files changed, 153 insertions(+), 50 deletions(-) create mode 100644 packages/verser/src/lib/tecemux/tecemux.ts diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 2af72925d..8269f0d0a 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -1,6 +1,7 @@ import { ObjLogger } from "@scramjet/obj-logger"; import { Duplex, Transform, TransformCallback, TransformOptions } from "stream"; import { HEADER_LENGTH, toHex } from "../utils"; +import { frameFlags } from "."; export class FrameDecoder extends Transform { buff: Buffer; @@ -9,8 +10,9 @@ export class FrameDecoder extends Transform { _streams = new Map(); + constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { - super(Object.assign({}, { readableObjectMode: true }, opts)); + super(Object.assign({}, { readableObjectMode: true, emitClose: false }, opts)); this.buff = Buffer.alloc(64 * 1024, 0, undefined); //@TODO: optimize this.logger = new ObjLogger(params.name); @@ -20,6 +22,11 @@ export class FrameDecoder extends Transform { }); } + parseFlags(byte: number) { + return frameFlags.filter((_flag, index) => byte >>> index & 1) + .reduce((p, c) => ({ ...p, [c]: true }), {}); + } + _transform(chunk: Buffer, encoding: BufferEncoding, callback: TransformCallback): void { this.logger.trace("Decoding frame...", toHex(chunk), "Size:", chunk.length); @@ -40,15 +47,16 @@ export class FrameDecoder extends Transform { const payload = { sourceAddress: [this.buff.readInt8(0), this.buff.readInt8(1), this.buff.readInt8(2), this.buff.readInt8(3)], destinationAddress: [this.buff.readInt8(4), this.buff.readInt8(5), this.buff.readInt8(6), this.buff.readInt8(7)], - destinationPort: this.buff.readInt8(9), chunk: this.buff.subarray(32, this.buff.readInt32LE(10)), + flags: this.parseFlags(this.buff.readInt8(25)), + sourcePort: this.buff.readInt16LE(12), + destinationPort: this.buff.readInt16LE(14), dataLength: frameSize - HEADER_LENGTH, chunkLength: frameSize, stringified: this.buff.subarray(32, frameSize).toString() }; this.push(JSON.stringify(payload) + "\n"); - //this.push(payload); this.size = 0; this.buff.fill(0); diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index b1fe54f36..b8ec9a96b 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -1,13 +1,16 @@ import { Transform, TransformCallback, TransformOptions } from "stream"; import { FrameTarget, HEADER_LENGTH, toHex as getHexString } from "../utils"; import { ObjLogger } from "@scramjet/obj-logger"; - +import { binaryFlags, frameFlags } from "."; export class FrameEncoder extends Transform { sequenceNumber = 0; logger = new ObjLogger("FrameEncoder",); - constructor(private frameTarget: FrameTarget, opts?: TransformOptions, params: { name: string } = { name: "FrameEncoder" }) { + constructor(private frameTarget: FrameTarget, opts: TransformOptions = {}, params: { name: string } = { name: "FrameEncoder" }) { + opts.emitClose = false; + super(opts); + this.logger = new ObjLogger(params.name, { id: this.frameTarget.toString() }); this.on("pipe", () => { @@ -15,9 +18,15 @@ export class FrameEncoder extends Transform { }); } - createChunkFrame(chunk: any) { - const checksum = 255; + setFlag(flag: keyof typeof binaryFlags, flags: Uint8Array = new Uint8Array([0])) { + this.logger.debug("settingFlag", flag); + flags[0] |= 1 << frameFlags.indexOf(flag); + + return flags; + } + setChannel() { + const checksum = this.getChecksum(); const buffer = Buffer.concat([ // 0: source address 0 - 3 new Uint8Array([10, 0, 0, 1]), @@ -27,7 +36,7 @@ export class FrameEncoder extends Transform { // 64: zeroes (8bit), protocol (8bit), 8 - 9 new Uint8Array([0, 1]), // tcp length (16bit) 10 - 11 - new Uint8Array(new Uint16Array([chunk.length + HEADER_LENGTH]).buffer), + new Uint8Array(new Uint16Array([0 + HEADER_LENGTH]).buffer), // 96: Source port, destination port 12 - 15 new Uint8Array(new Uint16Array([0, this.frameTarget]).buffer), @@ -41,7 +50,52 @@ export class FrameEncoder extends Transform { // 192: data offset (4bit), reserved (4bit), 24 new Uint8Array([0b00000000]), // 224: flags (8bit), 25 + this.setFlag("ECE"), + // window(16bit) 26 - 27 + new Uint8Array(new Uint16Array([0]).buffer), + + // checksum(16bit) 28 - 29 + new Uint8Array(new Uint16Array([checksum]).buffer), + // pointer (16bit) 30 - 31 + new Uint8Array(new Uint16Array([checksum]).buffer), + + // 256: data 32 - + new Uint8Array([]) + ]); + + this.push(buffer, undefined); + } + + getChecksum() { + return 255; + } + + createChunkFrame(chunk: any) { + const checksum = this.getChecksum(); + const buffer = Buffer.concat([ + // 0: source address 0 - 3 + new Uint8Array([10, 0, 0, 1]), + // 32: destination address 4 - 7 + new Uint8Array([10, 0, 0, 2]), + + // 64: zeroes (8bit), protocol (8bit), 8 - 9 + new Uint8Array([0, 1]), + // tcp length (16bit) 10 - 11 + new Uint8Array(new Uint16Array([chunk.length + HEADER_LENGTH]).buffer), + + // 96: Source port, destination port 12 - 15 + new Uint8Array(new Uint16Array([0, this.frameTarget]).buffer), + + // 128: sequenceNumber(32 bit, acnowledge number) 16 - 19 + new Uint8Array(new Uint32Array([this.sequenceNumber++]).buffer), + + // 160: Acknowledgement number 20-23 + new Uint8Array([0, 0, 0, 0]), + + // 192: data offset (4bit), reserved (4bit), 24 new Uint8Array([0b00000000]), + // 224: flags (8bit), 25 + this.setFlag("PSH"), // window(16bit) 26 - 27 new Uint8Array(new Uint16Array([0]).buffer), diff --git a/packages/verser/src/lib/tecemux/codecs/index.ts b/packages/verser/src/lib/tecemux/codecs/index.ts index 2e0d50d40..f2556a8ea 100644 --- a/packages/verser/src/lib/tecemux/codecs/index.ts +++ b/packages/verser/src/lib/tecemux/codecs/index.ts @@ -1,50 +1,21 @@ -import { Socket } from "net"; import { Duplex } from "stream"; -import { FrameEncoder } from "./frame-encoder"; -import { FrameTarget } from "../utils"; -import { TypedEmitter } from "@scramjet/utility"; -import { FrameDecoder } from "./frame-decoder"; export * from "./frame-decoder"; export * from "./frame-encoder"; -type TeceMuxEvents = { +export type TeceMuxEvents = { channel(socket: Duplex): void; + error(error: any): void; } -export class TeceMux extends TypedEmitter{ - socket: Duplex; - socketCount = 0; - decoder = new FrameDecoder(); - - channels: Duplex[] = []; - - constructor(socket: Socket) { - super(); - this.socket = socket; - - this.decoder.on("data", (chunk) => { - const frame = JSON.parse(chunk); - - const channel = frame.destinationPort; - - if (!this.channels[channel]) { - this.channels[channel] = new Duplex({ }); - this.channels[channel].pipe(new FrameEncoder(channel).pipe(this.socket)); // ? lot of encoders / encoder.writeToChannel ?; - this.emit("channel", this.channels[channel]); - } - - this.channels[channel].write(frame.chunk.data); - }); - - socket.pipe(this.decoder); - } - - multiplex() { - const multiplex = new Socket(); - - multiplex.pipe(new FrameEncoder(FrameTarget.API)).pipe(this.socket); - - return multiplex; - } +export const frameFlags = ["FIN", "SYN", "RST", "PSH", "ACK", "URG", "ECE", "CWR"]; +export const binaryFlags = { + FIN: 0b00000001, + SYN: 0b00000010, + RST: 0b00000100, + PSH: 0b00001000, + ACK: 0b00010000, + URG: 0b00100000, + ECE: 0b01000000, + CWR: 0b10000000 } diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/src/lib/tecemux/playground.ts index 41d7894cc..cc7429c8a 100644 --- a/packages/verser/src/lib/tecemux/playground.ts +++ b/packages/verser/src/lib/tecemux/playground.ts @@ -81,7 +81,7 @@ import { Socket } from "net"; ).on("data", (d) => { const parsed = JSON.parse(d) as DecodedFrame; - logger.debug(`Echo from server [${i++}]`, parsed.chunk, parsed.chunkLength); + logger.debug(`Echo from server [${i++}]`, parsed.chunk, parsed.dataLength, parsed.chunkLength); }); response.on("data", (d) => console.log("plain response", d)); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts new file mode 100644 index 000000000..45ff67d10 --- /dev/null +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -0,0 +1,70 @@ +import { TypedEmitter } from "@scramjet/utility"; +import { FrameDecoder, FrameEncoder, TeceMuxEvents } from "./codecs"; +import { Duplex } from "stream"; +import { Socket } from "net"; + +export type TeceMuxChannel = Duplex & { _id: number }; + +export class TeceMux extends TypedEmitter{ + carrierSocket: Duplex; + channelCount = 0; + decoder = new FrameDecoder(); + + channels: Duplex[] = []; + + private createChannel(): TeceMuxChannel { + const channel: TeceMuxChannel = Object.assign(new Duplex({ }), { _id: this.channelCount }); + + const encoder = new FrameEncoder(this.channelCount); + + channel.pipe(encoder).pipe(this.carrierSocket); + channel.on("error", (error) => { + this.emit("error", { error, source: channel }) + }); + + encoder.setChannel(); + + return channel; + } + + constructor(socket: Socket) { + super(); + this.carrierSocket = socket; + + this.main().catch((error) => { + this.emit("error", error); + }); + + socket.pipe(this.decoder); + } + + async main() { + for await (const chunk of this.decoder) { + const frame = JSON.parse(chunk); + const channel = frame.destinationPort; + + if (!this.channels[channel]) { + const channel = this.createChannel(); + + this.addChannel(channel); + } + + this.channels[channel].write(frame.chunk.data); + } + } + + addChannel(channel: TeceMuxChannel) { + this.channels[this.channelCount] = channel; + this.channelCount++; + + this.emit("channel", channel); + } + + multiplex(): TeceMuxChannel { + const channel = this.createChannel(); + + this.addChannel(channel); + + return channel; + } +} From e8796c61790c5dcda1f1fa550482ddecd8365dd7 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Fri, 13 Jan 2023 22:54:35 +0000 Subject: [PATCH 006/231] Flags utils --- .../src/lib/tecemux/codecs/frame-decoder.ts | 9 +-- .../src/lib/tecemux/codecs/frame-encoder.ts | 60 ++++++++++++++++--- .../verser/src/lib/tecemux/codecs/index.ts | 16 +++++ packages/verser/src/lib/tecemux/playground.ts | 6 +- packages/verser/src/lib/tecemux/tecemux.ts | 36 ++++++++--- .../verser/src/lib/tecemux/utils/index.ts | 13 ++-- 6 files changed, 110 insertions(+), 30 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 8269f0d0a..0657af2eb 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -1,7 +1,7 @@ import { ObjLogger } from "@scramjet/obj-logger"; import { Duplex, Transform, TransformCallback, TransformOptions } from "stream"; import { HEADER_LENGTH, toHex } from "../utils"; -import { frameFlags } from "."; +import { frameFlags, parseFlags } from "."; export class FrameDecoder extends Transform { buff: Buffer; @@ -22,11 +22,6 @@ export class FrameDecoder extends Transform { }); } - parseFlags(byte: number) { - return frameFlags.filter((_flag, index) => byte >>> index & 1) - .reduce((p, c) => ({ ...p, [c]: true }), {}); - } - _transform(chunk: Buffer, encoding: BufferEncoding, callback: TransformCallback): void { this.logger.trace("Decoding frame...", toHex(chunk), "Size:", chunk.length); @@ -48,7 +43,7 @@ export class FrameDecoder extends Transform { sourceAddress: [this.buff.readInt8(0), this.buff.readInt8(1), this.buff.readInt8(2), this.buff.readInt8(3)], destinationAddress: [this.buff.readInt8(4), this.buff.readInt8(5), this.buff.readInt8(6), this.buff.readInt8(7)], chunk: this.buff.subarray(32, this.buff.readInt32LE(10)), - flags: this.parseFlags(this.buff.readInt8(25)), + flags: parseFlags(this.buff.readInt8(25)), sourcePort: this.buff.readInt16LE(12), destinationPort: this.buff.readInt16LE(14), dataLength: frameSize - HEADER_LENGTH, diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index b8ec9a96b..e196c67b2 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -1,5 +1,5 @@ import { Transform, TransformCallback, TransformOptions } from "stream"; -import { FrameTarget, HEADER_LENGTH, toHex as getHexString } from "../utils"; +import { FrameData, FrameTarget, HEADER_LENGTH, toHex as getHexString } from "../utils"; import { ObjLogger } from "@scramjet/obj-logger"; import { binaryFlags, frameFlags } from "."; export class FrameEncoder extends Transform { @@ -18,9 +18,11 @@ export class FrameEncoder extends Transform { }); } - setFlag(flag: keyof typeof binaryFlags, flags: Uint8Array = new Uint8Array([0])) { - this.logger.debug("settingFlag", flag); - flags[0] |= 1 << frameFlags.indexOf(flag); + setFlags(flag: (keyof typeof binaryFlags)[] = [], flags: Uint8Array = new Uint8Array([0])) { + for (const f in flag) { + this.logger.debug("settingFlag", flag); + flags[0] |= 1 << frameFlags.indexOf(flag[f]); + } return flags; } @@ -50,7 +52,7 @@ export class FrameEncoder extends Transform { // 192: data offset (4bit), reserved (4bit), 24 new Uint8Array([0b00000000]), // 224: flags (8bit), 25 - this.setFlag("ECE"), + this.setFlags(["ECE"]), // window(16bit) 26 - 27 new Uint8Array(new Uint16Array([0]).buffer), @@ -70,6 +72,49 @@ export class FrameEncoder extends Transform { return 255; } + createFrame(chunk: any = new Uint8Array([]), frame: Partial) { + const checksum = this.getChecksum(); + const buffer = Buffer.concat([ + // 0: source address 0 - 3 + new Uint8Array([10, 0, 0, 1]), + // 32: destination address 4 - 7 + new Uint8Array([10, 0, 0, 2]), + + // 64: zeroes (8bit), protocol (8bit), 8 - 9 + new Uint8Array([0, 1]), + + // tcp length (16bit) 10 - 11 + new Uint8Array(new Uint16Array([chunk.length + HEADER_LENGTH]).buffer), + + // 96: Source port, destination port 12 - 15 + new Uint8Array(new Uint16Array([0, frame.destinationPort || this.frameTarget]).buffer), + + // 128: sequenceNumber(32 bit, acnowledge number) 16 - 19 + new Uint8Array(new Uint32Array([frame.sequenceNumber || this.sequenceNumber++]).buffer), + + // 160: Acknowledgement number 20-23 + new Uint8Array(new Uint32Array([frame.acknowledgeNumber || 0]).buffer), + + // 192: data offset (4bit), reserved (4bit), 24 + new Uint8Array([0b00000000]), + + // 224: flags (8bit), 25 + this.setFlags(frame.flagsArray, new Uint8Array([0b00000000])), + // window(16bit) 26 - 27 + new Uint8Array(new Uint16Array([0]).buffer), + + // checksum(16bit) 28 - 29 + new Uint8Array(new Uint16Array([checksum]).buffer), + // pointer (16bit) 30 - 31 + new Uint8Array(new Uint16Array([checksum]).buffer), + + // 256: data 32 - + new Uint8Array(chunk) + ]); + + return buffer; + } + createChunkFrame(chunk: any) { const checksum = this.getChecksum(); const buffer = Buffer.concat([ @@ -95,7 +140,7 @@ export class FrameEncoder extends Transform { // 192: data offset (4bit), reserved (4bit), 24 new Uint8Array([0b00000000]), // 224: flags (8bit), 25 - this.setFlag("PSH"), + this.setFlags(["PSH"]), // window(16bit) 26 - 27 new Uint8Array(new Uint16Array([0]).buffer), @@ -112,7 +157,8 @@ export class FrameEncoder extends Transform { } _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void { - const buffer = this.createChunkFrame(chunk); + //const buffer = this.createChunkFrame(chunk); + const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); this.logger.trace("Encoded frame", getHexString(buffer), "Size: ", buffer.length, "Pushing"); diff --git a/packages/verser/src/lib/tecemux/codecs/index.ts b/packages/verser/src/lib/tecemux/codecs/index.ts index f2556a8ea..de478859d 100644 --- a/packages/verser/src/lib/tecemux/codecs/index.ts +++ b/packages/verser/src/lib/tecemux/codecs/index.ts @@ -19,3 +19,19 @@ export const binaryFlags = { ECE: 0b01000000, CWR: 0b10000000 } + +export type flagsObjectType = Partial<{ + FIN: boolean, + SYN: boolean, + RST: boolean, + PSH: boolean, + ACK: boolean, + URG: boolean, + ECE: boolean, + CWR: boolean +}> + +export const parseFlags = (byte: number): flagsObjectType => { + return frameFlags.filter((_flag, index) => byte >>> index & 1) + .reduce((p, c) => ({ ...p, [c]: true }), {}); +} diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/src/lib/tecemux/playground.ts index cc7429c8a..81ce13e25 100644 --- a/packages/verser/src/lib/tecemux/playground.ts +++ b/packages/verser/src/lib/tecemux/playground.ts @@ -5,7 +5,7 @@ import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; import { IncomingMessage, createServer, request } from "http"; import { DataStream } from "scramjet"; -import { DecodedFrame, FrameTarget } from "./utils"; +import { FrameData, FrameTarget } from "./utils"; import { FrameDecoder, FrameEncoder } from "./codecs"; import { Socket } from "net"; @@ -42,7 +42,7 @@ import { Socket } from "net"; socket.pipe(serverFrameDecoder).pipe(process.stdout); serverFrameDecoder.on("data", (d: any) => { - const parsed = JSON.parse(d) as DecodedFrame; + const parsed = JSON.parse(d) as FrameData; logger.debug(`Server on request data [${i++}]`, parsed.chunk, parsed.dataLength); serverFrameEncoder.write(Buffer.from(new Uint8Array([255, 255, 255, 255]))); @@ -79,7 +79,7 @@ import { Socket } from "net"; socket.pipe( responseFrameDecoder ).on("data", (d) => { - const parsed = JSON.parse(d) as DecodedFrame; + const parsed = JSON.parse(d) as FrameData; logger.debug(`Echo from server [${i++}]`, parsed.chunk, parsed.dataLength, parsed.chunkLength); }); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 45ff67d10..10a04a8dd 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -2,6 +2,8 @@ import { TypedEmitter } from "@scramjet/utility"; import { FrameDecoder, FrameEncoder, TeceMuxEvents } from "./codecs"; import { Duplex } from "stream"; import { Socket } from "net"; +import { ObjLogger } from "@scramjet/obj-logger"; +import { FrameData } from "./utils"; export type TeceMuxChannel = Duplex & { _id: number }; @@ -12,6 +14,9 @@ export class TeceMux extends TypedEmitter{ channels: Duplex[] = []; + logger = new ObjLogger(this); + commonEncoder = new FrameEncoder(0); + private createChannel(): TeceMuxChannel { const channel: TeceMuxChannel = Object.assign(new Duplex({ }), { _id: this.channelCount }); @@ -40,21 +45,36 @@ export class TeceMux extends TypedEmitter{ async main() { for await (const chunk of this.decoder) { - const frame = JSON.parse(chunk); - const channel = frame.destinationPort; - - if (!this.channels[channel]) { - const channel = this.createChannel(); + const frame = JSON.parse(chunk) as FrameData; + const { flags, sequenceNumber, dataLength, destinationPort } = frame; - this.addChannel(channel); + if (flags.ACK) { + this.logger.trace("ACKNOWLEDGE", sequenceNumber); + // acknowledge received (confirm packet) + return; } - this.channels[channel].write(frame.chunk.data); + if (flags.PSH) { + if (!this.channels[destinationPort]) { + const channel = this.createChannel(); + + this.addChannel(channel); + this.commonEncoder.createFrame(undefined, { + flagsArray: ["ACK"], + destinationPort, + acknowledgeNumber: sequenceNumber + }) + } + + if (dataLength) { + this.channels[destinationPort].write(frame.chunk); + } + } } } addChannel(channel: TeceMuxChannel) { - this.channels[this.channelCount] = channel; + this.channels[this.channelCount] = channel; // wait for SYN reply? this.channelCount++; this.emit("channel", channel); diff --git a/packages/verser/src/lib/tecemux/utils/index.ts b/packages/verser/src/lib/tecemux/utils/index.ts index 631dba5d0..08cad4374 100644 --- a/packages/verser/src/lib/tecemux/utils/index.ts +++ b/packages/verser/src/lib/tecemux/utils/index.ts @@ -1,3 +1,5 @@ +import { frameFlags, binaryFlags, flagsObjectType } from "../codecs"; + export function toHex(chunk: Buffer) { return chunk.toString("hex").match(/../g)?.join(" "); } @@ -9,15 +11,16 @@ export enum FrameTarget { INPUT = 1001 } -export type DecodedFrame = { +export type FrameData = { sourceAddress: [number, number, number, number]; destinationAddress: [number, number, number, number]; destinationPort: number; - chunk: { - type: string; - data: any; - }; + sequenceNumber: number; + acknowledgeNumber: number; + chunk: Buffer; dataLength: number; chunkLength: number; stringified: string; + flags: flagsObjectType; + flagsArray: (keyof typeof binaryFlags)[]; }; From 9c2baf82b680218c23ed7725fc5a15deb5249ae5 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Sun, 15 Jan 2023 22:38:50 +0000 Subject: [PATCH 007/231] push data to channel --- package.json | 2 +- .../src/lib/tecemux/codecs/frame-decoder.ts | 7 +- .../src/lib/tecemux/codecs/frame-encoder.ts | 60 ++++----- .../src/lib/tecemux/playground-tecemux.ts | 115 ++++++++++++++++++ packages/verser/src/lib/tecemux/tecemux.ts | 68 +++++++++-- yarn.lock | 13 +- 6 files changed, 204 insertions(+), 61 deletions(-) create mode 100644 packages/verser/src/lib/tecemux/playground-tecemux.ts diff --git a/package.json b/package.json index fab95ff1f..f12137d1e 100644 --- a/package.json +++ b/package.json @@ -77,7 +77,7 @@ "devDependencies": { "@istanbuljs/nyc-config-typescript": "^1.0.2", "@npmcli/run-script": "4.2.1", - "@types/node": "15.12.5", + "@types/node": "18.11.18", "@typescript-eslint/eslint-plugin": "^5.41.0", "@typescript-eslint/parser": "^5.41.0", "build-if-changed": "^1.5.5", diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 0657af2eb..2bc15bd23 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -1,7 +1,7 @@ import { ObjLogger } from "@scramjet/obj-logger"; import { Duplex, Transform, TransformCallback, TransformOptions } from "stream"; -import { HEADER_LENGTH, toHex } from "../utils"; -import { frameFlags, parseFlags } from "."; +import { FrameData, HEADER_LENGTH, toHex } from "../utils"; +import { parseFlags } from "."; export class FrameDecoder extends Transform { buff: Buffer; @@ -10,7 +10,6 @@ export class FrameDecoder extends Transform { _streams = new Map(); - constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { super(Object.assign({}, { readableObjectMode: true, emitClose: false }, opts)); @@ -49,7 +48,7 @@ export class FrameDecoder extends Transform { dataLength: frameSize - HEADER_LENGTH, chunkLength: frameSize, stringified: this.buff.subarray(32, frameSize).toString() - }; + } as Partial; this.push(JSON.stringify(payload) + "\n"); diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index e196c67b2..f32e9911c 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -2,6 +2,7 @@ import { Transform, TransformCallback, TransformOptions } from "stream"; import { FrameData, FrameTarget, HEADER_LENGTH, toHex as getHexString } from "../utils"; import { ObjLogger } from "@scramjet/obj-logger"; import { binaryFlags, frameFlags } from "."; + export class FrameEncoder extends Transform { sequenceNumber = 0; logger = new ObjLogger("FrameEncoder",); @@ -16,6 +17,14 @@ export class FrameEncoder extends Transform { this.on("pipe", () => { this.logger.debug("onPipe"); }); + + /*const orgPush = this.push; + this.push = (chunk: any, encoding: BufferEncoding | undefined) => { + this.logger.debug("Pushing", chunk) + //return orgPush.call(this, chunk, encoding); + return true; + } + */ } setFlags(flag: (keyof typeof binaryFlags)[] = [], flags: Uint8Array = new Uint8Array([0])) { @@ -27,45 +36,15 @@ export class FrameEncoder extends Transform { return flags; } - setChannel() { - const checksum = this.getChecksum(); - const buffer = Buffer.concat([ - // 0: source address 0 - 3 - new Uint8Array([10, 0, 0, 1]), - // 32: destination address 4 - 7 - new Uint8Array([10, 0, 0, 2]), - - // 64: zeroes (8bit), protocol (8bit), 8 - 9 - new Uint8Array([0, 1]), - // tcp length (16bit) 10 - 11 - new Uint8Array(new Uint16Array([0 + HEADER_LENGTH]).buffer), + setChannel(channelCount: number) { + this.logger.debug("Set channel command", channelCount); - // 96: Source port, destination port 12 - 15 - new Uint8Array(new Uint16Array([0, this.frameTarget]).buffer), - - // 128: sequenceNumber(32 bit, acnowledge number) 16 - 19 - new Uint8Array(new Uint32Array([this.sequenceNumber++]).buffer), - - // 160: Acknowledgement number 20-23 - new Uint8Array([0, 0, 0, 0]), - - // 192: data offset (4bit), reserved (4bit), 24 - new Uint8Array([0b00000000]), - // 224: flags (8bit), 25 - this.setFlags(["ECE"]), - // window(16bit) 26 - 27 - new Uint8Array(new Uint16Array([0]).buffer), - - // checksum(16bit) 28 - 29 - new Uint8Array(new Uint16Array([checksum]).buffer), - // pointer (16bit) 30 - 31 - new Uint8Array(new Uint16Array([checksum]).buffer), - - // 256: data 32 - - new Uint8Array([]) - ]); + //const checksum = this.getChecksum(); - this.push(buffer, undefined); + this.push(this.createFrame([], { + flagsArray: ["PSH"], + destinationPort: channelCount + })); } getChecksum() { @@ -158,11 +137,14 @@ export class FrameEncoder extends Transform { _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void { //const buffer = this.createChunkFrame(chunk); + this.logger.debug("TRANSFORM IN", chunk, chunk.length); const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); this.logger.trace("Encoded frame", getHexString(buffer), "Size: ", buffer.length, "Pushing"); - this.push(buffer, undefined); - callback(null); + //this.push(buffer, undefined); + this.logger.debug("TRANSFORM OUT", getHexString(buffer), buffer.length); + this.push(buffer); + callback(); } } diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts new file mode 100644 index 000000000..891766b4f --- /dev/null +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -0,0 +1,115 @@ +/*eslint no-unused-vars: ["error", { "args": "none" }]*/ +/* eslint-disable @typescript-eslint/no-floating-promises */ +/* eslint-disable no-console */ + +import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; +import { IncomingMessage, createServer, request } from "http"; +import { DataStream } from "scramjet"; +import { FrameData } from "./utils"; + +import { Socket } from "net"; +import { TeceMux, TeceMuxChannel } from "./tecemux"; + +(async () => { + const logger = new ObjLogger("Sandbox"); + + logger.pipe(new DataStream().map(prettyPrint({ colors: true }))).pipe(process.stdout); + + const PORT = 6660; + const server = createServer(); + + server.setTimeout(0); + server.requestTimeout = 0; + + server.on("connect", async (req: IncomingMessage, socket: Socket) => { + //socket.setNoDelay(true); + //socket.write(`HTTP/1.1 ${200} \r\n\r\n`); + socket.write('HTTP/1.1 200 Connection Established\r\n' + + 'Proxy-agent: Node.js-Proxy\r\n' + + '\r\n'); + logger.debug("on connect"); + socket.write("dddd\r\n"); + //socket.pipe(process.stdout) + + const tcmux = new TeceMux(socket, "Server Side"); + tcmux.on("error", (err) => { + logger.error("TCMUX err", err); + }); + + tcmux.logger.pipe(logger); + + const channel = tcmux.multiplex(); + + let i = 0; + + req.on("pause", () => { logger.warn("Request paused"); }); + + setInterval(() => { + logger.warn("writing to server response") + channel.write("som\n"); + }, 250) + + for await (const chunk of channel) { + const parsed = JSON.parse(chunk) as FrameData; + logger.debug(`Server on request data [${i++}]`, parsed.chunk, parsed.dataLength); + + channel.write(new Uint8Array([1,2,3,4])); + //channel.write(Buffer.from(new Uint8Array([255, 255, 255, 255]))); + }; + }); + + server.listen(PORT); + + const req = request({ + hostname: "0.0.0.0", + method: "connect", + port: PORT, + headers: { "Content-Type": "application/octet-stream", "Transfer-Encoding": "chunked" } + }); + + req.on("connect", (response, socket, head) => { + const reqLogger = new ObjLogger("Req", { id: "Request"}); + reqLogger.pipe(logger); + + + //socket.setNoDelay(true); + + reqLogger.debug("Response. Head:", head.toString()); + + let i = 0; + let m = 0; + response.on("data", (d) => { + console.log("req data in", d); + }).pause(); + + + const tcmux = new TeceMux(socket); + + socket.resume(); + tcmux.logger.updateBaseLog({ id: "Client side"}); + + tcmux.logger.pipe(logger); + + tcmux.on("channel", (channel: TeceMuxChannel) => { + reqLogger.debug("New channel", channel._id); + channel.pipe(process.stdout); + + setInterval(async () => { + reqLogger.debug(`Writing [${++m}]..`); + + channel.write(Buffer.from(new Uint8Array([0x61, 0x62, 0x63, 0x64]))); + }, 500); + }); + /*setInterval(async () => { + reqLogger.debug(`Writing [${++m}]..`); + tcmux.multiplex().write(Buffer.from(new Uint8Array([0x61, 0x62, 0x63, 0x64]))); + }, 250);*/ + req.on("error", (err) => { + console.error(err); + }); + }); + + req.flushHeaders(); + + await new Promise((_res, _rej) => {}); +})(); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 10a04a8dd..e3185c524 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -1,50 +1,87 @@ import { TypedEmitter } from "@scramjet/utility"; import { FrameDecoder, FrameEncoder, TeceMuxEvents } from "./codecs"; -import { Duplex } from "stream"; +import { Duplex, PassThrough, pipeline } from "stream"; import { Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; import { FrameData } from "./utils"; +import { IObjectLogger } from "@scramjet/types"; export type TeceMuxChannel = Duplex & { _id: number }; export class TeceMux extends TypedEmitter{ + id: string; carrierSocket: Duplex; channelCount = 0; decoder = new FrameDecoder(); channels: Duplex[] = []; - logger = new ObjLogger(this); + logger: IObjectLogger; commonEncoder = new FrameEncoder(0); private createChannel(): TeceMuxChannel { - const channel: TeceMuxChannel = Object.assign(new Duplex({ }), { _id: this.channelCount }); + this.logger.debug("Create Channel", this.channelCount); const encoder = new FrameEncoder(this.channelCount); + encoder.logger.updateBaseLog({ id: this.id }); + encoder.logger.pipe(this.logger); + + const w = new PassThrough().on("data", (d) => { this.logger.warn("writeable DATA", d); }).pause(); + //w.pipe(process.stdout).pause() + + w.pipe(encoder); + + const channel: TeceMuxChannel = Object.assign( + Duplex.from({ + readable: new PassThrough().on("data", (d) => { this.logger.warn("readable DATA", d); }).pause(), + writable: w, + } as unknown as Iterable) as TeceMuxChannel, + { _id: this.channelCount } + ); + + + //channel.pipe(encoder).pipe(this.carrierSocket); + //channel.pipe(process.stdout); - channel.pipe(encoder).pipe(this.carrierSocket); channel.on("error", (error) => { this.emit("error", { error, source: channel }) }); - encoder.setChannel(); + encoder.setChannel(this.channelCount); + //channel.pipe(w); + //channel.pipe(this.carrierSocket); + encoder.pipe(this.carrierSocket); return channel; } - constructor(socket: Socket) { + constructor(socket: Socket, id = "") { super(); + this.id = id; + this.logger = new ObjLogger(this, { id: this.id }) this.carrierSocket = socket; - this.main().catch((error) => { - this.emit("error", error); - }); + this.decoder.logger.pipe(this.logger); - socket.pipe(this.decoder); + this.carrierSocket.pipe(this.decoder); + this.commonEncoder.pipe(this.carrierSocket); + + this.carrierSocket.on("data", (d) => { console.warn("carrier socket ", d); }); + this.commonEncoder.on("data", (d) => { this.logger.warn("to socket", d); }); + + this.main();//.catch((error) => { + //this.emit("error", error); + //}); + //this.carrierSocket.pipe(process.stdout); } async main() { + this.commonEncoder.logger.updateBaseLog({ id: "Commn" + this.commonEncoder.logger.baseLog.id }) + this.commonEncoder.logger.pipe(this.logger); + for await (const chunk of this.decoder) { + this.logger.debug("Decoded", JSON.parse(chunk)); + const frame = JSON.parse(chunk) as FrameData; const { flags, sequenceNumber, dataLength, destinationPort } = frame; @@ -67,20 +104,25 @@ export class TeceMux extends TypedEmitter{ } if (dataLength) { - this.channels[destinationPort].write(frame.chunk); + this.channels[destinationPort].write(new Uint8Array((frame.chunk) as any)); } } } } addChannel(channel: TeceMuxChannel) { + this.logger.debug("adding channel", channel._id); this.channels[this.channelCount] = channel; // wait for SYN reply? this.channelCount++; - - this.emit("channel", channel); + this.commonEncoder.createFrame(undefined, { + flagsArray: ["PSH"], + destinationPort: channel._id + }) + //this.emit("channel", channel); } multiplex(): TeceMuxChannel { + this.logger.trace("Multiplex") const channel = this.createChannel(); this.addChannel(channel); diff --git a/yarn.lock b/yarn.lock index 6c8a47843..c96e1bb09 100644 --- a/yarn.lock +++ b/yarn.lock @@ -959,16 +959,21 @@ "@types/node" "*" form-data "^3.0.0" -"@types/node@*", "@types/node@>=13.7.0", "@types/node@^18.11.18": - version "18.14.2" - resolved "https://registry.yarnpkg.com/@types/node/-/node-18.14.2.tgz#c076ed1d7b6095078ad3cf21dfeea951842778b1" - integrity sha512-1uEQxww3DaghA0RxqHx0O0ppVlo43pJhepY51OxuQIKHpjbnYLA7vcdwioNPzIqmC2u3I/dmylcqjlh0e7AyUA== +"@types/node@*", "@types/node@18.11.18", "@types/node@^18.11.18": + version "18.11.18" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f" + integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA== "@types/node@15.12.5": version "15.12.5" resolved "https://registry.yarnpkg.com/@types/node/-/node-15.12.5.tgz#9a78318a45d75c9523d2396131bd3cca54b2d185" integrity sha512-se3yX7UHv5Bscf8f1ERKvQOD6sTyycH3hdaoozvaLxgUiY5lIGEeH37AD0G0Qi9kPqihPn0HOfd2yaIEN9VwEg== +"@types/node@>=13.7.0": + version "18.11.3" + resolved "https://registry.npmjs.org/@types/node/-/node-18.11.3.tgz" + integrity sha512-fNjDQzzOsZeKZu5NATgXUPsaFaTxeRgFXoosrHivTl8RGeV733OLawXsGfEk9a8/tySyZUyiZ6E8LcjPFZ2y1A== + "@types/normalize-package-data@^2.4.0": version "2.4.1" resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301" From ef4060ed0b49d3d74eb085b21227b835231ce3e1 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 16 Jan 2023 17:08:47 +0000 Subject: [PATCH 008/231] Duplex communication --- .../src/lib/tecemux/codecs/frame-decoder.ts | 3 +- .../src/lib/tecemux/codecs/frame-encoder.ts | 28 ++-- .../src/lib/tecemux/playground-tecemux.ts | 120 +++++++++--------- packages/verser/src/lib/tecemux/tecemux.ts | 71 ++++++----- 4 files changed, 109 insertions(+), 113 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 2bc15bd23..9c60fac1c 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -35,7 +35,7 @@ export class FrameDecoder extends Transform { this.size += chunk.length; - if (this.size >= 10 && this.buff.readInt32LE(10) === this.size) { + if (this.size >= 10 ) {//&& this.buff.readInt32LE(10) === this.size) { const frameSize = this.buff.readInt32LE(10); const payload = { @@ -47,6 +47,7 @@ export class FrameDecoder extends Transform { destinationPort: this.buff.readInt16LE(14), dataLength: frameSize - HEADER_LENGTH, chunkLength: frameSize, + sequenceNumber: this.buff.readInt32LE(16), stringified: this.buff.subarray(32, frameSize).toString() } as Partial; diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index f32e9911c..5aec8f657 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -1,11 +1,14 @@ -import { Transform, TransformCallback, TransformOptions } from "stream"; -import { FrameData, FrameTarget, HEADER_LENGTH, toHex as getHexString } from "../utils"; +import { PassThrough, Transform, TransformCallback, TransformOptions } from "stream"; +import { FrameData, FrameTarget, HEADER_LENGTH, toHex as getHexString, toHex } from "../utils"; import { ObjLogger } from "@scramjet/obj-logger"; import { binaryFlags, frameFlags } from "."; export class FrameEncoder extends Transform { sequenceNumber = 0; logger = new ObjLogger("FrameEncoder",); + out = new PassThrough({ readableObjectMode: true }).on("data", (data) => { + this.logger.trace("outcome", toHex(data), data.length); + }); constructor(private frameTarget: FrameTarget, opts: TransformOptions = {}, params: { name: string } = { name: "FrameEncoder" }) { opts.emitClose = false; @@ -18,21 +21,16 @@ export class FrameEncoder extends Transform { this.logger.debug("onPipe"); }); - /*const orgPush = this.push; - this.push = (chunk: any, encoding: BufferEncoding | undefined) => { - this.logger.debug("Pushing", chunk) - //return orgPush.call(this, chunk, encoding); - return true; - } - */ + this.pipe(this.out); } setFlags(flag: (keyof typeof binaryFlags)[] = [], flags: Uint8Array = new Uint8Array([0])) { for (const f in flag) { - this.logger.debug("settingFlag", flag); flags[0] |= 1 << frameFlags.indexOf(flag[f]); } + this.logger.debug("settingFlag", flag, flags[0].toString(2).padStart(8, "0")); + return flags; } @@ -41,7 +39,7 @@ export class FrameEncoder extends Transform { //const checksum = this.getChecksum(); - this.push(this.createFrame([], { + this.out.write(this.createFrame([], { flagsArray: ["PSH"], destinationPort: channelCount })); @@ -137,14 +135,12 @@ export class FrameEncoder extends Transform { _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void { //const buffer = this.createChunkFrame(chunk); - this.logger.debug("TRANSFORM IN", chunk, chunk.length); + this.logger.debug("TRANSFORM IN", toHex(chunk), chunk.length); const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); - this.logger.trace("Encoded frame", getHexString(buffer), "Size: ", buffer.length, "Pushing"); - //this.push(buffer, undefined); - this.logger.debug("TRANSFORM OUT", getHexString(buffer), buffer.length); - this.push(buffer); + this.logger.debug("TRANSFORM OUT", getHexString(buffer), "Size: ", buffer.length); + this.push(buffer, undefined); callback(); } } diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index 891766b4f..8fcf7f363 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -3,18 +3,22 @@ /* eslint-disable no-console */ import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; -import { IncomingMessage, createServer, request } from "http"; +import { IncomingMessage, createServer } from "http"; import { DataStream } from "scramjet"; -import { FrameData } from "./utils"; -import { Socket } from "net"; +import { Socket, createConnection } from "net"; import { TeceMux, TeceMuxChannel } from "./tecemux"; +import { FrameData } from "./utils"; (async () => { const logger = new ObjLogger("Sandbox"); logger.pipe(new DataStream().map(prettyPrint({ colors: true }))).pipe(process.stdout); + /**********************************************/ + /* SERVER + /**********************************************/ + const PORT = 6660; const server = createServer(); @@ -22,94 +26,88 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; server.requestTimeout = 0; server.on("connect", async (req: IncomingMessage, socket: Socket) => { - //socket.setNoDelay(true); - //socket.write(`HTTP/1.1 ${200} \r\n\r\n`); - socket.write('HTTP/1.1 200 Connection Established\r\n' + - 'Proxy-agent: Node.js-Proxy\r\n' + - '\r\n'); - logger.debug("on connect"); - socket.write("dddd\r\n"); - //socket.pipe(process.stdout) - - const tcmux = new TeceMux(socket, "Server Side"); - tcmux.on("error", (err) => { - logger.error("TCMUX err", err); - }); + socket.setNoDelay(true); + + logger.info("Incoming request", req.method, req.headers); + + const tcmux = new TeceMux(socket, "Server Side") + .on("error", (err) => { + logger.error("TCMUX err", err); + }); tcmux.logger.pipe(logger); const channel = tcmux.multiplex(); - let i = 0; - req.on("pause", () => { logger.warn("Request paused"); }); - setInterval(() => { - logger.warn("writing to server response") - channel.write("som\n"); - }, 250) + logger.warn("writing to server response") + channel.write("som\n"); for await (const chunk of channel) { const parsed = JSON.parse(chunk) as FrameData; - logger.debug(`Server on request data [${i++}]`, parsed.chunk, parsed.dataLength); + logger.debug(`Server on request data [C: ${parsed.sequenceNumber}, SN: ${parsed.sequenceNumber}]`, parsed.chunk, parsed.dataLength); - channel.write(new Uint8Array([1,2,3,4])); + //channel.write(new Uint8Array([1,2,3,4])); //channel.write(Buffer.from(new Uint8Array([255, 255, 255, 255]))); }; + /* + channel.on("data", (d) => { + logger.error("Server on request data", d); + }); + */ + + //channel.pipe(process.stdout); }); - server.listen(PORT); + server.listen(PORT, "0.0.0.0"); - const req = request({ - hostname: "0.0.0.0", - method: "connect", - port: PORT, - headers: { "Content-Type": "application/octet-stream", "Transfer-Encoding": "chunked" } - }); + /**********************************************/ + /* CLIENT + /**********************************************/ - req.on("connect", (response, socket, head) => { - const reqLogger = new ObjLogger("Req", { id: "Request"}); - reqLogger.pipe(logger); + const socket = createConnection({ port: PORT, allowHalfOpen: true, host: "0.0.0.0" }, () => {}); + socket.setNoDelay(true); - //socket.setNoDelay(true); + const reqLogger = new ObjLogger("Req", { id: "Request"}); + reqLogger.pipe(logger); - reqLogger.debug("Response. Head:", head.toString()); - let i = 0; - let m = 0; - response.on("data", (d) => { - console.log("req data in", d); - }).pause(); + socket.write("CONNECT HTTP/1.1\r\n\r\n\r\n"); + socket.on("error", (error) => { + reqLogger.error("ERROR", error); + }); - const tcmux = new TeceMux(socket); + reqLogger.info('connected to server!'); - socket.resume(); - tcmux.logger.updateBaseLog({ id: "Client side"}); + const tcmux = new TeceMux(socket); - tcmux.logger.pipe(logger); + tcmux.logger.updateBaseLog({ id: "Client side"}); - tcmux.on("channel", (channel: TeceMuxChannel) => { - reqLogger.debug("New channel", channel._id); - channel.pipe(process.stdout); + tcmux.logger.pipe(logger); - setInterval(async () => { - reqLogger.debug(`Writing [${++m}]..`); + tcmux.on("channel", async (channel: TeceMuxChannel) => { + reqLogger.debug("New channel", channel._id); - channel.write(Buffer.from(new Uint8Array([0x61, 0x62, 0x63, 0x64]))); - }, 500); - }); - /*setInterval(async () => { - reqLogger.debug(`Writing [${++m}]..`); - tcmux.multiplex().write(Buffer.from(new Uint8Array([0x61, 0x62, 0x63, 0x64]))); - }, 250);*/ - req.on("error", (err) => { - console.error(err); - }); + //channel.write(Buffer.from(new Uint8Array([0x61, 0x62, 0x63, 0x64]))); + channel.pipe(process.stdout); + + for await (const chunk of channel) { + reqLogger.info("request on response data", chunk); + } }); + /*setInterval(async () => { + reqLogger.debug(`Writing [${++m}]..`); + tcmux.multiplex().write(Buffer.from(new Uint8Array([0x61, 0x62, 0x63, 0x64]))); + }, 250);*/ + socket.on("error", (err) => { + console.error(err); + }); + - req.flushHeaders(); + //socket.flushHeaders(); await new Promise((_res, _rej) => {}); })(); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index e3185c524..9a7913dc1 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -1,6 +1,6 @@ import { TypedEmitter } from "@scramjet/utility"; import { FrameDecoder, FrameEncoder, TeceMuxEvents } from "./codecs"; -import { Duplex, PassThrough, pipeline } from "stream"; +import { Duplex, PassThrough } from "stream"; import { Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; import { FrameData } from "./utils"; @@ -11,16 +11,16 @@ export type TeceMuxChannel = Duplex & { _id: number }; export class TeceMux extends TypedEmitter{ id: string; carrierSocket: Duplex; - channelCount = 0; + channelCount = 1; decoder = new FrameDecoder(); - channels: Duplex[] = []; + channels: TeceMuxChannel[] = []; logger: IObjectLogger; commonEncoder = new FrameEncoder(0); - private createChannel(): TeceMuxChannel { - this.logger.debug("Create Channel", this.channelCount); + private createChannel(destinationPort?: number): TeceMuxChannel { + this.logger.debug("Create Channel", destinationPort || this.channelCount); const encoder = new FrameEncoder(this.channelCount); encoder.logger.updateBaseLog({ id: this.id }); @@ -33,13 +33,12 @@ export class TeceMux extends TypedEmitter{ const channel: TeceMuxChannel = Object.assign( Duplex.from({ - readable: new PassThrough().on("data", (d) => { this.logger.warn("readable DATA", d); }).pause(), + readable: new PassThrough().on("data", (d) => { this.logger.warn("readable DATA", d); }), writable: w, } as unknown as Iterable) as TeceMuxChannel, - { _id: this.channelCount } + { _id: destinationPort || this.channelCount } ); - //channel.pipe(encoder).pipe(this.carrierSocket); //channel.pipe(process.stdout); @@ -47,10 +46,10 @@ export class TeceMux extends TypedEmitter{ this.emit("error", { error, source: channel }) }); - encoder.setChannel(this.channelCount); + encoder.setChannel(destinationPort || this.channelCount); //channel.pipe(w); //channel.pipe(this.carrierSocket); - encoder.pipe(this.carrierSocket); + encoder.out.pipe(this.carrierSocket); return channel; } @@ -64,15 +63,12 @@ export class TeceMux extends TypedEmitter{ this.decoder.logger.pipe(this.logger); this.carrierSocket.pipe(this.decoder); - this.commonEncoder.pipe(this.carrierSocket); + this.commonEncoder.out.pipe(this.carrierSocket); - this.carrierSocket.on("data", (d) => { console.warn("carrier socket ", d); }); - this.commonEncoder.on("data", (d) => { this.logger.warn("to socket", d); }); - this.main();//.catch((error) => { - //this.emit("error", error); - //}); - //this.carrierSocket.pipe(process.stdout); + this.main().catch((error) => { + this.emit("error", error); + }); } async main() { @@ -92,41 +88,46 @@ export class TeceMux extends TypedEmitter{ } if (flags.PSH) { - if (!this.channels[destinationPort]) { - const channel = this.createChannel(); + this.logger.trace(`Received PSH command [C: ${destinationPort}]`, dataLength, frame.chunk, !!this.channels[destinationPort]); + let channel = this.channels[destinationPort] + + if (!channel) { + this.logger.warn("NEW CHANNEL"); + channel = this.createChannel(destinationPort); this.addChannel(channel); - this.commonEncoder.createFrame(undefined, { - flagsArray: ["ACK"], - destinationPort, - acknowledgeNumber: sequenceNumber - }) } if (dataLength) { - this.channels[destinationPort].write(new Uint8Array((frame.chunk) as any)); + this.logger.warn("writing DATA LENGHT REC", dataLength); + channel.push(new Uint8Array(((frame.chunk as any).data) as any)); } } + + this.commonEncoder.out.write( + this.commonEncoder.createFrame(undefined, { + flagsArray: ["ACK"], + sequenceNumber, + destinationPort + }) + ); } } addChannel(channel: TeceMuxChannel) { - this.logger.debug("adding channel", channel._id); - this.channels[this.channelCount] = channel; // wait for SYN reply? - this.channelCount++; - this.commonEncoder.createFrame(undefined, { - flagsArray: ["PSH"], - destinationPort: channel._id - }) - //this.emit("channel", channel); + this.logger.debug("adding channel", ); + this.channels[channel._id] = channel; // wait for SYN reply? + + this.emit("channel", channel); } multiplex(): TeceMuxChannel { - this.logger.trace("Multiplex") - const channel = this.createChannel(); + const channel = this.createChannel(this.channelCount); + this.logger.trace("Multiplex", channel._id); this.addChannel(channel); + this.channelCount++; return channel; } } From 0bb7f96dbff8fba9121f0c07791670a02046a397 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 16 Jan 2023 18:12:36 +0000 Subject: [PATCH 009/231] Send stdin over frames --- .../src/lib/tecemux/playground-tecemux.ts | 33 +++++-------------- packages/verser/src/lib/tecemux/tecemux.ts | 7 ++-- 2 files changed, 14 insertions(+), 26 deletions(-) diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index 8fcf7f363..fc3635f8e 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -42,22 +42,16 @@ import { FrameData } from "./utils"; req.on("pause", () => { logger.warn("Request paused"); }); logger.warn("writing to server response") - channel.write("som\n"); + //channel.write("som\n"); + + + process.stdout.pipe(channel); + for await (const chunk of channel) { const parsed = JSON.parse(chunk) as FrameData; logger.debug(`Server on request data [C: ${parsed.sequenceNumber}, SN: ${parsed.sequenceNumber}]`, parsed.chunk, parsed.dataLength); - - //channel.write(new Uint8Array([1,2,3,4])); - //channel.write(Buffer.from(new Uint8Array([255, 255, 255, 255]))); }; - /* - channel.on("data", (d) => { - logger.error("Server on request data", d); - }); - */ - - //channel.pipe(process.stdout); }); server.listen(PORT, "0.0.0.0"); @@ -70,7 +64,7 @@ import { FrameData } from "./utils"; socket.setNoDelay(true); - const reqLogger = new ObjLogger("Req", { id: "Request"}); + const reqLogger = new ObjLogger("Req", { id: "Client Side"}); reqLogger.pipe(logger); @@ -82,7 +76,7 @@ import { FrameData } from "./utils"; reqLogger.info('connected to server!'); - const tcmux = new TeceMux(socket); + const tcmux = new TeceMux(socket, "Request"); tcmux.logger.updateBaseLog({ id: "Client side"}); @@ -91,23 +85,14 @@ import { FrameData } from "./utils"; tcmux.on("channel", async (channel: TeceMuxChannel) => { reqLogger.debug("New channel", channel._id); - //channel.write(Buffer.from(new Uint8Array([0x61, 0x62, 0x63, 0x64]))); - channel.pipe(process.stdout); - for await (const chunk of channel) { - reqLogger.info("request on response data", chunk); + reqLogger.info("Data from server", chunk.toString()); } }); - /*setInterval(async () => { - reqLogger.debug(`Writing [${++m}]..`); - tcmux.multiplex().write(Buffer.from(new Uint8Array([0x61, 0x62, 0x63, 0x64]))); - }, 250);*/ + socket.on("error", (err) => { console.error(err); }); - - //socket.flushHeaders(); - await new Promise((_res, _rej) => {}); })(); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 9a7913dc1..4e15bb36e 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -60,6 +60,7 @@ export class TeceMux extends TypedEmitter{ this.logger = new ObjLogger(this, { id: this.id }) this.carrierSocket = socket; + this.decoder.logger.updateBaseLog({ id: this.id }); this.decoder.logger.pipe(this.logger); this.carrierSocket.pipe(this.decoder); @@ -92,7 +93,7 @@ export class TeceMux extends TypedEmitter{ let channel = this.channels[destinationPort] if (!channel) { - this.logger.warn("NEW CHANNEL"); + this.logger.warn("Unknown channel"); channel = this.createChannel(destinationPort); this.addChannel(channel); @@ -122,11 +123,13 @@ export class TeceMux extends TypedEmitter{ } multiplex(): TeceMuxChannel { + this.logger.trace("Multiplex"); + const channel = this.createChannel(this.channelCount); - this.logger.trace("Multiplex", channel._id); this.addChannel(channel); + this.logger.trace("Multiplex ready", channel._id); this.channelCount++; return channel; } From ea3cd119b7cfe1221260ab7c73b77787ea1e0aee Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 16 Jan 2023 22:31:47 +0000 Subject: [PATCH 010/231] Issue, stream paused --- .../src/lib/tecemux/codecs/frame-decoder.ts | 2 +- .../src/lib/tecemux/codecs/frame-encoder.ts | 16 ++++-- .../src/lib/tecemux/playground-tecemux.ts | 54 +++++++++++++++---- packages/verser/src/lib/tecemux/tecemux.ts | 49 ++++++++++------- 4 files changed, 87 insertions(+), 34 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 9c60fac1c..f6a62efe4 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -11,7 +11,7 @@ export class FrameDecoder extends Transform { _streams = new Map(); constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { - super(Object.assign({}, { readableObjectMode: true, emitClose: false }, opts)); + super(Object.assign({}, { writableObjectMode: true, readableObjectMode: true, emitClose: false }, opts)); this.buff = Buffer.alloc(64 * 1024, 0, undefined); //@TODO: optimize this.logger = new ObjLogger(params.name); diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index 5aec8f657..9d3acf6ee 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -6,9 +6,19 @@ import { binaryFlags, frameFlags } from "."; export class FrameEncoder extends Transform { sequenceNumber = 0; logger = new ObjLogger("FrameEncoder",); - out = new PassThrough({ readableObjectMode: true }).on("data", (data) => { - this.logger.trace("outcome", toHex(data), data.length); - }); + out = new PassThrough({ readableObjectMode: true }) + .on("data", (data) => { + this.logger.trace("output to socket:", toHex(data), data.length, this.readableFlowing); + }) + .on("pause", () => { + this.logger.trace("output to socket paused"); + }) + .on("end", () => { + this.logger.trace("output to socket ended"); + }) + .on("resume", () => { + this.logger.trace("output to socket resumed"); + }) constructor(private frameTarget: FrameTarget, opts: TransformOptions = {}, params: { name: string } = { name: "FrameEncoder" }) { opts.emitClose = false; diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index fc3635f8e..f08adab9d 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -30,23 +30,38 @@ import { FrameData } from "./utils"; logger.info("Incoming request", req.method, req.headers); - const tcmux = new TeceMux(socket, "Server Side") + socket + .on("data", (data) => { + console.log("SERVER socket ondata", data); + console.log("SOCKET TX RX", socket.bytesWritten, socket.bytesRead) + }) + .on("pause", () => { + logger.info("Socket paused"); + }) + .on("resume", () => { + logger.info("Socket resumed"); + }) + + const tcmux = new TeceMux(socket, "Server") .on("error", (err) => { logger.error("TCMUX err", err); }); + + tcmux.logger.pipe(logger); const channel = tcmux.multiplex(); req.on("pause", () => { logger.warn("Request paused"); }); - logger.warn("writing to server response") - //channel.write("som\n"); - + logger.warn("Waiting for stdin..."); - process.stdout.pipe(channel); + process.stdin.pipe(channel); + const somePayload = "smth\n"; + logger.info("writing some payload to channel", somePayload); + channel.write(somePayload); for await (const chunk of channel) { const parsed = JSON.parse(chunk) as FrameData; @@ -64,10 +79,9 @@ import { FrameData } from "./utils"; socket.setNoDelay(true); - const reqLogger = new ObjLogger("Req", { id: "Client Side"}); + const reqLogger = new ObjLogger("Req", { id: "Client"}); reqLogger.pipe(logger); - socket.write("CONNECT HTTP/1.1\r\n\r\n\r\n"); socket.on("error", (error) => { @@ -78,16 +92,34 @@ import { FrameData } from "./utils"; const tcmux = new TeceMux(socket, "Request"); - tcmux.logger.updateBaseLog({ id: "Client side"}); + tcmux.logger.updateBaseLog({ id: reqLogger.baseLog.id }); tcmux.logger.pipe(logger); tcmux.on("channel", async (channel: TeceMuxChannel) => { reqLogger.debug("New channel", channel._id); - for await (const chunk of channel) { - reqLogger.info("Data from server", chunk.toString()); - } + // for await (const chunk of channel) { + // reqLogger.info("Data from server", chunk.toString()); + + // await new Promise((resolve, reject) => { + // setTimeout(() => { + // //channel.encoder.write("Echo\n"); + // resolve(); + // }, 2000); + // }); + // } + + channel.on("data", async (chunk) => { + reqLogger.info("SERVER->CLIENT->CHANNEL", chunk.toString()); + + await new Promise((resolve, reject) => { + setTimeout(() => { + //channel.encoder.write("Echo\n"); + resolve(); + }, 2000); + }); + }) }); socket.on("error", (err) => { diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 4e15bb36e..92d024a49 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -6,37 +6,41 @@ import { ObjLogger } from "@scramjet/obj-logger"; import { FrameData } from "./utils"; import { IObjectLogger } from "@scramjet/types"; -export type TeceMuxChannel = Duplex & { _id: number }; +export type TeceMuxChannel = Duplex & { _id: number, encoder: FrameEncoder }; export class TeceMux extends TypedEmitter{ id: string; carrierSocket: Duplex; channelCount = 1; - decoder = new FrameDecoder(); + decoder = new FrameDecoder().resume(); channels: TeceMuxChannel[] = []; logger: IObjectLogger; - commonEncoder = new FrameEncoder(0); + commonEncoder = new FrameEncoder(0).resume(); - private createChannel(destinationPort?: number): TeceMuxChannel { + private createChannel(destinationPort?: number, emit?: boolean): TeceMuxChannel { this.logger.debug("Create Channel", destinationPort || this.channelCount); - const encoder = new FrameEncoder(this.channelCount); + const encoder = new FrameEncoder(this.channelCount, { encoding: undefined }); + encoder.logger.updateBaseLog({ id: this.id }); encoder.logger.pipe(this.logger); - const w = new PassThrough().on("data", (d) => { this.logger.warn("writeable DATA", d); }).pause(); + const w = new PassThrough({ encoding: undefined }).on("data", (d) => { this.logger.warn("channel writeable on DATA", d); }); //w.pipe(process.stdout).pause() w.pipe(encoder); const channel: TeceMuxChannel = Object.assign( Duplex.from({ - readable: new PassThrough().on("data", (d) => { this.logger.warn("readable DATA", d); }), + readable: new PassThrough({ encoding: undefined }).on("data", (d) => { this.logger.warn("channel readable on DATA", d); }), writable: w, } as unknown as Iterable) as TeceMuxChannel, - { _id: destinationPort || this.channelCount } + { + _id: destinationPort || this.channelCount, + encoder + } ); //channel.pipe(encoder).pipe(this.carrierSocket); @@ -46,7 +50,10 @@ export class TeceMux extends TypedEmitter{ this.emit("error", { error, source: channel }) }); - encoder.setChannel(destinationPort || this.channelCount); + if (emit) { + encoder.setChannel(destinationPort || this.channelCount); + } + //channel.pipe(w); //channel.pipe(this.carrierSocket); encoder.out.pipe(this.carrierSocket); @@ -66,7 +73,6 @@ export class TeceMux extends TypedEmitter{ this.carrierSocket.pipe(this.decoder); this.commonEncoder.out.pipe(this.carrierSocket); - this.main().catch((error) => { this.emit("error", error); }); @@ -83,7 +89,7 @@ export class TeceMux extends TypedEmitter{ const { flags, sequenceNumber, dataLength, destinationPort } = frame; if (flags.ACK) { - this.logger.trace("ACKNOWLEDGE", sequenceNumber); + this.logger.trace("ACKNOWLEDGE frame received for sequenceNumber", sequenceNumber); // acknowledge received (confirm packet) return; } @@ -94,17 +100,18 @@ export class TeceMux extends TypedEmitter{ if (!channel) { this.logger.warn("Unknown channel"); - channel = this.createChannel(destinationPort); + channel = this.createChannel(destinationPort, false); - this.addChannel(channel); + this.addChannel(channel, true); } if (dataLength) { - this.logger.warn("writing DATA LENGHT REC", dataLength); + this.logger.warn("writing to channel [channel, length]", channel._id, dataLength); channel.push(new Uint8Array(((frame.chunk as any).data) as any)); } } + this.logger.debug("Write acknowledge frame for sequenceNumber", sequenceNumber); this.commonEncoder.out.write( this.commonEncoder.createFrame(undefined, { flagsArray: ["ACK"], @@ -115,19 +122,23 @@ export class TeceMux extends TypedEmitter{ } } - addChannel(channel: TeceMuxChannel) { - this.logger.debug("adding channel", ); + addChannel(channel: TeceMuxChannel, emit: boolean) { + this.logger.debug("adding channel", channel._id); this.channels[channel._id] = channel; // wait for SYN reply? - this.emit("channel", channel); + if (emit) { + this.emit("channel", channel); + this.logger.debug("channel event emitted", channel._id); + this.channelCount++; + } } multiplex(): TeceMuxChannel { this.logger.trace("Multiplex"); - const channel = this.createChannel(this.channelCount); + const channel = this.createChannel(this.channelCount, true); - this.addChannel(channel); + this.addChannel(channel, false); this.logger.trace("Multiplex ready", channel._id); this.channelCount++; From 258d9db4f188dc939bdee8a9362d4f3e4a1f017d Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 17 Jan 2023 21:37:41 +0000 Subject: [PATCH 011/231] Debug AbortError --- .../src/lib/tecemux/codecs/frame-decoder.ts | 70 +++++++++++-------- .../src/lib/tecemux/codecs/frame-encoder.ts | 49 ++----------- .../src/lib/tecemux/playground-tecemux.ts | 45 +++++++++--- packages/verser/src/lib/tecemux/tecemux.ts | 37 ++++++---- 4 files changed, 100 insertions(+), 101 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index f6a62efe4..803d27c57 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -11,54 +11,62 @@ export class FrameDecoder extends Transform { _streams = new Map(); constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { - super(Object.assign({}, { writableObjectMode: true, readableObjectMode: true, emitClose: false }, opts)); + super(Object.assign({}, opts, { writableObjectMode: true, readableObjectMode: true, emitClose: false })); this.buff = Buffer.alloc(64 * 1024, 0, undefined); //@TODO: optimize this.logger = new ObjLogger(params.name); this.on("pipe", () => { this.logger.debug("onPipe"); - }); + }).on("close", () => { + this.logger.debug("onClose"); + }) } _transform(chunk: Buffer, encoding: BufferEncoding, callback: TransformCallback): void { - this.logger.trace("Decoding frame...", toHex(chunk), "Size:", chunk.length); + try { + this.logger.trace("Decoding frame...", toHex(chunk), "Size:", chunk.length); - if (!Buffer.isBuffer(chunk)) { - this.push(JSON.stringify({ error: "not a buffer" }), undefined); - callback(); + if (!Buffer.isBuffer(chunk)) { + this.push(JSON.stringify({ error: "not a buffer" }), undefined); + callback(); - return; - } + return; + } - chunk.copy(this.buff, this.size, 0, chunk.length); + if (Buffer.isBuffer(chunk)) { + chunk.copy(this.buff, this.size, 0, chunk.length); + } - this.size += chunk.length; + this.size += chunk.length; - if (this.size >= 10 ) {//&& this.buff.readInt32LE(10) === this.size) { - const frameSize = this.buff.readInt32LE(10); + if (this.size >= 10 ) {//&& this.buff.readInt32LE(10) === this.size) { + const frameSize = this.buff.readInt32LE(10); - const payload = { - sourceAddress: [this.buff.readInt8(0), this.buff.readInt8(1), this.buff.readInt8(2), this.buff.readInt8(3)], - destinationAddress: [this.buff.readInt8(4), this.buff.readInt8(5), this.buff.readInt8(6), this.buff.readInt8(7)], - chunk: this.buff.subarray(32, this.buff.readInt32LE(10)), - flags: parseFlags(this.buff.readInt8(25)), - sourcePort: this.buff.readInt16LE(12), - destinationPort: this.buff.readInt16LE(14), - dataLength: frameSize - HEADER_LENGTH, - chunkLength: frameSize, - sequenceNumber: this.buff.readInt32LE(16), - stringified: this.buff.subarray(32, frameSize).toString() - } as Partial; + const payload = { + sourceAddress: [this.buff.readInt8(0), this.buff.readInt8(1), this.buff.readInt8(2), this.buff.readInt8(3)], + destinationAddress: [this.buff.readInt8(4), this.buff.readInt8(5), this.buff.readInt8(6), this.buff.readInt8(7)], + chunk: this.buff.subarray(32, this.buff.readInt32LE(10)), + flags: parseFlags(this.buff.readInt8(25)), + sourcePort: this.buff.readInt16LE(12), + destinationPort: this.buff.readInt16LE(14), + dataLength: frameSize - HEADER_LENGTH, + chunkLength: frameSize, + sequenceNumber: this.buff.readInt32LE(16), + stringified: this.buff.subarray(32, frameSize).toString() + } as Partial; - this.push(JSON.stringify(payload) + "\n"); + this.push(JSON.stringify(payload) + "\n"); - this.size = 0; - this.buff.fill(0); - } else { - this.logger.error("too few data", this.size, this.buff.readInt32LE(10)); - } + this.size = 0; + this.buff.fill(0); + } else { + this.logger.error("too few data", this.size, this.buff.readInt32LE(10)); + } - callback(); + callback(); + } catch(err) { + this.logger.error("ERROR", err) + } } } diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index 9d3acf6ee..c3c0dd571 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -8,7 +8,7 @@ export class FrameEncoder extends Transform { logger = new ObjLogger("FrameEncoder",); out = new PassThrough({ readableObjectMode: true }) .on("data", (data) => { - this.logger.trace("output to socket:", toHex(data), data.length, this.readableFlowing); + this.logger.trace("output to socket: " + (data.length == HEADER_LENGTH ? "HEADER ONLY" : ""), toHex(data), data.length, this.readableFlowing); }) .on("pause", () => { this.logger.trace("output to socket paused"); @@ -19,9 +19,12 @@ export class FrameEncoder extends Transform { .on("resume", () => { this.logger.trace("output to socket resumed"); }) + .on("error", (error) => { + this.logger.error("output to socket paused", error); + }) constructor(private frameTarget: FrameTarget, opts: TransformOptions = {}, params: { name: string } = { name: "FrameEncoder" }) { - opts.emitClose = false; + //opts.emitClose = false; super(opts); @@ -102,53 +105,11 @@ export class FrameEncoder extends Transform { return buffer; } - createChunkFrame(chunk: any) { - const checksum = this.getChecksum(); - const buffer = Buffer.concat([ - // 0: source address 0 - 3 - new Uint8Array([10, 0, 0, 1]), - // 32: destination address 4 - 7 - new Uint8Array([10, 0, 0, 2]), - - // 64: zeroes (8bit), protocol (8bit), 8 - 9 - new Uint8Array([0, 1]), - // tcp length (16bit) 10 - 11 - new Uint8Array(new Uint16Array([chunk.length + HEADER_LENGTH]).buffer), - - // 96: Source port, destination port 12 - 15 - new Uint8Array(new Uint16Array([0, this.frameTarget]).buffer), - - // 128: sequenceNumber(32 bit, acnowledge number) 16 - 19 - new Uint8Array(new Uint32Array([this.sequenceNumber++]).buffer), - - // 160: Acknowledgement number 20-23 - new Uint8Array([0, 0, 0, 0]), - - // 192: data offset (4bit), reserved (4bit), 24 - new Uint8Array([0b00000000]), - // 224: flags (8bit), 25 - this.setFlags(["PSH"]), - // window(16bit) 26 - 27 - new Uint8Array(new Uint16Array([0]).buffer), - - // checksum(16bit) 28 - 29 - new Uint8Array(new Uint16Array([checksum]).buffer), - // pointer (16bit) 30 - 31 - new Uint8Array(new Uint16Array([checksum]).buffer), - - // 256: data 32 - - new Uint8Array(chunk) - ]); - - return buffer; - } - _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void { //const buffer = this.createChunkFrame(chunk); this.logger.debug("TRANSFORM IN", toHex(chunk), chunk.length); const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); - //this.push(buffer, undefined); this.logger.debug("TRANSFORM OUT", getHexString(buffer), "Size: ", buffer.length); this.push(buffer, undefined); callback(); diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index f08adab9d..59e157221 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -22,7 +22,9 @@ import { FrameData } from "./utils"; const PORT = 6660; const server = createServer(); - server.setTimeout(0); + server.on("timeout", (socket) => { + logger.warn("Server on timeout"); + }); server.requestTimeout = 0; server.on("connect", async (req: IncomingMessage, socket: Socket) => { @@ -32,14 +34,31 @@ import { FrameData } from "./utils"; socket .on("data", (data) => { - console.log("SERVER socket ondata", data); - console.log("SOCKET TX RX", socket.bytesWritten, socket.bytesRead) + logger.info("SERVER Carrier socket ondata", data); + logger.info("SOCKET Carrier TX RX", socket.bytesWritten, socket.bytesRead) + }) + .on("pipe", () => { + logger.info("Carrier Socket piped"); + }) + .on("unpipe", () => { + logger.info("Carrier Socket unpiped"); }) .on("pause", () => { - logger.info("Socket paused"); + //socket.resume(); + logger.fatal("Carrier Socket paused"); + //debugger; }) .on("resume", () => { - logger.info("Socket resumed"); + logger.info("Carrier Socket resumed"); + }) + .on("error", (error) => { + logger.error("Carrier Socket error", error); + }) + .on("close", () => { + logger.info("Carrier Socket closed"); + }) + .on("timeout", () => { + logger.info("Carrier Socket timeout"); }) const tcmux = new TeceMux(socket, "Server") @@ -47,8 +66,6 @@ import { FrameData } from "./utils"; logger.error("TCMUX err", err); }); - - tcmux.logger.pipe(logger); const channel = tcmux.multiplex(); @@ -59,9 +76,9 @@ import { FrameData } from "./utils"; process.stdin.pipe(channel); - const somePayload = "smth\n"; - logger.info("writing some payload to channel", somePayload); - channel.write(somePayload); + // const somePayload = "smth\n"; + // logger.info("writing some payload to channel", somePayload); + // channel.write(somePayload); for await (const chunk of channel) { const parsed = JSON.parse(chunk) as FrameData; @@ -77,6 +94,12 @@ import { FrameData } from "./utils"; const socket = createConnection({ port: PORT, allowHalfOpen: true, host: "0.0.0.0" }, () => {}); + await new Promise((resolve, reject) => { + socket + .on("connect", resolve) + .on("error", reject); + }); + socket.setNoDelay(true); const reqLogger = new ObjLogger("Req", { id: "Client"}); @@ -111,7 +134,7 @@ import { FrameData } from "./utils"; // } channel.on("data", async (chunk) => { - reqLogger.info("SERVER->CLIENT->CHANNEL", chunk.toString()); + reqLogger.info("SERVER->CLIENT->CHANNEL", channel._id, chunk.toString()); await new Promise((resolve, reject) => { setTimeout(() => { diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 92d024a49..6632e4d02 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -12,7 +12,20 @@ export class TeceMux extends TypedEmitter{ id: string; carrierSocket: Duplex; channelCount = 1; - decoder = new FrameDecoder().resume(); + decoder = new FrameDecoder({ emitClose: false }) + .on("pause", () => { + this.logger.warn("Decoder paused"); + }) + .on("close", () => { + this.logger.warn("Decoder closed"); + }) + .on("end", () => { + this.logger.warn("Decoder ended"); + }) + .on("error", (error) => { + this.logger.error("Decoder error", error); + //debugger; + }) channels: TeceMuxChannel[] = []; @@ -27,10 +40,9 @@ export class TeceMux extends TypedEmitter{ encoder.logger.updateBaseLog({ id: this.id }); encoder.logger.pipe(this.logger); - const w = new PassThrough({ encoding: undefined }).on("data", (d) => { this.logger.warn("channel writeable on DATA", d); }); - //w.pipe(process.stdout).pause() + const w = new PassThrough({ encoding: undefined, readableObjectMode: true }).on("data", (d) => { this.logger.warn("channel writeable on DATA", d); }); - w.pipe(encoder); + w.pipe(encoder).out.pipe(this.carrierSocket); const channel: TeceMuxChannel = Object.assign( Duplex.from({ @@ -43,9 +55,6 @@ export class TeceMux extends TypedEmitter{ } ); - //channel.pipe(encoder).pipe(this.carrierSocket); - //channel.pipe(process.stdout); - channel.on("error", (error) => { this.emit("error", { error, source: channel }) }); @@ -54,10 +63,6 @@ export class TeceMux extends TypedEmitter{ encoder.setChannel(destinationPort || this.channelCount); } - //channel.pipe(w); - //channel.pipe(this.carrierSocket); - encoder.out.pipe(this.carrierSocket); - return channel; } @@ -70,8 +75,9 @@ export class TeceMux extends TypedEmitter{ this.decoder.logger.updateBaseLog({ id: this.id }); this.decoder.logger.pipe(this.logger); - this.carrierSocket.pipe(this.decoder); - this.commonEncoder.out.pipe(this.carrierSocket); + this.carrierSocket.pipe(this.decoder, { end: false }); + this.carrierSocket.on("data", (data) => { this.logger.info("CARRIER DATA", data); }) + this.commonEncoder.out.pipe(this.carrierSocket, { end: false }); this.main().catch((error) => { this.emit("error", error); @@ -79,7 +85,7 @@ export class TeceMux extends TypedEmitter{ } async main() { - this.commonEncoder.logger.updateBaseLog({ id: "Commn" + this.commonEncoder.logger.baseLog.id }) + this.commonEncoder.logger.updateBaseLog({ id: "Comm" + this.commonEncoder.logger.baseLog.id }) this.commonEncoder.logger.pipe(this.logger); for await (const chunk of this.decoder) { @@ -129,8 +135,9 @@ export class TeceMux extends TypedEmitter{ if (emit) { this.emit("channel", channel); this.logger.debug("channel event emitted", channel._id); - this.channelCount++; } + + this.channelCount++; } multiplex(): TeceMuxChannel { From 6c90a6c869ea36056f0ca909320bfd522993ee3e Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 18 Jan 2023 13:41:26 +0000 Subject: [PATCH 012/231] Add handlers --- .../src/lib/tecemux/codecs/frame-decoder.ts | 2 +- .../src/lib/tecemux/playground-tecemux.ts | 17 +-- packages/verser/src/lib/tecemux/tecemux.ts | 103 ++++++++++++------ 3 files changed, 81 insertions(+), 41 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 803d27c57..c49f6cb42 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -11,7 +11,7 @@ export class FrameDecoder extends Transform { _streams = new Map(); constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { - super(Object.assign({}, opts, { writableObjectMode: true, readableObjectMode: true, emitClose: false })); + super(Object.assign({}, opts, { writableObjectMode: false, readableObjectMode: false })); this.buff = Buffer.alloc(64 * 1024, 0, undefined); //@TODO: optimize this.logger = new ObjLogger(params.name); diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index 59e157221..b38480821 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -20,7 +20,7 @@ import { FrameData } from "./utils"; /**********************************************/ const PORT = 6660; - const server = createServer(); + const server = createServer({}); server.on("timeout", (socket) => { logger.warn("Server on timeout"); @@ -33,10 +33,10 @@ import { FrameData } from "./utils"; logger.info("Incoming request", req.method, req.headers); socket - .on("data", (data) => { - logger.info("SERVER Carrier socket ondata", data); - logger.info("SOCKET Carrier TX RX", socket.bytesWritten, socket.bytesRead) - }) + // .on("data", (data) => { + // logger.info("SERVER Carrier socket ondata", data); + // logger.info("SOCKET Carrier TX RX", socket.bytesWritten, socket.bytesRead) + // }) .on("pipe", () => { logger.info("Carrier Socket piped"); }) @@ -60,6 +60,7 @@ import { FrameData } from "./utils"; .on("timeout", () => { logger.info("Carrier Socket timeout"); }) + .pause() const tcmux = new TeceMux(socket, "Server") .on("error", (err) => { @@ -81,8 +82,7 @@ import { FrameData } from "./utils"; // channel.write(somePayload); for await (const chunk of channel) { - const parsed = JSON.parse(chunk) as FrameData; - logger.debug(`Server on request data [C: ${parsed.sequenceNumber}, SN: ${parsed.sequenceNumber}]`, parsed.chunk, parsed.dataLength); + logger.debug(`reading CHANNEL chunk`, chunk.toString());// [C: ${parsed.sequenceNumber}, SN: ${parsed.sequenceNumber}]`, parsed.chunk, parsed.dataLength); }; }); @@ -133,12 +133,13 @@ import { FrameData } from "./utils"; // }); // } + //for await (const chunk of channel) { channel.on("data", async (chunk) => { reqLogger.info("SERVER->CLIENT->CHANNEL", channel._id, chunk.toString()); await new Promise((resolve, reject) => { setTimeout(() => { - //channel.encoder.write("Echo\n"); + channel.write("XEcho\n"); resolve(); }, 2000); }); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 6632e4d02..cc43ebe25 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -4,7 +4,6 @@ import { Duplex, PassThrough } from "stream"; import { Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; import { FrameData } from "./utils"; -import { IObjectLogger } from "@scramjet/types"; export type TeceMuxChannel = Duplex & { _id: number, encoder: FrameEncoder }; @@ -12,25 +11,12 @@ export class TeceMux extends TypedEmitter{ id: string; carrierSocket: Duplex; channelCount = 1; - decoder = new FrameDecoder({ emitClose: false }) - .on("pause", () => { - this.logger.warn("Decoder paused"); - }) - .on("close", () => { - this.logger.warn("Decoder closed"); - }) - .on("end", () => { - this.logger.warn("Decoder ended"); - }) - .on("error", (error) => { - this.logger.error("Decoder error", error); - //debugger; - }) + decoder: FrameDecoder; channels: TeceMuxChannel[] = []; - logger: IObjectLogger; - commonEncoder = new FrameEncoder(0).resume(); + logger: ObjLogger; + commonEncoder = new FrameEncoder(0); private createChannel(destinationPort?: number, emit?: boolean): TeceMuxChannel { this.logger.debug("Create Channel", destinationPort || this.channelCount); @@ -40,15 +26,31 @@ export class TeceMux extends TypedEmitter{ encoder.logger.updateBaseLog({ id: this.id }); encoder.logger.pipe(this.logger); - const w = new PassThrough({ encoding: undefined, readableObjectMode: true }).on("data", (d) => { this.logger.warn("channel writeable on DATA", d); }); + const w = new PassThrough({ encoding: undefined }) + + process.nextTick(() => { + w.on("data", (d) => { this.logger.warn("channel writeable on DATA", d); }); + }); w.pipe(encoder).out.pipe(this.carrierSocket); + const duplex = new Duplex({ + write: (chunk, encoding, next) => { + this.logger.trace("WRITE channel", channel._id, chunk ); + w.write(chunk); + next(); + }, + read: (size) => { + this.logger.trace("READ channel", channel._id ); + //setTimeout(() => (channel as unknown as Duplex).write("a"), 1000); + } + }); const channel: TeceMuxChannel = Object.assign( - Duplex.from({ - readable: new PassThrough({ encoding: undefined }).on("data", (d) => { this.logger.warn("channel readable on DATA", d); }), - writable: w, - } as unknown as Iterable) as TeceMuxChannel, + // Duplex.from({ + // readable: new PassThrough({ encoding: undefined }).on("data", (d) => { this.logger.warn("channel readable on DATA", d); }), + // writable: w, + // } as unknown as Iterable) as TeceMuxChannel, + duplex, { _id: destinationPort || this.channelCount, encoder @@ -56,8 +58,14 @@ export class TeceMux extends TypedEmitter{ ); channel.on("error", (error) => { - this.emit("error", { error, source: channel }) - }); + this.logger.error("CHANNEL ERROR", error) + //this.emit("error", { error, source: channel }) + }).on("destroy", () => { + this.logger.trace("channel on DESTROY ", channel._id ); + }) + .on("abort", () => { + this.logger.trace("channel on DESTROY ", channel._id ); + }) if (emit) { encoder.setChannel(destinationPort || this.channelCount); @@ -72,20 +80,48 @@ export class TeceMux extends TypedEmitter{ this.logger = new ObjLogger(this, { id: this.id }) this.carrierSocket = socket; + this.decoder = new FrameDecoder({ emitClose: false }) + .on("pause", () => { + this.logger.warn("Decoder paused"); + }) + .on("close", () => { + this.logger.warn("Decoder closed"); + }) + .on("end", () => { + this.logger.warn("Decoder ended"); + }) + .on("error", (error) => { + this.logger.error("Decoder error", error); + //debugger; + }) + .on("abort", (error) => { + this.logger.error("Decoder abort", error); + //debugger; + }) + .on("destroy", (error) => { + this.logger.error("Decoder destroy", error); + //debugger; + }); + this.decoder.logger.updateBaseLog({ id: this.id }); this.decoder.logger.pipe(this.logger); this.carrierSocket.pipe(this.decoder, { end: false }); - this.carrierSocket.on("data", (data) => { this.logger.info("CARRIER DATA", data); }) + + + process.nextTick(() => { + //this.carrierSocket.on("data", (data) => { this.logger.info("CARRIER DATA", data); }) + }); + this.commonEncoder.out.pipe(this.carrierSocket, { end: false }); - this.main().catch((error) => { + this.main().then(() => 1);/*.catch((error) => { this.emit("error", error); - }); + });*/ } async main() { - this.commonEncoder.logger.updateBaseLog({ id: "Comm" + this.commonEncoder.logger.baseLog.id }) + this.commonEncoder.logger.updateBaseLog({ id: "CMN " + this.logger.baseLog.id }) this.commonEncoder.logger.pipe(this.logger); for await (const chunk of this.decoder) { @@ -113,18 +149,21 @@ export class TeceMux extends TypedEmitter{ if (dataLength) { this.logger.warn("writing to channel [channel, length]", channel._id, dataLength); - channel.push(new Uint8Array(((frame.chunk as any).data) as any)); + + const written = channel.push(new Uint8Array(((frame.chunk as any).data) as any)); + + this.logger.info("Bytes written to channel [writeResult, channel, length]", written, destinationPort, dataLength); } } - this.logger.debug("Write acknowledge frame for sequenceNumber", sequenceNumber); - this.commonEncoder.out.write( + this.logger.debug("OFF Write acknowledge frame for sequenceNumber", sequenceNumber); + /*this.commonEncoder.push( this.commonEncoder.createFrame(undefined, { flagsArray: ["ACK"], sequenceNumber, destinationPort }) - ); + );*/ } } From 1f6bb57f0dbe7593407c416cb72b484135621c48 Mon Sep 17 00:00:00 2001 From: Budleigh Salterton Date: Wed, 18 Jan 2023 20:27:11 +0100 Subject: [PATCH 013/231] Ack on, alternate stdin to cli/srv --- .../verser/src/lib/tecemux/playground-tecemux.ts | 16 +++++++++------- packages/verser/src/lib/tecemux/tecemux.ts | 4 ++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index b38480821..fb2e026d1 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -75,7 +75,7 @@ import { FrameData } from "./utils"; logger.warn("Waiting for stdin..."); - process.stdin.pipe(channel); + DataStream.from(process.stdin).filter((x: Buffer) => (x[0] % 2 !== 0)).pipe(channel); // const somePayload = "smth\n"; // logger.info("writing some payload to channel", somePayload); @@ -133,16 +133,18 @@ import { FrameData } from "./utils"; // }); // } + DataStream.from(process.stdin).filter((x: Buffer) => (x[0] % 2 !== 1)).pipe(channel); + //for await (const chunk of channel) { channel.on("data", async (chunk) => { reqLogger.info("SERVER->CLIENT->CHANNEL", channel._id, chunk.toString()); - await new Promise((resolve, reject) => { - setTimeout(() => { - channel.write("XEcho\n"); - resolve(); - }, 2000); - }); + // await new Promise((resolve, reject) => { + // setTimeout(() => { + // channel.write("abcde\n"); + // resolve(); + // }, 2000); + // }); }) }); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index cc43ebe25..7b4eacb47 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -157,13 +157,13 @@ export class TeceMux extends TypedEmitter{ } this.logger.debug("OFF Write acknowledge frame for sequenceNumber", sequenceNumber); - /*this.commonEncoder.push( + this.commonEncoder.push( this.commonEncoder.createFrame(undefined, { flagsArray: ["ACK"], sequenceNumber, destinationPort }) - );*/ + ); } } From 9becfad6dff5242bb8733849b530879d07221e1c Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 18 Jan 2023 22:05:00 +0000 Subject: [PATCH 014/231] Issue. Decoder receives more than 1 frame --- .../src/lib/tecemux/codecs/frame-decoder.ts | 48 +++++++++---------- .../src/lib/tecemux/codecs/frame-encoder.ts | 19 ++++++-- .../src/lib/tecemux/playground-tecemux.ts | 40 +++++++++------- packages/verser/src/lib/tecemux/tecemux.ts | 14 ++---- 4 files changed, 68 insertions(+), 53 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index c49f6cb42..124b6630d 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -11,7 +11,7 @@ export class FrameDecoder extends Transform { _streams = new Map(); constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { - super(Object.assign({}, opts, { writableObjectMode: false, readableObjectMode: false })); + super(Object.assign({}, opts, { writableObjectMode: true, readableObjectMode: true, readableHighWaterMark: 2 })); this.buff = Buffer.alloc(64 * 1024, 0, undefined); //@TODO: optimize this.logger = new ObjLogger(params.name); @@ -36,33 +36,33 @@ export class FrameDecoder extends Transform { if (Buffer.isBuffer(chunk)) { chunk.copy(this.buff, this.size, 0, chunk.length); + } else { + this.emit("error", "Chunk is not a buffer"); + callback(); } this.size += chunk.length; - if (this.size >= 10 ) {//&& this.buff.readInt32LE(10) === this.size) { - const frameSize = this.buff.readInt32LE(10); - - const payload = { - sourceAddress: [this.buff.readInt8(0), this.buff.readInt8(1), this.buff.readInt8(2), this.buff.readInt8(3)], - destinationAddress: [this.buff.readInt8(4), this.buff.readInt8(5), this.buff.readInt8(6), this.buff.readInt8(7)], - chunk: this.buff.subarray(32, this.buff.readInt32LE(10)), - flags: parseFlags(this.buff.readInt8(25)), - sourcePort: this.buff.readInt16LE(12), - destinationPort: this.buff.readInt16LE(14), - dataLength: frameSize - HEADER_LENGTH, - chunkLength: frameSize, - sequenceNumber: this.buff.readInt32LE(16), - stringified: this.buff.subarray(32, frameSize).toString() - } as Partial; - - this.push(JSON.stringify(payload) + "\n"); - - this.size = 0; - this.buff.fill(0); - } else { - this.logger.error("too few data", this.size, this.buff.readInt32LE(10)); - } + const frameSize = this.buff.readInt32LE(10); + + const payload = { + sourceAddress: [this.buff.readInt8(0), this.buff.readInt8(1), this.buff.readInt8(2), this.buff.readInt8(3)], + destinationAddress: [this.buff.readInt8(4), this.buff.readInt8(5), this.buff.readInt8(6), this.buff.readInt8(7)], + chunk: this.buff.subarray(32, this.buff.readInt32LE(10)), + flags: parseFlags(this.buff.readInt8(25)), + sourcePort: this.buff.readInt16LE(12), + destinationPort: this.buff.readInt16LE(14), + dataLength: frameSize - HEADER_LENGTH, + chunkLength: frameSize, + sequenceNumber: this.buff.readInt32LE(16), + stringified: this.buff.subarray(32, frameSize).toString() + } as Partial; + + this.push(JSON.stringify(payload) + "\n"); + + this.size = 0; + this.buff.fill(0); + callback(); } catch(err) { diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index c3c0dd571..1a5f8c63c 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -25,8 +25,12 @@ export class FrameEncoder extends Transform { constructor(private frameTarget: FrameTarget, opts: TransformOptions = {}, params: { name: string } = { name: "FrameEncoder" }) { //opts.emitClose = false; - - super(opts); + //opts.readableObjectMode = true; + super(Object.assign(opts, { + writableObjectMode: true, + readableObjectMode: true, + readableHighWaterMark: 0 + })); this.logger = new ObjLogger(params.name, { id: this.frameTarget.toString() }); @@ -111,7 +115,16 @@ export class FrameEncoder extends Transform { const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); this.logger.debug("TRANSFORM OUT", getHexString(buffer), "Size: ", buffer.length); - this.push(buffer, undefined); + + if (!this.push(buffer, undefined)) { + this.once("drain", () => { + this.push(buffer, undefined); + }); + }; + this.read(0); + callback(); } + + } diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index fb2e026d1..1c8b939c3 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -69,21 +69,30 @@ import { FrameData } from "./utils"; tcmux.logger.pipe(logger); - const channel = tcmux.multiplex(); + const channel1 = tcmux.multiplex(); + const channel2 = tcmux.multiplex(); req.on("pause", () => { logger.warn("Request paused"); }); logger.warn("Waiting for stdin..."); - DataStream.from(process.stdin).filter((x: Buffer) => (x[0] % 2 !== 0)).pipe(channel); - + process.stdin.pipe(channel1); + process.stdin.pipe(channel2); // const somePayload = "smth\n"; // logger.info("writing some payload to channel", somePayload); // channel.write(somePayload); - for await (const chunk of channel) { - logger.debug(`reading CHANNEL chunk`, chunk.toString());// [C: ${parsed.sequenceNumber}, SN: ${parsed.sequenceNumber}]`, parsed.chunk, parsed.dataLength); - }; + (async () => { + for await (const chunk of channel1) { + logger.debug(`reading CHANNEL1 chunk`, chunk.toString());// [C: ${parsed.sequenceNumber}, SN: ${parsed.sequenceNumber}]`, parsed.chunk, parsed.dataLength); + }; + })(); + + (async () => { + for await (const chunk of channel2) { + logger.debug(`reading CHANNEL2 chunk`, chunk.toString());// [C: ${parsed.sequenceNumber}, SN: ${parsed.sequenceNumber}]`, parsed.chunk, parsed.dataLength); + }; + })(); }); server.listen(PORT, "0.0.0.0"); @@ -133,19 +142,18 @@ import { FrameData } from "./utils"; // }); // } - DataStream.from(process.stdin).filter((x: Buffer) => (x[0] % 2 !== 1)).pipe(channel); + //DataStream.from(process.stdin).filter((x: Buffer) => (x[0] % 2 !== 1)).pipe(channel); - //for await (const chunk of channel) { - channel.on("data", async (chunk) => { + for await (const chunk of channel) { reqLogger.info("SERVER->CLIENT->CHANNEL", channel._id, chunk.toString()); - // await new Promise((resolve, reject) => { - // setTimeout(() => { - // channel.write("abcde\n"); - // resolve(); - // }, 2000); - // }); - }) + await new Promise((resolve, reject) => { + setTimeout(() => { + channel.write("abcde\n"); + resolve(); + }, 2000); + }); + } }); socket.on("error", (err) => { diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 7b4eacb47..2c17754de 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -108,16 +108,11 @@ export class TeceMux extends TypedEmitter{ this.carrierSocket.pipe(this.decoder, { end: false }); - - process.nextTick(() => { - //this.carrierSocket.on("data", (data) => { this.logger.info("CARRIER DATA", data); }) - }); - this.commonEncoder.out.pipe(this.carrierSocket, { end: false }); - this.main().then(() => 1);/*.catch((error) => { + this.main().catch((error) => { this.emit("error", error); - });*/ + }); } async main() { @@ -133,7 +128,7 @@ export class TeceMux extends TypedEmitter{ if (flags.ACK) { this.logger.trace("ACKNOWLEDGE frame received for sequenceNumber", sequenceNumber); // acknowledge received (confirm packet) - return; + continue; } if (flags.PSH) { @@ -156,7 +151,7 @@ export class TeceMux extends TypedEmitter{ } } - this.logger.debug("OFF Write acknowledge frame for sequenceNumber", sequenceNumber); + this.logger.debug("Write acknowledge frame for sequenceNumber", sequenceNumber); this.commonEncoder.push( this.commonEncoder.createFrame(undefined, { flagsArray: ["ACK"], @@ -187,7 +182,6 @@ export class TeceMux extends TypedEmitter{ this.addChannel(channel, false); this.logger.trace("Multiplex ready", channel._id); - this.channelCount++; return channel; } } From 5deef5a1f42f8f8378756788f1348300f7e135c8 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 18 Jan 2023 22:53:34 +0000 Subject: [PATCH 015/231] Read buffer of frames to decode --- .../src/lib/tecemux/codecs/frame-decoder.ts | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 124b6630d..bc4c3a936 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -13,7 +13,7 @@ export class FrameDecoder extends Transform { constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { super(Object.assign({}, opts, { writableObjectMode: true, readableObjectMode: true, readableHighWaterMark: 2 })); - this.buff = Buffer.alloc(64 * 1024, 0, undefined); //@TODO: optimize + this.buff = Buffer.alloc(0);// Buffer.alloc(64 * 1024, 0, undefined); //@TODO: optimize this.logger = new ObjLogger(params.name); this.on("pipe", () => { @@ -25,7 +25,6 @@ export class FrameDecoder extends Transform { _transform(chunk: Buffer, encoding: BufferEncoding, callback: TransformCallback): void { try { - this.logger.trace("Decoding frame...", toHex(chunk), "Size:", chunk.length); if (!Buffer.isBuffer(chunk)) { this.push(JSON.stringify({ error: "not a buffer" }), undefined); @@ -35,15 +34,24 @@ export class FrameDecoder extends Transform { } if (Buffer.isBuffer(chunk)) { - chunk.copy(this.buff, this.size, 0, chunk.length); + this.buff = Buffer.concat([this.buff, chunk]); + //chunk.copy(this.buff, this.size, 0, chunk.length); } else { + this.logger.error("Decoding buffer...", chunk); this.emit("error", "Chunk is not a buffer"); callback(); } - this.size += chunk.length; + this.logger.trace("Decoding buffer...", toHex(this.buff), "Size:", this.buff.length); - const frameSize = this.buff.readInt32LE(10); + let frameSize = 0; + + if (this.buff.length >= HEADER_LENGTH) { + frameSize = this.buff.readInt32LE(10); + } else { + this.logger.trace("To few data"); + callback(); + } const payload = { sourceAddress: [this.buff.readInt8(0), this.buff.readInt8(1), this.buff.readInt8(2), this.buff.readInt8(3)], @@ -60,11 +68,16 @@ export class FrameDecoder extends Transform { this.push(JSON.stringify(payload) + "\n"); - this.size = 0; - this.buff.fill(0); + this.buff = this.buff.subarray(frameSize); + if (this.buff.length === 0) { + this.logger.info("No remaining data!") + callback(); + return; + } - callback(); + this.logger.trace("More than one frame in chunk. processing", this.buff.length); + this._transform(Buffer.alloc(0), encoding, callback); } catch(err) { this.logger.error("ERROR", err) } From 097e632bd9c8313be32d3ef21b7b65732ba3bad0 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 19 Jan 2023 13:07:03 +0000 Subject: [PATCH 016/231] Cleanup, common sequenceNumber for TeceMux instance --- .../src/lib/tecemux/codecs/frame-decoder.ts | 42 ++++--- .../src/lib/tecemux/codecs/frame-encoder.ts | 10 +- .../verser/src/lib/tecemux/codecs/index.ts | 3 +- .../src/lib/tecemux/playground-tecemux.ts | 29 +---- packages/verser/src/lib/tecemux/playground.ts | 103 ------------------ packages/verser/src/lib/tecemux/tecemux.ts | 17 +-- 6 files changed, 36 insertions(+), 168 deletions(-) delete mode 100644 packages/verser/src/lib/tecemux/playground.ts diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index bc4c3a936..42bcc12a5 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -4,8 +4,7 @@ import { FrameData, HEADER_LENGTH, toHex } from "../utils"; import { parseFlags } from "."; export class FrameDecoder extends Transform { - buff: Buffer; - size = 0; + buffer: Buffer; logger: ObjLogger; _streams = new Map(); @@ -13,7 +12,7 @@ export class FrameDecoder extends Transform { constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { super(Object.assign({}, opts, { writableObjectMode: true, readableObjectMode: true, readableHighWaterMark: 2 })); - this.buff = Buffer.alloc(0);// Buffer.alloc(64 * 1024, 0, undefined); //@TODO: optimize + this.buffer = Buffer.alloc(0); this.logger = new ObjLogger(params.name); this.on("pipe", () => { @@ -25,7 +24,6 @@ export class FrameDecoder extends Transform { _transform(chunk: Buffer, encoding: BufferEncoding, callback: TransformCallback): void { try { - if (!Buffer.isBuffer(chunk)) { this.push(JSON.stringify({ error: "not a buffer" }), undefined); callback(); @@ -34,52 +32,52 @@ export class FrameDecoder extends Transform { } if (Buffer.isBuffer(chunk)) { - this.buff = Buffer.concat([this.buff, chunk]); - //chunk.copy(this.buff, this.size, 0, chunk.length); + this.buffer = Buffer.concat([this.buffer, chunk]); } else { this.logger.error("Decoding buffer...", chunk); this.emit("error", "Chunk is not a buffer"); callback(); } - this.logger.trace("Decoding buffer...", toHex(this.buff), "Size:", this.buff.length); + this.logger.trace("Decoding buffer...", toHex(this.buffer), "Size:", this.buffer.length); let frameSize = 0; - if (this.buff.length >= HEADER_LENGTH) { - frameSize = this.buff.readInt32LE(10); + if (this.buffer.length >= HEADER_LENGTH) { + frameSize = this.buffer.readInt32LE(10); } else { this.logger.trace("To few data"); callback(); } const payload = { - sourceAddress: [this.buff.readInt8(0), this.buff.readInt8(1), this.buff.readInt8(2), this.buff.readInt8(3)], - destinationAddress: [this.buff.readInt8(4), this.buff.readInt8(5), this.buff.readInt8(6), this.buff.readInt8(7)], - chunk: this.buff.subarray(32, this.buff.readInt32LE(10)), - flags: parseFlags(this.buff.readInt8(25)), - sourcePort: this.buff.readInt16LE(12), - destinationPort: this.buff.readInt16LE(14), + sourceAddress: [this.buffer.readInt8(0), this.buffer.readInt8(1), this.buffer.readInt8(2), this.buffer.readInt8(3)], + destinationAddress: [this.buffer.readInt8(4), this.buffer.readInt8(5), this.buffer.readInt8(6), this.buffer.readInt8(7)], + chunk: this.buffer.subarray(32, this.buffer.readInt32LE(10)), + flags: parseFlags(this.buffer.readInt8(25)), + sourcePort: this.buffer.readInt16LE(12), + destinationPort: this.buffer.readInt16LE(14), dataLength: frameSize - HEADER_LENGTH, chunkLength: frameSize, - sequenceNumber: this.buff.readInt32LE(16), - stringified: this.buff.subarray(32, frameSize).toString() + sequenceNumber: this.buffer.readInt32LE(16), + stringified: this.buffer.subarray(32, frameSize).toString() } as Partial; this.push(JSON.stringify(payload) + "\n"); - this.buff = this.buff.subarray(frameSize); + this.buffer = this.buffer.subarray(frameSize); - if (this.buff.length === 0) { + if (this.buffer.length === 0) { this.logger.info("No remaining data!") callback(); return; } - this.logger.trace("More than one frame in chunk. processing", this.buff.length); + this.logger.trace("More than one frame in chunk. processing", this.buffer.length); this._transform(Buffer.alloc(0), encoding, callback); - } catch(err) { - this.logger.error("ERROR", err) + } catch(error) { + this.logger.error("ERROR", error); + this.emit("error", error); } } } diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index 1a5f8c63c..f4b7483fd 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -2,9 +2,10 @@ import { PassThrough, Transform, TransformCallback, TransformOptions } from "str import { FrameData, FrameTarget, HEADER_LENGTH, toHex as getHexString, toHex } from "../utils"; import { ObjLogger } from "@scramjet/obj-logger"; import { binaryFlags, frameFlags } from "."; +import { TeceMux } from "../tecemux"; export class FrameEncoder extends Transform { - sequenceNumber = 0; + tecemux: TeceMux; logger = new ObjLogger("FrameEncoder",); out = new PassThrough({ readableObjectMode: true }) .on("data", (data) => { @@ -21,9 +22,9 @@ export class FrameEncoder extends Transform { }) .on("error", (error) => { this.logger.error("output to socket paused", error); - }) + }).pause(); - constructor(private frameTarget: FrameTarget, opts: TransformOptions = {}, params: { name: string } = { name: "FrameEncoder" }) { + constructor(private frameTarget: FrameTarget, tecemux: TeceMux, opts: TransformOptions = {}, params: { name: string } = { name: "FrameEncoder" }) { //opts.emitClose = false; //opts.readableObjectMode = true; super(Object.assign(opts, { @@ -32,6 +33,7 @@ export class FrameEncoder extends Transform { readableHighWaterMark: 0 })); + this.tecemux = tecemux; this.logger = new ObjLogger(params.name, { id: this.frameTarget.toString() }); this.on("pipe", () => { @@ -84,7 +86,7 @@ export class FrameEncoder extends Transform { new Uint8Array(new Uint16Array([0, frame.destinationPort || this.frameTarget]).buffer), // 128: sequenceNumber(32 bit, acnowledge number) 16 - 19 - new Uint8Array(new Uint32Array([frame.sequenceNumber || this.sequenceNumber++]).buffer), + new Uint8Array(new Uint32Array([this.tecemux.sequenceNumber++]).buffer), // 160: Acknowledgement number 20-23 new Uint8Array(new Uint32Array([frame.acknowledgeNumber || 0]).buffer), diff --git a/packages/verser/src/lib/tecemux/codecs/index.ts b/packages/verser/src/lib/tecemux/codecs/index.ts index de478859d..245e3e9fa 100644 --- a/packages/verser/src/lib/tecemux/codecs/index.ts +++ b/packages/verser/src/lib/tecemux/codecs/index.ts @@ -8,7 +8,6 @@ export type TeceMuxEvents = { error(error: any): void; } -export const frameFlags = ["FIN", "SYN", "RST", "PSH", "ACK", "URG", "ECE", "CWR"]; export const binaryFlags = { FIN: 0b00000001, SYN: 0b00000010, @@ -20,6 +19,8 @@ export const binaryFlags = { CWR: 0b10000000 } +export const frameFlags = Object.keys(binaryFlags); + export type flagsObjectType = Partial<{ FIN: boolean, SYN: boolean, diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index 1c8b939c3..fe3ec0cf1 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -8,7 +8,6 @@ import { DataStream } from "scramjet"; import { Socket, createConnection } from "net"; import { TeceMux, TeceMuxChannel } from "./tecemux"; -import { FrameData } from "./utils"; (async () => { const logger = new ObjLogger("Sandbox"); @@ -33,10 +32,6 @@ import { FrameData } from "./utils"; logger.info("Incoming request", req.method, req.headers); socket - // .on("data", (data) => { - // logger.info("SERVER Carrier socket ondata", data); - // logger.info("SOCKET Carrier TX RX", socket.bytesWritten, socket.bytesRead) - // }) .on("pipe", () => { logger.info("Carrier Socket piped"); }) @@ -76,21 +71,18 @@ import { FrameData } from "./utils"; logger.warn("Waiting for stdin..."); - process.stdin.pipe(channel1); - process.stdin.pipe(channel2); - // const somePayload = "smth\n"; - // logger.info("writing some payload to channel", somePayload); - // channel.write(somePayload); + DataStream.from(process.stdin).filter((x: Buffer) => !(parseInt(x[0].toString()) % 2)).pipe(channel1); + DataStream.from(process.stdin).filter((x: Buffer) => !!(parseInt(x[0].toString()) % 2)).pipe(channel2); (async () => { for await (const chunk of channel1) { - logger.debug(`reading CHANNEL1 chunk`, chunk.toString());// [C: ${parsed.sequenceNumber}, SN: ${parsed.sequenceNumber}]`, parsed.chunk, parsed.dataLength); + logger.debug(`reading CHANNEL1 chunk`, chunk.toString()); }; })(); (async () => { for await (const chunk of channel2) { - logger.debug(`reading CHANNEL2 chunk`, chunk.toString());// [C: ${parsed.sequenceNumber}, SN: ${parsed.sequenceNumber}]`, parsed.chunk, parsed.dataLength); + logger.debug(`reading CHANNEL2 chunk`, chunk.toString()); }; })(); }); @@ -131,19 +123,6 @@ import { FrameData } from "./utils"; tcmux.on("channel", async (channel: TeceMuxChannel) => { reqLogger.debug("New channel", channel._id); - // for await (const chunk of channel) { - // reqLogger.info("Data from server", chunk.toString()); - - // await new Promise((resolve, reject) => { - // setTimeout(() => { - // //channel.encoder.write("Echo\n"); - // resolve(); - // }, 2000); - // }); - // } - - //DataStream.from(process.stdin).filter((x: Buffer) => (x[0] % 2 !== 1)).pipe(channel); - for await (const chunk of channel) { reqLogger.info("SERVER->CLIENT->CHANNEL", channel._id, chunk.toString()); diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/src/lib/tecemux/playground.ts deleted file mode 100644 index 81ce13e25..000000000 --- a/packages/verser/src/lib/tecemux/playground.ts +++ /dev/null @@ -1,103 +0,0 @@ -/*eslint no-unused-vars: ["error", { "args": "none" }]*/ -/* eslint-disable @typescript-eslint/no-floating-promises */ -/* eslint-disable no-console */ - -import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; -import { IncomingMessage, createServer, request } from "http"; -import { DataStream } from "scramjet"; -import { FrameData, FrameTarget } from "./utils"; -import { FrameDecoder, FrameEncoder } from "./codecs"; -import { Socket } from "net"; - -(async () => { - const logger = new ObjLogger("Sandbox"); - - logger.pipe(new DataStream().map(prettyPrint({ colors: true }))).pipe(process.stdout); - - const PORT = 6660; - const server = createServer(); - - server.setTimeout(0); - server.requestTimeout = 0; - - server.on("connect", (req: IncomingMessage, socket: Socket) => { - socket.setNoDelay(true); - socket.write(`HTTP/1.1 ${200} \r\n\r\n`); - - logger.debug("on connect"); - - const serverFrameEncoder = new FrameEncoder(FrameTarget.API, { encoding: undefined }, { name: "ServerEncoder" }); - - serverFrameEncoder.logger.pipe(logger); - - serverFrameEncoder.pipe(socket); - let i = 0; - - req.on("pause", () => { logger.warn("Request paused"); }); - - const serverFrameDecoder = new FrameDecoder({ highWaterMark: 60 * 1024, encoding: undefined, emitClose: false }, { name: "ServerDecoder" }); - - serverFrameDecoder.logger.pipe(logger); - - socket.pipe(serverFrameDecoder).pipe(process.stdout); - - serverFrameDecoder.on("data", (d: any) => { - const parsed = JSON.parse(d) as FrameData; - - logger.debug(`Server on request data [${i++}]`, parsed.chunk, parsed.dataLength); - serverFrameEncoder.write(Buffer.from(new Uint8Array([255, 255, 255, 255]))); - }); - }); - - server.listen(PORT); - - const req = request({ - hostname: "0.0.0.0", - method: "connect", - port: PORT, - headers: { "Content-Type": "application/octet-stream", "Transfer-Encoding": "chunked" } - }); - - req.on("connect", (response, socket, head) => { - socket.setNoDelay(true); - - logger.debug("Response. Head:", head.toString()); - - let i = 0; - let m = 0; - - const frameEncoder = new FrameEncoder(FrameTarget.API, { encoding: undefined, emitClose: false }, { name: "RequestEncoder" }); - - frameEncoder.logger.pipe(logger); - - frameEncoder.pipe(socket); - - const responseFrameDecoder = new FrameDecoder({ highWaterMark: 60 * 1024, encoding: undefined, emitClose: false }, { name: "RequestDecoder" }); - - responseFrameDecoder.logger.pipe(logger); - - socket.pipe( - responseFrameDecoder - ).on("data", (d) => { - const parsed = JSON.parse(d) as FrameData; - - logger.debug(`Echo from server [${i++}]`, parsed.chunk, parsed.dataLength, parsed.chunkLength); - }); - - response.on("data", (d) => console.log("plain response", d)); - - req.on("error", (err) => { - console.error(err); - }); - - setInterval(async () => { - logger.debug(`Writing [${++m}]..`); - - frameEncoder.write(Buffer.from(new Uint8Array([0x61, 0x62, 0x63, 0x64]))); - }, 500); - }); - - req.flushHeaders(); - - await new Promise((_res, _rej) => {}); -})(); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 2c17754de..c159ba7e5 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -12,16 +12,16 @@ export class TeceMux extends TypedEmitter{ carrierSocket: Duplex; channelCount = 1; decoder: FrameDecoder; - + sequenceNumber = 0; channels: TeceMuxChannel[] = []; logger: ObjLogger; - commonEncoder = new FrameEncoder(0); + commonEncoder = new FrameEncoder(0, this); private createChannel(destinationPort?: number, emit?: boolean): TeceMuxChannel { this.logger.debug("Create Channel", destinationPort || this.channelCount); - const encoder = new FrameEncoder(this.channelCount, { encoding: undefined }); + const encoder = new FrameEncoder(this.channelCount, this, { encoding: undefined }); encoder.logger.updateBaseLog({ id: this.id }); encoder.logger.pipe(this.logger); @@ -42,14 +42,9 @@ export class TeceMux extends TypedEmitter{ }, read: (size) => { this.logger.trace("READ channel", channel._id ); - //setTimeout(() => (channel as unknown as Duplex).write("a"), 1000); } }); const channel: TeceMuxChannel = Object.assign( - // Duplex.from({ - // readable: new PassThrough({ encoding: undefined }).on("data", (d) => { this.logger.warn("channel readable on DATA", d); }), - // writable: w, - // } as unknown as Iterable) as TeceMuxChannel, duplex, { _id: destinationPort || this.channelCount, @@ -59,7 +54,7 @@ export class TeceMux extends TypedEmitter{ channel.on("error", (error) => { this.logger.error("CHANNEL ERROR", error) - //this.emit("error", { error, source: channel }) + this.emit("error", { error, source: channel }) }).on("destroy", () => { this.logger.trace("channel on DESTROY ", channel._id ); }) @@ -92,15 +87,12 @@ export class TeceMux extends TypedEmitter{ }) .on("error", (error) => { this.logger.error("Decoder error", error); - //debugger; }) .on("abort", (error) => { this.logger.error("Decoder abort", error); - //debugger; }) .on("destroy", (error) => { this.logger.error("Decoder destroy", error); - //debugger; }); this.decoder.logger.updateBaseLog({ id: this.id }); @@ -127,7 +119,6 @@ export class TeceMux extends TypedEmitter{ if (flags.ACK) { this.logger.trace("ACKNOWLEDGE frame received for sequenceNumber", sequenceNumber); - // acknowledge received (confirm packet) continue; } From dedf56de04ec52d7508760bcb263889e4de98e1c Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 19 Jan 2023 19:57:07 +0000 Subject: [PATCH 017/231] Send FIN --- .../src/lib/tecemux/codecs/frame-encoder.ts | 33 ++++-- .../src/lib/tecemux/playground-tecemux.ts | 64 +++++++++-- packages/verser/src/lib/tecemux/tecemux.ts | 105 +++++++++++++----- 3 files changed, 154 insertions(+), 48 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index f4b7483fd..fc7bc9643 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -7,22 +7,26 @@ import { TeceMux } from "../tecemux"; export class FrameEncoder extends Transform { tecemux: TeceMux; logger = new ObjLogger("FrameEncoder",); - out = new PassThrough({ readableObjectMode: true }) + out = Object.assign(new PassThrough({ readableObjectMode: true }) .on("data", (data) => { this.logger.trace("output to socket: " + (data.length == HEADER_LENGTH ? "HEADER ONLY" : ""), toHex(data), data.length, this.readableFlowing); }) .on("pause", () => { - this.logger.trace("output to socket paused"); + this.logger.trace("output to socket paused!"); }) .on("end", () => { - this.logger.trace("output to socket ended"); + this.logger.trace("output to socket ended!", this.frameTarget); + //this.tecemux.sendFIN(this.frameTarget); }) .on("resume", () => { this.logger.trace("output to socket resumed"); }) .on("error", (error) => { - this.logger.error("output to socket paused", error); - }).pause(); + this.logger.error("output to socket error", error); + }).pause() + ,{ + _id: this.frameTarget + }); constructor(private frameTarget: FrameTarget, tecemux: TeceMux, opts: TransformOptions = {}, params: { name: string } = { name: "FrameEncoder" }) { //opts.emitClose = false; @@ -36,9 +40,14 @@ export class FrameEncoder extends Transform { this.tecemux = tecemux; this.logger = new ObjLogger(params.name, { id: this.frameTarget.toString() }); - this.on("pipe", () => { - this.logger.debug("onPipe"); - }); + this + .on("pipe", () => { + this.logger.debug("onPipe"); + }) + .on("end", () => { + this.logger.debug("onEnd"); + }) + this.pipe(this.out); } @@ -64,6 +73,14 @@ export class FrameEncoder extends Transform { })); } + onChannelEnd(channelId: number) { + this.logger.debug("sending FIN for channel", channelId); + this.out.write(this.createFrame([], { + flagsArray: ["FIN"], + destinationPort: channelId + })); + } + getChecksum() { return 255; } diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index fe3ec0cf1..8054f1ce9 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -12,7 +12,18 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; (async () => { const logger = new ObjLogger("Sandbox"); - logger.pipe(new DataStream().map(prettyPrint({ colors: true }))).pipe(process.stdout); + logger + .pipe( + new DataStream() + .map(prettyPrint({ colors: true })) + .map((chunk: string) => ( + chunk.replace( + /(:?FIN|SYN|RST|PSH|ACK|URG|ECE|CWR)|^$]/, + "\x1b[41m" + "$&" + "\x1b[0m" + ) + )) + ) + .pipe(process.stdout); /**********************************************/ /* SERVER @@ -35,8 +46,8 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; .on("pipe", () => { logger.info("Carrier Socket piped"); }) - .on("unpipe", () => { - logger.info("Carrier Socket unpiped"); + .on("unpipe", (c: any) => { + logger.info("Carrier Socket unpiped", c); }) .on("pause", () => { //socket.resume(); @@ -75,16 +86,37 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; DataStream.from(process.stdin).filter((x: Buffer) => !!(parseInt(x[0].toString()) % 2)).pipe(channel2); (async () => { - for await (const chunk of channel1) { - logger.debug(`reading CHANNEL1 chunk`, chunk.toString()); - }; + try { + for await (const chunk of channel1) { + console.log("CHHUNK", chunk); + logger.debug(`reading CHANNEL1 chunk`, chunk.toString()); + }; + } catch (error) { + logger.error(`reading CHANNEL1 ERROR`, error); + } + + logger.debug(`reading CHANNEL1 END`); })(); (async () => { - for await (const chunk of channel2) { - logger.debug(`reading CHANNEL2 chunk`, chunk.toString()); - }; + try { + for await (const chunk of channel2) { + logger.debug(`reading CHANNEL2 chunk`, chunk.toString()); + }; + } catch (error) { + logger.error(`reading CHANNEL2 ERROR`, error); + } + + logger.debug(`reading CHANNEL2 END`); })(); + + + setTimeout(() => { + console.log("\n\n\n\n"); + logger.trace("Ending channels"); + channel1.push(null); + //channel2.end(); + }, 4000); }); server.listen(PORT, "0.0.0.0"); @@ -123,12 +155,24 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; tcmux.on("channel", async (channel: TeceMuxChannel) => { reqLogger.debug("New channel", channel._id); + channel + .on("finish", () => { + tcmux.logger.info("Channel finish", channel._id) + }) + .on("end", () => { + tcmux.logger.info("Channel end", channel._id) + }); + for await (const chunk of channel) { + console.log(chunk); + if (chunk === null) { + break; + } reqLogger.info("SERVER->CLIENT->CHANNEL", channel._id, chunk.toString()); await new Promise((resolve, reject) => { setTimeout(() => { - channel.write("abcde\n"); + //channel.write("abcde\n"); resolve(); }, 2000); }); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index c159ba7e5..aa9d24a05 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -5,7 +5,11 @@ import { Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; import { FrameData } from "./utils"; -export type TeceMuxChannel = Duplex & { _id: number, encoder: FrameEncoder }; +export type TeceMuxChannel = Duplex & { + _id: number, + encoder: FrameEncoder, + closedByFIN: boolean +}; export class TeceMux extends TypedEmitter{ id: string; @@ -26,41 +30,55 @@ export class TeceMux extends TypedEmitter{ encoder.logger.updateBaseLog({ id: this.id }); encoder.logger.pipe(this.logger); - const w = new PassThrough({ encoding: undefined }) + const w = new PassThrough({ encoding: undefined, allowHalfOpen: true }); - process.nextTick(() => { - w.on("data", (d) => { this.logger.warn("channel writeable on DATA", d); }); - }); - - w.pipe(encoder).out.pipe(this.carrierSocket); + w.pipe(encoder, { end: false }).out.pipe(this.carrierSocket, { end: false }); const duplex = new Duplex({ write: (chunk, encoding, next) => { - this.logger.trace("WRITE channel", channel._id, chunk ); - w.write(chunk); - next(); + this.logger.trace("WRITE channel", channel._id, chunk, chunk.toString() ); + + if (!w.push(chunk)) { + w.once("drain", next); + } else { + next(); + } }, read: (size) => { this.logger.trace("READ channel", channel._id ); - } + }, + allowHalfOpen: true }); const channel: TeceMuxChannel = Object.assign( duplex, { _id: destinationPort || this.channelCount, - encoder + encoder, + closedByFIN: false } ); - channel.on("error", (error) => { - this.logger.error("CHANNEL ERROR", error) - this.emit("error", { error, source: channel }) - }).on("destroy", () => { + process.nextTick(() => { + w.on("data", (d) => { this.logger.warn("channel writeable on DATA", d); }); + }); + + channel + .on("error", (error) => { + this.logger.error("CHANNEL ERROR", error) + //this.emit("error", { error, source: channel }) + }) + .on("destroy", () => { this.logger.trace("channel on DESTROY ", channel._id ); }) .on("abort", () => { - this.logger.trace("channel on DESTROY ", channel._id ); + this.logger.trace("channel on ABORT ", channel._id ); }) + .on("end", () => { + this.logger.info("CHANNEL end", channel._id); + if (!channel.closedByFIN) { + this.sendFIN(channel._id); + } + }); if (emit) { encoder.setChannel(destinationPort || this.channelCount); @@ -102,29 +120,42 @@ export class TeceMux extends TypedEmitter{ this.commonEncoder.out.pipe(this.carrierSocket, { end: false }); + this.commonEncoder.logger.updateBaseLog({ id }) + this.commonEncoder.logger.pipe(this.logger); + this.main().catch((error) => { this.emit("error", error); }); } async main() { - this.commonEncoder.logger.updateBaseLog({ id: "CMN " + this.logger.baseLog.id }) - this.commonEncoder.logger.pipe(this.logger); - for await (const chunk of this.decoder) { this.logger.debug("Decoded", JSON.parse(chunk)); const frame = JSON.parse(chunk) as FrameData; const { flags, sequenceNumber, dataLength, destinationPort } = frame; + let channel = this.channels[destinationPort] + if (flags.ACK) { - this.logger.trace("ACKNOWLEDGE frame received for sequenceNumber", sequenceNumber); + this.logger.trace("ACK frame received for sequenceNumber", sequenceNumber); continue; } + if (flags.FIN) { + this.logger.trace(`Received FIN command [C: ${destinationPort}]`, dataLength, frame.chunk, !!this.channels[destinationPort]); + + if (channel) { + //channel.end(); + channel.closedByFIN = true; + channel.push(null) + } else { + this.logger.error("FIN for unknown channel"); + } + } + if (flags.PSH) { this.logger.trace(`Received PSH command [C: ${destinationPort}]`, dataLength, frame.chunk, !!this.channels[destinationPort]); - let channel = this.channels[destinationPort] if (!channel) { this.logger.warn("Unknown channel"); @@ -142,17 +173,21 @@ export class TeceMux extends TypedEmitter{ } } - this.logger.debug("Write acknowledge frame for sequenceNumber", sequenceNumber); - this.commonEncoder.push( - this.commonEncoder.createFrame(undefined, { - flagsArray: ["ACK"], - sequenceNumber, - destinationPort - }) - ); + this.sendACK(sequenceNumber, destinationPort); } } + sendACK(sequenceNumber: number, channel: number) { + this.logger.debug("Write acknowledge frame for sequenceNumber", sequenceNumber); + this.commonEncoder.push( + this.commonEncoder.createFrame(undefined, { + flagsArray: ["ACK"], + sequenceNumber, + destinationPort: channel + }) + ); + } + addChannel(channel: TeceMuxChannel, emit: boolean) { this.logger.debug("adding channel", channel._id); this.channels[channel._id] = channel; // wait for SYN reply? @@ -165,6 +200,16 @@ export class TeceMux extends TypedEmitter{ this.channelCount++; } + sendFIN(channel: number) { + this.logger.debug("Write acknowledge frame for sequenceNumber"); + this.commonEncoder.push( + this.commonEncoder.createFrame(undefined, { + flagsArray: ["FIN"], + destinationPort: channel + }) + ); + } + multiplex(): TeceMuxChannel { this.logger.trace("Multiplex"); From d6a636ef310609174a059b1497c4b52313691ced Mon Sep 17 00:00:00 2001 From: patuwwy Date: Fri, 20 Jan 2023 12:40:22 +0000 Subject: [PATCH 018/231] Replace BPMux with TeCeMux --- packages/verser/package.json | 3 +- packages/verser/src/lib/tecemux/tecemux.ts | 4 +-- .../verser/src/lib/tecemux/utils/index.ts | 2 +- packages/verser/src/lib/verser-client.ts | 13 +++++---- packages/verser/src/lib/verser-connection.ts | 29 ++++++++++--------- 5 files changed, 26 insertions(+), 25 deletions(-) diff --git a/packages/verser/package.json b/packages/verser/package.json index 203fff311..4c29af24c 100644 --- a/packages/verser/package.json +++ b/packages/verser/package.json @@ -15,8 +15,7 @@ "license": "AGPL-3.0", "dependencies": { "@scramjet/obj-logger": "^0.33.5", - "@scramjet/utility": "^0.33.5", - "bpmux": "^8.2.1" + "@scramjet/utility": "^0.33.5" }, "devDependencies": { "@scramjet/api-server": "^0.33.5", diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index aa9d24a05..8b3a9d2cd 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -210,10 +210,10 @@ export class TeceMux extends TypedEmitter{ ); } - multiplex(): TeceMuxChannel { + multiplex(opts: { channel?: number } = {} ): TeceMuxChannel { this.logger.trace("Multiplex"); - const channel = this.createChannel(this.channelCount, true); + const channel = this.createChannel(opts.channel || this.channelCount, true); this.addChannel(channel, false); diff --git a/packages/verser/src/lib/tecemux/utils/index.ts b/packages/verser/src/lib/tecemux/utils/index.ts index 08cad4374..8528167e9 100644 --- a/packages/verser/src/lib/tecemux/utils/index.ts +++ b/packages/verser/src/lib/tecemux/utils/index.ts @@ -1,4 +1,4 @@ -import { frameFlags, binaryFlags, flagsObjectType } from "../codecs"; +import { binaryFlags, flagsObjectType } from "../codecs"; export function toHex(chunk: Buffer) { return chunk.toString("hex").match(/../g)?.join(" "); diff --git a/packages/verser/src/lib/verser-client.ts b/packages/verser/src/lib/verser-client.ts index a33fad356..ef072c93c 100644 --- a/packages/verser/src/lib/verser-client.ts +++ b/packages/verser/src/lib/verser-client.ts @@ -8,8 +8,7 @@ import { createConnection, Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; import { defaultVerserClientOptions } from "./verser-client-default-config"; import { URL } from "url"; - -const BPMux = require("bpmux").BPMux; +import { TeceMux } from "./tecemux/tecemux"; type Events = { error: (err: Error) => void; @@ -24,7 +23,7 @@ export class VerserClient extends TypedEmitter { /** * BPMux instance. */ - private bpmux: any; + private teceMux: any; /** * VerserClient options. @@ -126,8 +125,8 @@ export class VerserClient extends TypedEmitter { * otherwise stream will be passed to the server. */ private mux() { - this.bpmux = new BPMux(this.socket) - .on("peer_multiplex", async (mSocket: Duplex & { _chan: number }) => { + this.teceMux = new TeceMux(this.socket!) + .on("channel", async (mSocket: Duplex & { _chan: number }) => { const registeredChannelCallback = this.registeredChannels.get(mSocket._chan); if (registeredChannelCallback) { @@ -140,13 +139,15 @@ export class VerserClient extends TypedEmitter { this.emit("error", err); }); + this.teceMux.logger.pipe(this.logger); + this._verserAgent = new HttpAgent() as HttpAgent & { createConnection: typeof createConnection }; // lack of types? this._verserAgent.createConnection = () => { try { - const socket = this.bpmux!.multiplex() as Socket; + const socket = this.teceMux!.multiplex() as Socket; socket.on("error", () => { this.logger.error("Muxed stream error"); diff --git a/packages/verser/src/lib/verser-connection.ts b/packages/verser/src/lib/verser-connection.ts index 2e5b9045a..08146d012 100644 --- a/packages/verser/src/lib/verser-connection.ts +++ b/packages/verser/src/lib/verser-connection.ts @@ -10,8 +10,7 @@ import { import { ObjLogger } from "@scramjet/obj-logger"; import { createConnection, Socket } from "net"; import { VerserRequestResult } from "../types"; - -const BPMux = require("bpmux").BPMux; +import { TeceMux } from "./tecemux/tecemux"; /** * VerserConnection class. @@ -22,21 +21,21 @@ export class VerserConnection { logger = new ObjLogger(this); private request: IncomingMessage; - private bpmux?: { [key: string]: any }; + private teceMux?: TeceMux; - private _socket: Duplex; + private _socket: Socket; private agent?: Agent & { createConnection: typeof createConnection }; private channelListeners: ((socket: Duplex, data?: any) => any)[] = []; get connected() { - return !(this._socket.destroyed && this.bpmux); + return !(this._socket.destroyed && this.teceMux); } - get socket(): Duplex { + get socket(): Socket { return this._socket; } - constructor(request: IncomingMessage, socket: Duplex) { + constructor(request: IncomingMessage, socket: Socket) { this.request = request; this._socket = socket; @@ -102,7 +101,7 @@ export class VerserConnection { async forward(req: IncomingMessage, res: ServerResponse) { if (!this.connected) throw new Error("BPMux not connected"); - const channel = this.bpmux?.multiplex() as Duplex; + const channel = this.teceMux?.multiplex() as Duplex; channel .on("error", (error: Error) => { @@ -193,22 +192,24 @@ export class VerserConnection { * @returns Duplex stream. */ createChannel(id: number): Duplex { - if (!this.bpmux) throw new Error("BPMux not connected"); + if (!this.teceMux) throw new Error("TeCeMux not connected"); - return this.bpmux.multiplex({ channel: id }); + return this.teceMux.multiplex({ channel: id }); } reconnect() { this.logger.debug("Reconnecting..."); - this.bpmux = this.bpmux = new BPMux(this.socket).on("error", (error: Error) => { + this.teceMux = new TeceMux(this.socket).on("error", (error: Error) => { this.logger.error("BPMux Error", error.message); // TODO: Error handling? }); + this.teceMux.logger.pipe(this.logger); + this.agent = new Agent() as Agent & { createConnection: typeof createConnection }; // lack of types? this.agent.createConnection = () => { try { - const socket = this.bpmux!.multiplex() as Socket; + const socket = this.teceMux!.multiplex() as unknown as Socket; socket.on("error", () => { this.logger.error("Muxed stream error"); @@ -229,9 +230,9 @@ export class VerserConnection { } }; - this.bpmux!.on("peer_multiplex", (socket: Duplex, data: any) => { + this.teceMux!.on("channel", (socket: Duplex) => { this.channelListeners.forEach((listener) => { - listener(socket, data); + listener(socket); }); }); } From d4d9cea5741cf400e9e229119297b223a2c5f475 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Fri, 20 Jan 2023 14:06:57 +0000 Subject: [PATCH 019/231] Cleanup --- package.json | 2 +- packages/obj-logger/src/obj-logger.ts | 4 +- .../src/lib/tecemux/codecs/frame-decoder.ts | 26 +++++--- .../src/lib/tecemux/codecs/frame-encoder.ts | 26 ++++---- .../verser/src/lib/tecemux/codecs/index.ts | 36 ----------- packages/verser/src/lib/tecemux/constants.ts | 19 ++++++ .../src/lib/tecemux/playground-tecemux.ts | 47 +++++++------- packages/verser/src/lib/tecemux/tecemux.ts | 64 +++++++++---------- packages/verser/src/lib/tecemux/types.ts | 31 +++++++++ .../verser/src/lib/tecemux/utils/index.ts | 15 ++--- packages/verser/src/lib/verser-client.ts | 13 ++-- packages/verser/src/lib/verser-connection.ts | 4 +- yarn.lock | 30 +++++++-- 13 files changed, 175 insertions(+), 142 deletions(-) create mode 100644 packages/verser/src/lib/tecemux/constants.ts create mode 100644 packages/verser/src/lib/tecemux/types.ts diff --git a/package.json b/package.json index f12137d1e..fab95ff1f 100644 --- a/package.json +++ b/package.json @@ -77,7 +77,7 @@ "devDependencies": { "@istanbuljs/nyc-config-typescript": "^1.0.2", "@npmcli/run-script": "4.2.1", - "@types/node": "18.11.18", + "@types/node": "15.12.5", "@typescript-eslint/eslint-plugin": "^5.41.0", "@typescript-eslint/parser": "^5.41.0", "build-if-changed": "^1.5.5", diff --git a/packages/obj-logger/src/obj-logger.ts b/packages/obj-logger/src/obj-logger.ts index ef2f77b76..5222d3998 100644 --- a/packages/obj-logger/src/obj-logger.ts +++ b/packages/obj-logger/src/obj-logger.ts @@ -165,9 +165,11 @@ export class ObjLogger implements IObjectLogger { private _stringifiedOutput?: StringStream; get stringifiedOutput(): StringStream { - if (!this._stringifiedOutput) + if (!this._stringifiedOutput) { // eslint-disable-next-line no-console this._stringifiedOutput = this.output.JSONStringify().catch((e: any) => { console.error(e); }); + } + return this._stringifiedOutput; } diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 42bcc12a5..3382646c8 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -1,7 +1,7 @@ import { ObjLogger } from "@scramjet/obj-logger"; import { Duplex, Transform, TransformCallback, TransformOptions } from "stream"; -import { FrameData, HEADER_LENGTH, toHex } from "../utils"; -import { parseFlags } from "."; +import { FrameData, parseFlags, toHex } from "../utils"; +import { HEADER_LENGTH } from "../constants"; export class FrameDecoder extends Transform { buffer: Buffer; @@ -10,7 +10,11 @@ export class FrameDecoder extends Transform { _streams = new Map(); constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { - super(Object.assign({}, opts, { writableObjectMode: true, readableObjectMode: true, readableHighWaterMark: 2 })); + super(Object.assign({}, opts, { + writableObjectMode: true, + readableObjectMode: true, + readableHighWaterMark: 2 + })); this.buffer = Buffer.alloc(0); this.logger = new ObjLogger(params.name); @@ -19,7 +23,7 @@ export class FrameDecoder extends Transform { this.logger.debug("onPipe"); }).on("close", () => { this.logger.debug("onClose"); - }) + }); } _transform(chunk: Buffer, encoding: BufferEncoding, callback: TransformCallback): void { @@ -51,8 +55,12 @@ export class FrameDecoder extends Transform { } const payload = { - sourceAddress: [this.buffer.readInt8(0), this.buffer.readInt8(1), this.buffer.readInt8(2), this.buffer.readInt8(3)], - destinationAddress: [this.buffer.readInt8(4), this.buffer.readInt8(5), this.buffer.readInt8(6), this.buffer.readInt8(7)], + sourceAddress: [ + this.buffer.readInt8(0), this.buffer.readInt8(1), this.buffer.readInt8(2), this.buffer.readInt8(3) + ], + destinationAddress: [ + this.buffer.readInt8(4), this.buffer.readInt8(5), this.buffer.readInt8(6), this.buffer.readInt8(7) + ], chunk: this.buffer.subarray(32, this.buffer.readInt32LE(10)), flags: parseFlags(this.buffer.readInt8(25)), sourcePort: this.buffer.readInt16LE(12), @@ -67,15 +75,15 @@ export class FrameDecoder extends Transform { this.buffer = this.buffer.subarray(frameSize); - if (this.buffer.length === 0) { - this.logger.info("No remaining data!") + if (this.buffer.length === 0) { + this.logger.info("No remaining data!"); callback(); return; } this.logger.trace("More than one frame in chunk. processing", this.buffer.length); this._transform(Buffer.alloc(0), encoding, callback); - } catch(error) { + } catch (error) { this.logger.error("ERROR", error); this.emit("error", error); } diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index fc7bc9643..5c43d4d92 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -1,15 +1,16 @@ import { PassThrough, Transform, TransformCallback, TransformOptions } from "stream"; -import { FrameData, FrameTarget, HEADER_LENGTH, toHex as getHexString, toHex } from "../utils"; +import { FrameData, toHex as getHexString, toHex } from "../utils"; import { ObjLogger } from "@scramjet/obj-logger"; -import { binaryFlags, frameFlags } from "."; -import { TeceMux } from "../tecemux"; + +import { FrameTarget, HEADER_LENGTH, binaryFlags, frameFlags } from "../constants"; +import { ITeCeMux } from "../types"; export class FrameEncoder extends Transform { - tecemux: TeceMux; + tecemux: ITeCeMux; logger = new ObjLogger("FrameEncoder",); out = Object.assign(new PassThrough({ readableObjectMode: true }) .on("data", (data) => { - this.logger.trace("output to socket: " + (data.length == HEADER_LENGTH ? "HEADER ONLY" : ""), toHex(data), data.length, this.readableFlowing); + this.logger.trace("output to socket: " + (data.length === HEADER_LENGTH ? "HEADER ONLY" : ""), toHex(data), data.length, this.readableFlowing); }) .on("pause", () => { this.logger.trace("output to socket paused!"); @@ -24,11 +25,11 @@ export class FrameEncoder extends Transform { .on("error", (error) => { this.logger.error("output to socket error", error); }).pause() - ,{ + , { _id: this.frameTarget }); - constructor(private frameTarget: FrameTarget, tecemux: TeceMux, opts: TransformOptions = {}, params: { name: string } = { name: "FrameEncoder" }) { + constructor(private frameTarget: FrameTarget, tecemux: ITeCeMux, opts: TransformOptions = {}, params: { name: string } = { name: "FrameEncoder" }) { //opts.emitClose = false; //opts.readableObjectMode = true; super(Object.assign(opts, { @@ -46,8 +47,7 @@ export class FrameEncoder extends Transform { }) .on("end", () => { this.logger.debug("onEnd"); - }) - + }); this.pipe(this.out); } @@ -96,10 +96,10 @@ export class FrameEncoder extends Transform { // 64: zeroes (8bit), protocol (8bit), 8 - 9 new Uint8Array([0, 1]), - // tcp length (16bit) 10 - 11 + // tcp length (16bit) 10 - 11 new Uint8Array(new Uint16Array([chunk.length + HEADER_LENGTH]).buffer), - // 96: Source port, destination port 12 - 15 + // 96: Source port, destination port 12 - 15 new Uint8Array(new Uint16Array([0, frame.destinationPort || this.frameTarget]).buffer), // 128: sequenceNumber(32 bit, acnowledge number) 16 - 19 @@ -139,11 +139,9 @@ export class FrameEncoder extends Transform { this.once("drain", () => { this.push(buffer, undefined); }); - }; + } this.read(0); callback(); } - - } diff --git a/packages/verser/src/lib/tecemux/codecs/index.ts b/packages/verser/src/lib/tecemux/codecs/index.ts index 245e3e9fa..2a0fcd378 100644 --- a/packages/verser/src/lib/tecemux/codecs/index.ts +++ b/packages/verser/src/lib/tecemux/codecs/index.ts @@ -1,38 +1,2 @@ -import { Duplex } from "stream"; - export * from "./frame-decoder"; export * from "./frame-encoder"; - -export type TeceMuxEvents = { - channel(socket: Duplex): void; - error(error: any): void; -} - -export const binaryFlags = { - FIN: 0b00000001, - SYN: 0b00000010, - RST: 0b00000100, - PSH: 0b00001000, - ACK: 0b00010000, - URG: 0b00100000, - ECE: 0b01000000, - CWR: 0b10000000 -} - -export const frameFlags = Object.keys(binaryFlags); - -export type flagsObjectType = Partial<{ - FIN: boolean, - SYN: boolean, - RST: boolean, - PSH: boolean, - ACK: boolean, - URG: boolean, - ECE: boolean, - CWR: boolean -}> - -export const parseFlags = (byte: number): flagsObjectType => { - return frameFlags.filter((_flag, index) => byte >>> index & 1) - .reduce((p, c) => ({ ...p, [c]: true }), {}); -} diff --git a/packages/verser/src/lib/tecemux/constants.ts b/packages/verser/src/lib/tecemux/constants.ts new file mode 100644 index 000000000..4e4aa9f38 --- /dev/null +++ b/packages/verser/src/lib/tecemux/constants.ts @@ -0,0 +1,19 @@ +export const binaryFlags = { + FIN: 0b00000001, + SYN: 0b00000010, + RST: 0b00000100, + PSH: 0b00001000, + ACK: 0b00010000, + URG: 0b00100000, + ECE: 0b01000000, + CWR: 0b10000000 +}; + +export const frameFlags = Object.keys(binaryFlags); + +export const HEADER_LENGTH = 32; + +export enum FrameTarget { + API, + INPUT = 1001 +} diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index 8054f1ce9..7cca2a4c3 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -7,7 +7,8 @@ import { IncomingMessage, createServer } from "http"; import { DataStream } from "scramjet"; import { Socket, createConnection } from "net"; -import { TeceMux, TeceMuxChannel } from "./tecemux"; +import { TeceMux } from "./tecemux"; +import { TeceMuxChannel } from "./types"; (async () => { const logger = new ObjLogger("Sandbox"); @@ -16,12 +17,12 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; .pipe( new DataStream() .map(prettyPrint({ colors: true })) - .map((chunk: string) => ( + .map((chunk: string) => chunk.replace( /(:?FIN|SYN|RST|PSH|ACK|URG|ECE|CWR)|^$]/, - "\x1b[41m" + "$&" + "\x1b[0m" + "\x1b[41m\$&\x1b[0m" ) - )) + ) ) .pipe(process.stdout); @@ -32,7 +33,7 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; const PORT = 6660; const server = createServer({}); - server.on("timeout", (socket) => { + server.on("timeout", (_socket) => { logger.warn("Server on timeout"); }); server.requestTimeout = 0; @@ -66,7 +67,7 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; .on("timeout", () => { logger.info("Carrier Socket timeout"); }) - .pause() + .pause(); const tcmux = new TeceMux(socket, "Server") .on("error", (err) => { @@ -82,35 +83,34 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; logger.warn("Waiting for stdin..."); - DataStream.from(process.stdin).filter((x: Buffer) => !(parseInt(x[0].toString()) % 2)).pipe(channel1); - DataStream.from(process.stdin).filter((x: Buffer) => !!(parseInt(x[0].toString()) % 2)).pipe(channel2); + //DataStream.from(process.stdin).filter((x: Buffer) => !(parseInt(x[0].toString(), 10) % 2)).pipe(channel1); + //DataStream.from(process.stdin).filter((x: Buffer) => !!(parseInt(x[0].toString(), 10) % 2)).pipe(channel2); (async () => { try { for await (const chunk of channel1) { console.log("CHHUNK", chunk); - logger.debug(`reading CHANNEL1 chunk`, chunk.toString()); - }; + logger.debug("reading CHANNEL1 chunk", chunk.toString()); + } } catch (error) { - logger.error(`reading CHANNEL1 ERROR`, error); + logger.error("reading CHANNEL1 ERROR", error); } - logger.debug(`reading CHANNEL1 END`); + logger.debug("reading CHANNEL1 END"); })(); (async () => { try { for await (const chunk of channel2) { - logger.debug(`reading CHANNEL2 chunk`, chunk.toString()); - }; + logger.debug("reading CHANNEL2 chunk", chunk.toString()); + } } catch (error) { - logger.error(`reading CHANNEL2 ERROR`, error); + logger.error("reading CHANNEL2 ERROR", error); } - logger.debug(`reading CHANNEL2 END`); + logger.debug("reading CHANNEL2 END"); })(); - setTimeout(() => { console.log("\n\n\n\n"); logger.trace("Ending channels"); @@ -135,7 +135,8 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; socket.setNoDelay(true); - const reqLogger = new ObjLogger("Req", { id: "Client"}); + const reqLogger = new ObjLogger("Req", { id: "Client" }); + reqLogger.pipe(logger); socket.write("CONNECT HTTP/1.1\r\n\r\n\r\n"); @@ -144,7 +145,7 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; reqLogger.error("ERROR", error); }); - reqLogger.info('connected to server!'); + reqLogger.info("connected to server!"); const tcmux = new TeceMux(socket, "Request"); @@ -157,10 +158,10 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; channel .on("finish", () => { - tcmux.logger.info("Channel finish", channel._id) + tcmux.logger.info("Channel finish", channel._id); }) .on("end", () => { - tcmux.logger.info("Channel end", channel._id) + tcmux.logger.info("Channel end", channel._id); }); for await (const chunk of channel) { @@ -170,12 +171,12 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; } reqLogger.info("SERVER->CLIENT->CHANNEL", channel._id, chunk.toString()); - await new Promise((resolve, reject) => { + /*await new Promise((resolve, _reject) => { setTimeout(() => { //channel.write("abcde\n"); resolve(); }, 2000); - }); + });*/ } }); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 8b3a9d2cd..aaaf122da 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -1,17 +1,12 @@ import { TypedEmitter } from "@scramjet/utility"; -import { FrameDecoder, FrameEncoder, TeceMuxEvents } from "./codecs"; +import { FrameDecoder, FrameEncoder } from "./codecs"; import { Duplex, PassThrough } from "stream"; import { Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; import { FrameData } from "./utils"; +import { TeceMuxChannel, TeceMuxEvents } from "./types"; -export type TeceMuxChannel = Duplex & { - _id: number, - encoder: FrameEncoder, - closedByFIN: boolean -}; - -export class TeceMux extends TypedEmitter{ +export class TeceMux extends TypedEmitter { id: string; carrierSocket: Duplex; channelCount = 1; @@ -34,23 +29,22 @@ export class TeceMux extends TypedEmitter{ w.pipe(encoder, { end: false }).out.pipe(this.carrierSocket, { end: false }); - const duplex = new Duplex({ - write: (chunk, encoding, next) => { - this.logger.trace("WRITE channel", channel._id, chunk, chunk.toString() ); - - if (!w.push(chunk)) { - w.once("drain", next); - } else { - next(); - } - }, - read: (size) => { - this.logger.trace("READ channel", channel._id ); - }, - allowHalfOpen: true - }); const channel: TeceMuxChannel = Object.assign( - duplex, + new Duplex({ + write: (chunk, encoding, next) => { + this.logger.trace("WRITE channel", channel._id, chunk, chunk.toString()); + + if (!w.push(chunk)) { + w.once("drain", next); + } else { + next(); + } + }, + read: (_size) => { + this.logger.trace("READ channel", channel._id); + }, + allowHalfOpen: true + }), { _id: destinationPort || this.channelCount, encoder, @@ -64,14 +58,14 @@ export class TeceMux extends TypedEmitter{ channel .on("error", (error) => { - this.logger.error("CHANNEL ERROR", error) + this.logger.error("CHANNEL ERROR", error); //this.emit("error", { error, source: channel }) }) .on("destroy", () => { - this.logger.trace("channel on DESTROY ", channel._id ); + this.logger.trace("channel on DESTROY ", channel._id); }) .on("abort", () => { - this.logger.trace("channel on ABORT ", channel._id ); + this.logger.trace("channel on ABORT ", channel._id); }) .on("end", () => { this.logger.info("CHANNEL end", channel._id); @@ -90,7 +84,7 @@ export class TeceMux extends TypedEmitter{ constructor(socket: Socket, id = "") { super(); this.id = id; - this.logger = new ObjLogger(this, { id: this.id }) + this.logger = new ObjLogger(this, { id: this.id }); this.carrierSocket = socket; this.decoder = new FrameDecoder({ emitClose: false }) @@ -120,7 +114,7 @@ export class TeceMux extends TypedEmitter{ this.commonEncoder.out.pipe(this.carrierSocket, { end: false }); - this.commonEncoder.logger.updateBaseLog({ id }) + this.commonEncoder.logger.updateBaseLog({ id }); this.commonEncoder.logger.pipe(this.logger); this.main().catch((error) => { @@ -135,7 +129,7 @@ export class TeceMux extends TypedEmitter{ const frame = JSON.parse(chunk) as FrameData; const { flags, sequenceNumber, dataLength, destinationPort } = frame; - let channel = this.channels[destinationPort] + let channel = this.channels[destinationPort]; if (flags.ACK) { this.logger.trace("ACK frame received for sequenceNumber", sequenceNumber); @@ -148,14 +142,14 @@ export class TeceMux extends TypedEmitter{ if (channel) { //channel.end(); channel.closedByFIN = true; - channel.push(null) + channel.push(null); } else { this.logger.error("FIN for unknown channel"); } } if (flags.PSH) { - this.logger.trace(`Received PSH command [C: ${destinationPort}]`, dataLength, frame.chunk, !!this.channels[destinationPort]); + this.logger.trace(`Received PSH command [C: ${destinationPort}]`, dataLength); if (!channel) { this.logger.warn("Unknown channel"); @@ -201,7 +195,7 @@ export class TeceMux extends TypedEmitter{ } sendFIN(channel: number) { - this.logger.debug("Write acknowledge frame for sequenceNumber"); + this.logger.debug("Write FIN frame for channel", channel); this.commonEncoder.push( this.commonEncoder.createFrame(undefined, { flagsArray: ["FIN"], @@ -210,7 +204,7 @@ export class TeceMux extends TypedEmitter{ ); } - multiplex(opts: { channel?: number } = {} ): TeceMuxChannel { + multiplex(opts: { channel?: number } = {}): TeceMuxChannel { this.logger.trace("Multiplex"); const channel = this.createChannel(opts.channel || this.channelCount, true); @@ -221,3 +215,5 @@ export class TeceMux extends TypedEmitter{ return channel; } } +export { TeceMuxChannel }; + diff --git a/packages/verser/src/lib/tecemux/types.ts b/packages/verser/src/lib/tecemux/types.ts new file mode 100644 index 000000000..7d8c041f8 --- /dev/null +++ b/packages/verser/src/lib/tecemux/types.ts @@ -0,0 +1,31 @@ +import { Duplex } from "stream"; + +export type TeceMuxEvents = { + channel(socket: Duplex): void; + error(error: any): void; +} + +export type flagsObjectType = Partial<{ + FIN: boolean, + SYN: boolean, + RST: boolean, + PSH: boolean, + ACK: boolean, + URG: boolean, + ECE: boolean, + CWR: boolean +}> + +export interface ITeCeMux { + sequenceNumber: number; +} + +export interface IFrameEncoder { + +} + +export type TeceMuxChannel = Duplex & { + _id: number, + encoder: IFrameEncoder, + closedByFIN: boolean +}; diff --git a/packages/verser/src/lib/tecemux/utils/index.ts b/packages/verser/src/lib/tecemux/utils/index.ts index 8528167e9..dde25f260 100644 --- a/packages/verser/src/lib/tecemux/utils/index.ts +++ b/packages/verser/src/lib/tecemux/utils/index.ts @@ -1,16 +1,10 @@ -import { binaryFlags, flagsObjectType } from "../codecs"; +import { binaryFlags, frameFlags } from "../constants"; +import { flagsObjectType } from "../types"; export function toHex(chunk: Buffer) { return chunk.toString("hex").match(/../g)?.join(" "); } -export const HEADER_LENGTH = 32; - -export enum FrameTarget { - API, - INPUT = 1001 -} - export type FrameData = { sourceAddress: [number, number, number, number]; destinationAddress: [number, number, number, number]; @@ -24,3 +18,8 @@ export type FrameData = { flags: flagsObjectType; flagsArray: (keyof typeof binaryFlags)[]; }; + +export const parseFlags = (byte: number): flagsObjectType => { + return frameFlags.filter((_flag, index) => byte >>> index & 1) + .reduce((p, c) => ({ ...p, [c]: true }), {}); +}; diff --git a/packages/verser/src/lib/verser-client.ts b/packages/verser/src/lib/verser-client.ts index ef072c93c..3421ae57b 100644 --- a/packages/verser/src/lib/verser-client.ts +++ b/packages/verser/src/lib/verser-client.ts @@ -3,12 +3,11 @@ import { Agent as HttpsAgent, request } from "https"; import { merge, TypedEmitter } from "@scramjet/utility"; import { IObjectLogger } from "@scramjet/types"; import { VerserClientOptions, VerserClientConnection, RegisteredChannels, RegisteredChannelCallback } from "../types"; -import { Duplex } from "stream"; import { createConnection, Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; import { defaultVerserClientOptions } from "./verser-client-default-config"; import { URL } from "url"; -import { TeceMux } from "./tecemux/tecemux"; +import { TeceMux, TeceMuxChannel } from "./tecemux/tecemux"; type Events = { error: (err: Error) => void; @@ -125,14 +124,14 @@ export class VerserClient extends TypedEmitter { * otherwise stream will be passed to the server. */ private mux() { - this.teceMux = new TeceMux(this.socket!) - .on("channel", async (mSocket: Duplex & { _chan: number }) => { - const registeredChannelCallback = this.registeredChannels.get(mSocket._chan); + this.teceMux = new TeceMux(this.socket!, "client") + .on("channel", async (channel: TeceMuxChannel) => { + const registeredChannelCallback = this.registeredChannels.get(channel._id); if (registeredChannelCallback) { - await registeredChannelCallback(mSocket); + await registeredChannelCallback(channel); } else { - this.opts.server?.emit("connection", mSocket); + this.opts.server?.emit("connection", channel); } }) .on("error", (err: Error) => { diff --git a/packages/verser/src/lib/verser-connection.ts b/packages/verser/src/lib/verser-connection.ts index 08146d012..4928b4531 100644 --- a/packages/verser/src/lib/verser-connection.ts +++ b/packages/verser/src/lib/verser-connection.ts @@ -199,8 +199,8 @@ export class VerserConnection { reconnect() { this.logger.debug("Reconnecting..."); - this.teceMux = new TeceMux(this.socket).on("error", (error: Error) => { - this.logger.error("BPMux Error", error.message); + this.teceMux = new TeceMux(this.socket, "client").on("error", (error: Error) => { + this.logger.error("TeCeMux Error", error.message); // TODO: Error handling? }); diff --git a/yarn.lock b/yarn.lock index c96e1bb09..8e08e5d7f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -814,6 +814,22 @@ resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== +"@scramjet/obj-logger@^0.31.4": + version "0.31.4" + resolved "https://registry.yarnpkg.com/@scramjet/obj-logger/-/obj-logger-0.31.4.tgz#9c755403afa544afa31cc30dbca58913b95c8388" + integrity sha512-6sfS45IWkRMmdiG+2egr7FTyjpNW5Dc+n2WvzWbk6M/37/B30lRUtVkYGKTpIBbw/arp6lUuH0w/QjlC+QAiDQ== + dependencies: + "@scramjet/utility" "^0.31.4" + scramjet "^4.36.9" + +"@scramjet/utility@^0.31.4": + version "0.31.4" + resolved "https://registry.yarnpkg.com/@scramjet/utility/-/utility-0.31.4.tgz#9e13f30ef66636e77a7a065d1496f35e1dfc7017" + integrity sha512-BFwoYu+u5SbuJX5QgzrSgb+6Vak+cCAu141C5zasoAMplM4AmvqPWQAZ7CQ6Du7KQA7HsJqgD0xCHHky+mgoEA== + dependencies: + normalize-url "^5.3.1" + yaml "^2.1.3" + "@sindresorhus/is@^0.14.0": version "0.14.0" resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea" @@ -959,14 +975,9 @@ "@types/node" "*" form-data "^3.0.0" -"@types/node@*", "@types/node@18.11.18", "@types/node@^18.11.18": - version "18.11.18" - resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f" - integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA== - -"@types/node@15.12.5": +"@types/node@*", "@types/node@15.12.5": version "15.12.5" - resolved "https://registry.yarnpkg.com/@types/node/-/node-15.12.5.tgz#9a78318a45d75c9523d2396131bd3cca54b2d185" + resolved "https://registry.npmjs.org/@types/node/-/node-15.12.5.tgz" integrity sha512-se3yX7UHv5Bscf8f1ERKvQOD6sTyycH3hdaoozvaLxgUiY5lIGEeH37AD0G0Qi9kPqihPn0HOfd2yaIEN9VwEg== "@types/node@>=13.7.0": @@ -974,6 +985,11 @@ resolved "https://registry.npmjs.org/@types/node/-/node-18.11.3.tgz" integrity sha512-fNjDQzzOsZeKZu5NATgXUPsaFaTxeRgFXoosrHivTl8RGeV733OLawXsGfEk9a8/tySyZUyiZ6E8LcjPFZ2y1A== +"@types/node@^18.11.18": + version "18.11.18" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f" + integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA== + "@types/normalize-package-data@^2.4.0": version "2.4.1" resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301" From ae9f94bdff142d336e3384a307ce00233f2c188a Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 25 Jan 2023 13:02:13 +0000 Subject: [PATCH 020/231] Playground pass file --- .../src/lib/tecemux/codecs/frame-decoder.ts | 42 ++++++--- .../src/lib/tecemux/codecs/frame-encoder.ts | 73 ++++++++++----- .../verser/src/lib/tecemux/frames-keeper.ts | 36 +++++++ .../src/lib/tecemux/playground-tecemux.ts | 78 ++++++++++------ packages/verser/src/lib/tecemux/playground.ts | 93 +++++++++++++++++++ packages/verser/src/lib/tecemux/tecemux.ts | 82 +++++++++++----- yarn.lock | 5 + 7 files changed, 323 insertions(+), 86 deletions(-) create mode 100644 packages/verser/src/lib/tecemux/frames-keeper.ts create mode 100644 packages/verser/src/lib/tecemux/playground.ts diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 3382646c8..5d7da89a0 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -10,11 +10,7 @@ export class FrameDecoder extends Transform { _streams = new Map(); constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { - super(Object.assign({}, opts, { - writableObjectMode: true, - readableObjectMode: true, - readableHighWaterMark: 2 - })); + super(Object.assign({}, opts, { readableObjectMode: true})); this.buffer = Buffer.alloc(0); this.logger = new ObjLogger(params.name); @@ -29,9 +25,9 @@ export class FrameDecoder extends Transform { _transform(chunk: Buffer, encoding: BufferEncoding, callback: TransformCallback): void { try { if (!Buffer.isBuffer(chunk)) { - this.push(JSON.stringify({ error: "not a buffer" }), undefined); - callback(); + this.push(JSON.stringify({ error: "not a buffer" }), undefined); + callback(); return; } @@ -40,18 +36,32 @@ export class FrameDecoder extends Transform { } else { this.logger.error("Decoding buffer...", chunk); this.emit("error", "Chunk is not a buffer"); + callback(); + return; } - this.logger.trace("Decoding buffer...", toHex(this.buffer), "Size:", this.buffer.length); - let frameSize = 0; + this.logger.trace("Decoding buffer...", /* toHex(this.buffer) */ "Size:", this.buffer.length); + + if (this.buffer.length === 0) { + callback(null); + return; + } + if (this.buffer.length >= HEADER_LENGTH) { frameSize = this.buffer.readInt32LE(10); } else { this.logger.trace("To few data"); - callback(); + callback(null); + return; + } + + if (this.buffer.length < frameSize) { + this.logger.trace("To few data"); + callback(null); + return; } const payload = { @@ -68,13 +78,21 @@ export class FrameDecoder extends Transform { dataLength: frameSize - HEADER_LENGTH, chunkLength: frameSize, sequenceNumber: this.buffer.readInt32LE(16), - stringified: this.buffer.subarray(32, frameSize).toString() + acknowledgeNumber: this.buffer.readInt32LE(20), + //stringified: this.buffer.subarray(32, frameSize).toString() } as Partial; - this.push(JSON.stringify(payload) + "\n"); + if (payload.dataLength && payload.dataLength < 0) { + this.emit("error", "Data length incorrect"); + return; + } + + this.push(JSON.stringify(payload), "utf-8") this.buffer = this.buffer.subarray(frameSize); + this.logger.trace("Decoded", { ...payload, stringified: "--not-displayed--" }); + if (this.buffer.length === 0) { this.logger.info("No remaining data!"); callback(); diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index 5c43d4d92..d2e4a481c 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -7,35 +7,38 @@ import { ITeCeMux } from "../types"; export class FrameEncoder extends Transform { tecemux: ITeCeMux; - logger = new ObjLogger("FrameEncoder",); + total = 0; + logger = new ObjLogger("FrameEncoder"); out = Object.assign(new PassThrough({ readableObjectMode: true }) - .on("data", (data) => { + /*.on("data", (data) => { this.logger.trace("output to socket: " + (data.length === HEADER_LENGTH ? "HEADER ONLY" : ""), toHex(data), data.length, this.readableFlowing); - }) + })*/ .on("pause", () => { - this.logger.trace("output to socket paused!"); + this.logger.trace("OUT paused!"); }) .on("end", () => { - this.logger.trace("output to socket ended!", this.frameTarget); + this.logger.trace("OUT ended!", this.frameTarget); //this.tecemux.sendFIN(this.frameTarget); }) .on("resume", () => { - this.logger.trace("output to socket resumed"); + this.logger.trace("OUT resumed"); }) .on("error", (error) => { - this.logger.error("output to socket error", error); - }).pause() + this.logger.error("OUT error", error); + })//.pause() , { _id: this.frameTarget }); - constructor(private frameTarget: FrameTarget, tecemux: ITeCeMux, opts: TransformOptions = {}, params: { name: string } = { name: "FrameEncoder" }) { - //opts.emitClose = false; - //opts.readableObjectMode = true; + constructor( + private frameTarget: FrameTarget, + tecemux: ITeCeMux, + opts: TransformOptions = {}, + params: { name: string } = { name: "FrameEncoder" + }) { super(Object.assign(opts, { - writableObjectMode: true, readableObjectMode: true, - readableHighWaterMark: 0 + writableObjectMode: true })); this.tecemux = tecemux; @@ -47,6 +50,12 @@ export class FrameEncoder extends Transform { }) .on("end", () => { this.logger.debug("onEnd"); + }) + .on("pause", () => { + this.logger.debug("onPause"); + }) + .on("resume", () => { + this.logger.debug("onResume"); }); this.pipe(this.out); @@ -103,7 +112,7 @@ export class FrameEncoder extends Transform { new Uint8Array(new Uint16Array([0, frame.destinationPort || this.frameTarget]).buffer), // 128: sequenceNumber(32 bit, acnowledge number) 16 - 19 - new Uint8Array(new Uint32Array([this.tecemux.sequenceNumber++]).buffer), + new Uint8Array(new Uint32Array([this.tecemux.sequenceNumber]).buffer), // 160: Acknowledgement number 20-23 new Uint8Array(new Uint32Array([frame.acknowledgeNumber || 0]).buffer), @@ -129,19 +138,35 @@ export class FrameEncoder extends Transform { } _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void { - //const buffer = this.createChunkFrame(chunk); - this.logger.debug("TRANSFORM IN", toHex(chunk), chunk.length); - const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); + const MAX_CHUNK_SIZE = 10 * 1024 - HEADER_LENGTH; - this.logger.debug("TRANSFORM OUT", getHexString(buffer), "Size: ", buffer.length); + if (chunk) { - if (!this.push(buffer, undefined)) { - this.once("drain", () => { - this.push(buffer, undefined); - }); } - this.read(0); - callback(); + this.total += chunk.length; + + this.logger.debug("TRANSFORM IN", /* toHex(chunk), */ chunk.length, this.total); + + let remaining = Buffer.alloc(0); + + if (chunk.length > MAX_CHUNK_SIZE) { + this.logger.debug("TRANSFORM big chunk, splitting", chunk.length); + remaining = (chunk as Buffer).subarray(MAX_CHUNK_SIZE); + chunk = chunk.subarray(0, MAX_CHUNK_SIZE); + this.logger.debug("TRANSFORM processing part/remaining", chunk.length, remaining.length); + } + + const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); + this.tecemux.sequenceNumber++; + + this.logger.debug("TRANSFORM OUT", /*getHexString(buffer), */ "Size: ", buffer.length); + + if (remaining.length) { + this.push(buffer); + this._transform(remaining, encoding, callback); + } else { + callback(null, buffer); + } } } diff --git a/packages/verser/src/lib/tecemux/frames-keeper.ts b/packages/verser/src/lib/tecemux/frames-keeper.ts new file mode 100644 index 000000000..1aece983e --- /dev/null +++ b/packages/verser/src/lib/tecemux/frames-keeper.ts @@ -0,0 +1,36 @@ +import { ObjLogger } from "@scramjet/obj-logger"; +import { PassThrough } from "stream"; + +export class FramesKeeper extends PassThrough { + logger = new ObjLogger(this); + archive = new Map(); + lastSequenceNumberSent: number = -1; + + _write(chunk: any, encoding: BufferEncoding, cb: ((error: Error | null | undefined) => void) | undefined) { + const sequenceNumber = chunk.readInt32LE(16); + this.archive.set(sequenceNumber, chunk); + this.logger.debug(`sequenceNumber ${sequenceNumber} stored, size: ${chunk.length}`); + + this.push(chunk, encoding); + if (cb) cb(undefined); + + this.lastSequenceNumberSent = sequenceNumber; + + // await new Promise((resolve, reject) => { + // while (this.archive.get(sequenceNumber)) { + // this.logger.debug("wait for ack for", sequenceNumber) + // } + // resolve(); + // }); + + return true + } + + _read(size: number) { + } + + onACK(sequenceNumber: number) { + this.logger.debug("onACK", sequenceNumber); + this.archive.delete(sequenceNumber); + } +} diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index 7cca2a4c3..2c5686597 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -9,6 +9,9 @@ import { DataStream } from "scramjet"; import { Socket, createConnection } from "net"; import { TeceMux } from "./tecemux"; import { TeceMuxChannel } from "./types"; +import { createReadStream, createWriteStream } from "fs"; +import path from "path"; +import { Readable } from "stream"; (async () => { const logger = new ObjLogger("Sandbox"); @@ -67,7 +70,7 @@ import { TeceMuxChannel } from "./types"; .on("timeout", () => { logger.info("Carrier Socket timeout"); }) - .pause(); + const tcmux = new TeceMux(socket, "Server") .on("error", (err) => { @@ -77,19 +80,22 @@ import { TeceMuxChannel } from "./types"; tcmux.logger.pipe(logger); const channel1 = tcmux.multiplex(); - const channel2 = tcmux.multiplex(); + //const channel2 = tcmux.multiplex(); + + createReadStream(path.join(__dirname, "../../../../forever.tar.gz")).pipe(channel1); + //Readable.from(Buffer.alloc(1024 * 100)).pipe(channel1, { end: false }); req.on("pause", () => { logger.warn("Request paused"); }); - logger.warn("Waiting for stdin..."); + //logger.warn("Waiting for stdin..."); //DataStream.from(process.stdin).filter((x: Buffer) => !(parseInt(x[0].toString(), 10) % 2)).pipe(channel1); //DataStream.from(process.stdin).filter((x: Buffer) => !!(parseInt(x[0].toString(), 10) % 2)).pipe(channel2); - + /* (async () => { try { for await (const chunk of channel1) { - console.log("CHHUNK", chunk); + console.log("CHUNK", chunk); logger.debug("reading CHANNEL1 chunk", chunk.toString()); } } catch (error) { @@ -98,25 +104,27 @@ import { TeceMuxChannel } from "./types"; logger.debug("reading CHANNEL1 END"); })(); - - (async () => { - try { - for await (const chunk of channel2) { - logger.debug("reading CHANNEL2 chunk", chunk.toString()); - } - } catch (error) { - logger.error("reading CHANNEL2 ERROR", error); - } - - logger.debug("reading CHANNEL2 END"); - })(); - +*/ + // (async () => { + // try { + // for await (const chunk of channel2) { + // logger.debug("reading CHANNEL2 chunk", chunk.toString()); + // } + // } catch (error) { + // logger.error("reading CHANNEL2 ERROR", error); + // } + + // logger.debug("reading CHANNEL2 END"); + // })(); + + /* setTimeout(() => { console.log("\n\n\n\n"); logger.trace("Ending channels"); channel1.push(null); //channel2.end(); }, 4000); + */ }); server.listen(PORT, "0.0.0.0"); @@ -164,20 +172,38 @@ import { TeceMuxChannel } from "./types"; tcmux.logger.info("Channel end", channel._id); }); - for await (const chunk of channel) { + channel + //.pipe(process.stdout); + .pipe(createWriteStream(path.join(__dirname, "output.tar.gz"))); + + let total = 0; + channel + .on("data", (d) => { + total += d.length; + tcmux.logger.info("-------------------- data", channel._id, d.length, total); + + }) + .on("pause", () => { + tcmux.logger.info("-------------------- paused", channel._id); + }) + .on("resume", () => { + tcmux.logger.info("-------------------- resumed", channel._id); + }) + /*for await (const chunk of channel) { console.log(chunk); if (chunk === null) { break; } reqLogger.info("SERVER->CLIENT->CHANNEL", channel._id, chunk.toString()); - /*await new Promise((resolve, _reject) => { - setTimeout(() => { - //channel.write("abcde\n"); - resolve(); - }, 2000); - });*/ - } + // await new Promise((resolve, _reject) => { + // setTimeout(() => { + // //channel.write("abcde\n"); + // resolve(); + // }, 2000); + // }); + }*/ + }); socket.on("error", (err) => { diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/src/lib/tecemux/playground.ts new file mode 100644 index 000000000..33e3adf3a --- /dev/null +++ b/packages/verser/src/lib/tecemux/playground.ts @@ -0,0 +1,93 @@ +import { DataStream } from "scramjet"; +import { FrameEncoder } from "./codecs"; +import { FrameDecoder } from "./codecs"; +import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; +import { createReadStream, createWriteStream } from "fs"; +import path from "path"; +import { Duplex, PassThrough, Transform } from "stream"; +import { wrap } from "module"; + +const tcm = { + sequenceNumber: 0 +}; + +const logger = new ObjLogger("Sandbox"); + +logger.pipe( + new DataStream() + .map(prettyPrint({ colors: true })) + .map((chunk: string) => + chunk.replace( + /(:?FIN|SYN|RST|PSH|ACK|URG|ECE|CWR)|^$]/, + "\x1b[41m\$&\x1b[0m" + ) + ) +) +.pipe(process.stdout); + +const encoder = new FrameEncoder(0, tcm); +const decoder = new FrameDecoder(); + +encoder.logger.pipe(logger); +decoder.logger.pipe(logger); + +/* +process.stdin + .pipe(encoder).out + .pipe(decoder) + //.pipe(process.stdout); +*/ +let i = 0; +let t = 0; + +const ws = createWriteStream(path.join(__dirname, "output.tar.gz")) + +const delayedPassThrough = () => new PassThrough({ + async transform(chunk, encoding, callback) { + await new Promise((res,_rej)=> { + setTimeout(res, 1000); + }); + this.push(chunk, encoding) + callback(null) + }, +}); + +const dh = new Transform({ writableObjectMode: true, transform: (chunk, encoding, callback) => { + try { + console.log(i++, t); + + if (chunk && chunk.length) { + t += chunk.length; + + console.log({ ...JSON.parse(chunk.toString()).chunk, }); + if (!ws.write(new Uint8Array(JSON.parse(chunk.toString()).chunk.data))) { + dh.pause(); + ws.once("drain", () => { + callback(); + dh.resume(); + }) + } else { + callback(null); + } + } + + //callback(null); + } catch (e) { + debugger; + } + +}}) + +createReadStream(path.join(__dirname, "../../../../forever.tar.gz"), { encoding: undefined }) + .pipe(encoder, { end: false}).out + + .pipe(delayedPassThrough()) + .pipe(decoder) + .pipe(delayedPassThrough()) + .pipe(dh) + + // .pipe( + // ws + // ) + + diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index aaaf122da..13b06e479 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -1,16 +1,18 @@ import { TypedEmitter } from "@scramjet/utility"; import { FrameDecoder, FrameEncoder } from "./codecs"; -import { Duplex, PassThrough } from "stream"; +import { Duplex, PassThrough, Readable, ReadableOptions } from "stream"; import { Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; import { FrameData } from "./utils"; import { TeceMuxChannel, TeceMuxEvents } from "./types"; +import { FramesKeeper } from "./frames-keeper"; export class TeceMux extends TypedEmitter { id: string; carrierSocket: Duplex; channelCount = 1; decoder: FrameDecoder; + framesKeeper = new FramesKeeper(); sequenceNumber = 0; channels: TeceMuxChannel[] = []; @@ -25,20 +27,16 @@ export class TeceMux extends TypedEmitter { encoder.logger.updateBaseLog({ id: this.id }); encoder.logger.pipe(this.logger); - const w = new PassThrough({ encoding: undefined, allowHalfOpen: true }); + //const w = new PassThrough().on("error", (error) => { this.logger.error("W error", error)}); - w.pipe(encoder, { end: false }).out.pipe(this.carrierSocket, { end: false }); + //w.pipe(encoder) const channel: TeceMuxChannel = Object.assign( new Duplex({ write: (chunk, encoding, next) => { - this.logger.trace("WRITE channel", channel._id, chunk, chunk.toString()); + this.logger.trace("WRITE channel", channel._id, chunk); - if (!w.push(chunk)) { - w.once("drain", next); - } else { - next(); - } + return encoder.write(chunk, encoding, next); }, read: (_size) => { this.logger.trace("READ channel", channel._id); @@ -52,13 +50,14 @@ export class TeceMux extends TypedEmitter { } ); - process.nextTick(() => { - w.on("data", (d) => { this.logger.warn("channel writeable on DATA", d); }); - }); + encoder.out + .pipe(this.carrierSocket, { end: false }); channel .on("error", (error) => { this.logger.error("CHANNEL ERROR", error); + + debugger; //this.emit("error", { error, source: channel }) }) .on("destroy", () => { @@ -97,9 +96,9 @@ export class TeceMux extends TypedEmitter { .on("end", () => { this.logger.warn("Decoder ended"); }) - .on("error", (error) => { + /*.on("error", (error) => { this.logger.error("Decoder error", error); - }) + })*/ .on("abort", (error) => { this.logger.error("Decoder abort", error); }) @@ -117,22 +116,36 @@ export class TeceMux extends TypedEmitter { this.commonEncoder.logger.updateBaseLog({ id }); this.commonEncoder.logger.pipe(this.logger); + this.framesKeeper.logger.pipe(this.logger); + this.main().catch((error) => { this.emit("error", error); }); } async main() { + let t = 0; for await (const chunk of this.decoder) { - this.logger.debug("Decoded", JSON.parse(chunk)); + //console.log(chunk); + let frame: FrameData; + + try { + frame = JSON.parse(chunk) as FrameData; + } catch (error) { + console.debug( chunk.toString()) + this.logger.error("error Parsing data from decoder", error, chunk, chunk.length, chunk.toString()); + continue; + } + + //this.logger.debug("Decoded", { ...frame, stringified: "--not-displayed--" }); - const frame = JSON.parse(chunk) as FrameData; - const { flags, sequenceNumber, dataLength, destinationPort } = frame; + const { flags, sequenceNumber, dataLength, destinationPort, acknowledgeNumber } = frame; let channel = this.channels[destinationPort]; if (flags.ACK) { - this.logger.trace("ACK frame received for sequenceNumber", sequenceNumber); + this.logger.trace("ACK frame received for sequenceNumber", acknowledgeNumber); + //this.framesKeeper.onACK(sequenceNumber); continue; } @@ -146,10 +159,13 @@ export class TeceMux extends TypedEmitter { } else { this.logger.error("FIN for unknown channel"); } + + this.sendACK(sequenceNumber, destinationPort); + continue; } if (flags.PSH) { - this.logger.trace(`Received PSH command [C: ${destinationPort}]`, dataLength); + this.logger.trace(`Received PSH command [C: ${destinationPort}, SIZE: ${dataLength}]`); if (!channel) { this.logger.warn("Unknown channel"); @@ -160,14 +176,32 @@ export class TeceMux extends TypedEmitter { if (dataLength) { this.logger.warn("writing to channel [channel, length]", channel._id, dataLength); + this.logger.warn("writing to channel [flowing, isPaused]", channel.readableFlowing, channel.isPaused()); + + //Readable.from(chunk).pipe(channel, { end: false });//channel.write(new Uint8Array(((frame.chunk as any).data) as any)); + //channel.push(frame.chunk, undefined); + channel.push(new Uint8Array((frame.chunk as any).data), undefined); + + t += (frame.chunk as any).data.length; + this.logger.info("Writen", t) - const written = channel.push(new Uint8Array(((frame.chunk as any).data) as any)); - this.logger.info("Bytes written to channel [writeResult, channel, length]", written, destinationPort, dataLength); + /* + await new Promise((resolve, reject) => { + this.logger.debug("waiting for drain!") + channel.on("drain", () => { + //channel.push(new Uint8Array(((frame.chunk as any).data) as any)); + this.logger.debug("Drained!") + resolve(); + }); + }); + }*/ + + //this.logger.info("Bytes written to channel [writeResult, channel, length]", written, destinationPort, dataLength); } - } - this.sendACK(sequenceNumber, destinationPort); + this.sendACK(sequenceNumber, destinationPort); + } } } @@ -176,7 +210,7 @@ export class TeceMux extends TypedEmitter { this.commonEncoder.push( this.commonEncoder.createFrame(undefined, { flagsArray: ["ACK"], - sequenceNumber, + acknowledgeNumber: sequenceNumber, destinationPort: channel }) ); diff --git a/yarn.lock b/yarn.lock index 8e08e5d7f..d6a6a59e9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -980,6 +980,11 @@ resolved "https://registry.npmjs.org/@types/node/-/node-15.12.5.tgz" integrity sha512-se3yX7UHv5Bscf8f1ERKvQOD6sTyycH3hdaoozvaLxgUiY5lIGEeH37AD0G0Qi9kPqihPn0HOfd2yaIEN9VwEg== +"@types/node@18.11.18": + version "18.11.18" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f" + integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA== + "@types/node@>=13.7.0": version "18.11.3" resolved "https://registry.npmjs.org/@types/node/-/node-18.11.3.tgz" From a3a8b2736f35134e0ee709a72cfe7a285a2c0726 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 25 Jan 2023 18:47:39 +0000 Subject: [PATCH 021/231] Pass big file --- packages/verser/package.json | 2 +- .../src/lib/tecemux/codecs/frame-decoder.ts | 6 +- .../src/lib/tecemux/codecs/frame-encoder.ts | 9 ++- .../verser/src/lib/tecemux/frames-keeper.ts | 5 +- .../src/lib/tecemux/playground-tecemux.ts | 24 +------ packages/verser/src/lib/tecemux/playground.ts | 69 ++++++++----------- packages/verser/src/lib/tecemux/tecemux.ts | 29 ++------ packages/verser/src/lib/verser-client.ts | 2 +- packages/verser/src/lib/verser-connection.ts | 2 +- yarn.lock | 10 --- 10 files changed, 50 insertions(+), 108 deletions(-) diff --git a/packages/verser/package.json b/packages/verser/package.json index 4c29af24c..f2c426a17 100644 --- a/packages/verser/package.json +++ b/packages/verser/package.json @@ -25,7 +25,7 @@ "ts-node": "^10.9.1", "typedoc": "^0.23.17", "typedoc-plugin-markdown": "^3.13.6", - "typescript": "~4.9.4" + "typescript": "~4.7.4" }, "ava": { "extensions": [ diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 5d7da89a0..a4452891b 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -1,6 +1,6 @@ import { ObjLogger } from "@scramjet/obj-logger"; import { Duplex, Transform, TransformCallback, TransformOptions } from "stream"; -import { FrameData, parseFlags, toHex } from "../utils"; +import { FrameData, parseFlags } from "../utils"; import { HEADER_LENGTH } from "../constants"; export class FrameDecoder extends Transform { @@ -10,7 +10,7 @@ export class FrameDecoder extends Transform { _streams = new Map(); constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { - super(Object.assign({}, opts, { readableObjectMode: true})); + super(Object.assign({}, opts, { readableObjectMode: true })); this.buffer = Buffer.alloc(0); this.logger = new ObjLogger(params.name); @@ -87,7 +87,7 @@ export class FrameDecoder extends Transform { return; } - this.push(JSON.stringify(payload), "utf-8") + this.push(JSON.stringify(payload), "utf-8"); this.buffer = this.buffer.subarray(frameSize); diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index d2e4a481c..d31cf0138 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -1,5 +1,5 @@ import { PassThrough, Transform, TransformCallback, TransformOptions } from "stream"; -import { FrameData, toHex as getHexString, toHex } from "../utils"; +import { FrameData } from "../utils"; import { ObjLogger } from "@scramjet/obj-logger"; import { FrameTarget, HEADER_LENGTH, binaryFlags, frameFlags } from "../constants"; @@ -140,10 +140,6 @@ export class FrameEncoder extends Transform { _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void { const MAX_CHUNK_SIZE = 10 * 1024 - HEADER_LENGTH; - if (chunk) { - - } - this.total += chunk.length; this.logger.debug("TRANSFORM IN", /* toHex(chunk), */ chunk.length, this.total); @@ -152,12 +148,15 @@ export class FrameEncoder extends Transform { if (chunk.length > MAX_CHUNK_SIZE) { this.logger.debug("TRANSFORM big chunk, splitting", chunk.length); + remaining = (chunk as Buffer).subarray(MAX_CHUNK_SIZE); chunk = chunk.subarray(0, MAX_CHUNK_SIZE); + this.logger.debug("TRANSFORM processing part/remaining", chunk.length, remaining.length); } const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); + this.tecemux.sequenceNumber++; this.logger.debug("TRANSFORM OUT", /*getHexString(buffer), */ "Size: ", buffer.length); diff --git a/packages/verser/src/lib/tecemux/frames-keeper.ts b/packages/verser/src/lib/tecemux/frames-keeper.ts index 1aece983e..dc402e5db 100644 --- a/packages/verser/src/lib/tecemux/frames-keeper.ts +++ b/packages/verser/src/lib/tecemux/frames-keeper.ts @@ -8,6 +8,7 @@ export class FramesKeeper extends PassThrough { _write(chunk: any, encoding: BufferEncoding, cb: ((error: Error | null | undefined) => void) | undefined) { const sequenceNumber = chunk.readInt32LE(16); + this.archive.set(sequenceNumber, chunk); this.logger.debug(`sequenceNumber ${sequenceNumber} stored, size: ${chunk.length}`); @@ -23,10 +24,10 @@ export class FramesKeeper extends PassThrough { // resolve(); // }); - return true + return true; } - _read(size: number) { + _read(_size: number) { } onACK(sequenceNumber: number) { diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index 2c5686597..f66cbf171 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -11,7 +11,6 @@ import { TeceMux } from "./tecemux"; import { TeceMuxChannel } from "./types"; import { createReadStream, createWriteStream } from "fs"; import path from "path"; -import { Readable } from "stream"; (async () => { const logger = new ObjLogger("Sandbox"); @@ -69,8 +68,7 @@ import { Readable } from "stream"; }) .on("timeout", () => { logger.info("Carrier Socket timeout"); - }) - + }); const tcmux = new TeceMux(socket, "Server") .on("error", (err) => { @@ -173,37 +171,21 @@ import { Readable } from "stream"; }); channel - //.pipe(process.stdout); .pipe(createWriteStream(path.join(__dirname, "output.tar.gz"))); let total = 0; + channel .on("data", (d) => { total += d.length; tcmux.logger.info("-------------------- data", channel._id, d.length, total); - }) .on("pause", () => { tcmux.logger.info("-------------------- paused", channel._id); }) .on("resume", () => { tcmux.logger.info("-------------------- resumed", channel._id); - }) - /*for await (const chunk of channel) { - console.log(chunk); - if (chunk === null) { - break; - } - reqLogger.info("SERVER->CLIENT->CHANNEL", channel._id, chunk.toString()); - - // await new Promise((resolve, _reject) => { - // setTimeout(() => { - // //channel.write("abcde\n"); - // resolve(); - // }, 2000); - // }); - }*/ - + }); }); socket.on("error", (err) => { diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/src/lib/tecemux/playground.ts index 33e3adf3a..badd45084 100644 --- a/packages/verser/src/lib/tecemux/playground.ts +++ b/packages/verser/src/lib/tecemux/playground.ts @@ -4,8 +4,7 @@ import { FrameDecoder } from "./codecs"; import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; import { createReadStream, createWriteStream } from "fs"; import path from "path"; -import { Duplex, PassThrough, Transform } from "stream"; -import { wrap } from "module"; +import { PassThrough, Transform } from "stream"; const tcm = { sequenceNumber: 0 @@ -37,57 +36,49 @@ process.stdin .pipe(decoder) //.pipe(process.stdout); */ -let i = 0; -let t = 0; - -const ws = createWriteStream(path.join(__dirname, "output.tar.gz")) +const ws = createWriteStream(path.join(__dirname, "output.tar.gz")); const delayedPassThrough = () => new PassThrough({ async transform(chunk, encoding, callback) { - await new Promise((res,_rej)=> { + await new Promise((res,_rej) => { setTimeout(res, 1000); }); - this.push(chunk, encoding) - callback(null) + + this.push(chunk, encoding); + callback(null); }, }); -const dh = new Transform({ writableObjectMode: true, transform: (chunk, encoding, callback) => { - try { - console.log(i++, t); - - if (chunk && chunk.length) { - t += chunk.length; +let t = 0; - console.log({ ...JSON.parse(chunk.toString()).chunk, }); - if (!ws.write(new Uint8Array(JSON.parse(chunk.toString()).chunk.data))) { - dh.pause(); - ws.once("drain", () => { - callback(); - dh.resume(); - }) - } else { - callback(null); +const dh = new Transform({ + writableObjectMode: true, + transform: (chunk, encoding, callback) => { + try { + if (chunk && chunk.length) { + t = t + chunk.length; + + if (!ws.write(new Uint8Array(JSON.parse(chunk.toString()).chunk.data))) { + dh.pause(); + ws.once("drain", () => { + callback(); + dh.resume(); + }); + } else { + callback(null); + } } - } - //callback(null); - } catch (e) { - debugger; + //callback(null); + } catch (e) { + logger.error("dh error", e); + } } - -}}) +}); createReadStream(path.join(__dirname, "../../../../forever.tar.gz"), { encoding: undefined }) - .pipe(encoder, { end: false}).out - + .pipe(encoder, { end: false }).out .pipe(delayedPassThrough()) .pipe(decoder) .pipe(delayedPassThrough()) - .pipe(dh) - - // .pipe( - // ws - // ) - - + .pipe(dh); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 13b06e479..588ab7468 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -1,6 +1,6 @@ import { TypedEmitter } from "@scramjet/utility"; import { FrameDecoder, FrameEncoder } from "./codecs"; -import { Duplex, PassThrough, Readable, ReadableOptions } from "stream"; +import { Duplex } from "stream"; import { Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; import { FrameData } from "./utils"; @@ -56,9 +56,7 @@ export class TeceMux extends TypedEmitter { channel .on("error", (error) => { this.logger.error("CHANNEL ERROR", error); - - debugger; - //this.emit("error", { error, source: channel }) + this.emit("error", { error, source: channel }); }) .on("destroy", () => { this.logger.trace("channel on DESTROY ", channel._id); @@ -125,20 +123,17 @@ export class TeceMux extends TypedEmitter { async main() { let t = 0; + for await (const chunk of this.decoder) { - //console.log(chunk); let frame: FrameData; try { frame = JSON.parse(chunk) as FrameData; } catch (error) { - console.debug( chunk.toString()) this.logger.error("error Parsing data from decoder", error, chunk, chunk.length, chunk.toString()); continue; } - //this.logger.debug("Decoded", { ...frame, stringified: "--not-displayed--" }); - const { flags, sequenceNumber, dataLength, destinationPort, acknowledgeNumber } = frame; let channel = this.channels[destinationPort]; @@ -178,26 +173,10 @@ export class TeceMux extends TypedEmitter { this.logger.warn("writing to channel [channel, length]", channel._id, dataLength); this.logger.warn("writing to channel [flowing, isPaused]", channel.readableFlowing, channel.isPaused()); - //Readable.from(chunk).pipe(channel, { end: false });//channel.write(new Uint8Array(((frame.chunk as any).data) as any)); - //channel.push(frame.chunk, undefined); channel.push(new Uint8Array((frame.chunk as any).data), undefined); t += (frame.chunk as any).data.length; - this.logger.info("Writen", t) - - - /* - await new Promise((resolve, reject) => { - this.logger.debug("waiting for drain!") - channel.on("drain", () => { - //channel.push(new Uint8Array(((frame.chunk as any).data) as any)); - this.logger.debug("Drained!") - resolve(); - }); - }); - }*/ - - //this.logger.info("Bytes written to channel [writeResult, channel, length]", written, destinationPort, dataLength); + this.logger.info("Writen", t); } this.sendACK(sequenceNumber, destinationPort); diff --git a/packages/verser/src/lib/verser-client.ts b/packages/verser/src/lib/verser-client.ts index 3421ae57b..2ed5235f2 100644 --- a/packages/verser/src/lib/verser-client.ts +++ b/packages/verser/src/lib/verser-client.ts @@ -138,7 +138,7 @@ export class VerserClient extends TypedEmitter { this.emit("error", err); }); - this.teceMux.logger.pipe(this.logger); + //this.teceMux.logger.pipe(this.logger); this._verserAgent = new HttpAgent() as HttpAgent & { createConnection: typeof createConnection diff --git a/packages/verser/src/lib/verser-connection.ts b/packages/verser/src/lib/verser-connection.ts index 4928b4531..c8c10e470 100644 --- a/packages/verser/src/lib/verser-connection.ts +++ b/packages/verser/src/lib/verser-connection.ts @@ -204,7 +204,7 @@ export class VerserConnection { // TODO: Error handling? }); - this.teceMux.logger.pipe(this.logger); + //this.teceMux.logger.pipe(this.logger); this.agent = new Agent() as Agent & { createConnection: typeof createConnection }; // lack of types? this.agent.createConnection = () => { diff --git a/yarn.lock b/yarn.lock index d6a6a59e9..1bbc97518 100644 --- a/yarn.lock +++ b/yarn.lock @@ -980,11 +980,6 @@ resolved "https://registry.npmjs.org/@types/node/-/node-15.12.5.tgz" integrity sha512-se3yX7UHv5Bscf8f1ERKvQOD6sTyycH3hdaoozvaLxgUiY5lIGEeH37AD0G0Qi9kPqihPn0HOfd2yaIEN9VwEg== -"@types/node@18.11.18": - version "18.11.18" - resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f" - integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA== - "@types/node@>=13.7.0": version "18.11.3" resolved "https://registry.npmjs.org/@types/node/-/node-18.11.3.tgz" @@ -7392,11 +7387,6 @@ typescript@~4.7.4: resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.7.4.tgz#1a88596d1cf47d59507a1bcdfb5b9dfe4d488235" integrity sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ== -typescript@~4.9.4: - version "4.9.4" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.4.tgz#a2a3d2756c079abda241d75f149df9d561091e78" - integrity sha512-Uz+dTXYzxXXbsFpM86Wh3dKCxrQqUcVMxwU54orwlJjOpO3ao8L7j5lH+dWfTwgCwIuM9GQ2kvVotzYJMXTBZg== - uglify-js@^3.1.4: version "3.17.4" resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.17.4.tgz#61678cf5fa3f5b7eb789bb345df29afb8257c22c" From 26079ec38f9ca92efb5098548d68d4a04c216b58 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Sat, 28 Jan 2023 15:25:28 +0000 Subject: [PATCH 022/231] Calc/check checksum --- .../src/lib/tecemux/codecs/frame-decoder.ts | 15 ++++++++ .../src/lib/tecemux/codecs/frame-encoder.ts | 16 +++++---- .../verser/src/lib/tecemux/codecs/utils.ts | 25 +++++++++++++ packages/verser/src/lib/tecemux/tecemux.ts | 11 ++++-- .../verser/src/lib/tecemux/utils/index.ts | 1 + yarn.lock | 36 +++++++++++++++++++ 6 files changed, 95 insertions(+), 9 deletions(-) create mode 100644 packages/verser/src/lib/tecemux/codecs/utils.ts diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index a4452891b..1aa7df77b 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -2,6 +2,7 @@ import { ObjLogger } from "@scramjet/obj-logger"; import { Duplex, Transform, TransformCallback, TransformOptions } from "stream"; import { FrameData, parseFlags } from "../utils"; import { HEADER_LENGTH } from "../constants"; +import { calculateChecksum, getChecksum } from "./utils"; export class FrameDecoder extends Transform { buffer: Buffer; @@ -64,6 +65,10 @@ export class FrameDecoder extends Transform { return; } + const checksum = getChecksum(this.buffer.subarray(0, frameSize)); + + this.logger.debug("getChecksum", checksum.toString()); + const payload = { sourceAddress: [ this.buffer.readInt8(0), this.buffer.readInt8(1), this.buffer.readInt8(2), this.buffer.readInt8(3) @@ -79,9 +84,19 @@ export class FrameDecoder extends Transform { chunkLength: frameSize, sequenceNumber: this.buffer.readInt32LE(16), acknowledgeNumber: this.buffer.readInt32LE(20), + checksum //stringified: this.buffer.subarray(32, frameSize).toString() } as Partial; + const expectedChecksum = calculateChecksum(this.buffer.subarray(0, frameSize)); + + if (checksum !== expectedChecksum) { + this.emit("error", { code: "INVALID_CHECKSUM", payload, expectedChecksum }); + payload.error = "checksum"; + } else { + this.logger.info("Checksum match!", expectedChecksum); + } + if (payload.dataLength && payload.dataLength < 0) { this.emit("error", "Data length incorrect"); return; diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index d31cf0138..6f16e9875 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -4,6 +4,7 @@ import { ObjLogger } from "@scramjet/obj-logger"; import { FrameTarget, HEADER_LENGTH, binaryFlags, frameFlags } from "../constants"; import { ITeCeMux } from "../types"; +import { calculateChecksum } from "./utils"; export class FrameEncoder extends Transform { tecemux: ITeCeMux; @@ -34,8 +35,8 @@ export class FrameEncoder extends Transform { private frameTarget: FrameTarget, tecemux: ITeCeMux, opts: TransformOptions = {}, - params: { name: string } = { name: "FrameEncoder" - }) { + params: { name: string } = { name: "FrameEncoder" } + ) { super(Object.assign(opts, { readableObjectMode: true, writableObjectMode: true @@ -95,7 +96,6 @@ export class FrameEncoder extends Transform { } createFrame(chunk: any = new Uint8Array([]), frame: Partial) { - const checksum = this.getChecksum(); const buffer = Buffer.concat([ // 0: source address 0 - 3 new Uint8Array([10, 0, 0, 1]), @@ -122,18 +122,22 @@ export class FrameEncoder extends Transform { // 224: flags (8bit), 25 this.setFlags(frame.flagsArray, new Uint8Array([0b00000000])), - // window(16bit) 26 - 27 + // window(16bit) 26 - 27, ZEROes before calculation new Uint8Array(new Uint16Array([0]).buffer), // checksum(16bit) 28 - 29 - new Uint8Array(new Uint16Array([checksum]).buffer), + new Uint8Array(new Uint16Array([0]).buffer), // pointer (16bit) 30 - 31 - new Uint8Array(new Uint16Array([checksum]).buffer), + new Uint8Array(new Uint16Array([0]).buffer), // 256: data 32 - new Uint8Array(chunk) ]); + buffer.writeUInt16LE( + calculateChecksum(buffer), 28 + ); + return buffer; } diff --git a/packages/verser/src/lib/tecemux/codecs/utils.ts b/packages/verser/src/lib/tecemux/codecs/utils.ts new file mode 100644 index 000000000..98db2e4bf --- /dev/null +++ b/packages/verser/src/lib/tecemux/codecs/utils.ts @@ -0,0 +1,25 @@ +export const calculateChecksum = (buffer: Buffer) => { + let tempFrame = Buffer.concat([buffer, Buffer.alloc(0)]); + + if (buffer.length % 1) { + tempFrame = Buffer.concat([buffer, Buffer.alloc(1, 0)]); + } + + let checksum = 0; + let i = 0; + + while (i <= tempFrame.length - 2) { + if (i !== 28) { + checksum += tempFrame.readUInt16LE(i); + } + + i += 2; + } + + return checksum % 0x10000; +}; + +export const getChecksum = (buffer: Buffer) => { + return buffer.readUInt16LE(28); +}; + diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 588ab7468..c7c4ea150 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -129,12 +129,17 @@ export class TeceMux extends TypedEmitter { try { frame = JSON.parse(chunk) as FrameData; - } catch (error) { - this.logger.error("error Parsing data from decoder", error, chunk, chunk.length, chunk.toString()); + } catch (err) { + this.logger.error("error Parsing data from decoder", err, chunk, chunk.length, chunk.toString()); continue; } - const { flags, sequenceNumber, dataLength, destinationPort, acknowledgeNumber } = frame; + const { flags, sequenceNumber, dataLength, destinationPort, acknowledgeNumber, error } = frame; + + if (error) { + this.emit("error", frame); + continue; + } let channel = this.channels[destinationPort]; diff --git a/packages/verser/src/lib/tecemux/utils/index.ts b/packages/verser/src/lib/tecemux/utils/index.ts index dde25f260..91dc00170 100644 --- a/packages/verser/src/lib/tecemux/utils/index.ts +++ b/packages/verser/src/lib/tecemux/utils/index.ts @@ -17,6 +17,7 @@ export type FrameData = { stringified: string; flags: flagsObjectType; flagsArray: (keyof typeof binaryFlags)[]; + error?: "checksum" }; export const parseFlags = (byte: number): flagsObjectType => { diff --git a/yarn.lock b/yarn.lock index 1bbc97518..df5227dda 100644 --- a/yarn.lock +++ b/yarn.lock @@ -814,6 +814,29 @@ resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== +"@scramjet/api-server@^0.31.4": + version "0.31.4" + resolved "https://registry.yarnpkg.com/@scramjet/api-server/-/api-server-0.31.4.tgz#a2855de120c1093d6cfd96b50447babd62116823" + integrity sha512-ghhKbK6omrTJIT4xgRhT5bLUE/Hk+ShmM3/J9XwcVwxGgltqtEC10k1G6Dm42vqYnUxLgW3iIsmW9OMCNh2jfw== + dependencies: + "0http" "^3.4.1" + "@scramjet/model" "^0.31.4" + "@scramjet/obj-logger" "^0.31.4" + "@scramjet/symbols" "^0.31.4" + "@scramjet/utility" "^0.31.4" + http-status-codes "^2.2.0" + scramjet "^4.36.9" + +"@scramjet/model@^0.31.4": + version "0.31.4" + resolved "https://registry.yarnpkg.com/@scramjet/model/-/model-0.31.4.tgz#2cc4e9a9855abeee0e450c7a9d66525bfdb03301" + integrity sha512-NkM/6DhVpN0tsE/1aCY9v2EOSxq3IecBtwtnaz9EYEAb0XTt78hkseALevzvOLDvc4G+F0BqmZKeqHWxbAJFgg== + dependencies: + "@scramjet/obj-logger" "^0.31.4" + "@scramjet/symbols" "^0.31.4" + scramjet "^4.36.9" + uuid "^8.3.2" + "@scramjet/obj-logger@^0.31.4": version "0.31.4" resolved "https://registry.yarnpkg.com/@scramjet/obj-logger/-/obj-logger-0.31.4.tgz#9c755403afa544afa31cc30dbca58913b95c8388" @@ -822,6 +845,19 @@ "@scramjet/utility" "^0.31.4" scramjet "^4.36.9" +"@scramjet/symbols@^0.31.4": + version "0.31.4" + resolved "https://registry.yarnpkg.com/@scramjet/symbols/-/symbols-0.31.4.tgz#f1ccaeee59b26530caf47fe021bf45727f6da4f0" + integrity sha512-dJWBd7VgMU627SlXVg27n3F/aBzvUHTEGMCTq5N46W91OeU/YhjRT+dO5NyJhAxpyTHdlMFCHKAFC9FrGzdoMw== + +"@scramjet/types@^0.31.4": + version "0.31.4" + resolved "https://registry.yarnpkg.com/@scramjet/types/-/types-0.31.4.tgz#73e298bbcf01abf7a378b26fa0da9a9093d45341" + integrity sha512-RxJzbRD+q4w5RrlaT5cg6GdlMzlXnDrqBSKaAeDdzu/2OgApHJOBE4roP1hqtwQkgtQvXMNeF8uUJM0IFCsjSQ== + dependencies: + "@scramjet/symbols" "^0.31.4" + http-status-codes "^2.2.0" + "@scramjet/utility@^0.31.4": version "0.31.4" resolved "https://registry.yarnpkg.com/@scramjet/utility/-/utility-0.31.4.tgz#9e13f30ef66636e77a7a065d1496f35e1dfc7017" From 8300d049f2f87931aad70738b0360b5bf3914dfc Mon Sep 17 00:00:00 2001 From: patuwwy Date: Sun, 29 Jan 2023 21:33:55 +0000 Subject: [PATCH 023/231] FramesKeeper --- .../src/lib/tecemux/codecs/frame-encoder.ts | 30 +++++---- .../verser/src/lib/tecemux/frames-keeper.ts | 63 +++++++++++-------- .../src/lib/tecemux/playground-tecemux.ts | 2 +- packages/verser/src/lib/tecemux/playground.ts | 7 +-- packages/verser/src/lib/tecemux/tecemux.ts | 23 +++---- .../verser/src/lib/tecemux/utils/index.ts | 3 +- 6 files changed, 72 insertions(+), 56 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index 6f16e9875..98cadb502 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -12,7 +12,9 @@ export class FrameEncoder extends Transform { logger = new ObjLogger("FrameEncoder"); out = Object.assign(new PassThrough({ readableObjectMode: true }) /*.on("data", (data) => { - this.logger.trace("output to socket: " + (data.length === HEADER_LENGTH ? "HEADER ONLY" : ""), toHex(data), data.length, this.readableFlowing); + this.logger.trace( + "output to socket: " + (data.length === HEADER_LENGTH ? "HEADER ONLY" : ""), + toHex(data), data.length, this.readableFlowing); })*/ .on("pause", () => { this.logger.trace("OUT paused!"); @@ -64,7 +66,9 @@ export class FrameEncoder extends Transform { setFlags(flag: (keyof typeof binaryFlags)[] = [], flags: Uint8Array = new Uint8Array([0])) { for (const f in flag) { - flags[0] |= 1 << frameFlags.indexOf(flag[f]); + if (Object.prototype.hasOwnProperty.call(flag, f)) { + flags[0] |= 1 << frameFlags.indexOf(flag[f]); + } } this.logger.debug("settingFlag", flag, flags[0].toString(2).padStart(8, "0")); @@ -75,20 +79,22 @@ export class FrameEncoder extends Transform { setChannel(channelCount: number) { this.logger.debug("Set channel command", channelCount); - //const checksum = this.getChecksum(); - - this.out.write(this.createFrame([], { - flagsArray: ["PSH"], - destinationPort: channelCount - })); + this.out.write( + this.createFrame([], { + flagsArray: ["PSH"], + destinationPort: channelCount + }) + ); } onChannelEnd(channelId: number) { this.logger.debug("sending FIN for channel", channelId); - this.out.write(this.createFrame([], { - flagsArray: ["FIN"], - destinationPort: channelId - })); + this.out.write( + this.createFrame([], { + flagsArray: ["FIN"], + destinationPort: channelId + }) + ); } getChecksum() { diff --git a/packages/verser/src/lib/tecemux/frames-keeper.ts b/packages/verser/src/lib/tecemux/frames-keeper.ts index dc402e5db..40dae5918 100644 --- a/packages/verser/src/lib/tecemux/frames-keeper.ts +++ b/packages/verser/src/lib/tecemux/frames-keeper.ts @@ -1,37 +1,46 @@ import { ObjLogger } from "@scramjet/obj-logger"; -import { PassThrough } from "stream"; - -export class FramesKeeper extends PassThrough { - logger = new ObjLogger(this); - archive = new Map(); - lastSequenceNumberSent: number = -1; - - _write(chunk: any, encoding: BufferEncoding, cb: ((error: Error | null | undefined) => void) | undefined) { - const sequenceNumber = chunk.readInt32LE(16); - - this.archive.set(sequenceNumber, chunk); - this.logger.debug(`sequenceNumber ${sequenceNumber} stored, size: ${chunk.length}`); +import { TransformOptions, Writable } from "stream"; +import { IObjectLogger } from "@scramjet/types"; + +export class FramesKeeper extends Writable { + logger: IObjectLogger; + framesSent = new Map(); + lastSequenceSent: number = -1; + lastSequenceReceived: number = -1; + + constructor( + opts: TransformOptions = {}, + params: { name: string } = { name: "Keeper" } + ) { + super(Object.assign(opts, { + readableObjectMode: true, + writableObjectMode: true + })); + + this.logger = new ObjLogger(params.name); + } - this.push(chunk, encoding); - if (cb) cb(undefined); + _write(chunk: any, encoding: BufferEncoding, cb: ((error: Error | null | undefined) => void)) { + if (Buffer.isBuffer(chunk)) { + this.logger.info("transform buffer"); + const sequenceNumber = chunk.readInt32LE(16); - this.lastSequenceNumberSent = sequenceNumber; + this.framesSent.set(sequenceNumber, { buffer: chunk, received: false, sequenceNumber }); - // await new Promise((resolve, reject) => { - // while (this.archive.get(sequenceNumber)) { - // this.logger.debug("wait for ack for", sequenceNumber) - // } - // resolve(); - // }); + this.lastSequenceSent = sequenceNumber; + this.logger.debug(`lastSequenceSent ${sequenceNumber}, size: ${chunk.length}`); + } - return true; + cb(undefined); } - _read(_size: number) { - } + onACK(acknowledgeNumber: number,) { + this.logger.debug("onACK", acknowledgeNumber); + + const storedFrame = this.framesSent.get(acknowledgeNumber); - onACK(sequenceNumber: number) { - this.logger.debug("onACK", sequenceNumber); - this.archive.delete(sequenceNumber); + if (storedFrame) { + this.framesSent.set(acknowledgeNumber, { ...storedFrame, received: true }); + } } } diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index f66cbf171..7b3586db3 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -89,7 +89,7 @@ import path from "path"; //DataStream.from(process.stdin).filter((x: Buffer) => !(parseInt(x[0].toString(), 10) % 2)).pipe(channel1); //DataStream.from(process.stdin).filter((x: Buffer) => !!(parseInt(x[0].toString(), 10) % 2)).pipe(channel2); - /* + /* (async () => { try { for await (const chunk of channel1) { diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/src/lib/tecemux/playground.ts index badd45084..304316f47 100644 --- a/packages/verser/src/lib/tecemux/playground.ts +++ b/packages/verser/src/lib/tecemux/playground.ts @@ -1,6 +1,5 @@ import { DataStream } from "scramjet"; -import { FrameEncoder } from "./codecs"; -import { FrameDecoder } from "./codecs"; +import { FrameDecoder, FrameEncoder } from "./codecs"; import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; import { createReadStream, createWriteStream } from "fs"; import path from "path"; @@ -22,7 +21,7 @@ logger.pipe( ) ) ) -.pipe(process.stdout); + .pipe(process.stdout); const encoder = new FrameEncoder(0, tcm); const decoder = new FrameDecoder(); @@ -40,7 +39,7 @@ const ws = createWriteStream(path.join(__dirname, "output.tar.gz")); const delayedPassThrough = () => new PassThrough({ async transform(chunk, encoding, callback) { - await new Promise((res,_rej) => { + await new Promise((res, _rej) => { setTimeout(res, 1000); }); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index c7c4ea150..dd60a0b4d 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -11,7 +11,7 @@ export class TeceMux extends TypedEmitter { id: string; carrierSocket: Duplex; channelCount = 1; - decoder: FrameDecoder; + carrierDecoder: FrameDecoder; framesKeeper = new FramesKeeper(); sequenceNumber = 0; channels: TeceMuxChannel[] = []; @@ -27,10 +27,6 @@ export class TeceMux extends TypedEmitter { encoder.logger.updateBaseLog({ id: this.id }); encoder.logger.pipe(this.logger); - //const w = new PassThrough().on("error", (error) => { this.logger.error("W error", error)}); - - //w.pipe(encoder) - const channel: TeceMuxChannel = Object.assign( new Duplex({ write: (chunk, encoding, next) => { @@ -50,6 +46,9 @@ export class TeceMux extends TypedEmitter { } ); + encoder.out + .pipe(this.framesKeeper); + encoder.out .pipe(this.carrierSocket, { end: false }); @@ -84,7 +83,7 @@ export class TeceMux extends TypedEmitter { this.logger = new ObjLogger(this, { id: this.id }); this.carrierSocket = socket; - this.decoder = new FrameDecoder({ emitClose: false }) + this.carrierDecoder = new FrameDecoder({ emitClose: false }) .on("pause", () => { this.logger.warn("Decoder paused"); }) @@ -104,10 +103,10 @@ export class TeceMux extends TypedEmitter { this.logger.error("Decoder destroy", error); }); - this.decoder.logger.updateBaseLog({ id: this.id }); - this.decoder.logger.pipe(this.logger); + this.carrierDecoder.logger.updateBaseLog({ id: this.id }); + this.carrierDecoder.logger.pipe(this.logger); - this.carrierSocket.pipe(this.decoder, { end: false }); + this.carrierSocket.pipe(this.carrierDecoder, { end: false }); this.commonEncoder.out.pipe(this.carrierSocket, { end: false }); @@ -124,7 +123,9 @@ export class TeceMux extends TypedEmitter { async main() { let t = 0; - for await (const chunk of this.decoder) { + //this.carrierDecoder.pipe(this.framesKeeper); + + for await (const chunk of this.carrierDecoder) { let frame: FrameData; try { @@ -145,7 +146,7 @@ export class TeceMux extends TypedEmitter { if (flags.ACK) { this.logger.trace("ACK frame received for sequenceNumber", acknowledgeNumber); - //this.framesKeeper.onACK(sequenceNumber); + this.framesKeeper.onACK(acknowledgeNumber); continue; } diff --git a/packages/verser/src/lib/tecemux/utils/index.ts b/packages/verser/src/lib/tecemux/utils/index.ts index 91dc00170..b90056eb7 100644 --- a/packages/verser/src/lib/tecemux/utils/index.ts +++ b/packages/verser/src/lib/tecemux/utils/index.ts @@ -17,7 +17,8 @@ export type FrameData = { stringified: string; flags: flagsObjectType; flagsArray: (keyof typeof binaryFlags)[]; - error?: "checksum" + error?: "checksum"; + received: boolean; }; export const parseFlags = (byte: number): flagsObjectType => { From 1408c720be1e7a8f5ce0c37178f31c4e20b51d61 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 30 Jan 2023 13:19:32 +0000 Subject: [PATCH 024/231] Check for ACK of last frames send --- .../src/lib/tecemux/codecs/frame-encoder.ts | 26 +++++++++++++--- packages/verser/src/lib/tecemux/constants.ts | 2 ++ .../verser/src/lib/tecemux/frames-keeper.ts | 31 ++++++++++++++++--- packages/verser/src/lib/tecemux/playground.ts | 13 +++----- packages/verser/src/lib/tecemux/types.ts | 15 +++++++++ 5 files changed, 71 insertions(+), 16 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index 98cadb502..d11f9d32d 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -2,7 +2,7 @@ import { PassThrough, Transform, TransformCallback, TransformOptions } from "str import { FrameData } from "../utils"; import { ObjLogger } from "@scramjet/obj-logger"; -import { FrameTarget, HEADER_LENGTH, binaryFlags, frameFlags } from "../constants"; +import { ACK_FRAME_DELTA, FrameTarget, HEADER_LENGTH, binaryFlags, frameFlags } from "../constants"; import { ITeCeMux } from "../types"; import { calculateChecksum } from "./utils"; @@ -147,7 +147,7 @@ export class FrameEncoder extends Transform { return buffer; } - _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void { + async _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): Promise { const MAX_CHUNK_SIZE = 10 * 1024 - HEADER_LENGTH; this.total += chunk.length; @@ -167,13 +167,31 @@ export class FrameEncoder extends Transform { const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); - this.tecemux.sequenceNumber++; + if (this.tecemux.sequenceNumber > -ACK_FRAME_DELTA) { + // eslint-disable-next-line no-loop-func + while (!await new Promise((res, _rej) => { + setImmediate(() => { + const frame = this.tecemux.framesKeeper.getFrame(this.tecemux.sequenceNumber + ACK_FRAME_DELTA); + + if (frame) { + const rec = frame?.received || frame?.flags.ACK; + + this.logger.info(`Sending ${this.tecemux.sequenceNumber}, ${frame?.sequenceNumber} ${frame?.received}`); + res(!!rec); + } else { + _rej("frame not exists"); + } + }); + })); + } + + this.tecemux.sequenceNumber++; this.logger.debug("TRANSFORM OUT", /*getHexString(buffer), */ "Size: ", buffer.length); if (remaining.length) { this.push(buffer); - this._transform(remaining, encoding, callback); + await this._transform(remaining, encoding, callback); } else { callback(null, buffer); } diff --git a/packages/verser/src/lib/tecemux/constants.ts b/packages/verser/src/lib/tecemux/constants.ts index 4e4aa9f38..17e8362a3 100644 --- a/packages/verser/src/lib/tecemux/constants.ts +++ b/packages/verser/src/lib/tecemux/constants.ts @@ -17,3 +17,5 @@ export enum FrameTarget { API, INPUT = 1001 } + +export const ACK_FRAME_DELTA = -5; diff --git a/packages/verser/src/lib/tecemux/frames-keeper.ts b/packages/verser/src/lib/tecemux/frames-keeper.ts index 40dae5918..27146824e 100644 --- a/packages/verser/src/lib/tecemux/frames-keeper.ts +++ b/packages/verser/src/lib/tecemux/frames-keeper.ts @@ -1,10 +1,13 @@ import { ObjLogger } from "@scramjet/obj-logger"; -import { TransformOptions, Writable } from "stream"; import { IObjectLogger } from "@scramjet/types"; -export class FramesKeeper extends Writable { +import { TransformOptions, Writable } from "stream"; +import { parseFlags } from "./utils"; +import { IFramesKeeper, FramesKeeperFrame } from "./types"; + +export class FramesKeeper extends Writable implements IFramesKeeper { logger: IObjectLogger; - framesSent = new Map(); + framesSent = new Map(); lastSequenceSent: number = -1; lastSequenceReceived: number = -1; @@ -24,8 +27,13 @@ export class FramesKeeper extends Writable { if (Buffer.isBuffer(chunk)) { this.logger.info("transform buffer"); const sequenceNumber = chunk.readInt32LE(16); + const destinationPort = chunk.readInt16LE(14); + const flags = parseFlags(chunk.readInt8(25)); - this.framesSent.set(sequenceNumber, { buffer: chunk, received: false, sequenceNumber }); + this.framesSent.set( + sequenceNumber, { + buffer: chunk, received: false, sequenceNumber, destinationPort, flags + }); this.lastSequenceSent = sequenceNumber; this.logger.debug(`lastSequenceSent ${sequenceNumber}, size: ${chunk.length}`); @@ -43,4 +51,19 @@ export class FramesKeeper extends Writable { this.framesSent.set(acknowledgeNumber, { ...storedFrame, received: true }); } } + + isReceived(sequenceNumber: number) { + const frame = this.framesSent.get(sequenceNumber); + + // received or not stored + return frame === undefined || !!this.framesSent.get(sequenceNumber)?.received; + } + + getFrame(sequenceNumber: number) { + return this.framesSent.get(sequenceNumber); + } + + getDestinationPort(sequenceNumber: number) { + return this.framesSent.get(sequenceNumber)?.received; + } } diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/src/lib/tecemux/playground.ts index 304316f47..f51d43ecd 100644 --- a/packages/verser/src/lib/tecemux/playground.ts +++ b/packages/verser/src/lib/tecemux/playground.ts @@ -4,10 +4,13 @@ import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; import { createReadStream, createWriteStream } from "fs"; import path from "path"; import { PassThrough, Transform } from "stream"; +import { FramesKeeper } from "./frames-keeper"; +import { ITeCeMux } from "./types"; const tcm = { - sequenceNumber: 0 -}; + sequenceNumber: 0, + framesKeeper: new FramesKeeper() +} as ITeCeMux; const logger = new ObjLogger("Sandbox"); @@ -29,12 +32,6 @@ const decoder = new FrameDecoder(); encoder.logger.pipe(logger); decoder.logger.pipe(logger); -/* -process.stdin - .pipe(encoder).out - .pipe(decoder) - //.pipe(process.stdout); -*/ const ws = createWriteStream(path.join(__dirname, "output.tar.gz")); const delayedPassThrough = () => new PassThrough({ diff --git a/packages/verser/src/lib/tecemux/types.ts b/packages/verser/src/lib/tecemux/types.ts index 7d8c041f8..9b04efc2f 100644 --- a/packages/verser/src/lib/tecemux/types.ts +++ b/packages/verser/src/lib/tecemux/types.ts @@ -16,8 +16,23 @@ export type flagsObjectType = Partial<{ CWR: boolean }> +export type FramesKeeperFrame = { + buffer: Buffer; + received: boolean; + sequenceNumber: number; + destinationPort: number; + flags: any +}; + +export interface IFramesKeeper { + onACK(acknowledgeNumber: number): void; + isReceived(sequenceNumber: number): boolean; + getFrame(sequenceNumber: number): FramesKeeperFrame | undefined +} + export interface ITeCeMux { sequenceNumber: number; + framesKeeper: IFramesKeeper; } export interface IFrameEncoder { From 63f10574fb898d09cf60493df1f2a0760a3e6ad1 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 31 Jan 2023 11:43:53 +0000 Subject: [PATCH 025/231] Fix send FIN --- .../src/lib/tecemux/playground-tecemux.ts | 5 ++++- packages/verser/src/lib/tecemux/tecemux.ts | 18 +++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index 7b3586db3..69199e99b 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -80,7 +80,10 @@ import path from "path"; const channel1 = tcmux.multiplex(); //const channel2 = tcmux.multiplex(); - createReadStream(path.join(__dirname, "../../../../forever.tar.gz")).pipe(channel1); + createReadStream(path.join(__dirname, "../../../../forever.tar.gz")).on("end", () => { + logger.info("FILE END"); + }).pipe(channel1); + //Readable.from(Buffer.alloc(1024 * 100)).pipe(channel1, { end: false }); req.on("pause", () => { logger.warn("Request paused"); }); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index dd60a0b4d..6a0dc4eaa 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -32,11 +32,20 @@ export class TeceMux extends TypedEmitter { write: (chunk, encoding, next) => { this.logger.trace("WRITE channel", channel._id, chunk); + if (chunk === null) { + this.logger.info("NULL ON CHANNEL"); + channel.end(); + return false; + } + return encoder.write(chunk, encoding, next); }, read: (_size) => { this.logger.trace("READ channel", channel._id); }, + final() { + //channel.emit("end"); + }, allowHalfOpen: true }), { @@ -65,6 +74,7 @@ export class TeceMux extends TypedEmitter { }) .on("end", () => { this.logger.info("CHANNEL end", channel._id); + if (!channel.closedByFIN) { this.sendFIN(channel._id); } @@ -154,8 +164,14 @@ export class TeceMux extends TypedEmitter { this.logger.trace(`Received FIN command [C: ${destinationPort}]`, dataLength, frame.chunk, !!this.channels[destinationPort]); if (channel) { - //channel.end(); channel.closedByFIN = true; + + if (channel.readableEnded) { + channel.once("end", () => { + this.logger.info("channel --------- ENDED"); + }); + } + channel.push(null); } else { this.logger.error("FIN for unknown channel"); From 21842ea277eafeb31efee0860e2d865583ac9024 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 31 Jan 2023 22:19:43 +0000 Subject: [PATCH 026/231] Send file both directions --- .../src/lib/tecemux/codecs/frame-encoder.ts | 3 +- .../verser/src/lib/tecemux/frames-keeper.ts | 2 +- .../src/lib/tecemux/playground-tecemux.ts | 16 +++--- packages/verser/src/lib/tecemux/playground.ts | 3 +- packages/verser/src/lib/tecemux/tecemux.ts | 27 +++++----- packages/verser/src/lib/tecemux/types.ts | 1 + yarn.lock | 52 ------------------- 7 files changed, 29 insertions(+), 75 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index d11f9d32d..f9370dc1f 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -167,7 +167,7 @@ export class FrameEncoder extends Transform { const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); - if (this.tecemux.sequenceNumber > -ACK_FRAME_DELTA) { + if (this.tecemux.framesSent > -ACK_FRAME_DELTA) { // eslint-disable-next-line no-loop-func while (!await new Promise((res, _rej) => { setImmediate(() => { @@ -186,6 +186,7 @@ export class FrameEncoder extends Transform { })); } + this.tecemux.framesSent++; this.tecemux.sequenceNumber++; this.logger.debug("TRANSFORM OUT", /*getHexString(buffer), */ "Size: ", buffer.length); diff --git a/packages/verser/src/lib/tecemux/frames-keeper.ts b/packages/verser/src/lib/tecemux/frames-keeper.ts index 27146824e..d0e9bf8a2 100644 --- a/packages/verser/src/lib/tecemux/frames-keeper.ts +++ b/packages/verser/src/lib/tecemux/frames-keeper.ts @@ -36,7 +36,7 @@ export class FramesKeeper extends Writable implements IFramesKeeper { }); this.lastSequenceSent = sequenceNumber; - this.logger.debug(`lastSequenceSent ${sequenceNumber}, size: ${chunk.length}`); + this.logger.debug(`lastSequenceSent ${sequenceNumber}, size: ${chunk.length} total frames sent ${this.framesSent.size}`,); } cb(undefined); diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index 69199e99b..b75bf78d7 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -80,9 +80,13 @@ import path from "path"; const channel1 = tcmux.multiplex(); //const channel2 = tcmux.multiplex(); - createReadStream(path.join(__dirname, "../../../../forever.tar.gz")).on("end", () => { - logger.info("FILE END"); - }).pipe(channel1); + channel1.pipe(createWriteStream(path.join(__dirname, "output-server.tar.gz"))); + + createReadStream(path.join(__dirname, "../../../../forever.tar.gz")) + .on("end", () => { + logger.info("FILE END"); + }) + .pipe(channel1); //Readable.from(Buffer.alloc(1024 * 100)).pipe(channel1, { end: false }); @@ -170,11 +174,11 @@ import path from "path"; tcmux.logger.info("Channel finish", channel._id); }) .on("end", () => { - tcmux.logger.info("Channel end", channel._id); + tcmux.logger.info("Channel readable end", channel._id, channel.readableEnded, channel.writableEnded); }); - channel - .pipe(createWriteStream(path.join(__dirname, "output.tar.gz"))); + createReadStream(path.join(__dirname, "../../../../forever.tar.gz")).pipe(channel, { end: false }); + channel.pipe(createWriteStream(path.join(__dirname, "output-client.tar.gz"))); let total = 0; diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/src/lib/tecemux/playground.ts index f51d43ecd..694ad1a45 100644 --- a/packages/verser/src/lib/tecemux/playground.ts +++ b/packages/verser/src/lib/tecemux/playground.ts @@ -9,7 +9,8 @@ import { ITeCeMux } from "./types"; const tcm = { sequenceNumber: 0, - framesKeeper: new FramesKeeper() + framesKeeper: new FramesKeeper(), + framesSent: 0 } as ITeCeMux; const logger = new ObjLogger("Sandbox"); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 6a0dc4eaa..e37595768 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -11,9 +11,10 @@ export class TeceMux extends TypedEmitter { id: string; carrierSocket: Duplex; channelCount = 1; + framesSent = 0; carrierDecoder: FrameDecoder; framesKeeper = new FramesKeeper(); - sequenceNumber = 0; + sequenceNumber = Math.abs((Math.random() * (2 ** 32)) | 0); channels: TeceMuxChannel[] = []; logger: ObjLogger; @@ -34,6 +35,7 @@ export class TeceMux extends TypedEmitter { if (chunk === null) { this.logger.info("NULL ON CHANNEL"); + channel.end(); return false; } @@ -74,10 +76,7 @@ export class TeceMux extends TypedEmitter { }) .on("end", () => { this.logger.info("CHANNEL end", channel._id); - - if (!channel.closedByFIN) { - this.sendFIN(channel._id); - } + this.sendFIN(channel._id); }); if (emit) { @@ -149,30 +148,30 @@ export class TeceMux extends TypedEmitter { if (error) { this.emit("error", frame); - continue; + break; } let channel = this.channels[destinationPort]; if (flags.ACK) { - this.logger.trace("ACK frame received for sequenceNumber", acknowledgeNumber); + this.logger.trace("Received ACK flag for sequenceNumber", acknowledgeNumber); this.framesKeeper.onACK(acknowledgeNumber); continue; } if (flags.FIN) { - this.logger.trace(`Received FIN command [C: ${destinationPort}]`, dataLength, frame.chunk, !!this.channels[destinationPort]); + this.logger.trace(`Received FIN flag [C: ${destinationPort}]`, dataLength, frame.chunk, !!this.channels[destinationPort], channel._id); if (channel) { channel.closedByFIN = true; - - if (channel.readableEnded) { - channel.once("end", () => { - this.logger.info("channel --------- ENDED"); - }); + if (!channel.writableEnded) { + channel.push(null); } - channel.push(null); + if (channel.writableEnded && channel.readableEnded) { + channel.destroy(); + this.logger.info("Channel destroy"); + } } else { this.logger.error("FIN for unknown channel"); } diff --git a/packages/verser/src/lib/tecemux/types.ts b/packages/verser/src/lib/tecemux/types.ts index 9b04efc2f..67a5a6443 100644 --- a/packages/verser/src/lib/tecemux/types.ts +++ b/packages/verser/src/lib/tecemux/types.ts @@ -32,6 +32,7 @@ export interface IFramesKeeper { export interface ITeCeMux { sequenceNumber: number; + framesSent: number; framesKeeper: IFramesKeeper; } diff --git a/yarn.lock b/yarn.lock index df5227dda..07dea5cdb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -814,58 +814,6 @@ resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== -"@scramjet/api-server@^0.31.4": - version "0.31.4" - resolved "https://registry.yarnpkg.com/@scramjet/api-server/-/api-server-0.31.4.tgz#a2855de120c1093d6cfd96b50447babd62116823" - integrity sha512-ghhKbK6omrTJIT4xgRhT5bLUE/Hk+ShmM3/J9XwcVwxGgltqtEC10k1G6Dm42vqYnUxLgW3iIsmW9OMCNh2jfw== - dependencies: - "0http" "^3.4.1" - "@scramjet/model" "^0.31.4" - "@scramjet/obj-logger" "^0.31.4" - "@scramjet/symbols" "^0.31.4" - "@scramjet/utility" "^0.31.4" - http-status-codes "^2.2.0" - scramjet "^4.36.9" - -"@scramjet/model@^0.31.4": - version "0.31.4" - resolved "https://registry.yarnpkg.com/@scramjet/model/-/model-0.31.4.tgz#2cc4e9a9855abeee0e450c7a9d66525bfdb03301" - integrity sha512-NkM/6DhVpN0tsE/1aCY9v2EOSxq3IecBtwtnaz9EYEAb0XTt78hkseALevzvOLDvc4G+F0BqmZKeqHWxbAJFgg== - dependencies: - "@scramjet/obj-logger" "^0.31.4" - "@scramjet/symbols" "^0.31.4" - scramjet "^4.36.9" - uuid "^8.3.2" - -"@scramjet/obj-logger@^0.31.4": - version "0.31.4" - resolved "https://registry.yarnpkg.com/@scramjet/obj-logger/-/obj-logger-0.31.4.tgz#9c755403afa544afa31cc30dbca58913b95c8388" - integrity sha512-6sfS45IWkRMmdiG+2egr7FTyjpNW5Dc+n2WvzWbk6M/37/B30lRUtVkYGKTpIBbw/arp6lUuH0w/QjlC+QAiDQ== - dependencies: - "@scramjet/utility" "^0.31.4" - scramjet "^4.36.9" - -"@scramjet/symbols@^0.31.4": - version "0.31.4" - resolved "https://registry.yarnpkg.com/@scramjet/symbols/-/symbols-0.31.4.tgz#f1ccaeee59b26530caf47fe021bf45727f6da4f0" - integrity sha512-dJWBd7VgMU627SlXVg27n3F/aBzvUHTEGMCTq5N46W91OeU/YhjRT+dO5NyJhAxpyTHdlMFCHKAFC9FrGzdoMw== - -"@scramjet/types@^0.31.4": - version "0.31.4" - resolved "https://registry.yarnpkg.com/@scramjet/types/-/types-0.31.4.tgz#73e298bbcf01abf7a378b26fa0da9a9093d45341" - integrity sha512-RxJzbRD+q4w5RrlaT5cg6GdlMzlXnDrqBSKaAeDdzu/2OgApHJOBE4roP1hqtwQkgtQvXMNeF8uUJM0IFCsjSQ== - dependencies: - "@scramjet/symbols" "^0.31.4" - http-status-codes "^2.2.0" - -"@scramjet/utility@^0.31.4": - version "0.31.4" - resolved "https://registry.yarnpkg.com/@scramjet/utility/-/utility-0.31.4.tgz#9e13f30ef66636e77a7a065d1496f35e1dfc7017" - integrity sha512-BFwoYu+u5SbuJX5QgzrSgb+6Vak+cCAu141C5zasoAMplM4AmvqPWQAZ7CQ6Du7KQA7HsJqgD0xCHHky+mgoEA== - dependencies: - normalize-url "^5.3.1" - yaml "^2.1.3" - "@sindresorhus/is@^0.14.0": version "0.14.0" resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea" From 540dedf1ef22357df9190f57697c9cae00a9ad13 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 2 Feb 2023 23:27:35 +0000 Subject: [PATCH 027/231] Debug w/o FrameKeeper --- packages/verser/src/lib/tecemux/codecs/frame-decoder.ts | 2 +- packages/verser/src/lib/tecemux/codecs/frame-encoder.ts | 9 +++++---- packages/verser/src/lib/tecemux/frames-keeper.ts | 5 +++-- packages/verser/src/lib/tecemux/tecemux.ts | 5 +++-- packages/verser/src/lib/verser.ts | 4 ++++ 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 1aa7df77b..5c81b7378 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -106,7 +106,7 @@ export class FrameDecoder extends Transform { this.buffer = this.buffer.subarray(frameSize); - this.logger.trace("Decoded", { ...payload, stringified: "--not-displayed--" }); + this.logger.trace("Decoded", { ...payload, stringified: payload.chunk?.toString() }); if (this.buffer.length === 0) { this.logger.info("No remaining data!"); diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index f9370dc1f..a7f1c1de8 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -2,7 +2,7 @@ import { PassThrough, Transform, TransformCallback, TransformOptions } from "str import { FrameData } from "../utils"; import { ObjLogger } from "@scramjet/obj-logger"; -import { ACK_FRAME_DELTA, FrameTarget, HEADER_LENGTH, binaryFlags, frameFlags } from "../constants"; +import { FrameTarget, HEADER_LENGTH, binaryFlags, frameFlags } from "../constants"; import { ITeCeMux } from "../types"; import { calculateChecksum } from "./utils"; @@ -167,7 +167,7 @@ export class FrameEncoder extends Transform { const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); - if (this.tecemux.framesSent > -ACK_FRAME_DELTA) { + /*if (this.tecemux.framesSent > -ACK_FRAME_DELTA) { // eslint-disable-next-line no-loop-func while (!await new Promise((res, _rej) => { setImmediate(() => { @@ -180,11 +180,12 @@ export class FrameEncoder extends Transform { res(!!rec); } else { - _rej("frame not exists"); + console.log((this.tecemux.framesKeeper as any)["framesSent"]) + _rej("frame not exists" + (this.tecemux.sequenceNumber + ACK_FRAME_DELTA)); } }); })); - } + }*/ this.tecemux.framesSent++; this.tecemux.sequenceNumber++; diff --git a/packages/verser/src/lib/tecemux/frames-keeper.ts b/packages/verser/src/lib/tecemux/frames-keeper.ts index d0e9bf8a2..418ed1661 100644 --- a/packages/verser/src/lib/tecemux/frames-keeper.ts +++ b/packages/verser/src/lib/tecemux/frames-keeper.ts @@ -17,7 +17,8 @@ export class FramesKeeper extends Writable implements IFramesKeeper { ) { super(Object.assign(opts, { readableObjectMode: true, - writableObjectMode: true + writableObjectMode: true, + writableHighWaterMark: 0 })); this.logger = new ObjLogger(params.name); @@ -25,11 +26,11 @@ export class FramesKeeper extends Writable implements IFramesKeeper { _write(chunk: any, encoding: BufferEncoding, cb: ((error: Error | null | undefined) => void)) { if (Buffer.isBuffer(chunk)) { - this.logger.info("transform buffer"); const sequenceNumber = chunk.readInt32LE(16); const destinationPort = chunk.readInt16LE(14); const flags = parseFlags(chunk.readInt8(25)); + this.logger.info("transform buffer", sequenceNumber); this.framesSent.set( sequenceNumber, { buffer: chunk, received: false, sequenceNumber, destinationPort, flags diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index e37595768..18badd578 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -117,6 +117,7 @@ export class TeceMux extends TypedEmitter { this.carrierSocket.pipe(this.carrierDecoder, { end: false }); + this.commonEncoder.out.pipe(this.framesKeeper); this.commonEncoder.out.pipe(this.carrierSocket, { end: false }); this.commonEncoder.logger.updateBaseLog({ id }); @@ -204,9 +205,9 @@ export class TeceMux extends TypedEmitter { } } } - + i = 0; sendACK(sequenceNumber: number, channel: number) { - this.logger.debug("Write acknowledge frame for sequenceNumber", sequenceNumber); + this.logger.debug("Write acknowledge frame for sequenceNumber", sequenceNumber, this.i++); this.commonEncoder.push( this.commonEncoder.createFrame(undefined, { flagsArray: ["ACK"], diff --git a/packages/verser/src/lib/verser.ts b/packages/verser/src/lib/verser.ts index 9f2fab0da..35f3cfc82 100644 --- a/packages/verser/src/lib/verser.ts +++ b/packages/verser/src/lib/verser.ts @@ -24,11 +24,15 @@ export class Verser extends TypedEmitter { super(); this.server = server; + this.server.timeout = 0; + this.server.on("connect", (req, socket: Socket) => { this.logger.info("New connection:", req.url); const connection = new VerserConnection(req, socket); + connection.logger.pipe(this.logger); + this.connections.push(connection); this.logger.info("Total connections:", this.connections.length); From 08b0b54d6d04fe794dc309aa460250cf0dbd0640 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 9 Feb 2023 21:35:24 +0000 Subject: [PATCH 028/231] FramesKeeper generator --- .../src/lib/tecemux/codecs/frame-decoder.ts | 2 +- .../src/lib/tecemux/codecs/frame-encoder.ts | 24 ++---- .../verser/src/lib/tecemux/frames-keeper.ts | 80 +++++++++---------- packages/verser/src/lib/tecemux/playground.ts | 2 +- packages/verser/src/lib/tecemux/tecemux.ts | 21 +++-- packages/verser/src/lib/tecemux/types.ts | 16 ++-- 6 files changed, 61 insertions(+), 84 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 5c81b7378..65ef3f5ab 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -106,7 +106,7 @@ export class FrameDecoder extends Transform { this.buffer = this.buffer.subarray(frameSize); - this.logger.trace("Decoded", { ...payload, stringified: payload.chunk?.toString() }); + this.logger.trace("Decoded", { ...payload, stringified: "N/A" });//payload.chunk?.toString() }); if (this.buffer.length === 0) { this.logger.info("No remaining data!"); diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index a7f1c1de8..6f6a56108 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -167,25 +167,11 @@ export class FrameEncoder extends Transform { const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); - /*if (this.tecemux.framesSent > -ACK_FRAME_DELTA) { - // eslint-disable-next-line no-loop-func - while (!await new Promise((res, _rej) => { - setImmediate(() => { - const frame = this.tecemux.framesKeeper.getFrame(this.tecemux.sequenceNumber + ACK_FRAME_DELTA); - - if (frame) { - const rec = frame?.received || frame?.flags.ACK; - - this.logger.info(`Sending ${this.tecemux.sequenceNumber}, ${frame?.sequenceNumber} ${frame?.received}`); - - res(!!rec); - } else { - console.log((this.tecemux.framesKeeper as any)["framesSent"]) - _rej("frame not exists" + (this.tecemux.sequenceNumber + ACK_FRAME_DELTA)); - } - }); - })); - }*/ + this.logger.debug("Awaiting keeper"); + await this.tecemux.framesKeeper.generator.next(this.tecemux.sequenceNumber); + this.logger.debug("Awaiting keeper DONE"); + + this.tecemux.framesKeeper.handlePSH(this.tecemux.sequenceNumber); this.tecemux.framesSent++; this.tecemux.sequenceNumber++; diff --git a/packages/verser/src/lib/tecemux/frames-keeper.ts b/packages/verser/src/lib/tecemux/frames-keeper.ts index 418ed1661..9ed00cf10 100644 --- a/packages/verser/src/lib/tecemux/frames-keeper.ts +++ b/packages/verser/src/lib/tecemux/frames-keeper.ts @@ -1,70 +1,62 @@ import { ObjLogger } from "@scramjet/obj-logger"; -import { IObjectLogger } from "@scramjet/types"; +import { EventEmitter } from "stream"; +import { FramesKeeperFrame, IFramesKeeper } from "./types"; -import { TransformOptions, Writable } from "stream"; -import { parseFlags } from "./utils"; -import { IFramesKeeper, FramesKeeperFrame } from "./types"; +export class FramesKeeper extends EventEmitter implements IFramesKeeper { + #MAX_FRAMES_DIFFERENCE = 5; + + #framesSent = new Map(); -export class FramesKeeper extends Writable implements IFramesKeeper { - logger: IObjectLogger; - framesSent = new Map(); lastSequenceSent: number = -1; lastSequenceReceived: number = -1; - constructor( - opts: TransformOptions = {}, - params: { name: string } = { name: "Keeper" } - ) { - super(Object.assign(opts, { - readableObjectMode: true, - writableObjectMode: true, - writableHighWaterMark: 0 - })); - - this.logger = new ObjLogger(params.name); - } - - _write(chunk: any, encoding: BufferEncoding, cb: ((error: Error | null | undefined) => void)) { - if (Buffer.isBuffer(chunk)) { - const sequenceNumber = chunk.readInt32LE(16); - const destinationPort = chunk.readInt16LE(14); - const flags = parseFlags(chunk.readInt8(25)); - - this.logger.info("transform buffer", sequenceNumber); - this.framesSent.set( - sequenceNumber, { - buffer: chunk, received: false, sequenceNumber, destinationPort, flags + logger = new ObjLogger(this); + + generator: AsyncGenerator = (async function* (this: FramesKeeper) { + while (true) { + if (this.lastSequenceSent - this.lastSequenceReceived < this.#MAX_FRAMES_DIFFERENCE) { + this.logger.info("Write allowed"); + yield Promise.resolve(this.lastSequenceReceived); + continue; + } + + this.logger.warn("Write NOT allowed"); + yield new Promise((res) => { + this.logger.info("waiting for ACK..."); + this.once("ack", (acknowledgeNumber: number) => { + this.logger.info("ACK processed"); + res(acknowledgeNumber); }); - - this.lastSequenceSent = sequenceNumber; - this.logger.debug(`lastSequenceSent ${sequenceNumber}, size: ${chunk.length} total frames sent ${this.framesSent.size}`,); + }); } + }).apply(this); - cb(undefined); + handlePSH(sequenceNumber: number) { + this.lastSequenceSent = sequenceNumber; } - onACK(acknowledgeNumber: number,) { + handleACK(acknowledgeNumber: number) { this.logger.debug("onACK", acknowledgeNumber); - const storedFrame = this.framesSent.get(acknowledgeNumber); + const storedFrame = this.#framesSent.get(acknowledgeNumber); if (storedFrame) { - this.framesSent.set(acknowledgeNumber, { ...storedFrame, received: true }); + this.#framesSent.set(acknowledgeNumber, { ...storedFrame, received: true }); } + + this.lastSequenceReceived = acknowledgeNumber; + this.emit("ack", acknowledgeNumber); } isReceived(sequenceNumber: number) { - const frame = this.framesSent.get(sequenceNumber); + const frame = this.#framesSent.get(sequenceNumber); // received or not stored - return frame === undefined || !!this.framesSent.get(sequenceNumber)?.received; + return frame === undefined || !!this.#framesSent.get(sequenceNumber)?.received; } getFrame(sequenceNumber: number) { - return this.framesSent.get(sequenceNumber); - } - - getDestinationPort(sequenceNumber: number) { - return this.framesSent.get(sequenceNumber)?.received; + return this.#framesSent.get(sequenceNumber); } } + diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/src/lib/tecemux/playground.ts index 694ad1a45..1218fceda 100644 --- a/packages/verser/src/lib/tecemux/playground.ts +++ b/packages/verser/src/lib/tecemux/playground.ts @@ -4,8 +4,8 @@ import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; import { createReadStream, createWriteStream } from "fs"; import path from "path"; import { PassThrough, Transform } from "stream"; -import { FramesKeeper } from "./frames-keeper"; import { ITeCeMux } from "./types"; +import { FramesKeeper } from "./frames-keeper"; const tcm = { sequenceNumber: 0, diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 18badd578..8705a4558 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -15,7 +15,7 @@ export class TeceMux extends TypedEmitter { carrierDecoder: FrameDecoder; framesKeeper = new FramesKeeper(); sequenceNumber = Math.abs((Math.random() * (2 ** 32)) | 0); - channels: TeceMuxChannel[] = []; + channels = new Map(); logger: ObjLogger; commonEncoder = new FrameEncoder(0, this); @@ -57,9 +57,6 @@ export class TeceMux extends TypedEmitter { } ); - encoder.out - .pipe(this.framesKeeper); - encoder.out .pipe(this.carrierSocket, { end: false }); @@ -117,7 +114,6 @@ export class TeceMux extends TypedEmitter { this.carrierSocket.pipe(this.carrierDecoder, { end: false }); - this.commonEncoder.out.pipe(this.framesKeeper); this.commonEncoder.out.pipe(this.carrierSocket, { end: false }); this.commonEncoder.logger.updateBaseLog({ id }); @@ -152,16 +148,16 @@ export class TeceMux extends TypedEmitter { break; } - let channel = this.channels[destinationPort]; + let channel = this.channels.get(destinationPort); if (flags.ACK) { this.logger.trace("Received ACK flag for sequenceNumber", acknowledgeNumber); - this.framesKeeper.onACK(acknowledgeNumber); + this.framesKeeper.handleACK(acknowledgeNumber); continue; } if (flags.FIN) { - this.logger.trace(`Received FIN flag [C: ${destinationPort}]`, dataLength, frame.chunk, !!this.channels[destinationPort], channel._id); + this.logger.trace(`Received FIN flag [C: ${destinationPort}]`, dataLength, frame.chunk, !!channel, channel?._id); if (channel) { channel.closedByFIN = true; @@ -205,10 +201,10 @@ export class TeceMux extends TypedEmitter { } } } - i = 0; + sendACK(sequenceNumber: number, channel: number) { - this.logger.debug("Write acknowledge frame for sequenceNumber", sequenceNumber, this.i++); - this.commonEncoder.push( + this.logger.debug("Write acknowledge frame for sequenceNumber", sequenceNumber); + this.channels.get(channel)?.encoder?.push( this.commonEncoder.createFrame(undefined, { flagsArray: ["ACK"], acknowledgeNumber: sequenceNumber, @@ -219,7 +215,7 @@ export class TeceMux extends TypedEmitter { addChannel(channel: TeceMuxChannel, emit: boolean) { this.logger.debug("adding channel", channel._id); - this.channels[channel._id] = channel; // wait for SYN reply? + this.channels.set(channel._id, channel); // wait for SYN reply? if (emit) { this.emit("channel", channel); @@ -231,6 +227,7 @@ export class TeceMux extends TypedEmitter { sendFIN(channel: number) { this.logger.debug("Write FIN frame for channel", channel); + this.commonEncoder.push( this.commonEncoder.createFrame(undefined, { flagsArray: ["FIN"], diff --git a/packages/verser/src/lib/tecemux/types.ts b/packages/verser/src/lib/tecemux/types.ts index 67a5a6443..40174699f 100644 --- a/packages/verser/src/lib/tecemux/types.ts +++ b/packages/verser/src/lib/tecemux/types.ts @@ -1,4 +1,4 @@ -import { Duplex } from "stream"; +import { Duplex, Transform } from "stream"; export type TeceMuxEvents = { channel(socket: Duplex): void; @@ -25,9 +25,11 @@ export type FramesKeeperFrame = { }; export interface IFramesKeeper { - onACK(acknowledgeNumber: number): void; + handleACK(acknowledgeNumber: number): void; isReceived(sequenceNumber: number): boolean; - getFrame(sequenceNumber: number): FramesKeeperFrame | undefined + handlePSH(sequenceNumber: number): void; + getFrame(sequenceNumber: number): FramesKeeperFrame | undefined; + generator: AsyncGenerator; } export interface ITeCeMux { @@ -36,12 +38,12 @@ export interface ITeCeMux { framesKeeper: IFramesKeeper; } -export interface IFrameEncoder { +export interface IFrameEncoder extends Transform { } export type TeceMuxChannel = Duplex & { - _id: number, - encoder: IFrameEncoder, - closedByFIN: boolean + _id: number; + encoder: IFrameEncoder; + closedByFIN: boolean; }; From d991a6c971f4aabce5f166c9b67edf102cc096b8 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Fri, 10 Feb 2023 12:34:51 +0000 Subject: [PATCH 029/231] Cleanup --- .../src/lib/tecemux/codecs/frame-encoder.ts | 10 +-- .../verser/src/lib/tecemux/frames-keeper.ts | 7 +- .../src/lib/tecemux/playground-tecemux.ts | 65 +++---------------- packages/verser/src/lib/tecemux/playground.ts | 2 - packages/verser/src/lib/tecemux/tecemux.ts | 21 ++++-- packages/verser/src/lib/tecemux/types.ts | 4 ++ packages/verser/test/http-connection.spec.ts | 7 +- 7 files changed, 41 insertions(+), 75 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index 6f6a56108..6198fda38 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -7,6 +7,8 @@ import { ITeCeMux } from "../types"; import { calculateChecksum } from "./utils"; export class FrameEncoder extends Transform { + MAX_CHUNK_SIZE = 10 * 1024 - HEADER_LENGTH; + tecemux: ITeCeMux; total = 0; logger = new ObjLogger("FrameEncoder"); @@ -148,19 +150,17 @@ export class FrameEncoder extends Transform { } async _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): Promise { - const MAX_CHUNK_SIZE = 10 * 1024 - HEADER_LENGTH; - this.total += chunk.length; this.logger.debug("TRANSFORM IN", /* toHex(chunk), */ chunk.length, this.total); let remaining = Buffer.alloc(0); - if (chunk.length > MAX_CHUNK_SIZE) { + if (chunk.length > this.MAX_CHUNK_SIZE) { this.logger.debug("TRANSFORM big chunk, splitting", chunk.length); - remaining = (chunk as Buffer).subarray(MAX_CHUNK_SIZE); - chunk = chunk.subarray(0, MAX_CHUNK_SIZE); + remaining = (chunk as Buffer).subarray(this.MAX_CHUNK_SIZE); + chunk = chunk.subarray(0, this.MAX_CHUNK_SIZE); this.logger.debug("TRANSFORM processing part/remaining", chunk.length, remaining.length); } diff --git a/packages/verser/src/lib/tecemux/frames-keeper.ts b/packages/verser/src/lib/tecemux/frames-keeper.ts index 9ed00cf10..48126f77f 100644 --- a/packages/verser/src/lib/tecemux/frames-keeper.ts +++ b/packages/verser/src/lib/tecemux/frames-keeper.ts @@ -1,8 +1,8 @@ import { ObjLogger } from "@scramjet/obj-logger"; -import { EventEmitter } from "stream"; -import { FramesKeeperFrame, IFramesKeeper } from "./types"; +import { TypedEmitter } from "@scramjet/utility"; +import { FramesKeeperEvents, FramesKeeperFrame, IFramesKeeper } from "./types"; -export class FramesKeeper extends EventEmitter implements IFramesKeeper { +export class FramesKeeper extends TypedEmitter implements IFramesKeeper { #MAX_FRAMES_DIFFERENCE = 5; #framesSent = new Map(); @@ -23,6 +23,7 @@ export class FramesKeeper extends EventEmitter implements IFramesKeeper { this.logger.warn("Write NOT allowed"); yield new Promise((res) => { this.logger.info("waiting for ACK..."); + this.once("ack", (acknowledgeNumber: number) => { this.logger.info("ACK processed"); res(acknowledgeNumber); diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index b75bf78d7..7dfb9f827 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -53,9 +53,7 @@ import path from "path"; logger.info("Carrier Socket unpiped", c); }) .on("pause", () => { - //socket.resume(); logger.fatal("Carrier Socket paused"); - //debugger; }) .on("resume", () => { logger.info("Carrier Socket resumed"); @@ -78,7 +76,6 @@ import path from "path"; tcmux.logger.pipe(logger); const channel1 = tcmux.multiplex(); - //const channel2 = tcmux.multiplex(); channel1.pipe(createWriteStream(path.join(__dirname, "output-server.tar.gz"))); @@ -87,49 +84,6 @@ import path from "path"; logger.info("FILE END"); }) .pipe(channel1); - - //Readable.from(Buffer.alloc(1024 * 100)).pipe(channel1, { end: false }); - - req.on("pause", () => { logger.warn("Request paused"); }); - - //logger.warn("Waiting for stdin..."); - - //DataStream.from(process.stdin).filter((x: Buffer) => !(parseInt(x[0].toString(), 10) % 2)).pipe(channel1); - //DataStream.from(process.stdin).filter((x: Buffer) => !!(parseInt(x[0].toString(), 10) % 2)).pipe(channel2); - /* - (async () => { - try { - for await (const chunk of channel1) { - console.log("CHUNK", chunk); - logger.debug("reading CHANNEL1 chunk", chunk.toString()); - } - } catch (error) { - logger.error("reading CHANNEL1 ERROR", error); - } - - logger.debug("reading CHANNEL1 END"); - })(); -*/ - // (async () => { - // try { - // for await (const chunk of channel2) { - // logger.debug("reading CHANNEL2 chunk", chunk.toString()); - // } - // } catch (error) { - // logger.error("reading CHANNEL2 ERROR", error); - // } - - // logger.debug("reading CHANNEL2 END"); - // })(); - - /* - setTimeout(() => { - console.log("\n\n\n\n"); - logger.trace("Ending channels"); - channel1.push(null); - //channel2.end(); - }, 4000); - */ }); server.listen(PORT, "0.0.0.0"); @@ -169,20 +123,15 @@ import path from "path"; tcmux.on("channel", async (channel: TeceMuxChannel) => { reqLogger.debug("New channel", channel._id); + let total = 0; + channel .on("finish", () => { tcmux.logger.info("Channel finish", channel._id); }) .on("end", () => { - tcmux.logger.info("Channel readable end", channel._id, channel.readableEnded, channel.writableEnded); - }); - - createReadStream(path.join(__dirname, "../../../../forever.tar.gz")).pipe(channel, { end: false }); - channel.pipe(createWriteStream(path.join(__dirname, "output-client.tar.gz"))); - - let total = 0; - - channel + tcmux.logger.info("Channel readable end [id, readableEnded, writableEnded]", channel._id, channel.readableEnded, channel.writableEnded); + }) .on("data", (d) => { total += d.length; tcmux.logger.info("-------------------- data", channel._id, d.length, total); @@ -192,7 +141,11 @@ import path from "path"; }) .on("resume", () => { tcmux.logger.info("-------------------- resumed", channel._id); - }); + }) + .pause(); + + createReadStream(path.join(__dirname, "../../../../forever.tar.gz")).pipe(channel); + channel.pipe(createWriteStream(path.join(__dirname, "output-client.tar.gz"))); }); socket.on("error", (err) => { diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/src/lib/tecemux/playground.ts index 1218fceda..557dd6ed6 100644 --- a/packages/verser/src/lib/tecemux/playground.ts +++ b/packages/verser/src/lib/tecemux/playground.ts @@ -65,8 +65,6 @@ const dh = new Transform({ callback(null); } } - - //callback(null); } catch (e) { logger.error("dh error", e); } diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 8705a4558..c1187bee1 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -45,9 +45,6 @@ export class TeceMux extends TypedEmitter { read: (_size) => { this.logger.trace("READ channel", channel._id); }, - final() { - //channel.emit("end"); - }, allowHalfOpen: true }), { @@ -71,10 +68,22 @@ export class TeceMux extends TypedEmitter { .on("abort", () => { this.logger.trace("channel on ABORT ", channel._id); }) + .on("close", () => { + this.logger.info("CHANNEL close", channel._id); + this.sendFIN(channel._id); + }) .on("end", () => { this.logger.info("CHANNEL end", channel._id); + }) + .on("finish", () => { + this.logger.info("CHANNEL finish", channel._id); this.sendFIN(channel._id); - }); + }) + .on("data", (d) => { + if (d === null) { + this.logger.info("CHANNEL end", channel._id); + } + });//.pause(); if (emit) { encoder.setChannel(destinationPort || this.channelCount); @@ -99,9 +108,9 @@ export class TeceMux extends TypedEmitter { .on("end", () => { this.logger.warn("Decoder ended"); }) - /*.on("error", (error) => { + .on("error", (error) => { this.logger.error("Decoder error", error); - })*/ + }) .on("abort", (error) => { this.logger.error("Decoder abort", error); }) diff --git a/packages/verser/src/lib/tecemux/types.ts b/packages/verser/src/lib/tecemux/types.ts index 40174699f..81d38c7b3 100644 --- a/packages/verser/src/lib/tecemux/types.ts +++ b/packages/verser/src/lib/tecemux/types.ts @@ -5,6 +5,10 @@ export type TeceMuxEvents = { error(error: any): void; } +export type FramesKeeperEvents = { + ack: (acknowledgeNumber: number) => void; +} + export type flagsObjectType = Partial<{ FIN: boolean, SYN: boolean, diff --git a/packages/verser/test/http-connection.spec.ts b/packages/verser/test/http-connection.spec.ts index ba90dc211..c0c4b0a71 100644 --- a/packages/verser/test/http-connection.spec.ts +++ b/packages/verser/test/http-connection.spec.ts @@ -45,7 +45,7 @@ function getJSONResponseFromRequest(request: http.ClientRequest): Promise { test("Connect VerserClient A to Verser B and send HTTP GET Request to VerserClient A", async (t) => { const SERVER_B_PORT = 1999; - const apiB = createServer(); + const apiB = createServer({ }); apiB.server.listen(SERVER_B_PORT); @@ -56,7 +56,7 @@ test("Connect VerserClient A to Verser B and send HTTP GET Request to VerserClie const verserClientA = new VerserClient({ headers: { city: "Valencia" }, - verserUrl: `http://127.0.0.1:${SERVER_B_PORT}`, + verserUrl: `http://0.0.0.0:${SERVER_B_PORT}`, server: apiA.server }); @@ -123,7 +123,7 @@ test("Connect VerserClient A to Verser B over SSL and send HTTPS GET Request to ca: [readFileSync(path.join(certDir, "myCA.pem"))] }); - requestToAThroughB.end(); + requestToAThroughB.flushHeaders(); const responseFromAToB = await getJSONResponseFromRequest(requestToAThroughB); @@ -132,3 +132,4 @@ test("Connect VerserClient A to Verser B over SSL and send HTTPS GET Request to spawnSync("./cleanup-localhost-cert.sh", { cwd: certDir }); }); + From ca0a494b1f7362a48e1d87362128d3d6ac467823 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 14 Feb 2023 12:18:31 +0000 Subject: [PATCH 030/231] Tests --- .../src/lib/tecemux/codecs/frame-decoder.ts | 3 +- .../src/lib/tecemux/codecs/frame-encoder.ts | 4 +- packages/verser/src/lib/tecemux/tecemux.ts | 7 +- packages/verser/src/lib/tecemux/types.ts | 18 +++++ .../verser/src/lib/tecemux/utils/index.ts | 18 +---- .../playgrounds}/playground-tecemux.ts | 8 +- .../playgrounds}/playground.ts | 6 +- packages/verser/test/tecemux-transfer.spec.ts | 80 +++++++++++++++++++ packages/verser/tsconfig.json | 2 +- 9 files changed, 113 insertions(+), 33 deletions(-) rename packages/verser/{src/lib/tecemux => test/playgrounds}/playground-tecemux.ts (94%) rename packages/verser/{src/lib/tecemux => test/playgrounds}/playground.ts (91%) create mode 100644 packages/verser/test/tecemux-transfer.spec.ts diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 65ef3f5ab..5801390dc 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -1,8 +1,9 @@ import { ObjLogger } from "@scramjet/obj-logger"; import { Duplex, Transform, TransformCallback, TransformOptions } from "stream"; -import { FrameData, parseFlags } from "../utils"; +import { parseFlags } from "../utils"; import { HEADER_LENGTH } from "../constants"; import { calculateChecksum, getChecksum } from "./utils"; +import { FrameData } from "../types"; export class FrameDecoder extends Transform { buffer: Buffer; diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index 6198fda38..7aa386263 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -1,9 +1,8 @@ import { PassThrough, Transform, TransformCallback, TransformOptions } from "stream"; -import { FrameData } from "../utils"; import { ObjLogger } from "@scramjet/obj-logger"; import { FrameTarget, HEADER_LENGTH, binaryFlags, frameFlags } from "../constants"; -import { ITeCeMux } from "../types"; +import { FrameData, ITeCeMux } from "../types"; import { calculateChecksum } from "./utils"; export class FrameEncoder extends Transform { @@ -23,7 +22,6 @@ export class FrameEncoder extends Transform { }) .on("end", () => { this.logger.trace("OUT ended!", this.frameTarget); - //this.tecemux.sendFIN(this.frameTarget); }) .on("resume", () => { this.logger.trace("OUT resumed"); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index c1187bee1..96bb6ccd9 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -3,8 +3,7 @@ import { FrameDecoder, FrameEncoder } from "./codecs"; import { Duplex } from "stream"; import { Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; -import { FrameData } from "./utils"; -import { TeceMuxChannel, TeceMuxEvents } from "./types"; +import { FrameData, TeceMuxChannel, TeceMuxEvents } from "./types"; import { FramesKeeper } from "./frames-keeper"; export class TeceMux extends TypedEmitter { @@ -83,7 +82,7 @@ export class TeceMux extends TypedEmitter { if (d === null) { this.logger.info("CHANNEL end", channel._id); } - });//.pause(); + }); if (emit) { encoder.setChannel(destinationPort || this.channelCount); @@ -144,7 +143,7 @@ export class TeceMux extends TypedEmitter { let frame: FrameData; try { - frame = JSON.parse(chunk) as FrameData; + frame = JSON.parse(chunk); } catch (err) { this.logger.error("error Parsing data from decoder", err, chunk, chunk.length, chunk.toString()); continue; diff --git a/packages/verser/src/lib/tecemux/types.ts b/packages/verser/src/lib/tecemux/types.ts index 81d38c7b3..2cf55c247 100644 --- a/packages/verser/src/lib/tecemux/types.ts +++ b/packages/verser/src/lib/tecemux/types.ts @@ -9,6 +9,8 @@ export type FramesKeeperEvents = { ack: (acknowledgeNumber: number) => void; } +export type BinaryFlags = "FIN" | "SYN" | "RST" | "PSH" | "ACK" | "URG" | "ECE" | "CWR"; + export type flagsObjectType = Partial<{ FIN: boolean, SYN: boolean, @@ -51,3 +53,19 @@ export type TeceMuxChannel = Duplex & { encoder: IFrameEncoder; closedByFIN: boolean; }; + +export type FrameData = { + sourceAddress: [number, number, number, number]; + destinationAddress: [number, number, number, number]; + destinationPort: number; + sequenceNumber: number; + acknowledgeNumber: number; + chunk: Buffer; + dataLength: number; + chunkLength: number; + stringified: string; + flags: flagsObjectType; + flagsArray: BinaryFlags[]; + error?: "checksum"; + received: boolean; +}; diff --git a/packages/verser/src/lib/tecemux/utils/index.ts b/packages/verser/src/lib/tecemux/utils/index.ts index b90056eb7..86fa4a9cf 100644 --- a/packages/verser/src/lib/tecemux/utils/index.ts +++ b/packages/verser/src/lib/tecemux/utils/index.ts @@ -1,26 +1,10 @@ -import { binaryFlags, frameFlags } from "../constants"; +import { frameFlags } from "../constants"; import { flagsObjectType } from "../types"; export function toHex(chunk: Buffer) { return chunk.toString("hex").match(/../g)?.join(" "); } -export type FrameData = { - sourceAddress: [number, number, number, number]; - destinationAddress: [number, number, number, number]; - destinationPort: number; - sequenceNumber: number; - acknowledgeNumber: number; - chunk: Buffer; - dataLength: number; - chunkLength: number; - stringified: string; - flags: flagsObjectType; - flagsArray: (keyof typeof binaryFlags)[]; - error?: "checksum"; - received: boolean; -}; - export const parseFlags = (byte: number): flagsObjectType => { return frameFlags.filter((_flag, index) => byte >>> index & 1) .reduce((p, c) => ({ ...p, [c]: true }), {}); diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/test/playgrounds/playground-tecemux.ts similarity index 94% rename from packages/verser/src/lib/tecemux/playground-tecemux.ts rename to packages/verser/test/playgrounds/playground-tecemux.ts index 7dfb9f827..fa84cf427 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/test/playgrounds/playground-tecemux.ts @@ -7,8 +7,8 @@ import { IncomingMessage, createServer } from "http"; import { DataStream } from "scramjet"; import { Socket, createConnection } from "net"; -import { TeceMux } from "./tecemux"; -import { TeceMuxChannel } from "./types"; +import { TeceMux } from "../../src/lib/tecemux/tecemux"; +import { TeceMuxChannel } from "../../src/lib/tecemux/types"; import { createReadStream, createWriteStream } from "fs"; import path from "path"; @@ -79,7 +79,7 @@ import path from "path"; channel1.pipe(createWriteStream(path.join(__dirname, "output-server.tar.gz"))); - createReadStream(path.join(__dirname, "../../../../forever.tar.gz")) + createReadStream(path.join(__dirname, "../../../forever.tar.gz")) .on("end", () => { logger.info("FILE END"); }) @@ -144,7 +144,7 @@ import path from "path"; }) .pause(); - createReadStream(path.join(__dirname, "../../../../forever.tar.gz")).pipe(channel); + createReadStream(path.join(__dirname, "../../../forever.tar.gz")).pipe(channel); channel.pipe(createWriteStream(path.join(__dirname, "output-client.tar.gz"))); }); diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/test/playgrounds/playground.ts similarity index 91% rename from packages/verser/src/lib/tecemux/playground.ts rename to packages/verser/test/playgrounds/playground.ts index 557dd6ed6..d52e5f452 100644 --- a/packages/verser/src/lib/tecemux/playground.ts +++ b/packages/verser/test/playgrounds/playground.ts @@ -1,11 +1,11 @@ import { DataStream } from "scramjet"; -import { FrameDecoder, FrameEncoder } from "./codecs"; +import { FrameDecoder, FrameEncoder } from "../../src/lib/tecemux/codecs"; import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; import { createReadStream, createWriteStream } from "fs"; import path from "path"; import { PassThrough, Transform } from "stream"; -import { ITeCeMux } from "./types"; -import { FramesKeeper } from "./frames-keeper"; +import { ITeCeMux } from "../../src/lib/tecemux/types"; +import { FramesKeeper } from "../../src/lib/tecemux/frames-keeper"; const tcm = { sequenceNumber: 0, diff --git a/packages/verser/test/tecemux-transfer.spec.ts b/packages/verser/test/tecemux-transfer.spec.ts new file mode 100644 index 000000000..54c9c97d7 --- /dev/null +++ b/packages/verser/test/tecemux-transfer.spec.ts @@ -0,0 +1,80 @@ +/* eslint-disable no-console */ +import test from "ava"; +import { IncomingMessage, Server, createServer } from "http"; +import { TeceMux, TeceMuxChannel } from "../src/lib/tecemux/tecemux"; +import * as crypto from "crypto"; +import { Socket, createConnection } from "net"; +import { Readable } from "stream"; + +let serverTeceMux: TeceMux; + +const PORT = 6660; + +async function startServer() { + const server = createServer({}); + + return new Promise((resolve) => { + server.listen(PORT, () => { resolve(server); }); + }); +} + +test("Protocol send file over http connection", async (t) => { + const server = await startServer(); + + const hashReceived = crypto.createHash("md5"); + const hashSent = crypto.createHash("md5"); + + const serverSideChannelPromise = new Promise((resolve) => { + server.on("connect", async (req: IncomingMessage, socket: Socket) => { + socket.setNoDelay(true); + + serverTeceMux = new TeceMux(socket, "Server") + .on("error", (error) => { + console.error("TeceMux error", error); + }); + + resolve(serverTeceMux.multiplex()); + }); + }); + + const socket = createConnection({ port: PORT, allowHalfOpen: true, host: "0.0.0.0" }, () => {}); + + await new Promise((resolve, reject) => { + socket + .on("connect", resolve) + .on("error", reject); + }); + + socket.write("CONNECT HTTP/1.1\r\n\r\n\r\n"); + + const clientTeceMux = new TeceMux(socket, "Request"); + + await new Promise(resolve => { + clientTeceMux.on("channel", async (clientSideChannel: TeceMuxChannel) => { + function* gen() { + for (let i = 0; i < 1e3; i++) { + const str = crypto.randomBytes(1024).toString("hex"); + + hashSent.update(str); + yield str; + } + } + + Readable.from(gen()).pipe(clientSideChannel); + resolve(clientSideChannel); + }); + }); + + const serverSideChannel = await serverSideChannelPromise; + + for await (const d of serverSideChannel) { + hashReceived.update(d); + } + + console.dir({ + hashSent, + hashReceived + }); + + t.assert(hashReceived.digest("hex") === hashSent.digest("hex"), "Unequal hashes"); +}); diff --git a/packages/verser/tsconfig.json b/packages/verser/tsconfig.json index 6027e8072..55c8006d1 100644 --- a/packages/verser/tsconfig.json +++ b/packages/verser/tsconfig.json @@ -5,7 +5,7 @@ "extends": "../../tsconfig.base.json", "include": [ "./src/**/*.ts", - "./test/**/*.ts", + "./test/**/*.ts", "src/types" ], "exclude": [ From 2809cc5f8ef65ce3d8582079cd7c28e4aff6a423 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Fri, 20 Jan 2023 12:40:22 +0000 Subject: [PATCH 031/231] Replace BPMux with TeCeMux --- packages/verser/src/lib/verser-client.ts | 2 -- packages/verser/src/lib/verser-connection.ts | 2 -- 2 files changed, 4 deletions(-) diff --git a/packages/verser/src/lib/verser-client.ts b/packages/verser/src/lib/verser-client.ts index 2ed5235f2..bdab51ed2 100644 --- a/packages/verser/src/lib/verser-client.ts +++ b/packages/verser/src/lib/verser-client.ts @@ -138,8 +138,6 @@ export class VerserClient extends TypedEmitter { this.emit("error", err); }); - //this.teceMux.logger.pipe(this.logger); - this._verserAgent = new HttpAgent() as HttpAgent & { createConnection: typeof createConnection }; // lack of types? diff --git a/packages/verser/src/lib/verser-connection.ts b/packages/verser/src/lib/verser-connection.ts index c8c10e470..87edc0344 100644 --- a/packages/verser/src/lib/verser-connection.ts +++ b/packages/verser/src/lib/verser-connection.ts @@ -204,8 +204,6 @@ export class VerserConnection { // TODO: Error handling? }); - //this.teceMux.logger.pipe(this.logger); - this.agent = new Agent() as Agent & { createConnection: typeof createConnection }; // lack of types? this.agent.createConnection = () => { try { From 03bd54cdd41a01c329d610de6e7d0eb27ef13a29 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 23 Mar 2023 21:05:17 +0000 Subject: [PATCH 032/231] R->STH. WIP --- packages/host/src/lib/csi-controller.ts | 1 + packages/host/src/lib/socket-server.ts | 77 +++++++++++++--------- packages/runner/package.json | 1 + packages/runner/src/host-client.ts | 47 ++++++++----- packages/runner/src/runner-app-context.ts | 1 - packages/verser/src/index.ts | 1 + packages/verser/src/lib/tecemux/index.ts | 1 + packages/verser/src/lib/tecemux/tecemux.ts | 13 ++-- packages/verser/src/lib/tecemux/types.ts | 1 + 9 files changed, 89 insertions(+), 54 deletions(-) create mode 100644 packages/verser/src/lib/tecemux/index.ts diff --git a/packages/host/src/lib/csi-controller.ts b/packages/host/src/lib/csi-controller.ts index b92bfc095..2bb7e616c 100644 --- a/packages/host/src/lib/csi-controller.ts +++ b/packages/host/src/lib/csi-controller.ts @@ -480,6 +480,7 @@ export class CSIController extends TypedEmitter { }); this.communicationHandler.addMonitoringHandler(RunnerMessageCode.PANG, async (message) => { + this.logger.info("PANG"); const pangData = message[1]; this.provides ||= this.outputTopic || pangData.provides; diff --git a/packages/host/src/lib/socket-server.ts b/packages/host/src/lib/socket-server.ts index fbc729772..9551a3684 100644 --- a/packages/host/src/lib/socket-server.ts +++ b/packages/host/src/lib/socket-server.ts @@ -1,21 +1,24 @@ -import { IComponent, DownstreamStreamsConfig, IObjectLogger } from "@scramjet/types"; +import { IComponent, IObjectLogger } from "@scramjet/types"; -import net from "net"; +import net, { Socket } from "net"; import { isDefined, TypedEmitter } from "@scramjet/utility"; import { ObjLogger } from "@scramjet/obj-logger"; +import { TeceMux, TeceMuxChannel } from "@scramjet/verser"; + +type MaybeChannel = TeceMuxChannel | Socket | null; + +type RunnerChannels = [ + TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel +]; -type MaybeSocket = net.Socket | null type RunnerConnectionsInProgress = [ - MaybeSocket, MaybeSocket, MaybeSocket, MaybeSocket, MaybeSocket, MaybeSocket, MaybeSocket, MaybeSocket, MaybeSocket -] -type RunnerOpenConnections = [ - net.Socket, net.Socket, net.Socket, net.Socket, net.Socket, net.Socket, net.Socket, net.Socket, net.Socket -] + MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel +]; type Events = { - connect: (id: string, streams: DownstreamStreamsConfig) => void -} + connect: (id: string, streams: RunnerChannels) => void +}; /** * Server for incoming connections from Runners @@ -37,36 +40,47 @@ export class SocketServer extends TypedEmitter implements IComponent { async start(): Promise { this.server = net.createServer(); - this.server - .on("connection", async (connection) => { - connection.on("error", (err) => { - this.logger.error("Error on connection from runner", err); - }); + let protocol: TeceMux; - const id = await new Promise((resolve) => { - connection.once("readable", () => { - resolve(connection.read(36).toString()); - }); - }); + this.server.on("connection", async (connection: net.Socket) => { + connection.on("error", (err) => { + this.logger.error("Error on connection from runner", err); + }); - const channel = await new Promise((resolve) => { - connection.once("readable", () => { - resolve(parseInt(connection.read(1).toString(), 10)); + protocol = new TeceMux(connection); + + protocol.on("channel", async (channel: TeceMuxChannel) => { + const { instanceId, channelId } = + await new Promise<{ instanceId: string, channelId: number }>((resolve) => { + channel.pause(); + channel.once("readable", () => { + const payload = channel.read(37).toString(); + const instId = payload.substring(0, 36); + const chanId = parseInt(payload.substring(36, 37), 10); + + resolve({ + instanceId: instId, + channelId: chanId + }); + }); }); - }); - connection - .on("error", (err) => this.logger.error("Error on Instance in stream", id, channel, err)) - .on("end", () => this.logger.debug(`Channel [${id}:${channel}] ended`)); + + channel + .on("error", (err: any) => this.logger.error("Error on Instance in stream", instanceId, channelId, err)) + .on("end", () => this.logger.debug(`Channel [${instanceId}:${channelId}] ended`)); + try { - await this.handleConnection(id, channel, connection); + await this.handleConnection(instanceId, channelId, connection as Socket); } catch (err: any) { connection.destroy(); } - }); + }) - return new Promise((res, rej) => { + }); + + return new Promise((res, rej) => { this.server! .listen(this.port, this.hostname, () => { this.logger.info("SocketServer on", this.server?.address()); @@ -91,9 +105,10 @@ export class SocketServer extends TypedEmitter implements IComponent { } if (runner.every(isDefined)) { this.runnerConnectionsInProgress.delete(id); - this.emit("connect", id, runner as RunnerOpenConnections); + this.emit("connect", id, runner as RunnerChannels); } } + close() { this.server?.close((err: any) => { if (err) { diff --git a/packages/runner/package.json b/packages/runner/package.json index 9ec7c62ed..e796061b5 100644 --- a/packages/runner/package.json +++ b/packages/runner/package.json @@ -22,6 +22,7 @@ "@scramjet/obj-logger": "^0.33.5", "@scramjet/symbols": "^0.33.5", "@scramjet/utility": "^0.33.5", + "@scramjet/verser": "^0.33.5", "bpmux": "^8.2.1", "scramjet": "^4.36.9" }, diff --git a/packages/runner/src/host-client.ts b/packages/runner/src/host-client.ts index 9d1558425..c3e97fa69 100644 --- a/packages/runner/src/host-client.ts +++ b/packages/runner/src/host-client.ts @@ -4,9 +4,10 @@ import { CommunicationChannel as CC } from "@scramjet/symbols"; import net, { createConnection, Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; import { Agent } from "http"; +import { TeceMux, TeceMuxChannel } from "@scramjet/verser"; type HostOpenConnections = [ - net.Socket, net.Socket, net.Socket, net.Socket, net.Socket, net.Socket, net.Socket, net.Socket, net.Socket + TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel ] const BPMux = require("bpmux").BPMux; @@ -40,29 +41,41 @@ class HostClient implements IHostClient { } async init(id: string): Promise { + const tunnel = net.createConnection(this.instancesServerPort, this.instancesServerHost); + const protocol = new TeceMux(tunnel); + const openConnections = await Promise.all( Array.from(Array(9)) - .map(() => { + .map((_c, index) => { // Error handling for each connection is process crash for now - const connection = net.createConnection(this.instancesServerPort, this.instancesServerHost); + const channel = protocol.multiplex({ channel: index });//net.createConnection(this.instancesServerPort, this.instancesServerHost); + + // return new Promise(res => { + // channel.on("connect", () => res(channel)); + // }); - return new Promise(res => { - connection.on("connect", () => res(connection)); - }); + return Promise.resolve(channel); }) - .map((connPromised, index) => { - return connPromised.then((connection) => { - // Assuming id is exactly 36 bytes - connection.write(id); - // Assuming number is from 0-7, sending 1 byte - connection.write(index.toString()); - - return connection; - }); + ).then(async res => { + return Promise.all( + res.map(async (channel, index) => { + // Assuming id is exactly 36 bytes + channel.write(id + "" + index); + // eslint-disable-next-line no-console + console.log(channel._id); + // Assuming number is from 0-8, sending 1 byte + //channel.write(index.toString()); + + //await defer(500); + + return channel; }) - ); + ); + }); + + //await defer(500); - this._streams = openConnections as HostOpenConnections; + this._streams = await openConnections as HostOpenConnections; try { this.bpmux = new BPMux(this._streams[CC.PACKAGE]); diff --git a/packages/runner/src/runner-app-context.ts b/packages/runner/src/runner-app-context.ts index 6ccc35f3e..80249a14a 100644 --- a/packages/runner/src/runner-app-context.ts +++ b/packages/runner/src/runner-app-context.ts @@ -44,7 +44,6 @@ implements AppContext { this.hub = hostClient; this.instanceId = id; } - private handleSave(_state: any): void { throw new Error("Method not implemented."); } diff --git a/packages/verser/src/index.ts b/packages/verser/src/index.ts index 19bca6084..c56cde24d 100644 --- a/packages/verser/src/index.ts +++ b/packages/verser/src/index.ts @@ -1,3 +1,4 @@ export * from "./lib/verser"; export * from "./lib/verser-client"; export * from "./lib/verser-connection"; +export * from "./lib/tecemux"; diff --git a/packages/verser/src/lib/tecemux/index.ts b/packages/verser/src/lib/tecemux/index.ts new file mode 100644 index 000000000..802a3f900 --- /dev/null +++ b/packages/verser/src/lib/tecemux/index.ts @@ -0,0 +1 @@ +export * from "./tecemux"; diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 96bb6ccd9..b6752f033 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -20,9 +20,11 @@ export class TeceMux extends TypedEmitter { commonEncoder = new FrameEncoder(0, this); private createChannel(destinationPort?: number, emit?: boolean): TeceMuxChannel { - this.logger.debug("Create Channel", destinationPort || this.channelCount); + const port = destinationPort !== undefined ? destinationPort : this.channelCount; - const encoder = new FrameEncoder(this.channelCount, this, { encoding: undefined }); + this.logger.debug("Create Channel", port); + + const encoder = new FrameEncoder(port, this, { encoding: undefined }); encoder.logger.updateBaseLog({ id: this.id }); encoder.logger.pipe(this.logger); @@ -47,7 +49,7 @@ export class TeceMux extends TypedEmitter { allowHalfOpen: true }), { - _id: destinationPort || this.channelCount, + _id: port || this.channelCount, encoder, closedByFIN: false } @@ -193,6 +195,7 @@ export class TeceMux extends TypedEmitter { channel = this.createChannel(destinationPort, false); this.addChannel(channel, true); + //this.emit("peer", { channelId: destinationPort}) } if (dataLength) { @@ -247,7 +250,7 @@ export class TeceMux extends TypedEmitter { multiplex(opts: { channel?: number } = {}): TeceMuxChannel { this.logger.trace("Multiplex"); - const channel = this.createChannel(opts.channel || this.channelCount, true); + const channel = this.createChannel(opts.channel !== undefined ? opts.channel : this.channelCount, true); this.addChannel(channel, false); @@ -255,5 +258,5 @@ export class TeceMux extends TypedEmitter { return channel; } } -export { TeceMuxChannel }; +export { TeceMuxChannel }; diff --git a/packages/verser/src/lib/tecemux/types.ts b/packages/verser/src/lib/tecemux/types.ts index 2cf55c247..17ac39a18 100644 --- a/packages/verser/src/lib/tecemux/types.ts +++ b/packages/verser/src/lib/tecemux/types.ts @@ -3,6 +3,7 @@ import { Duplex, Transform } from "stream"; export type TeceMuxEvents = { channel(socket: Duplex): void; error(error: any): void; + peer(payload: { channelId: number }): void; } export type FramesKeeperEvents = { From 7d3be521fe6095d23ce2c78d0d70152cac795359 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 28 Mar 2023 14:06:15 +0000 Subject: [PATCH 033/231] R<->STH --- packages/runner/Dockerfile | 1 + packages/runner/src/host-client.ts | 12 +-------- packages/runner/src/runner.ts | 2 ++ .../src/lib/tecemux/codecs/frame-encoder.ts | 27 ++++++++++++++----- packages/verser/src/lib/tecemux/tecemux.ts | 10 +++---- packages/verser/src/lib/tecemux/types.ts | 2 ++ packages/verser/src/lib/verser-connection.ts | 6 ++--- 7 files changed, 34 insertions(+), 26 deletions(-) diff --git a/packages/runner/Dockerfile b/packages/runner/Dockerfile index 3d659fea6..7c3076035 100644 --- a/packages/runner/Dockerfile +++ b/packages/runner/Dockerfile @@ -26,6 +26,7 @@ COPY ./dist/symbols ./dist/symbols COPY ./dist/obj-logger ./dist/obj-logger COPY ./dist/model ./dist/model COPY ./dist/runner ./dist/runner +COPY ./dist/verser ./dist/verser COPY ./dist/package.json ./dist/package.json FROM target diff --git a/packages/runner/src/host-client.ts b/packages/runner/src/host-client.ts index c3e97fa69..fd38d9a58 100644 --- a/packages/runner/src/host-client.ts +++ b/packages/runner/src/host-client.ts @@ -45,17 +45,7 @@ class HostClient implements IHostClient { const protocol = new TeceMux(tunnel); const openConnections = await Promise.all( - Array.from(Array(9)) - .map((_c, index) => { - // Error handling for each connection is process crash for now - const channel = protocol.multiplex({ channel: index });//net.createConnection(this.instancesServerPort, this.instancesServerHost); - - // return new Promise(res => { - // channel.on("connect", () => res(channel)); - // }); - - return Promise.resolve(channel); - }) + Array.from(Array(9)).map((_c, index) => protocol.multiplex({ channel: index })) ).then(async res => { return Promise.all( res.map(async (channel, index) => { diff --git a/packages/runner/src/runner.ts b/packages/runner/src/runner.ts index 880b07c1e..323e764f7 100644 --- a/packages/runner/src/runner.ts +++ b/packages/runner/src/runner.ts @@ -300,6 +300,8 @@ export class Runner implements IComponent { this.logger.debug("Streams initialized"); + //await defer(15); + this.sendHandshakeMessage(); const { appConfig, args } = await this.waitForHandshakeResponse(); diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index 7aa386263..ae3b20c79 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -76,15 +76,28 @@ export class FrameEncoder extends Transform { return flags; } - setChannel(channelCount: number) { + async setChannel(channelCount: number) { this.logger.debug("Set channel command", channelCount); - this.out.write( - this.createFrame([], { - flagsArray: ["PSH"], - destinationPort: channelCount - }) - ); + const frame = this.createFrame([], { + flagsArray: ["PSH"], + destinationPort: channelCount + }); + + this.out.write(frame); + + const sn = +this.tecemux.sequenceNumber; + + return await new Promise((resolve, _reject) => { + const waiter = (sequenceNumber: number) => { + if (sequenceNumber === sn) { + this.tecemux.framesKeeper.off("ack", waiter); + resolve(); + } + }; + + this.tecemux.framesKeeper.on("ack", waiter); + }); } onChannelEnd(channelId: number) { diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index b6752f033..63610f400 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -19,7 +19,7 @@ export class TeceMux extends TypedEmitter { logger: ObjLogger; commonEncoder = new FrameEncoder(0, this); - private createChannel(destinationPort?: number, emit?: boolean): TeceMuxChannel { + private async createChannel(destinationPort?: number, emit?: boolean): Promise { const port = destinationPort !== undefined ? destinationPort : this.channelCount; this.logger.debug("Create Channel", port); @@ -87,7 +87,7 @@ export class TeceMux extends TypedEmitter { }); if (emit) { - encoder.setChannel(destinationPort || this.channelCount); + await encoder.setChannel(destinationPort || this.channelCount); } return channel; @@ -192,7 +192,7 @@ export class TeceMux extends TypedEmitter { if (!channel) { this.logger.warn("Unknown channel"); - channel = this.createChannel(destinationPort, false); + channel = await this.createChannel(destinationPort, false); this.addChannel(channel, true); //this.emit("peer", { channelId: destinationPort}) @@ -247,10 +247,10 @@ export class TeceMux extends TypedEmitter { ); } - multiplex(opts: { channel?: number } = {}): TeceMuxChannel { + async multiplex(opts: { channel?: number } = {}): Promise { this.logger.trace("Multiplex"); - const channel = this.createChannel(opts.channel !== undefined ? opts.channel : this.channelCount, true); + const channel = await this.createChannel(opts.channel !== undefined ? opts.channel : this.channelCount, true); this.addChannel(channel, false); diff --git a/packages/verser/src/lib/tecemux/types.ts b/packages/verser/src/lib/tecemux/types.ts index 17ac39a18..94964849f 100644 --- a/packages/verser/src/lib/tecemux/types.ts +++ b/packages/verser/src/lib/tecemux/types.ts @@ -37,6 +37,8 @@ export interface IFramesKeeper { handlePSH(sequenceNumber: number): void; getFrame(sequenceNumber: number): FramesKeeperFrame | undefined; generator: AsyncGenerator; + on(event: string, handler: (sequenceNumber: number) => void): void; + off(event: string, handler: (sequenceNumber: number) => void): void; } export interface ITeCeMux { diff --git a/packages/verser/src/lib/verser-connection.ts b/packages/verser/src/lib/verser-connection.ts index 87edc0344..16685aaeb 100644 --- a/packages/verser/src/lib/verser-connection.ts +++ b/packages/verser/src/lib/verser-connection.ts @@ -101,7 +101,7 @@ export class VerserConnection { async forward(req: IncomingMessage, res: ServerResponse) { if (!this.connected) throw new Error("BPMux not connected"); - const channel = this.teceMux?.multiplex() as Duplex; + const channel = await this.teceMux?.multiplex() as Duplex; channel .on("error", (error: Error) => { @@ -191,10 +191,10 @@ export class VerserConnection { * @param id {number} Channel id. * @returns Duplex stream. */ - createChannel(id: number): Duplex { + async createChannel(id: number): Promise { if (!this.teceMux) throw new Error("TeCeMux not connected"); - return this.teceMux.multiplex({ channel: id }); + return await this.teceMux.multiplex({ channel: id }); } reconnect() { From c285ad1cfe0c74e2dd5307692be1f56911e4f9ed Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 3 Apr 2023 21:12:12 +0000 Subject: [PATCH 034/231] TeCeMux wip --- packages/host/src/lib/socket-server.ts | 1 - .../src/lib/tecemux/codecs/frame-encoder.ts | 16 +++++++++++----- packages/verser/src/lib/tecemux/tecemux.ts | 14 ++++++++++---- packages/verser/src/lib/tecemux/types.ts | 2 +- packages/verser/src/lib/verser-connection.ts | 2 +- 5 files changed, 23 insertions(+), 12 deletions(-) diff --git a/packages/host/src/lib/socket-server.ts b/packages/host/src/lib/socket-server.ts index 9551a3684..042286c32 100644 --- a/packages/host/src/lib/socket-server.ts +++ b/packages/host/src/lib/socket-server.ts @@ -52,7 +52,6 @@ export class SocketServer extends TypedEmitter implements IComponent { protocol.on("channel", async (channel: TeceMuxChannel) => { const { instanceId, channelId } = await new Promise<{ instanceId: string, channelId: number }>((resolve) => { - channel.pause(); channel.once("readable", () => { const payload = channel.read(37).toString(); const instId = payload.substring(0, 36); diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index ae3b20c79..80f86768b 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -1,3 +1,5 @@ +/* eslint-disable no-console */ + import { PassThrough, Transform, TransformCallback, TransformOptions } from "stream"; import { ObjLogger } from "@scramjet/obj-logger"; @@ -76,9 +78,11 @@ export class FrameEncoder extends Transform { return flags; } - async setChannel(channelCount: number) { + async establishChannel(channelCount: number) { this.logger.debug("Set channel command", channelCount); + console.log("Set channel command", channelCount); + const frame = this.createFrame([], { flagsArray: ["PSH"], destinationPort: channelCount @@ -88,15 +92,17 @@ export class FrameEncoder extends Transform { const sn = +this.tecemux.sequenceNumber; - return await new Promise((resolve, _reject) => { - const waiter = (sequenceNumber: number) => { + return new Promise((resolve, _reject) => { + const ackTempHandler = (sequenceNumber: number) => { + console.log("ACK RECEIVED", sequenceNumber, sn); + if (sequenceNumber === sn) { - this.tecemux.framesKeeper.off("ack", waiter); + this.tecemux.framesKeeper.off("ack", ackTempHandler); resolve(); } }; - this.tecemux.framesKeeper.on("ack", waiter); + this.tecemux.framesKeeper.on("ack", ackTempHandler); }); } diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 63610f400..c6bd6bb0c 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -1,3 +1,5 @@ +/* eslint-disable no-console */ + import { TypedEmitter } from "@scramjet/utility"; import { FrameDecoder, FrameEncoder } from "./codecs"; import { Duplex } from "stream"; @@ -13,7 +15,7 @@ export class TeceMux extends TypedEmitter { framesSent = 0; carrierDecoder: FrameDecoder; framesKeeper = new FramesKeeper(); - sequenceNumber = Math.abs((Math.random() * (2 ** 32)) | 0); + sequenceNumber = Math.abs((Math.random() * (2 ** 32)) / 2 | 0); channels = new Map(); logger: ObjLogger; @@ -87,7 +89,7 @@ export class TeceMux extends TypedEmitter { }); if (emit) { - await encoder.setChannel(destinationPort || this.channelCount); + await encoder.establishChannel(port || this.channelCount); } return channel; @@ -249,10 +251,14 @@ export class TeceMux extends TypedEmitter { async multiplex(opts: { channel?: number } = {}): Promise { this.logger.trace("Multiplex"); + console.log("Multiplex", opts.channel); + + const id = opts.channel !== undefined ? opts.channel : this.channelCount; + const channel = await this.createChannel(id, true); - const channel = await this.createChannel(opts.channel !== undefined ? opts.channel : this.channelCount, true); + await channel.encoder.establishChannel(id); - this.addChannel(channel, false); + this.addChannel(channel, true); this.logger.trace("Multiplex ready", channel._id); return channel; diff --git a/packages/verser/src/lib/tecemux/types.ts b/packages/verser/src/lib/tecemux/types.ts index 94964849f..f1860077e 100644 --- a/packages/verser/src/lib/tecemux/types.ts +++ b/packages/verser/src/lib/tecemux/types.ts @@ -48,7 +48,7 @@ export interface ITeCeMux { } export interface IFrameEncoder extends Transform { - + establishChannel(id: number): Promise; } export type TeceMuxChannel = Duplex & { diff --git a/packages/verser/src/lib/verser-connection.ts b/packages/verser/src/lib/verser-connection.ts index 16685aaeb..79c1bd826 100644 --- a/packages/verser/src/lib/verser-connection.ts +++ b/packages/verser/src/lib/verser-connection.ts @@ -99,7 +99,7 @@ export class VerserConnection { * @param res {ServerResponse} Response object. */ async forward(req: IncomingMessage, res: ServerResponse) { - if (!this.connected) throw new Error("BPMux not connected"); + if (!this.connected) throw new Error("Not connected"); const channel = await this.teceMux?.multiplex() as Duplex; From d613203b87b91cfa7d4cd42c7d29751e92d5ba9c Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 6 Apr 2023 19:25:21 +0000 Subject: [PATCH 035/231] fix channels _id --- packages/host/src/lib/socket-server.ts | 45 +++++++++++++++++++ packages/runner/src/host-client.ts | 4 +- .../verser/src/lib/tecemux/frames-keeper.ts | 4 +- packages/verser/src/lib/tecemux/tecemux.ts | 12 +++-- 4 files changed, 55 insertions(+), 10 deletions(-) diff --git a/packages/host/src/lib/socket-server.ts b/packages/host/src/lib/socket-server.ts index 042286c32..d49c8328b 100644 --- a/packages/host/src/lib/socket-server.ts +++ b/packages/host/src/lib/socket-server.ts @@ -48,6 +48,7 @@ export class SocketServer extends TypedEmitter implements IComponent { }); protocol = new TeceMux(connection); + //protocol.logger.pipe(this.logger); protocol.on("channel", async (channel: TeceMuxChannel) => { const { instanceId, channelId } = @@ -64,16 +65,60 @@ export class SocketServer extends TypedEmitter implements IComponent { }); }); +<<<<<<< HEAD +||||||| constructed merge base + // const channelId = await new Promise((resolve) => { + // channel.once("readable", () => { + // resolve(parseInt(channel.read(1).toString(), 10)); + // }); + // }); + + this.logger.info("new channel", instanceId, channelId); + + let runner = this.runnerConnectionsInProgress.get(instanceId); + + if (!runner) { + runner = [null, null, null, null, null, null, null, null, null]; + this.runnerConnectionsInProgress.set(instanceId, runner); + } +======= + // const channelId = await new Promise((resolve) => { + // channel.once("readable", () => { + // resolve(parseInt(channel.read(1).toString(), 10)); + // }); + // }); + + this.logger.info("new channel", instanceId, channelId, channel._id); + + let runner = this.runnerConnectionsInProgress.get(instanceId); + + if (!runner) { + runner = [null, null, null, null, null, null, null, null, null]; + this.runnerConnectionsInProgress.set(instanceId, runner); + } +>>>>>>> fix channels _id channel .on("error", (err: any) => this.logger.error("Error on Instance in stream", instanceId, channelId, err)) .on("end", () => this.logger.debug(`Channel [${instanceId}:${channelId}] ended`)); +<<<<<<< HEAD try { await this.handleConnection(instanceId, channelId, connection as Socket); } catch (err: any) { connection.destroy(); +||||||| constructed merge base + if (runner.every(isDefined)) { + this.runnerConnectionsInProgress.delete(instanceId); + this.emit("connect", instanceId, runner as RunnerChannels); +======= + if (runner.every(isDefined)) { + // eslint-disable-next-line no-console + console.log(runner.map(r => r!._id)); + this.runnerConnectionsInProgress.delete(instanceId); + this.emit("connect", instanceId, runner as RunnerChannels); +>>>>>>> fix channels _id } }) diff --git a/packages/runner/src/host-client.ts b/packages/runner/src/host-client.ts index fd38d9a58..115fbefb2 100644 --- a/packages/runner/src/host-client.ts +++ b/packages/runner/src/host-client.ts @@ -44,6 +44,8 @@ class HostClient implements IHostClient { const tunnel = net.createConnection(this.instancesServerPort, this.instancesServerHost); const protocol = new TeceMux(tunnel); + //protocol.logger.pipe(this.logger); + const openConnections = await Promise.all( Array.from(Array(9)).map((_c, index) => protocol.multiplex({ channel: index })) ).then(async res => { @@ -52,7 +54,7 @@ class HostClient implements IHostClient { // Assuming id is exactly 36 bytes channel.write(id + "" + index); // eslint-disable-next-line no-console - console.log(channel._id); + console.log(id, index, channel._id); // Assuming number is from 0-8, sending 1 byte //channel.write(index.toString()); diff --git a/packages/verser/src/lib/tecemux/frames-keeper.ts b/packages/verser/src/lib/tecemux/frames-keeper.ts index 48126f77f..cc2fbd3b9 100644 --- a/packages/verser/src/lib/tecemux/frames-keeper.ts +++ b/packages/verser/src/lib/tecemux/frames-keeper.ts @@ -15,7 +15,7 @@ export class FramesKeeper extends TypedEmitter implements IF generator: AsyncGenerator = (async function* (this: FramesKeeper) { while (true) { if (this.lastSequenceSent - this.lastSequenceReceived < this.#MAX_FRAMES_DIFFERENCE) { - this.logger.info("Write allowed"); + this.logger.debug("Write allowed"); yield Promise.resolve(this.lastSequenceReceived); continue; } @@ -53,7 +53,7 @@ export class FramesKeeper extends TypedEmitter implements IF const frame = this.#framesSent.get(sequenceNumber); // received or not stored - return frame === undefined || !!this.#framesSent.get(sequenceNumber)?.received; + return frame === undefined || !!frame.received; } getFrame(sequenceNumber: number) { diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index c6bd6bb0c..6608e0dc5 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -34,7 +34,7 @@ export class TeceMux extends TypedEmitter { const channel: TeceMuxChannel = Object.assign( new Duplex({ write: (chunk, encoding, next) => { - this.logger.trace("WRITE channel", channel._id, chunk); + this.logger.debug("WRITE channel", channel._id, chunk); if (chunk === null) { this.logger.info("NULL ON CHANNEL"); @@ -46,12 +46,12 @@ export class TeceMux extends TypedEmitter { return encoder.write(chunk, encoding, next); }, read: (_size) => { - this.logger.trace("READ channel", channel._id); + this.logger.debug("READ channel", channel._id); }, allowHalfOpen: true }), { - _id: port || this.channelCount, + _id: port, encoder, closedByFIN: false } @@ -89,7 +89,7 @@ export class TeceMux extends TypedEmitter { }); if (emit) { - await encoder.establishChannel(port || this.channelCount); + await encoder.establishChannel(channel._id); } return channel; @@ -141,8 +141,6 @@ export class TeceMux extends TypedEmitter { async main() { let t = 0; - //this.carrierDecoder.pipe(this.framesKeeper); - for await (const chunk of this.carrierDecoder) { let frame: FrameData; @@ -228,7 +226,7 @@ export class TeceMux extends TypedEmitter { addChannel(channel: TeceMuxChannel, emit: boolean) { this.logger.debug("adding channel", channel._id); - this.channels.set(channel._id, channel); // wait for SYN reply? + this.channels.set(channel._id, channel); if (emit) { this.emit("channel", channel); From d89275a7a52a59b7eafa46cab66b546e82566f12 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 6 Apr 2023 21:47:02 +0000 Subject: [PATCH 036/231] Test defer --- packages/runner/src/runner.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/runner/src/runner.ts b/packages/runner/src/runner.ts index 323e764f7..29bd8639e 100644 --- a/packages/runner/src/runner.ts +++ b/packages/runner/src/runner.ts @@ -302,7 +302,7 @@ export class Runner implements IComponent { //await defer(15); - this.sendHandshakeMessage(); + await this.sendHandshakeMessage(); const { appConfig, args } = await this.waitForHandshakeResponse(); @@ -451,7 +451,8 @@ export class Runner implements IComponent { // TODO: what if it fails? } - sendHandshakeMessage() { + async sendHandshakeMessage() { + await defer(1000); MessageUtils.writeMessageOnStream([RunnerMessageCode.PING, {}], this.hostClient.monitorStream); this.logger.trace("Handshake sent"); From 3d4535d3e7a541346dd8b2faa26cc3b00febf3cf Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 11 Apr 2023 09:41:12 +0000 Subject: [PATCH 037/231] CI test --- packages/host/src/lib/csi-controller.ts | 5 +- packages/host/src/lib/socket-server.ts | 50 +------------------ .../src/lib/tecemux/codecs/frame-encoder.ts | 11 ++-- packages/verser/src/lib/tecemux/tecemux.ts | 7 ++- packages/verser/test/tecemux-transfer.spec.ts | 22 +++++--- 5 files changed, 26 insertions(+), 69 deletions(-) diff --git a/packages/host/src/lib/csi-controller.ts b/packages/host/src/lib/csi-controller.ts index 2bb7e616c..cad7ce566 100644 --- a/packages/host/src/lib/csi-controller.ts +++ b/packages/host/src/lib/csi-controller.ts @@ -539,8 +539,9 @@ export class CSIController extends TypedEmitter { const pongMsg: HandshakeAcknowledgeMessage = { msgCode: RunnerMessageCode.PONG, appConfig: this.appConfig, - args: this.args - }; + args: this.args, + b: Buffer.from(new Uint8Array(1024 * 1024).fill(2)).toString("hex") + } as HandshakeAcknowledgeMessage; await this.controlDataStream.whenWrote(MessageUtilities.serializeMessage(pongMsg)); } else { diff --git a/packages/host/src/lib/socket-server.ts b/packages/host/src/lib/socket-server.ts index d49c8328b..da707029d 100644 --- a/packages/host/src/lib/socket-server.ts +++ b/packages/host/src/lib/socket-server.ts @@ -48,7 +48,6 @@ export class SocketServer extends TypedEmitter implements IComponent { }); protocol = new TeceMux(connection); - //protocol.logger.pipe(this.logger); protocol.on("channel", async (channel: TeceMuxChannel) => { const { instanceId, channelId } = @@ -65,63 +64,16 @@ export class SocketServer extends TypedEmitter implements IComponent { }); }); -<<<<<<< HEAD -||||||| constructed merge base - // const channelId = await new Promise((resolve) => { - // channel.once("readable", () => { - // resolve(parseInt(channel.read(1).toString(), 10)); - // }); - // }); - - this.logger.info("new channel", instanceId, channelId); - - let runner = this.runnerConnectionsInProgress.get(instanceId); - - if (!runner) { - runner = [null, null, null, null, null, null, null, null, null]; - this.runnerConnectionsInProgress.set(instanceId, runner); - } -======= - // const channelId = await new Promise((resolve) => { - // channel.once("readable", () => { - // resolve(parseInt(channel.read(1).toString(), 10)); - // }); - // }); - - this.logger.info("new channel", instanceId, channelId, channel._id); - - let runner = this.runnerConnectionsInProgress.get(instanceId); - - if (!runner) { - runner = [null, null, null, null, null, null, null, null, null]; - this.runnerConnectionsInProgress.set(instanceId, runner); - } ->>>>>>> fix channels _id - channel .on("error", (err: any) => this.logger.error("Error on Instance in stream", instanceId, channelId, err)) .on("end", () => this.logger.debug(`Channel [${instanceId}:${channelId}] ended`)); - -<<<<<<< HEAD try { await this.handleConnection(instanceId, channelId, connection as Socket); } catch (err: any) { connection.destroy(); -||||||| constructed merge base - if (runner.every(isDefined)) { - this.runnerConnectionsInProgress.delete(instanceId); - this.emit("connect", instanceId, runner as RunnerChannels); -======= - if (runner.every(isDefined)) { - // eslint-disable-next-line no-console - console.log(runner.map(r => r!._id)); - this.runnerConnectionsInProgress.delete(instanceId); - this.emit("connect", instanceId, runner as RunnerChannels); ->>>>>>> fix channels _id } - }) - + }); }); return new Promise((res, rej) => { diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index 80f86768b..9c6f012e2 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -78,14 +78,12 @@ export class FrameEncoder extends Transform { return flags; } - async establishChannel(channelCount: number) { - this.logger.debug("Set channel command", channelCount); - - console.log("Set channel command", channelCount); + async establishChannel(channelId: number) { + this.logger.debug("Establishing channel", channelId); const frame = this.createFrame([], { flagsArray: ["PSH"], - destinationPort: channelCount + destinationPort: channelId }); this.out.write(frame); @@ -94,10 +92,11 @@ export class FrameEncoder extends Transform { return new Promise((resolve, _reject) => { const ackTempHandler = (sequenceNumber: number) => { - console.log("ACK RECEIVED", sequenceNumber, sn); + this.logger.debug("ACK RECEIVED", sequenceNumber, sn); if (sequenceNumber === sn) { this.tecemux.framesKeeper.off("ack", ackTempHandler); + this.logger.debug("channel established", channelId); resolve(); } }; diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 6608e0dc5..5f99fa3b5 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -248,11 +248,10 @@ export class TeceMux extends TypedEmitter { } async multiplex(opts: { channel?: number } = {}): Promise { - this.logger.trace("Multiplex"); - console.log("Multiplex", opts.channel); - const id = opts.channel !== undefined ? opts.channel : this.channelCount; - const channel = await this.createChannel(id, true); + const channel = await this.createChannel(id); + + this.logger.trace("Multiplex", id); await channel.encoder.establishChannel(id); diff --git a/packages/verser/test/tecemux-transfer.spec.ts b/packages/verser/test/tecemux-transfer.spec.ts index 54c9c97d7..326818ed7 100644 --- a/packages/verser/test/tecemux-transfer.spec.ts +++ b/packages/verser/test/tecemux-transfer.spec.ts @@ -18,7 +18,7 @@ async function startServer() { }); } -test("Protocol send file over http connection", async (t) => { +test.serial("Protocol send file over http connection", async (t) => { const server = await startServer(); const hashReceived = crypto.createHash("md5"); @@ -26,14 +26,15 @@ test("Protocol send file over http connection", async (t) => { const serverSideChannelPromise = new Promise((resolve) => { server.on("connect", async (req: IncomingMessage, socket: Socket) => { + console.log("server on connect!"); socket.setNoDelay(true); serverTeceMux = new TeceMux(socket, "Server") .on("error", (error) => { - console.error("TeceMux error", error); + console.error("TeceMux error", error.code); }); - resolve(serverTeceMux.multiplex()); + resolve(serverTeceMux.multiplex({ channel: 1 })); }); }); @@ -51,6 +52,7 @@ test("Protocol send file over http connection", async (t) => { await new Promise(resolve => { clientTeceMux.on("channel", async (clientSideChannel: TeceMuxChannel) => { + console.log("TEST: on channel", clientSideChannel._id); function* gen() { for (let i = 0; i < 1e3; i++) { const str = crypto.randomBytes(1024).toString("hex"); @@ -67,14 +69,18 @@ test("Protocol send file over http connection", async (t) => { const serverSideChannel = await serverSideChannelPromise; + console.log("Serverside channel", serverSideChannel._id); + for await (const d of serverSideChannel) { hashReceived.update(d); } - console.dir({ - hashSent, - hashReceived - }); + const res = { + txHash: hashSent.digest("hex"), + rxHash: hashReceived.digest("hex") + }; + + console.dir(res); - t.assert(hashReceived.digest("hex") === hashSent.digest("hex"), "Unequal hashes"); + t.assert(res.txHash === res.rxHash, "Unequal hashes"); }); From ab725938a088f4a7f2fb529d282bd6ee5972428c Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 11 Apr 2023 13:51:40 +0000 Subject: [PATCH 038/231] Remove debug listeners --- packages/host/src/lib/csi-controller.ts | 3 +- packages/runner/src/host-client.ts | 10 +--- packages/verser/src/lib/tecemux/tecemux.ts | 58 +++++++++++----------- 3 files changed, 31 insertions(+), 40 deletions(-) diff --git a/packages/host/src/lib/csi-controller.ts b/packages/host/src/lib/csi-controller.ts index cad7ce566..aaa2e8550 100644 --- a/packages/host/src/lib/csi-controller.ts +++ b/packages/host/src/lib/csi-controller.ts @@ -539,8 +539,7 @@ export class CSIController extends TypedEmitter { const pongMsg: HandshakeAcknowledgeMessage = { msgCode: RunnerMessageCode.PONG, appConfig: this.appConfig, - args: this.args, - b: Buffer.from(new Uint8Array(1024 * 1024).fill(2)).toString("hex") + args: this.args } as HandshakeAcknowledgeMessage; await this.controlDataStream.whenWrote(MessageUtilities.serializeMessage(pongMsg)); diff --git a/packages/runner/src/host-client.ts b/packages/runner/src/host-client.ts index 115fbefb2..e11a2d878 100644 --- a/packages/runner/src/host-client.ts +++ b/packages/runner/src/host-client.ts @@ -51,22 +51,14 @@ class HostClient implements IHostClient { ).then(async res => { return Promise.all( res.map(async (channel, index) => { - // Assuming id is exactly 36 bytes + // Assuming id is exactly 36 bytes + Assuming number is from 0-8, sending 1 byte channel.write(id + "" + index); - // eslint-disable-next-line no-console - console.log(id, index, channel._id); - // Assuming number is from 0-8, sending 1 byte - //channel.write(index.toString()); - - //await defer(500); return channel; }) ); }); - //await defer(500); - this._streams = await openConnections as HostOpenConnections; try { diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 5f99fa3b5..004e206ec 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -65,28 +65,28 @@ export class TeceMux extends TypedEmitter { this.logger.error("CHANNEL ERROR", error); this.emit("error", { error, source: channel }); }) - .on("destroy", () => { - this.logger.trace("channel on DESTROY ", channel._id); - }) - .on("abort", () => { - this.logger.trace("channel on ABORT ", channel._id); - }) + // .on("destroy", () => { + // this.logger.trace("channel on DESTROY ", channel._id); + // }) + // .on("abort", () => { + // this.logger.trace("channel on ABORT ", channel._id); + // }) .on("close", () => { this.logger.info("CHANNEL close", channel._id); this.sendFIN(channel._id); }) - .on("end", () => { - this.logger.info("CHANNEL end", channel._id); - }) + // .on("end", () => { + // this.logger.info("CHANNEL end", channel._id); + // }) .on("finish", () => { this.logger.info("CHANNEL finish", channel._id); this.sendFIN(channel._id); - }) - .on("data", (d) => { - if (d === null) { - this.logger.info("CHANNEL end", channel._id); - } }); + // .on("data", (d) => { + // if (d === null) { + // this.logger.info("CHANNEL end", channel._id); + // } + // }); if (emit) { await encoder.establishChannel(channel._id); @@ -102,24 +102,24 @@ export class TeceMux extends TypedEmitter { this.carrierSocket = socket; this.carrierDecoder = new FrameDecoder({ emitClose: false }) - .on("pause", () => { - this.logger.warn("Decoder paused"); - }) - .on("close", () => { - this.logger.warn("Decoder closed"); - }) - .on("end", () => { - this.logger.warn("Decoder ended"); - }) .on("error", (error) => { this.logger.error("Decoder error", error); - }) - .on("abort", (error) => { - this.logger.error("Decoder abort", error); - }) - .on("destroy", (error) => { - this.logger.error("Decoder destroy", error); }); + // .on("pause", () => { + // this.logger.warn("Decoder paused"); + // }) + // .on("close", () => { + // this.logger.warn("Decoder closed"); + // }) + // .on("end", () => { + // this.logger.warn("Decoder ended"); + // }) + // .on("abort", (error) => { + // this.logger.error("Decoder abort", error); + // }) + // .on("destroy", (error) => { + // this.logger.error("Decoder destroy", error); + // }); this.carrierDecoder.logger.updateBaseLog({ id: this.id }); this.carrierDecoder.logger.pipe(this.logger); From 78a7f7d634e997c91f7a91ecca6004398ff90573 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 12 Apr 2023 09:22:52 +0000 Subject: [PATCH 039/231] Runner host-client destroy --- packages/runner/src/host-client.ts | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/packages/runner/src/host-client.ts b/packages/runner/src/host-client.ts index e11a2d878..a7ad388d1 100644 --- a/packages/runner/src/host-client.ts +++ b/packages/runner/src/host-client.ts @@ -59,7 +59,7 @@ class HostClient implements IHostClient { ); }); - this._streams = await openConnections as HostOpenConnections; + this._streams = openConnections as HostOpenConnections; try { this.bpmux = new BPMux(this._streams[CC.PACKAGE]); @@ -102,22 +102,12 @@ class HostClient implements IHostClient { async disconnect() { this.logger.trace("Disconnecting from host"); - const streamsExitedPromised: Promise[] = this.streams.map((stream, i) => + const streamsExitedPromised: Promise[] = this.streams.map((stream, _i) => new Promise( (res) => { - if ("writable" in stream!) { - stream - .on("error", (e) => { - console.error("Error on stream", i, e.stack); - }) - .on("close", () => { - res(); - }) - .end(); - } else { - stream!.destroy(); - res(); - } + // now we have readable and writable always + stream!.destroy(); + res(); } )); From 25b58f799e8f9c1ad1ba6ba70935e2ddebdb9f82 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 12 Apr 2023 15:19:14 +0000 Subject: [PATCH 040/231] nextTick before sequence call --- packages/host/src/lib/socket-server.ts | 6 ++++-- packages/runner/src/host-client.ts | 2 +- packages/runner/src/runner.ts | 6 +++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/host/src/lib/socket-server.ts b/packages/host/src/lib/socket-server.ts index da707029d..a892ed263 100644 --- a/packages/host/src/lib/socket-server.ts +++ b/packages/host/src/lib/socket-server.ts @@ -9,11 +9,13 @@ import { TeceMux, TeceMuxChannel } from "@scramjet/verser"; type MaybeChannel = TeceMuxChannel | Socket | null; type RunnerChannels = [ - TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel + TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, + TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel ]; type RunnerConnectionsInProgress = [ - MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel + MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, + MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel ]; type Events = { diff --git a/packages/runner/src/host-client.ts b/packages/runner/src/host-client.ts index a7ad388d1..3338d8576 100644 --- a/packages/runner/src/host-client.ts +++ b/packages/runner/src/host-client.ts @@ -59,7 +59,7 @@ class HostClient implements IHostClient { ); }); - this._streams = openConnections as HostOpenConnections; + this._streams = await openConnections as HostOpenConnections; try { this.bpmux = new BPMux(this._streams[CC.PACKAGE]); diff --git a/packages/runner/src/runner.ts b/packages/runner/src/runner.ts index 29bd8639e..dbe4035ab 100644 --- a/packages/runner/src/runner.ts +++ b/packages/runner/src/runner.ts @@ -504,12 +504,16 @@ export class Runner implements IComponent { try { this.logger.debug("Processing function on index", sequence.length - itemsLeftInSequence - 1); + // eslint-disable-next-line no-loop-func + await new Promise((res) => { + process.nextTick(res); + }); + out = func.call( this.context, stream, ...args ); - this.logger.debug("Function called", sequence.length - itemsLeftInSequence - 1); } catch (error: any) { this.logger.error("Function errored", sequence.length - itemsLeftInSequence, error.stack); From 81ce3f7d83e00dc8b9d35da4015d5a1e17776f07 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 13 Apr 2023 00:11:47 +0000 Subject: [PATCH 041/231] TeceMuxChannel class --- .../src/lib/tecemux/codecs/frame-encoder.ts | 2 +- .../verser/src/lib/tecemux/tecemux-channel.ts | 60 +++++ packages/verser/src/lib/tecemux/tecemux.ts | 245 +++++++++--------- packages/verser/src/lib/tecemux/types.ts | 6 +- 4 files changed, 189 insertions(+), 124 deletions(-) create mode 100644 packages/verser/src/lib/tecemux/tecemux-channel.ts diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index 9c6f012e2..042f0bfbe 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -8,7 +8,7 @@ import { FrameData, ITeCeMux } from "../types"; import { calculateChecksum } from "./utils"; export class FrameEncoder extends Transform { - MAX_CHUNK_SIZE = 10 * 1024 - HEADER_LENGTH; + MAX_CHUNK_SIZE = 64 * 1024 - HEADER_LENGTH; tecemux: ITeCeMux; total = 0; diff --git a/packages/verser/src/lib/tecemux/tecemux-channel.ts b/packages/verser/src/lib/tecemux/tecemux-channel.ts new file mode 100644 index 000000000..36460e5b1 --- /dev/null +++ b/packages/verser/src/lib/tecemux/tecemux-channel.ts @@ -0,0 +1,60 @@ +import { ObjLogger } from "@scramjet/obj-logger"; +import { Duplex } from "stream"; +import { IFrameEncoder, ITeCeMux } from "./types"; +import { FrameEncoder } from "./codecs"; + +export class TeceMuxChannel extends Duplex { + _id: number; + logger: ObjLogger; + allowHalfOpen: boolean; + encoder: IFrameEncoder; + closedByFIN = false; + + constructor(duplexOptions: ConstructorParameters[0], id: number, teceMux: ITeCeMux) { + super(duplexOptions); + this.logger = new ObjLogger(this); + this.allowHalfOpen = true; + this._id = id; + this.encoder = new FrameEncoder(id, teceMux, { encoding: undefined }); + } + + _write(chunk: any, encoding: BufferEncoding, next: (error?: Error | null | undefined) => void) { + this.logger.debug("WRITE channel", this._id, chunk); + + if (chunk === null) { + this.logger.info("NULL ON CHANNEL"); + + this.end(); + return false; + } + + return this.encoder.write(chunk, encoding, next); + } + + _read(_size?: number) { + this.logger.debug("READ channel", this._id); + } + + sendACK(sequenceNumber: number) { + this.encoder.push( + this.encoder.createFrame(undefined, { + flagsArray: ["ACK"], + acknowledgeNumber: sequenceNumber, + destinationPort: this._id + }) + ); + } + + handlerFIN() { + this.closedByFIN = true; + + if (!this.writableEnded) { + this.push(null); + } + + if (this.writableEnded && this.readableEnded) { + this.destroy(); + this.logger.info("Channel destroy"); + } + } +} diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 004e206ec..9cedacf8f 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -1,12 +1,13 @@ /* eslint-disable no-console */ import { TypedEmitter } from "@scramjet/utility"; -import { FrameDecoder, FrameEncoder } from "./codecs"; +import { FrameDecoder } from "./codecs"; import { Duplex } from "stream"; import { Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; -import { FrameData, TeceMuxChannel, TeceMuxEvents } from "./types"; +import { FrameData, TeceMuxEvents } from "./types"; import { FramesKeeper } from "./frames-keeper"; +import { TeceMuxChannel } from "./tecemux-channel"; export class TeceMux extends TypedEmitter { id: string; @@ -19,45 +20,66 @@ export class TeceMux extends TypedEmitter { channels = new Map(); logger: ObjLogger; - commonEncoder = new FrameEncoder(0, this); - private async createChannel(destinationPort?: number, emit?: boolean): Promise { + private async createChannel(destinationPort?: number, establish?: boolean): Promise { const port = destinationPort !== undefined ? destinationPort : this.channelCount; this.logger.debug("Create Channel", port); - const encoder = new FrameEncoder(port, this, { encoding: undefined }); - - encoder.logger.updateBaseLog({ id: this.id }); - encoder.logger.pipe(this.logger); - - const channel: TeceMuxChannel = Object.assign( - new Duplex({ - write: (chunk, encoding, next) => { - this.logger.debug("WRITE channel", channel._id, chunk); - - if (chunk === null) { - this.logger.info("NULL ON CHANNEL"); - - channel.end(); - return false; - } - - return encoder.write(chunk, encoding, next); - }, - read: (_size) => { - this.logger.debug("READ channel", channel._id); - }, - allowHalfOpen: true - }), - { - _id: port, - encoder, - closedByFIN: false - } - ); - - encoder.out + //const encoder = new FrameEncoder(port, this, { encoding: undefined }); + + //encoder.logger.updateBaseLog({ id: this.id }); + //encoder.logger.pipe(this.logger); + + const channel = new TeceMuxChannel({ allowHalfOpen: true }, port, this); + // const channel: TeceMuxChannel = Object.assign( + // new Duplex({ + // write: (chunk, encoding, next) => { + // this.logger.debug("WRITE channel", channel._id, chunk); + + // if (chunk === null) { + // this.logger.info("NULL ON CHANNEL"); + + // channel.end(); + // return false; + // } + + // return encoder.write(chunk, encoding, next); + // }, + // read: (_size) => { + // this.logger.debug("READ channel", channel._id); + // }, + // allowHalfOpen: true + // }), + // { + // _id: port, + // encoder, + // closedByFIN: false, + // sendACK: (sequenceNumber: number) => { + // channel.encoder.push( + // channel.encoder.createFrame(undefined, { + // flagsArray: ["ACK"], + // acknowledgeNumber: sequenceNumber, + // destinationPort: port + // }) + // ); + // }, + // handlerFIN: () => { + // channel.closedByFIN = true; + + // if (!channel.writableEnded) { + // channel.push(null); + // } + + // if (channel.writableEnded && channel.readableEnded) { + // channel.destroy(); + // this.logger.info("Channel destroy"); + // } + // } + // } + // ); + + channel.encoder.out .pipe(this.carrierSocket, { end: false }); channel @@ -65,31 +87,17 @@ export class TeceMux extends TypedEmitter { this.logger.error("CHANNEL ERROR", error); this.emit("error", { error, source: channel }); }) - // .on("destroy", () => { - // this.logger.trace("channel on DESTROY ", channel._id); - // }) - // .on("abort", () => { - // this.logger.trace("channel on ABORT ", channel._id); - // }) .on("close", () => { this.logger.info("CHANNEL close", channel._id); this.sendFIN(channel._id); }) - // .on("end", () => { - // this.logger.info("CHANNEL end", channel._id); - // }) .on("finish", () => { this.logger.info("CHANNEL finish", channel._id); this.sendFIN(channel._id); }); - // .on("data", (d) => { - // if (d === null) { - // this.logger.info("CHANNEL end", channel._id); - // } - // }); - if (emit) { - await encoder.establishChannel(channel._id); + if (establish) { + await channel.encoder.establishChannel(channel._id); } return channel; @@ -126,11 +134,6 @@ export class TeceMux extends TypedEmitter { this.carrierSocket.pipe(this.carrierDecoder, { end: false }); - this.commonEncoder.out.pipe(this.carrierSocket, { end: false }); - - this.commonEncoder.logger.updateBaseLog({ id }); - this.commonEncoder.logger.pipe(this.logger); - this.framesKeeper.logger.pipe(this.logger); this.main().catch((error) => { @@ -139,87 +142,83 @@ export class TeceMux extends TypedEmitter { } async main() { - let t = 0; - for await (const chunk of this.carrierDecoder) { - let frame: FrameData; + if (await this.handleDecodedFrame(chunk)) break; + } + } - try { - frame = JSON.parse(chunk); - } catch (err) { - this.logger.error("error Parsing data from decoder", err, chunk, chunk.length, chunk.toString()); - continue; - } + async handleDecodedFrame(chunk: any) { + let frame: FrameData; - const { flags, sequenceNumber, dataLength, destinationPort, acknowledgeNumber, error } = frame; + try { + frame = JSON.parse(chunk); + } catch (err) { + this.logger.error("error Parsing data from decoder", err, chunk, chunk.length, chunk.toString()); + this.emit("error", { chunk }); + return 1; + } - if (error) { - this.emit("error", frame); - break; - } + const { flags, sequenceNumber, dataLength, destinationPort, acknowledgeNumber, error } = frame; - let channel = this.channels.get(destinationPort); + if (error) { + this.emit("error", { frame, chunk }); + return 2; + } - if (flags.ACK) { - this.logger.trace("Received ACK flag for sequenceNumber", acknowledgeNumber); - this.framesKeeper.handleACK(acknowledgeNumber); - continue; - } + if (flags.ACK) { + this.logger.trace("Received ACK flag for sequenceNumber", acknowledgeNumber); + this.framesKeeper.handleACK(acknowledgeNumber); - if (flags.FIN) { - this.logger.trace(`Received FIN flag [C: ${destinationPort}]`, dataLength, frame.chunk, !!channel, channel?._id); - - if (channel) { - channel.closedByFIN = true; - if (!channel.writableEnded) { - channel.push(null); - } - - if (channel.writableEnded && channel.readableEnded) { - channel.destroy(); - this.logger.info("Channel destroy"); - } - } else { - this.logger.error("FIN for unknown channel"); - } - - this.sendACK(sequenceNumber, destinationPort); - continue; - } + return 0; + } - if (flags.PSH) { - this.logger.trace(`Received PSH command [C: ${destinationPort}, SIZE: ${dataLength}]`); + let channel = this.channels.get(destinationPort); - if (!channel) { - this.logger.warn("Unknown channel"); - channel = await this.createChannel(destinationPort, false); + if (flags.FIN) { + this.logger.trace(`Received FIN flag [C: ${destinationPort}]`, dataLength, frame.chunk, !!channel, channel?._id); - this.addChannel(channel, true); - //this.emit("peer", { channelId: destinationPort}) - } + if (channel) { + channel.handlerFIN(); + channel.sendACK(sequenceNumber); + } else { + this.logger.error("FIN for unknown channel"); + } - if (dataLength) { - this.logger.warn("writing to channel [channel, length]", channel._id, dataLength); - this.logger.warn("writing to channel [flowing, isPaused]", channel.readableFlowing, channel.isPaused()); + return 0; + } - channel.push(new Uint8Array((frame.chunk as any).data), undefined); + if (flags.PSH) { + this.logger.trace(`Received PSH command [C: ${destinationPort}, SIZE: ${dataLength}]`); - t += (frame.chunk as any).data.length; - this.logger.info("Writen", t); - } + if (!channel) { + this.logger.warn("Unknown channel"); + channel = await this.createChannel(destinationPort, false); - this.sendACK(sequenceNumber, destinationPort); + this.addChannel(channel, true); } + + if (dataLength) { + this.logger.warn("writing to channel [channel, length]", channel._id, dataLength); + this.logger.warn("writing to channel [flowing, isPaused]", channel.readableFlowing, channel.isPaused()); + + channel.push(new Uint8Array((frame.chunk as any).data), undefined); + } + + channel.sendACK(sequenceNumber); } + + return 0; } - sendACK(sequenceNumber: number, channel: number) { + sendACK(sequenceNumber: number, channelId: number) { this.logger.debug("Write acknowledge frame for sequenceNumber", sequenceNumber); - this.channels.get(channel)?.encoder?.push( - this.commonEncoder.createFrame(undefined, { + const channel = this.channels.get(channelId); + + channel?.encoder.push( + channel.encoder.createFrame(undefined, { flagsArray: ["ACK"], acknowledgeNumber: sequenceNumber, - destinationPort: channel + destinationPort: channelId }) ); } @@ -236,13 +235,15 @@ export class TeceMux extends TypedEmitter { this.channelCount++; } - sendFIN(channel: number) { - this.logger.debug("Write FIN frame for channel", channel); + sendFIN(channelId: number) { + this.logger.debug("Write FIN frame for channel", channelId); + + const channel = this.channels.get(channelId)!; - this.commonEncoder.push( - this.commonEncoder.createFrame(undefined, { + channel.encoder.push( + channel.encoder.createFrame(undefined, { flagsArray: ["FIN"], - destinationPort: channel + destinationPort: channelId }) ); } diff --git a/packages/verser/src/lib/tecemux/types.ts b/packages/verser/src/lib/tecemux/types.ts index f1860077e..43321ec75 100644 --- a/packages/verser/src/lib/tecemux/types.ts +++ b/packages/verser/src/lib/tecemux/types.ts @@ -1,4 +1,4 @@ -import { Duplex, Transform } from "stream"; +import { Duplex, Readable, Transform } from "stream"; export type TeceMuxEvents = { channel(socket: Duplex): void; @@ -49,12 +49,16 @@ export interface ITeCeMux { export interface IFrameEncoder extends Transform { establishChannel(id: number): Promise; + createFrame(chunk: any, frame: Partial): Buffer; + out: Readable; } export type TeceMuxChannel = Duplex & { _id: number; encoder: IFrameEncoder; closedByFIN: boolean; + sendACK(sequenceNumber: number): void; + handlerFIN(): void; }; export type FrameData = { From 9ed55fa55a0bb559e320b39bb85d116dbaed7229 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 13 Apr 2023 20:56:19 +0000 Subject: [PATCH 042/231] Disable nagle in R-H --- packages/host/src/lib/socket-server.ts | 11 ++++++----- packages/runner/src/host-client.ts | 5 +++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/host/src/lib/socket-server.ts b/packages/host/src/lib/socket-server.ts index a892ed263..5415bf024 100644 --- a/packages/host/src/lib/socket-server.ts +++ b/packages/host/src/lib/socket-server.ts @@ -44,12 +44,13 @@ export class SocketServer extends TypedEmitter implements IComponent { let protocol: TeceMux; - this.server.on("connection", async (connection: net.Socket) => { - connection.on("error", (err) => { + this.server.on("connection", async (runnerConnection: net.Socket) => { + runnerConnection.setNoDelay(true); + runnerConnection.on("error", (err) => { this.logger.error("Error on connection from runner", err); }); - protocol = new TeceMux(connection); + protocol = new TeceMux(runnerConnection); protocol.on("channel", async (channel: TeceMuxChannel) => { const { instanceId, channelId } = @@ -71,9 +72,9 @@ export class SocketServer extends TypedEmitter implements IComponent { .on("end", () => this.logger.debug(`Channel [${instanceId}:${channelId}] ended`)); try { - await this.handleConnection(instanceId, channelId, connection as Socket); + await this.handleConnection(instanceId, channelId, channel as unknown as Socket); } catch (err: any) { - connection.destroy(); + channel.destroy(); } }); }); diff --git a/packages/runner/src/host-client.ts b/packages/runner/src/host-client.ts index 3338d8576..c98d26ebd 100644 --- a/packages/runner/src/host-client.ts +++ b/packages/runner/src/host-client.ts @@ -41,10 +41,11 @@ class HostClient implements IHostClient { } async init(id: string): Promise { - const tunnel = net.createConnection(this.instancesServerPort, this.instancesServerHost); - const protocol = new TeceMux(tunnel); + const hostSocket = net.createConnection(this.instancesServerPort, this.instancesServerHost); + const protocol = new TeceMux(hostSocket); //protocol.logger.pipe(this.logger); + hostSocket.setNoDelay(true); const openConnections = await Promise.all( Array.from(Array(9)).map((_c, index) => protocol.multiplex({ channel: index })) From b9124478aed8430f137b13a7e62cef91a2c56baf Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 19 Apr 2023 21:39:58 +0000 Subject: [PATCH 043/231] Simplify TecemuxChannel --- bdd/package.json | 10 +-- .../verser/src/lib/tecemux/tecemux-channel.ts | 50 +++++------ packages/verser/src/lib/tecemux/tecemux.ts | 84 ++----------------- packages/verser/src/lib/verser-connection.ts | 5 +- 4 files changed, 41 insertions(+), 108 deletions(-) diff --git a/bdd/package.json b/bdd/package.json index 308ce85ef..cbcb40455 100644 --- a/bdd/package.json +++ b/bdd/package.json @@ -4,10 +4,10 @@ "description": "As the \"problem scope\" of the business problem that our technology solves is quite complex, we decided to use the BDD practice to support the development process. BDD is a methodology of high automation and agility. It describes a cycle of interactions with well-defined outcomes. As a result of these activities, we obtain working, tested software that has a real value.", "main": "_cucumber.js", "dependencies": { - "@scramjet/api-client": "^0.33.3", - "@scramjet/logger": "^0.33.3", - "@scramjet/obj-logger": "^0.33.3", - "@scramjet/sth-config": "^0.33.3", + "@scramjet/api-client": "^0.33.4", + "@scramjet/logger": "^0.33.4", + "@scramjet/obj-logger": "^0.33.4", + "@scramjet/sth-config": "^0.33.4", "dockerode": "^3.3.4", "find-package-json": "^1.2.0", "freeport": "^1.0.5", @@ -17,7 +17,7 @@ "devDependencies": { "@cucumber/cucumber": "^7.3.2", "@cucumber/pretty-formatter": "^1.0.0", - "@scramjet/types": "^0.33.3" + "@scramjet/types": "^0.33.4" }, "scripts": { "build:bdd": "tsc -p tsconfig.json", diff --git a/packages/verser/src/lib/tecemux/tecemux-channel.ts b/packages/verser/src/lib/tecemux/tecemux-channel.ts index 36460e5b1..83144a074 100644 --- a/packages/verser/src/lib/tecemux/tecemux-channel.ts +++ b/packages/verser/src/lib/tecemux/tecemux-channel.ts @@ -1,5 +1,5 @@ import { ObjLogger } from "@scramjet/obj-logger"; -import { Duplex } from "stream"; +import { Duplex, PassThrough } from "stream"; import { IFrameEncoder, ITeCeMux } from "./types"; import { FrameEncoder } from "./codecs"; @@ -10,30 +10,8 @@ export class TeceMuxChannel extends Duplex { encoder: IFrameEncoder; closedByFIN = false; - constructor(duplexOptions: ConstructorParameters[0], id: number, teceMux: ITeCeMux) { - super(duplexOptions); - this.logger = new ObjLogger(this); - this.allowHalfOpen = true; - this._id = id; - this.encoder = new FrameEncoder(id, teceMux, { encoding: undefined }); - } - - _write(chunk: any, encoding: BufferEncoding, next: (error?: Error | null | undefined) => void) { - this.logger.debug("WRITE channel", this._id, chunk); - - if (chunk === null) { - this.logger.info("NULL ON CHANNEL"); - - this.end(); - return false; - } - - return this.encoder.write(chunk, encoding, next); - } - - _read(_size?: number) { - this.logger.debug("READ channel", this._id); - } + private __writable = new PassThrough(); + public __readable = new PassThrough(); sendACK(sequenceNumber: number) { this.encoder.push( @@ -57,4 +35,26 @@ export class TeceMuxChannel extends Duplex { this.logger.info("Channel destroy"); } } + + constructor(duplexOptions: ConstructorParameters[0], id: number, teceMux: ITeCeMux) { + super(duplexOptions); + + this.logger = new ObjLogger(this); + this.allowHalfOpen = true; + this._id = id; + this.encoder = new FrameEncoder(id, teceMux, { encoding: undefined }); + + this.__writable.pipe(this.encoder); + + return Object.assign( + Duplex.from({ + readable: this.__readable, + writable: this.__writable + } as unknown as AsyncIterable, duplexOptions), { + ...this, + sendACK: this.sendACK, + handlerFIN: this.handlerFIN + } + ); + } } diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 9cedacf8f..25a606754 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -21,63 +21,13 @@ export class TeceMux extends TypedEmitter { logger: ObjLogger; - private async createChannel(destinationPort?: number, establish?: boolean): Promise { + private createChannel(destinationPort?: number): TeceMuxChannel { const port = destinationPort !== undefined ? destinationPort : this.channelCount; this.logger.debug("Create Channel", port); - //const encoder = new FrameEncoder(port, this, { encoding: undefined }); - - //encoder.logger.updateBaseLog({ id: this.id }); - //encoder.logger.pipe(this.logger); - const channel = new TeceMuxChannel({ allowHalfOpen: true }, port, this); - // const channel: TeceMuxChannel = Object.assign( - // new Duplex({ - // write: (chunk, encoding, next) => { - // this.logger.debug("WRITE channel", channel._id, chunk); - - // if (chunk === null) { - // this.logger.info("NULL ON CHANNEL"); - - // channel.end(); - // return false; - // } - - // return encoder.write(chunk, encoding, next); - // }, - // read: (_size) => { - // this.logger.debug("READ channel", channel._id); - // }, - // allowHalfOpen: true - // }), - // { - // _id: port, - // encoder, - // closedByFIN: false, - // sendACK: (sequenceNumber: number) => { - // channel.encoder.push( - // channel.encoder.createFrame(undefined, { - // flagsArray: ["ACK"], - // acknowledgeNumber: sequenceNumber, - // destinationPort: port - // }) - // ); - // }, - // handlerFIN: () => { - // channel.closedByFIN = true; - - // if (!channel.writableEnded) { - // channel.push(null); - // } - - // if (channel.writableEnded && channel.readableEnded) { - // channel.destroy(); - // this.logger.info("Channel destroy"); - // } - // } - // } - // ); + channel.encoder.out .pipe(this.carrierSocket, { end: false }); @@ -96,10 +46,6 @@ export class TeceMux extends TypedEmitter { this.sendFIN(channel._id); }); - if (establish) { - await channel.encoder.establishChannel(channel._id); - } - return channel; } @@ -113,21 +59,6 @@ export class TeceMux extends TypedEmitter { .on("error", (error) => { this.logger.error("Decoder error", error); }); - // .on("pause", () => { - // this.logger.warn("Decoder paused"); - // }) - // .on("close", () => { - // this.logger.warn("Decoder closed"); - // }) - // .on("end", () => { - // this.logger.warn("Decoder ended"); - // }) - // .on("abort", (error) => { - // this.logger.error("Decoder abort", error); - // }) - // .on("destroy", (error) => { - // this.logger.error("Decoder destroy", error); - // }); this.carrierDecoder.logger.updateBaseLog({ id: this.id }); this.carrierDecoder.logger.pipe(this.logger); @@ -192,7 +123,7 @@ export class TeceMux extends TypedEmitter { if (!channel) { this.logger.warn("Unknown channel"); - channel = await this.createChannel(destinationPort, false); + channel = this.createChannel(destinationPort); this.addChannel(channel, true); } @@ -201,7 +132,7 @@ export class TeceMux extends TypedEmitter { this.logger.warn("writing to channel [channel, length]", channel._id, dataLength); this.logger.warn("writing to channel [flowing, isPaused]", channel.readableFlowing, channel.isPaused()); - channel.push(new Uint8Array((frame.chunk as any).data), undefined); + channel.__readable.push(new Uint8Array((frame.chunk as any).data), undefined); } channel.sendACK(sequenceNumber); @@ -248,15 +179,14 @@ export class TeceMux extends TypedEmitter { ); } - async multiplex(opts: { channel?: number } = {}): Promise { + multiplex(opts: { channel?: number } = {}): TeceMuxChannel { const id = opts.channel !== undefined ? opts.channel : this.channelCount; - const channel = await this.createChannel(id); + const channel = this.createChannel(id); this.logger.trace("Multiplex", id); - await channel.encoder.establishChannel(id); - this.addChannel(channel, true); + channel.encoder.establishChannel(id); this.logger.trace("Multiplex ready", channel._id); return channel; diff --git a/packages/verser/src/lib/verser-connection.ts b/packages/verser/src/lib/verser-connection.ts index 79c1bd826..1518f820e 100644 --- a/packages/verser/src/lib/verser-connection.ts +++ b/packages/verser/src/lib/verser-connection.ts @@ -194,7 +194,7 @@ export class VerserConnection { async createChannel(id: number): Promise { if (!this.teceMux) throw new Error("TeCeMux not connected"); - return await this.teceMux.multiplex({ channel: id }); + return this.teceMux.multiplex({ channel: id }); } reconnect() { @@ -205,6 +205,7 @@ export class VerserConnection { }); this.agent = new Agent() as Agent & { createConnection: typeof createConnection }; // lack of types? + this.agent.createConnection = () => { try { const socket = this.teceMux!.multiplex() as unknown as Socket; @@ -219,8 +220,10 @@ export class VerserConnection { socket.setTimeout ||= (_timeout: number, _callback?: () => void) => socket; this.logger.debug("Created new muxed stream"); + return socket; } catch (e) { + this.logger.error("Create connection error", e); const ret = new Socket(); setImmediate(() => ret.emit("error", e)); From 34636c8c8420e67fc26b559286a09c428429d29e Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 20 Apr 2023 09:04:04 +0000 Subject: [PATCH 044/231] Fix lint --- packages/verser/src/lib/tecemux/tecemux.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 25a606754..663657d4f 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -28,7 +28,6 @@ export class TeceMux extends TypedEmitter { const channel = new TeceMuxChannel({ allowHalfOpen: true }, port, this); - channel.encoder.out .pipe(this.carrierSocket, { end: false }); @@ -186,7 +185,12 @@ export class TeceMux extends TypedEmitter { this.logger.trace("Multiplex", id); this.addChannel(channel, true); - channel.encoder.establishChannel(id); + + channel.encoder.establishChannel(id).then(() => { + this.logger.debug("Channel established", id); + }, (err) => { + this.logger.error("Channel establish error", err); + }); this.logger.trace("Multiplex ready", channel._id); return channel; From 15841a972ba0e5d48ccda9635ca1d04d89fee868 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 20 Apr 2023 10:36:24 +0000 Subject: [PATCH 045/231] Restore TCM channel implementation for older nodejs --- .../verser/src/lib/tecemux/tecemux-channel.ts | 50 +++++++++---------- packages/verser/src/lib/tecemux/tecemux.ts | 2 +- .../test/playgrounds/playground-tecemux.ts | 16 +----- .../verser/test/playgrounds/playground.ts | 15 +----- 4 files changed, 30 insertions(+), 53 deletions(-) diff --git a/packages/verser/src/lib/tecemux/tecemux-channel.ts b/packages/verser/src/lib/tecemux/tecemux-channel.ts index 83144a074..36460e5b1 100644 --- a/packages/verser/src/lib/tecemux/tecemux-channel.ts +++ b/packages/verser/src/lib/tecemux/tecemux-channel.ts @@ -1,5 +1,5 @@ import { ObjLogger } from "@scramjet/obj-logger"; -import { Duplex, PassThrough } from "stream"; +import { Duplex } from "stream"; import { IFrameEncoder, ITeCeMux } from "./types"; import { FrameEncoder } from "./codecs"; @@ -10,8 +10,30 @@ export class TeceMuxChannel extends Duplex { encoder: IFrameEncoder; closedByFIN = false; - private __writable = new PassThrough(); - public __readable = new PassThrough(); + constructor(duplexOptions: ConstructorParameters[0], id: number, teceMux: ITeCeMux) { + super(duplexOptions); + this.logger = new ObjLogger(this); + this.allowHalfOpen = true; + this._id = id; + this.encoder = new FrameEncoder(id, teceMux, { encoding: undefined }); + } + + _write(chunk: any, encoding: BufferEncoding, next: (error?: Error | null | undefined) => void) { + this.logger.debug("WRITE channel", this._id, chunk); + + if (chunk === null) { + this.logger.info("NULL ON CHANNEL"); + + this.end(); + return false; + } + + return this.encoder.write(chunk, encoding, next); + } + + _read(_size?: number) { + this.logger.debug("READ channel", this._id); + } sendACK(sequenceNumber: number) { this.encoder.push( @@ -35,26 +57,4 @@ export class TeceMuxChannel extends Duplex { this.logger.info("Channel destroy"); } } - - constructor(duplexOptions: ConstructorParameters[0], id: number, teceMux: ITeCeMux) { - super(duplexOptions); - - this.logger = new ObjLogger(this); - this.allowHalfOpen = true; - this._id = id; - this.encoder = new FrameEncoder(id, teceMux, { encoding: undefined }); - - this.__writable.pipe(this.encoder); - - return Object.assign( - Duplex.from({ - readable: this.__readable, - writable: this.__writable - } as unknown as AsyncIterable, duplexOptions), { - ...this, - sendACK: this.sendACK, - handlerFIN: this.handlerFIN - } - ); - } } diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 663657d4f..2f7bd0e59 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -131,7 +131,7 @@ export class TeceMux extends TypedEmitter { this.logger.warn("writing to channel [channel, length]", channel._id, dataLength); this.logger.warn("writing to channel [flowing, isPaused]", channel.readableFlowing, channel.isPaused()); - channel.__readable.push(new Uint8Array((frame.chunk as any).data), undefined); + channel.push(new Uint8Array((frame.chunk as any).data), undefined); } channel.sendACK(sequenceNumber); diff --git a/packages/verser/test/playgrounds/playground-tecemux.ts b/packages/verser/test/playgrounds/playground-tecemux.ts index fa84cf427..5c028b26a 100644 --- a/packages/verser/test/playgrounds/playground-tecemux.ts +++ b/packages/verser/test/playgrounds/playground-tecemux.ts @@ -2,9 +2,8 @@ /* eslint-disable @typescript-eslint/no-floating-promises */ /* eslint-disable no-console */ -import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; +import { ObjLogger } from "@scramjet/obj-logger"; import { IncomingMessage, createServer } from "http"; -import { DataStream } from "scramjet"; import { Socket, createConnection } from "net"; import { TeceMux } from "../../src/lib/tecemux/tecemux"; @@ -15,18 +14,7 @@ import path from "path"; (async () => { const logger = new ObjLogger("Sandbox"); - logger - .pipe( - new DataStream() - .map(prettyPrint({ colors: true })) - .map((chunk: string) => - chunk.replace( - /(:?FIN|SYN|RST|PSH|ACK|URG|ECE|CWR)|^$]/, - "\x1b[41m\$&\x1b[0m" - ) - ) - ) - .pipe(process.stdout); + logger.pipe(process.stdout); /**********************************************/ /* SERVER diff --git a/packages/verser/test/playgrounds/playground.ts b/packages/verser/test/playgrounds/playground.ts index d52e5f452..897abfa43 100644 --- a/packages/verser/test/playgrounds/playground.ts +++ b/packages/verser/test/playgrounds/playground.ts @@ -1,6 +1,5 @@ -import { DataStream } from "scramjet"; import { FrameDecoder, FrameEncoder } from "../../src/lib/tecemux/codecs"; -import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; +import { ObjLogger } from "@scramjet/obj-logger"; import { createReadStream, createWriteStream } from "fs"; import path from "path"; import { PassThrough, Transform } from "stream"; @@ -15,17 +14,7 @@ const tcm = { const logger = new ObjLogger("Sandbox"); -logger.pipe( - new DataStream() - .map(prettyPrint({ colors: true })) - .map((chunk: string) => - chunk.replace( - /(:?FIN|SYN|RST|PSH|ACK|URG|ECE|CWR)|^$]/, - "\x1b[41m\$&\x1b[0m" - ) - ) -) - .pipe(process.stdout); +logger.pipe(process.stdout); const encoder = new FrameEncoder(0, tcm); const decoder = new FrameDecoder(); From 16c05fbfae08052c91f3fc6d84c7ef6f4acb4a01 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 26 Apr 2023 20:18:53 +0000 Subject: [PATCH 046/231] Fix M-S connection --- bdd/package.json | 10 +++++----- packages/host/src/lib/cpm-connector.ts | 2 +- packages/verser/src/lib/verser-client.ts | 2 +- packages/verser/src/lib/verser-connection.ts | 2 +- packages/verser/src/types/index.ts | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/bdd/package.json b/bdd/package.json index cbcb40455..44956bfda 100644 --- a/bdd/package.json +++ b/bdd/package.json @@ -4,10 +4,10 @@ "description": "As the \"problem scope\" of the business problem that our technology solves is quite complex, we decided to use the BDD practice to support the development process. BDD is a methodology of high automation and agility. It describes a cycle of interactions with well-defined outcomes. As a result of these activities, we obtain working, tested software that has a real value.", "main": "_cucumber.js", "dependencies": { - "@scramjet/api-client": "^0.33.4", - "@scramjet/logger": "^0.33.4", - "@scramjet/obj-logger": "^0.33.4", - "@scramjet/sth-config": "^0.33.4", + "@scramjet/api-client": "^0.33.5", + "@scramjet/logger": "^0.33.5", + "@scramjet/obj-logger": "^0.33.5", + "@scramjet/sth-config": "^0.33.5", "dockerode": "^3.3.4", "find-package-json": "^1.2.0", "freeport": "^1.0.5", @@ -17,7 +17,7 @@ "devDependencies": { "@cucumber/cucumber": "^7.3.2", "@cucumber/pretty-formatter": "^1.0.0", - "@scramjet/types": "^0.33.4" + "@scramjet/types": "^0.33.5" }, "scripts": { "build:bdd": "tsc -p tsconfig.json", diff --git a/packages/host/src/lib/cpm-connector.ts b/packages/host/src/lib/cpm-connector.ts index 9aaedaf0e..08cb29041 100644 --- a/packages/host/src/lib/cpm-connector.ts +++ b/packages/host/src/lib/cpm-connector.ts @@ -340,7 +340,7 @@ export class CPMConnector extends TypedEmitter { this.connected = true; this.connectionAttempts = 0; - connection.req.once("error", async (error: any) => { + connection.response.once("error", async (error: any) => { this.logger.error("Request error", error); try { diff --git a/packages/verser/src/lib/verser-client.ts b/packages/verser/src/lib/verser-client.ts index bdab51ed2..cdac390a2 100644 --- a/packages/verser/src/lib/verser-client.ts +++ b/packages/verser/src/lib/verser-client.ts @@ -110,7 +110,7 @@ export class VerserClient extends TypedEmitter { this.socket = socket; this.mux(); - resolve({ req: response, socket }); + resolve({ response, socket }); }); connectRequest.flushHeaders(); diff --git a/packages/verser/src/lib/verser-connection.ts b/packages/verser/src/lib/verser-connection.ts index 1518f820e..f93f8c783 100644 --- a/packages/verser/src/lib/verser-connection.ts +++ b/packages/verser/src/lib/verser-connection.ts @@ -191,7 +191,7 @@ export class VerserConnection { * @param id {number} Channel id. * @returns Duplex stream. */ - async createChannel(id: number): Promise { + createChannel(id: number) { if (!this.teceMux) throw new Error("TeCeMux not connected"); return this.teceMux.multiplex({ channel: id }); diff --git a/packages/verser/src/types/index.ts b/packages/verser/src/types/index.ts index 21e6496b4..9c9ecb1c9 100644 --- a/packages/verser/src/types/index.ts +++ b/packages/verser/src/types/index.ts @@ -40,7 +40,7 @@ export type VerserClientConnection = { /** * Connection request object. */ - req: IncomingMessage; + response: IncomingMessage; }; export type VerserRequestResult = { incomingMessage: IncomingMessage; clientRequest: ClientRequest } From daf4851e426adaf9c539cc648c2f2dab8be03d01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Wed, 17 May 2023 11:59:48 +0200 Subject: [PATCH 047/231] Initial changes --- packages/python-runner/runner-tecemux.py | 124 +++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 packages/python-runner/runner-tecemux.py diff --git a/packages/python-runner/runner-tecemux.py b/packages/python-runner/runner-tecemux.py new file mode 100644 index 000000000..e2df9f6d2 --- /dev/null +++ b/packages/python-runner/runner-tecemux.py @@ -0,0 +1,124 @@ +import asyncio +import sys +import codecs +import socket +from attrs import define +from logging_setup import LoggingSetup +from hardcoded_magic_values import CommunicationChannels as CC + +sequence_path = '/tmp/' +server_port = 8001 +server_host = 'localhost' +instance_id = '1234' + +@define +class TeceMuxChannelContext: + _name: str + _queue: asyncio.Queue + _reader: asyncio.StreamReader + _writer: asyncio.StreamWriter + + _task_reader: asyncio.coroutine = None + _task_writer: asyncio.coroutine = None + + + def get_name(self): + return self._name.name + +class Tecemux: + + def __init__(self, reader,writer): + + self._queue = asyncio.Queue() + self._reader = reader + self._writer = writer + self._channels = None + + async def prepare(self): + async def prepare_channel(name): + rsock, wsock = socket.socketpair() + reader, _ = await asyncio.open_unix_connection(sock=rsock) + _, writer = await asyncio.open_unix_connection(sock=wsock) + return TeceMuxChannelContext(name, reader,writer,asyncio.Queue()) + + self._channels = {channel : await prepare_channel(channel) for channel in CC } + + def get_channel(self, channel): + return self._channels[channel] + + async def loop(self): + + loop = asyncio.get_event_loop() + for channel in CC: + self._channels[channel]._task_reader = loop.create_task(Tecemux.channel_reader(self._channels[channel])) + self._channels[channel]._task_writer = loop.create_task(Tecemux.channel_writer(self._channels[channel])) + + channel_readers = asyncio.gather(*[ channel._task_reader for channel in self._channels.values()]) + + + protocol_reader = loop.create_task(Tecemux.protocol_reader(self._reader, self._queue)) + protocol_writer = loop.create_task(Tecemux.protocol_writer(self._writer, self._queue)) + + await asyncio.gather(*[protocol_reader, protocol_writer]) + print('End main loop') + + + @staticmethod + async def protocol_reader(reader, queue): + while True: + print(f'PROTOCOL: WRITE!') + await asyncio.sleep(0) + + @staticmethod + async def protocol_writer(writer, queue): + while True: + print(f'PROTOCOL: READ!') + await asyncio.sleep(0) + + @staticmethod + async def channel_reader(channel_context): + while True: + print(f'{channel_context.get_name()}: READ!') + await asyncio.sleep(0) + + @staticmethod + async def channel_writer(channel_context): + while True: + print(f'{channel_context.get_name()}: WRITE!') + await asyncio.sleep(0) + +class Runner: + def __init__(self, instance_id, sequence_path, log_setup) -> None: + self.instance_id = instance_id + self.seq_path = sequence_path + self._logging_setup = log_setup + + self.logger = log_setup.logger + + self.protocol = None + + def connect_stdio(self): + + sys.stdout = codecs.getwriter('utf-8')(self.protocol.get_channel(CC.STDOUT).writer) + sys.stderr = codecs.getwriter('utf-8')(self.protocol.get_channel(CC.STDERR).writer) + sys.stdout.flush = lambda: True + sys.stderr.flush = lambda: True + + + + async def main(self, server_host, server_port): + self.logger.info('Connecting to host...') + + self.logger.debug(f'Connecting to {server_host}:{server_port}...') + self.protocol = Tecemux(*await asyncio.open_connection(server_host, server_port)) + self.logger.debug('Connected.') + + await self.protocol.prepare() + + #self.connect_stdio() + + await self.protocol.loop() + + +runner = Runner(instance_id, sequence_path, LoggingSetup(sys.stdout)) +asyncio.run(runner.main(server_host, server_port)) From fbc48f4a8b7d249ecc713b902ed55865f8f94605 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Wed, 17 May 2023 13:15:32 +0200 Subject: [PATCH 048/231] Logger with coroutine support --- packages/python-runner/runner-tecemux.py | 110 +++++++++++++++-------- 1 file changed, 73 insertions(+), 37 deletions(-) diff --git a/packages/python-runner/runner-tecemux.py b/packages/python-runner/runner-tecemux.py index e2df9f6d2..c43c64c1b 100644 --- a/packages/python-runner/runner-tecemux.py +++ b/packages/python-runner/runner-tecemux.py @@ -2,50 +2,76 @@ import sys import codecs import socket -from attrs import define +from attrs import define, field + from logging_setup import LoggingSetup from hardcoded_magic_values import CommunicationChannels as CC +def get_logger(): + if not hasattr(get_logger, "log_setup"): + get_logger.log_setup = LoggingSetup(sys.stdout) + return get_logger.log_setup.logger + sequence_path = '/tmp/' server_port = 8001 server_host = 'localhost' instance_id = '1234' + @define -class TeceMuxChannelContext: - _name: str - _queue: asyncio.Queue - _reader: asyncio.StreamReader - _writer: asyncio.StreamWriter +class Tecemux: + @define + class ChannelContext: + _name: str + _queue: asyncio.Queue + _reader: asyncio.StreamReader + _writer: asyncio.StreamWriter - _task_reader: asyncio.coroutine = None - _task_writer: asyncio.coroutine = None + _task_reader: asyncio.coroutine = None + _task_writer: asyncio.coroutine = None - def get_name(self): - return self._name.name + def get_name(self): + return self._name.name -class Tecemux: - - def __init__(self, reader,writer): - - self._queue = asyncio.Queue() + _queue: asyncio.Queue = asyncio.Queue() + _reader: asyncio.StreamReader = field(default=None) + _writer: asyncio.StreamWriter = field(default=None) + _logger = field(default=get_logger()) + _channels = field(default={}) + + async def connect(self, reader, writer): self._reader = reader self._writer = writer - self._channels = None - async def prepare(self): - async def prepare_channel(name): - rsock, wsock = socket.socketpair() - reader, _ = await asyncio.open_unix_connection(sock=rsock) - _, writer = await asyncio.open_unix_connection(sock=wsock) - return TeceMuxChannelContext(name, reader,writer,asyncio.Queue()) - self._channels = {channel : await prepare_channel(channel) for channel in CC } + @staticmethod + async def prepare_tcp_connection(server_host, server_port): + return await asyncio.open_connection(server_host, server_port) + + @staticmethod + async def prepare_socket_connection(): + rsock, wsock = socket.socketpair() + reader, _ = await asyncio.open_unix_connection(sock=rsock) + _, writer = await asyncio.open_unix_connection(sock=wsock) + return reader,writer + + async def prepare(self): + self._channels = {channel : Tecemux.ChannelContext(channel, *await Tecemux.prepare_socket_connection(),asyncio.Queue()) for channel in CC } def get_channel(self, channel): return self._channels[channel] + async def sandbox(self): + self._logger.debug("Begin sandbox") + + await self._queue.put(b'test') + await asyncio.sleep(0) + + self._writer.write(b'test 2') + await self._writer.drain() + + self._logger.debug("End sandbox") async def loop(self): loop = asyncio.get_event_loop() @@ -58,42 +84,46 @@ async def loop(self): protocol_reader = loop.create_task(Tecemux.protocol_reader(self._reader, self._queue)) protocol_writer = loop.create_task(Tecemux.protocol_writer(self._writer, self._queue)) + sandbox = loop.create_task(self.sandbox()) - await asyncio.gather(*[protocol_reader, protocol_writer]) + await asyncio.gather(*[protocol_reader, protocol_writer, sandbox]) print('End main loop') @staticmethod async def protocol_reader(reader, queue): while True: - print(f'PROTOCOL: WRITE!') - await asyncio.sleep(0) + chunk = await reader.read(100) + get_logger().debug(f'Chunk received: {chunk}') + await queue.put(chunk) + get_logger().debug(f'Chunk redirected: {chunk}') @staticmethod async def protocol_writer(writer, queue): while True: - print(f'PROTOCOL: READ!') - await asyncio.sleep(0) + chunk = await queue.get() + get_logger().debug(f'PROTOCOL: Chunk waiting: {chunk}') + #writer.write(chunk) + queue.task_done() + get_logger().debug(f'PROTOCOL: Chunk consumed: {chunk}') @staticmethod async def channel_reader(channel_context): while True: - print(f'{channel_context.get_name()}: READ!') + #print(f'{channel_context.get_name()}: READ!') await asyncio.sleep(0) @staticmethod async def channel_writer(channel_context): while True: - print(f'{channel_context.get_name()}: WRITE!') + #print(f'{channel_context.get_name()}: WRITE!') await asyncio.sleep(0) class Runner: - def __init__(self, instance_id, sequence_path, log_setup) -> None: + def __init__(self, instance_id, sequence_path, logger) -> None: self.instance_id = instance_id self.seq_path = sequence_path - self._logging_setup = log_setup - - self.logger = log_setup.logger + self.logger = logger self.protocol = None @@ -109,8 +139,14 @@ def connect_stdio(self): async def main(self, server_host, server_port): self.logger.info('Connecting to host...') - self.logger.debug(f'Connecting to {server_host}:{server_port}...') - self.protocol = Tecemux(*await asyncio.open_connection(server_host, server_port)) + self.protocol = Tecemux() + + # self.logger.debug(f'Connecting to {server_host}:{server_port}...') + # await self.protocol.connect(*await Tecemux.prepare_tcp_connection(server_host, server_port)) + + self.logger.debug(f'Connecting locally via unix sockets') + await self.protocol.connect(*await Tecemux.prepare_socket_connection()) + self.logger.debug('Connected.') await self.protocol.prepare() @@ -120,5 +156,5 @@ async def main(self, server_host, server_port): await self.protocol.loop() -runner = Runner(instance_id, sequence_path, LoggingSetup(sys.stdout)) +runner = Runner(instance_id, sequence_path, get_logger()) asyncio.run(runner.main(server_host, server_port)) From af3bc1c10af7585b6f763abdffd9b2ab2e3f3eac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Wed, 17 May 2023 14:14:12 +0200 Subject: [PATCH 049/231] TCP/IP packet decode/encode class --- packages/python-runner/inet.py | 231 +++++++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 packages/python-runner/inet.py diff --git a/packages/python-runner/inet.py b/packages/python-runner/inet.py new file mode 100644 index 000000000..8d3916ebd --- /dev/null +++ b/packages/python-runner/inet.py @@ -0,0 +1,231 @@ +import array +import struct +from binascii import hexlify, unhexlify +from socket import inet_ntoa, inet_aton +from attrs import define,field + +@define +class TCPSegment: + class Flags: + FIN = 0x01 # end of data + SYN = 0x02 # synchronize sequence numbers + RST = 0x04 # reset connection + PSH = 0x08 # push + ACK = 0x10 # acknowledgment number set + URG = 0x20 # urgent pointer set + ECE = 0x40 # ECN echo, RFC 3168 + CWR = 0x80 # congestion window reduced + NS = 0x100 # nonce sum, RFC 3540 + + @staticmethod + def flags_to_str(val): + flags = [] + if val & TCPSegment.Flags.FIN: + flags.append('FIN') + if val & TCPSegment.Flags.SYN: + flags.append('SYN') + if val & TCPSegment.Flags.RST: + flags.append('RST') + if val & TCPSegment.Flags.PSH: + flags.append('PSH') + if val & TCPSegment.Flags.ACK: + flags.append('ACK') + if val & TCPSegment.Flags.URG: + flags.append('URG') + if val & TCPSegment.Flags.ECE: + flags.append('ECE') + if val & TCPSegment.Flags.CWR: + flags.append('CWR') + if val & TCPSegment.Flags.NS: + flags.append('NS') + return '+'.join(flags) + + @staticmethod + def parse_flags(value): + res = 0 + if value is None: + return 0 + if isinstance(value,int): + return value + + for flag in value: + res = res | getattr(TCPSegment.Flags, flag ) + return res + + + src_port: int = field(default = 0) + dst_port: int = field(default = 0) + seq: int = field(default = 0) + ack: int = field(default = 0) + offset: int = field(default = 0, converter = lambda value: value >> 4) + flags: int = field(default = 0, repr = lambda value: TCPSegment.Flags.flags_to_str(value), \ + converter = lambda value: TCPSegment.Flags.parse_flags(value)) + win: int = field(default = 0) + checksum: int = field(default = 0, repr = lambda value: hex(value)) + urp: int = field(default = 0) + data: bytes = field(default=b'', repr = lambda value: f'{value[0:5]}... ' \ + if len(value)>5 else f'{value}') + + @classmethod + def from_buffer(cls,buffer): + return cls(*struct.unpack("!HHIIBBHHH", buffer[0:20]), buffer[20:]) + + + def to_buffer(self): + return struct.pack('!HHIIBBHHH', \ + self.src_port,\ + self.dst_port,\ + self.seq,\ + self.ack,\ + self.offset << 4,\ + self.flags,\ + self.win,\ + self.checksum,\ + self.urp) + self.data + + def set_flags(self, list_of_flags): + self.flags = list_of_flags + return self + + def is_flag(self,flag): + return (self.flags & getattr(TCPSegment.Flags, flag)) > 0 + + def encapsulate(self, src_addr: str, dst_addr: str): + return IPPacket(src_addr=src_addr, dst_addr=dst_addr,segment=self) + +@define +class IPPacket: + class Flags: + RF = 0x4 # reserved + DF = 0x2 # don't fragment + MF = 0x1 # more fragments + + @staticmethod + def flags_to_str(val): + flags = [] + if val & IPPacket.Flags.RF == 0: + if val & IPPacket.Flags.MF != 0: + flags.append('MF') + if val & IPPacket.Flags.DF != 0: + flags.append('DF') + else: + flags.append('UNKNOWN') + return '+'.join(flags) + + @staticmethod + def parse_flags(value): + res = 0 + if value is None: + return 0 + if isinstance(value,int): + return value + + for flag in value: + res = res | getattr(IPPacket.Flags, flag ) + return res + + ihl: int = 5 + version: int = field( default=4, converter = lambda value: value >> 4) + tos: int = field(default=0) + len: int = field(default=None) + ids: int = field(default=None) + flags_offset: int = field(default = 0) + flags: int = field(default = 0, init= False, repr = lambda value: IPPacket.Flags.flags_to_str(value), \ + converter = lambda value: IPPacket.Flags.parse_flags(value)) + offset: int = field(default = 0, init=False) + ttl: int = field(default=255) + protocol: int = field(default=6) + checksum: int = field(default = 0, repr = lambda value: hex(value)) + src_addr: str = field(default='', converter = lambda value: inet_ntoa(value)) + dst_addr: str = field(default='', converter = lambda value: inet_ntoa(value)) + segment: TCPSegment = None + + def __attrs_post_init__(self): + self.offset = self.flags_offset & 0x1FFF + self.flags = self.flags_offset >> 13 + + @staticmethod + def calc_checksum(pkt: bytes) -> int: + #source: https://github.com/secdev/scapy + if len(pkt) % 2 == 1: + pkt += b"\0" + s = sum(array.array("H", pkt)) + s = (s >> 16) + (s & 0xffff) + s += s >> 16 + s = ~s + return (((s >> 8) & 0xff) | s << 8) & 0xffff + + @classmethod + def from_buffer(cls,buffer): + ihl = (buffer[0] & 0xf) + return cls(ihl, *struct.unpack("!BBHHHBBH4s4s", buffer[0:ihl*4]),TCPSegment.from_buffer(buffer[ihl*4:]) if len(buffer) > ihl*4 else None) + + def is_flag(self,flag): + return (self.flags & getattr(IPPacket.Flags, flag)) > 0 + + def to_buffer(self): + ihl_ver = (int(self.version) << 4) + int(self.ihl) + + return struct.pack('!BBHHHBBH4s4s', \ + ihl_ver, \ + self.tos, \ + self.len, \ + self.ids, + self.flags_offset, \ + self.ttl, \ + self.protocol, \ + self.checksum, \ + inet_aton(self.src_addr), \ + inet_aton(self.dst_addr)) + (self.get_segment().to_buffer() if self.segment else b'') + + def _validate_tcp(self): + + self.segment.checksum = 0 + + tcp_segment = self.segment.to_buffer() + + pseudo_hdr = struct.pack("!4s4sHH", inet_aton(self.src_addr), inet_aton(self.dst_addr), self.protocol, len(tcp_segment)) + + self.segment.checksum = IPPacket.calc_checksum(pseudo_hdr + tcp_segment) + + return self + + def _validate_ip(self): + + self.checksum = 0 + + self.checksum = IPPacket.calc_checksum(self.to_buffer()[0:self.ihl*4]) + + return self + + def build(self): + + if self.segment: + self._validate_tcp() + + self._validate_ip() + + return self + + def encapsulate(self): + return EthernetFrame(b'', b'', b'', self) + + def get_segment(self): + return self.segment + +@define +class EthernetFrame: + src_mac: str = field(converter = lambda value: hexlify(value)) + dst_mac: str = field(converter = lambda value: hexlify(value)) + eth_type: bytes = field(repr = lambda value: hexlify(value)) + packet: IPPacket + + @classmethod + def from_buffer(cls,buffer): + return cls(*struct.unpack("!6s6s2s", buffer[0:14]),IPPacket.from_buffer(buffer[14:])) + + def to_buffer(self): + return struct.pack("!6s6s2s", unhexlify(self.src_mac), unhexlify(self.dst_mac),self.eth_type) + self.get_packet().to_buffer() + + def get_packet(self): + return self.packet \ No newline at end of file From e3cb4b00392bdf01db1d607ce7a22fe3ea2a0608 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Wed, 17 May 2023 14:40:25 +0200 Subject: [PATCH 050/231] Tests for TCP/IP classes --- packages/python-runner/__init__.py | 0 packages/python-runner/test/__init__.py | 0 packages/python-runner/test/test_inet.py | 125 +++++++++++++++++++++++ 3 files changed, 125 insertions(+) create mode 100644 packages/python-runner/__init__.py create mode 100644 packages/python-runner/test/__init__.py create mode 100644 packages/python-runner/test/test_inet.py diff --git a/packages/python-runner/__init__.py b/packages/python-runner/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/python-runner/test/__init__.py b/packages/python-runner/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/python-runner/test/test_inet.py b/packages/python-runner/test/test_inet.py new file mode 100644 index 000000000..3c2bb6ea1 --- /dev/null +++ b/packages/python-runner/test/test_inet.py @@ -0,0 +1,125 @@ +import pytest +from inet import TCPSegment, IPPacket, EthernetFrame + +class TestIP: + def test_mf_df_flags(self): + + data = b'E\x00\x00\x14\x00\x01`\x00@\x00\x1c\xe7\x7f\x00\x00\x01\x7f\x00\x00\x01' + pkt = IPPacket.from_buffer(data) + assert pkt.flags == (IPPacket.Flags.MF | IPPacket.Flags.DF) & ~IPPacket.Flags.RF + assert pkt.is_flag('MF') == True + assert pkt.is_flag('DF') == True + assert pkt.is_flag('RF') == False + + def test_mf_flags(self): + + data = b'E\x00\x00\x14\x00\x01 \x00@\x00\\\xe7\x7f\x00\x00\x01\x7f\x00\x00\x01' + pkt = IPPacket.from_buffer(data) + assert pkt.flags == IPPacket.Flags.MF & ~IPPacket.Flags.RF + assert pkt.is_flag('MF') == True + assert pkt.is_flag('DF') == False + assert pkt.is_flag('RF') == False + + def test_df_flags(self): + data= (b'\x00\x23\x20\xd4\x2a\x8c\x00\x23\x20\xd4\x2a\x8c\x08\x00\x45\x00\x00\x54\x00\x00\x40\x00' + b'\x40\x01\x25\x8d\x0a\x00\x00\x8f\x0a\x00\x00\x8e\x08\x00\x2e\xa0\x01\xff\x23\x73\x20\x48' + b'\x4a\x4d\x00\x00\x00\x00\x78\x85\x02\x00\x00\x00\x00\x00\x10\x11\x12\x13\x14\x15\x16\x17' + b'\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d' + b'\x2e\x2f\x30\x31\x32\x33\x34\x35\x36\x37') + + pkt = EthernetFrame.from_buffer(data).get_packet() + + assert pkt.flags == IPPacket.Flags.DF + assert pkt.flags == (IPPacket.Flags.DF & (~IPPacket.Flags.MF & ~IPPacket.Flags.RF)) + + assert pkt.is_flag('DF') == True + assert pkt.is_flag('MF') == False + assert pkt.is_flag('RF') == False + + assert pkt.offset == 0 + + def test_checksum_calc(self): + + data = b'E\x00\x00\x14\x00\x01\x00\x00@\x00j\xd6\x01\x02\x03\x04\x05\x06\x07\x08' + checksum = 27350 + pkt = IPPacket.from_buffer(data) + pkt.checksum = 0 + + assert pkt.checksum == 0 + + pkt.build() + + assert pkt.checksum == checksum + +class TestTCP: + def test_tcp_offset(self): + data = b'\x01\xbb\xc0\xd7\xb6\x56\xa8\xb9\xd1\xac\xaa\xb1\x50\x18\x40\x00\x56\xf8\x00\x00' + segment = TCPSegment.from_buffer(data) + assert segment.offset == 5 + + + def test_prepare_segment_with_flags(self): + + segment = TCPSegment(flags=['FIN','SYN','ACK']) + + assert segment.is_flag('FIN') == True + assert segment.is_flag('SYN') == True + assert segment.is_flag('ACK') == True + assert segment.is_flag('PSH') == False + + segment = TCPSegment(flags=['SYN','PSH']) + + assert segment.is_flag('FIN') == False + assert segment.is_flag('SYN') == True + assert segment.is_flag('ACK') == False + assert segment.is_flag('PSH') == True + + segment = TCPSegment().set_flags(['FIN','SYN','ACK']) + + assert segment.is_flag('FIN') == True + assert segment.is_flag('SYN') == True + assert segment.is_flag('ACK') == True + assert segment.is_flag('PSH') == False + + segment = TCPSegment().set_flags(['SYN','PSH']) + + assert segment.is_flag('FIN') == False + assert segment.is_flag('SYN') == True + assert segment.is_flag('ACK') == False + assert segment.is_flag('PSH') == True + + + def test_basic_details(self): + data = b'E\x00\x00(\x00\x01@\x00@\x06,\xbe\x01\x02\x03\x04\x04\x05\x06\x07\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x81m\x00\x00' + pkt = IPPacket.from_buffer(data) + assert pkt.segment.dst_port == 80 + assert pkt.segment.flags & TCPSegment.Flags.SYN + assert pkt.segment.is_flag('SYN') == True + assert pkt.segment.is_flag('PSH') == False + + + def test_checksum(self): + data = b'E\x00\x00(\x00\x01@\x00@\x06,\xbe\x01\x02\x03\x04\x04\x05\x06\x07\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x81m\x00\x00' + pkt = IPPacket.from_buffer(data) + assert pkt.segment.checksum == 33133 + + def test_checksum_calc(self): + data = b'E\x00\x00*\x00\x01\x00\x00@\x06N\x9d\n\x0b\x0c\x0e\n\x0b\x0c\r\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x00\x00\x00\x00Hi' + pkt = IPPacket.from_buffer(data) + + assert pkt.segment.checksum == 0 + + pkt = IPPacket.from_buffer(data).build() + + assert pkt.segment.checksum == 6883 + + def test_unpack(self): + data = (b'\x00\x50\x0d\x2c\x11\x4c\x61\x8b\x38\xaf\xfe\x14\x70\x12\x16\xd0' + b'\x5b\xdc\x00\x00\x02\x04\x05\x64\x01\x01\x04\x02') + pkt = TCPSegment.from_buffer(data) + assert pkt.flags == (TCPSegment.Flags.SYN | TCPSegment.Flags.ACK) + assert pkt.offset == 7 + assert pkt.win == 5840 + assert pkt.dst_port == 3372 + assert pkt.seq == 290218379 + assert pkt.ack == 951057940 \ No newline at end of file From f6696e31c7721dc5f705d67c70eb4170b04a6d0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Wed, 17 May 2023 16:59:19 +0200 Subject: [PATCH 051/231] First python Runner tests --- packages/python-runner/pytest.ini | 3 ++ .../{runner-tecemux.py => runner_tecemux.py} | 13 +++++-- packages/python-runner/test/test_runner.py | 38 +++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 packages/python-runner/pytest.ini rename packages/python-runner/{runner-tecemux.py => runner_tecemux.py} (94%) create mode 100644 packages/python-runner/test/test_runner.py diff --git a/packages/python-runner/pytest.ini b/packages/python-runner/pytest.ini new file mode 100644 index 000000000..864639716 --- /dev/null +++ b/packages/python-runner/pytest.ini @@ -0,0 +1,3 @@ +# pytest.ini +[pytest] +asyncio_mode = auto diff --git a/packages/python-runner/runner-tecemux.py b/packages/python-runner/runner_tecemux.py similarity index 94% rename from packages/python-runner/runner-tecemux.py rename to packages/python-runner/runner_tecemux.py index c43c64c1b..93a87fe3f 100644 --- a/packages/python-runner/runner-tecemux.py +++ b/packages/python-runner/runner_tecemux.py @@ -4,6 +4,7 @@ import socket from attrs import define, field +from inet import TCPSegment,IPPacket from logging_setup import LoggingSetup from hardcoded_magic_values import CommunicationChannels as CC @@ -30,6 +31,12 @@ class ChannelContext: _task_reader: asyncio.coroutine = None _task_writer: asyncio.coroutine = None + + def encode(data): + pass + + def decode(data): + pass def get_name(self): return self._name.name @@ -155,6 +162,6 @@ async def main(self, server_host, server_port): await self.protocol.loop() - -runner = Runner(instance_id, sequence_path, get_logger()) -asyncio.run(runner.main(server_host, server_port)) +if __name__ == '__main__': + runner = Runner(instance_id, sequence_path, get_logger()) + asyncio.run(runner.main(server_host, server_port)) diff --git a/packages/python-runner/test/test_runner.py b/packages/python-runner/test/test_runner.py new file mode 100644 index 000000000..f02ac7e69 --- /dev/null +++ b/packages/python-runner/test/test_runner.py @@ -0,0 +1,38 @@ +import asyncio +import pytest +from runner_tecemux import Tecemux + + +@pytest.fixture() +async def local_socket_connection(): + protocol = Tecemux() + await protocol.connect(*await Tecemux.prepare_socket_connection()) + return protocol + +class TestTecemux: + def test_default_init(self): + protocol = Tecemux() + assert isinstance(protocol, Tecemux) + + @pytest.mark.asyncio + async def test_socket_connection(self): + + protocol = Tecemux() + + assert protocol._reader == None + assert protocol._writer == None + + await protocol.connect(*await Tecemux.prepare_socket_connection()) + + assert isinstance(protocol._reader,asyncio.StreamReader) + assert isinstance(protocol._writer,asyncio.StreamWriter) + + assert True + + @pytest.mark.asyncio + async def test_tecemux_pipe(self, local_socket_connection): + protocol = local_socket_connection + protocol._writer.write(b'dnkrozz') + await protocol._writer.drain() + + assert await protocol._reader.read(100) == b'dnkrozz' \ No newline at end of file From e3d4a5b7b8f0a6300c034fa4514264a93b5d4aae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Thu, 18 May 2023 13:49:31 +0200 Subject: [PATCH 052/231] Main loop for Tecemux --- packages/python-runner/runner_tecemux.py | 115 +++++++++++++++-------- 1 file changed, 78 insertions(+), 37 deletions(-) diff --git a/packages/python-runner/runner_tecemux.py b/packages/python-runner/runner_tecemux.py index 93a87fe3f..55357a863 100644 --- a/packages/python-runner/runner_tecemux.py +++ b/packages/python-runner/runner_tecemux.py @@ -44,12 +44,18 @@ def get_name(self): _queue: asyncio.Queue = asyncio.Queue() _reader: asyncio.StreamReader = field(default=None) _writer: asyncio.StreamWriter = field(default=None) + + _incoming_data_forwarder: asyncio.coroutine = field(default=None) + _outcoming_data_forwarder: asyncio.coroutine = field(default=None) + _logger = field(default=get_logger()) _channels = field(default={}) + _stop_event = asyncio.Event() async def connect(self, reader, writer): self._reader = reader self._writer = writer + self._stop_event.clear() @staticmethod @@ -69,62 +75,89 @@ async def prepare(self): def get_channel(self, channel): return self._channels[channel] - async def sandbox(self): - self._logger.debug("Begin sandbox") + async def stop(self): + await self._writer.drain() + self._writer.close() + await self._writer.wait_closed() + self._stop_event.set() - await self._queue.put(b'test') - await asyncio.sleep(0) + async def wait_until_end(self): + await self._stop_event.wait() - self._writer.write(b'test 2') - await self._writer.drain() + await asyncio.gather(*[ channel._task_reader for channel in self._channels.values()]) + await asyncio.gather(*[ channel._task_writer for channel in self._channels.values()]) + await asyncio.gather(*[self._incoming_data_forwarder, self._outcoming_data_forwarder]) + + get_logger().debug(f'Tecemux/MAIN: [-] Finished') - self._logger.debug("End sandbox") async def loop(self): loop = asyncio.get_event_loop() for channel in CC: - self._channels[channel]._task_reader = loop.create_task(Tecemux.channel_reader(self._channels[channel])) - self._channels[channel]._task_writer = loop.create_task(Tecemux.channel_writer(self._channels[channel])) + self._channels[channel]._task_reader = loop.create_task(Tecemux.channel_reader(self._channels[channel], self._stop_event)) + self._channels[channel]._task_writer = loop.create_task(Tecemux.channel_writer(self._channels[channel], self._stop_event)) - channel_readers = asyncio.gather(*[ channel._task_reader for channel in self._channels.values()]) + self._incoming_data_forwarder = loop.create_task(self.incoming_data_forward()) + self._outcoming_data_forwarder = loop.create_task(self.outcoming_data_forward()) + async def incoming_data_forward(self): + while not self._reader.at_eof(): + buf = await self._reader.read(1024) - protocol_reader = loop.create_task(Tecemux.protocol_reader(self._reader, self._queue)) - protocol_writer = loop.create_task(Tecemux.protocol_writer(self._writer, self._queue)) - sandbox = loop.create_task(self.sandbox()) + if not buf: + break - await asyncio.gather(*[protocol_reader, protocol_writer, sandbox]) - print('End main loop') - + if len(buf) < 20: + get_logger().warning(f'Tecemux/MAIN: [<] Too few data from Transform Hub received') + continue - @staticmethod - async def protocol_reader(reader, queue): - while True: - chunk = await reader.read(100) - get_logger().debug(f'Chunk received: {chunk}') - await queue.put(chunk) - get_logger().debug(f'Chunk redirected: {chunk}') + get_logger().debug(f'Tecemux/MAIN: [<] Incomming chunk from Transform Hub was received') - @staticmethod - async def protocol_writer(writer, queue): - while True: - chunk = await queue.get() - get_logger().debug(f'PROTOCOL: Chunk waiting: {chunk}') - #writer.write(chunk) - queue.task_done() - get_logger().debug(f'PROTOCOL: Chunk consumed: {chunk}') + pkt = TCPSegment().from_buffer(buf) + channel = CC(pkt.dst_port) + await self._channels[channel]._queue.put(pkt.data) + get_logger().debug(f'Tecemux/MAIN: [<] Chunk forwarded to {channel} steam') + + get_logger().debug(f'Tecemux/MAIN: Incomming data forwarder finished') + + async def outcoming_data_forward(self): + + while not self._reader.at_eof(): + + try: + chunk = await self._queue.get_nowait() + get_logger().debug(f'Tecemux/MAIN: [>] Outcoming chunk "{chunk}" is waiting to send to Transform Hub') + self._writer.write(chunk) + await self._writer.drain() + await self._queue.task_done() + get_logger().debug(f'Tecemux/MAIN: [>] Chunk "{chunk}" was sent to Transform Hub') + except asyncio.QueueEmpty: + await asyncio.sleep(0) + + get_logger().debug(f'Tecemux/MAIN: Outcoming data forwarder finished') @staticmethod - async def channel_reader(channel_context): - while True: - #print(f'{channel_context.get_name()}: READ!') + async def channel_reader(channel_context, stop_event): + + get_logger().debug(f'Tecemux/{channel_context.get_name()}: Channel reader started.') + + while not stop_event.is_set(): + #get_logger().debug(f'Tecemux/{channel_context.get_name()}: READ!') await asyncio.sleep(0) + + get_logger().debug(f'Tecemux/{channel_context.get_name()}: Channel writer finished.') @staticmethod - async def channel_writer(channel_context): - while True: - #print(f'{channel_context.get_name()}: WRITE!') + async def channel_writer(channel_context, stop_event): + + get_logger().debug(f'Tecemux/{channel_context.get_name()}: Channel writer started.') + + while not stop_event.is_set(): + #get_logger().debug(f'Tecemux/{channel_context.get_name()}: WRITE!') await asyncio.sleep(0) + + get_logger().debug(f'Tecemux/{channel_context.get_name()}: Channel writer finished.') + class Runner: def __init__(self, instance_id, sequence_path, logger) -> None: @@ -162,6 +195,14 @@ async def main(self, server_host, server_port): await self.protocol.loop() + self.logger.debug("Run sequence") + + await self.protocol.stop() + + await self.protocol.wait_until_end() + + self.logger.debug("End main") + if __name__ == '__main__': runner = Runner(instance_id, sequence_path, get_logger()) asyncio.run(runner.main(server_host, server_port)) From b36db371a3d1be5456770484fe9d36b4b7bc12c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Thu, 18 May 2023 13:59:54 +0200 Subject: [PATCH 053/231] Tecemux class in external file --- packages/python-runner/runner_tecemux.py | 148 +----------------- packages/python-runner/tecemux.py | 146 +++++++++++++++++ .../test/{test_runner.py => test_tecemux.py} | 3 +- 3 files changed, 150 insertions(+), 147 deletions(-) create mode 100644 packages/python-runner/tecemux.py rename packages/python-runner/test/{test_runner.py => test_tecemux.py} (94%) diff --git a/packages/python-runner/runner_tecemux.py b/packages/python-runner/runner_tecemux.py index 55357a863..21ad0283e 100644 --- a/packages/python-runner/runner_tecemux.py +++ b/packages/python-runner/runner_tecemux.py @@ -1,10 +1,7 @@ import asyncio import sys import codecs -import socket -from attrs import define, field - -from inet import TCPSegment,IPPacket +from tecemux import Tecemux from logging_setup import LoggingSetup from hardcoded_magic_values import CommunicationChannels as CC @@ -17,147 +14,6 @@ def get_logger(): server_port = 8001 server_host = 'localhost' instance_id = '1234' - - -@define -class Tecemux: - @define - class ChannelContext: - _name: str - _queue: asyncio.Queue - _reader: asyncio.StreamReader - _writer: asyncio.StreamWriter - - _task_reader: asyncio.coroutine = None - _task_writer: asyncio.coroutine = None - - - def encode(data): - pass - - def decode(data): - pass - - def get_name(self): - return self._name.name - - _queue: asyncio.Queue = asyncio.Queue() - _reader: asyncio.StreamReader = field(default=None) - _writer: asyncio.StreamWriter = field(default=None) - - _incoming_data_forwarder: asyncio.coroutine = field(default=None) - _outcoming_data_forwarder: asyncio.coroutine = field(default=None) - - _logger = field(default=get_logger()) - _channels = field(default={}) - _stop_event = asyncio.Event() - - async def connect(self, reader, writer): - self._reader = reader - self._writer = writer - self._stop_event.clear() - - - @staticmethod - async def prepare_tcp_connection(server_host, server_port): - return await asyncio.open_connection(server_host, server_port) - - @staticmethod - async def prepare_socket_connection(): - rsock, wsock = socket.socketpair() - reader, _ = await asyncio.open_unix_connection(sock=rsock) - _, writer = await asyncio.open_unix_connection(sock=wsock) - return reader,writer - - async def prepare(self): - self._channels = {channel : Tecemux.ChannelContext(channel, *await Tecemux.prepare_socket_connection(),asyncio.Queue()) for channel in CC } - - def get_channel(self, channel): - return self._channels[channel] - - async def stop(self): - await self._writer.drain() - self._writer.close() - await self._writer.wait_closed() - self._stop_event.set() - - async def wait_until_end(self): - await self._stop_event.wait() - - await asyncio.gather(*[ channel._task_reader for channel in self._channels.values()]) - await asyncio.gather(*[ channel._task_writer for channel in self._channels.values()]) - await asyncio.gather(*[self._incoming_data_forwarder, self._outcoming_data_forwarder]) - - get_logger().debug(f'Tecemux/MAIN: [-] Finished') - - async def loop(self): - - loop = asyncio.get_event_loop() - for channel in CC: - self._channels[channel]._task_reader = loop.create_task(Tecemux.channel_reader(self._channels[channel], self._stop_event)) - self._channels[channel]._task_writer = loop.create_task(Tecemux.channel_writer(self._channels[channel], self._stop_event)) - - self._incoming_data_forwarder = loop.create_task(self.incoming_data_forward()) - self._outcoming_data_forwarder = loop.create_task(self.outcoming_data_forward()) - - async def incoming_data_forward(self): - while not self._reader.at_eof(): - buf = await self._reader.read(1024) - - if not buf: - break - - if len(buf) < 20: - get_logger().warning(f'Tecemux/MAIN: [<] Too few data from Transform Hub received') - continue - - get_logger().debug(f'Tecemux/MAIN: [<] Incomming chunk from Transform Hub was received') - - pkt = TCPSegment().from_buffer(buf) - channel = CC(pkt.dst_port) - await self._channels[channel]._queue.put(pkt.data) - get_logger().debug(f'Tecemux/MAIN: [<] Chunk forwarded to {channel} steam') - - get_logger().debug(f'Tecemux/MAIN: Incomming data forwarder finished') - - async def outcoming_data_forward(self): - - while not self._reader.at_eof(): - - try: - chunk = await self._queue.get_nowait() - get_logger().debug(f'Tecemux/MAIN: [>] Outcoming chunk "{chunk}" is waiting to send to Transform Hub') - self._writer.write(chunk) - await self._writer.drain() - await self._queue.task_done() - get_logger().debug(f'Tecemux/MAIN: [>] Chunk "{chunk}" was sent to Transform Hub') - except asyncio.QueueEmpty: - await asyncio.sleep(0) - - get_logger().debug(f'Tecemux/MAIN: Outcoming data forwarder finished') - - @staticmethod - async def channel_reader(channel_context, stop_event): - - get_logger().debug(f'Tecemux/{channel_context.get_name()}: Channel reader started.') - - while not stop_event.is_set(): - #get_logger().debug(f'Tecemux/{channel_context.get_name()}: READ!') - await asyncio.sleep(0) - - get_logger().debug(f'Tecemux/{channel_context.get_name()}: Channel writer finished.') - - @staticmethod - async def channel_writer(channel_context, stop_event): - - get_logger().debug(f'Tecemux/{channel_context.get_name()}: Channel writer started.') - - while not stop_event.is_set(): - #get_logger().debug(f'Tecemux/{channel_context.get_name()}: WRITE!') - await asyncio.sleep(0) - - get_logger().debug(f'Tecemux/{channel_context.get_name()}: Channel writer finished.') - class Runner: def __init__(self, instance_id, sequence_path, logger) -> None: @@ -179,7 +35,7 @@ def connect_stdio(self): async def main(self, server_host, server_port): self.logger.info('Connecting to host...') - self.protocol = Tecemux() + self.protocol = Tecemux(logger=self.logger) # self.logger.debug(f'Connecting to {server_host}:{server_port}...') # await self.protocol.connect(*await Tecemux.prepare_tcp_connection(server_host, server_port)) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py new file mode 100644 index 000000000..af467e3cf --- /dev/null +++ b/packages/python-runner/tecemux.py @@ -0,0 +1,146 @@ +import asyncio +import socket +from attrs import define, field + +from inet import TCPSegment,IPPacket +from hardcoded_magic_values import CommunicationChannels as CC + +@define +class Tecemux: + @define + class ChannelContext: + _name: str + _queue: asyncio.Queue + _reader: asyncio.StreamReader + _writer: asyncio.StreamWriter + + _task_reader: asyncio.coroutine = None + _task_writer: asyncio.coroutine = None + + + def encode(data): + pass + + def decode(data): + pass + + def get_name(self): + return self._name.name + + _queue: asyncio.Queue = asyncio.Queue() + _reader: asyncio.StreamReader = field(default=None) + _writer: asyncio.StreamWriter = field(default=None) + + _incoming_data_forwarder: asyncio.coroutine = field(default=None) + _outcoming_data_forwarder: asyncio.coroutine = field(default=None) + + _logger = field(default=None) + _channels = field(default={}) + _stop_event = asyncio.Event() + + async def connect(self, reader, writer): + self._reader = reader + self._writer = writer + self._stop_event.clear() + + + @staticmethod + async def prepare_tcp_connection(server_host, server_port): + return await asyncio.open_connection(server_host, server_port) + + @staticmethod + async def prepare_socket_connection(): + rsock, wsock = socket.socketpair() + reader, _ = await asyncio.open_unix_connection(sock=rsock) + _, writer = await asyncio.open_unix_connection(sock=wsock) + return reader,writer + + async def prepare(self): + self._channels = {channel : Tecemux.ChannelContext(channel, *await Tecemux.prepare_socket_connection(),asyncio.Queue()) for channel in CC } + + def get_channel(self, channel): + return self._channels[channel] + + async def stop(self): + await self._writer.drain() + self._writer.close() + await self._writer.wait_closed() + self._stop_event.set() + + async def wait_until_end(self): + await self._stop_event.wait() + + await asyncio.gather(*[ channel._task_reader for channel in self._channels.values()]) + await asyncio.gather(*[ channel._task_writer for channel in self._channels.values()]) + await asyncio.gather(*[self._incoming_data_forwarder, self._outcoming_data_forwarder]) + + self._logger.debug(f'Tecemux/MAIN: [-] Finished') + + async def loop(self): + + loop = asyncio.get_event_loop() + for channel in CC: + self._channels[channel]._task_reader = loop.create_task(Tecemux.channel_reader(self._channels[channel], self._stop_event)) + self._channels[channel]._task_writer = loop.create_task(Tecemux.channel_writer(self._channels[channel], self._stop_event)) + + self._incoming_data_forwarder = loop.create_task(self.incoming_data_forward()) + self._outcoming_data_forwarder = loop.create_task(self.outcoming_data_forward()) + + async def incoming_data_forward(self): + while not self._reader.at_eof(): + buf = await self._reader.read(1024) + + if not buf: + break + + if len(buf) < 20: + self._logger.warning(f'Tecemux/MAIN: [<] Too few data from Transform Hub received') + continue + + self._logger.debug(f'Tecemux/MAIN: [<] Incomming chunk from Transform Hub was received') + + pkt = TCPSegment().from_buffer(buf) + channel = CC(pkt.dst_port) + await self._channels[channel]._queue.put(pkt.data) + self._logger.debug(f'Tecemux/MAIN: [<] Chunk forwarded to {channel} steam') + + self._logger.debug(f'Tecemux/MAIN: Incomming data forwarder finished') + + async def outcoming_data_forward(self): + + while not self._reader.at_eof(): + + try: + chunk = await self._queue.get_nowait() + self._logger.debug(f'Tecemux/MAIN: [>] Outcoming chunk "{chunk}" is waiting to send to Transform Hub') + self._writer.write(chunk) + await self._writer.drain() + await self._queue.task_done() + self._logger.debug(f'Tecemux/MAIN: [>] Chunk "{chunk}" was sent to Transform Hub') + except asyncio.QueueEmpty: + await asyncio.sleep(0) + + self._logger.debug(f'Tecemux/MAIN: Outcoming data forwarder finished') + + @staticmethod + async def channel_reader(channel_context, stop_event): + + #get_logger().debug(f'Tecemux/{channel_context.get_name()}: Channel reader started.') + + while not stop_event.is_set(): + #get_logger().debug(f'Tecemux/{channel_context.get_name()}: READ!') + await asyncio.sleep(0) + + #get_logger().debug(f'Tecemux/{channel_context.get_name()}: Channel writer finished.') + + @staticmethod + async def channel_writer(channel_context, stop_event): + + #get_logger().debug(f'Tecemux/{channel_context.get_name()}: Channel writer started.') + + while not stop_event.is_set(): + #get_logger().debug(f'Tecemux/{channel_context.get_name()}: WRITE!') + await asyncio.sleep(0) + + #get_logger().debug(f'Tecemux/{channel_context.get_name()}: Channel writer finished.') + diff --git a/packages/python-runner/test/test_runner.py b/packages/python-runner/test/test_tecemux.py similarity index 94% rename from packages/python-runner/test/test_runner.py rename to packages/python-runner/test/test_tecemux.py index f02ac7e69..997bf2d14 100644 --- a/packages/python-runner/test/test_runner.py +++ b/packages/python-runner/test/test_tecemux.py @@ -1,6 +1,7 @@ import asyncio import pytest -from runner_tecemux import Tecemux +from tecemux import Tecemux +from runner_tecemux import get_logger @pytest.fixture() From 8394d6b9dba21647e38f290ada136a3830829a79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 19 May 2023 13:23:22 +0200 Subject: [PATCH 054/231] First version of channel data forward --- packages/python-runner/inet.py | 10 +-- packages/python-runner/runner_tecemux.py | 15 +++- packages/python-runner/tecemux.py | 97 +++++++++++---------- packages/python-runner/test/test_inet.py | 9 ++ packages/python-runner/test/test_tecemux.py | 28 +++++- 5 files changed, 104 insertions(+), 55 deletions(-) diff --git a/packages/python-runner/inet.py b/packages/python-runner/inet.py index 8d3916ebd..6a547b36d 100644 --- a/packages/python-runner/inet.py +++ b/packages/python-runner/inet.py @@ -81,7 +81,7 @@ def to_buffer(self): self.flags,\ self.win,\ self.checksum,\ - self.urp) + self.data + self.urp) + self.data.encode("utf-8") def set_flags(self, list_of_flags): self.flags = list_of_flags @@ -127,8 +127,8 @@ def parse_flags(value): ihl: int = 5 version: int = field( default=4, converter = lambda value: value >> 4) tos: int = field(default=0) - len: int = field(default=None) - ids: int = field(default=None) + len: int = field(default=0) + ids: int = field(default=0) flags_offset: int = field(default = 0) flags: int = field(default = 0, init= False, repr = lambda value: IPPacket.Flags.flags_to_str(value), \ converter = lambda value: IPPacket.Flags.parse_flags(value)) @@ -136,8 +136,8 @@ def parse_flags(value): ttl: int = field(default=255) protocol: int = field(default=6) checksum: int = field(default = 0, repr = lambda value: hex(value)) - src_addr: str = field(default='', converter = lambda value: inet_ntoa(value)) - dst_addr: str = field(default='', converter = lambda value: inet_ntoa(value)) + src_addr: str = field(default='', converter = lambda value: inet_ntoa(value) if isinstance(value,bytes) else value) + dst_addr: str = field(default='', converter = lambda value: inet_ntoa(value) if isinstance(value,bytes) else value) segment: TCPSegment = None def __attrs_post_init__(self): diff --git a/packages/python-runner/runner_tecemux.py b/packages/python-runner/runner_tecemux.py index 21ad0283e..14a3a57c7 100644 --- a/packages/python-runner/runner_tecemux.py +++ b/packages/python-runner/runner_tecemux.py @@ -2,6 +2,7 @@ import sys import codecs from tecemux import Tecemux +from inet import TCPSegment, IPPacket from logging_setup import LoggingSetup from hardcoded_magic_values import CommunicationChannels as CC @@ -35,7 +36,8 @@ def connect_stdio(self): async def main(self, server_host, server_port): self.logger.info('Connecting to host...') - self.protocol = Tecemux(logger=self.logger) + self.protocol = Tecemux() + self.protocol.set_logger(get_logger()) # self.logger.debug(f'Connecting to {server_host}:{server_port}...') # await self.protocol.connect(*await Tecemux.prepare_tcp_connection(server_host, server_port)) @@ -52,6 +54,17 @@ async def main(self, server_host, server_port): await self.protocol.loop() self.logger.debug("Run sequence") + + pkt = IPPacket(src_addr='172.25.44.3',dst_addr='172.25.44.254',segment=TCPSegment(dst_port=int(CC.CONTROL.value),flags=['ACK'],data="ala ma kota")) + self.protocol._writer.write(pkt.to_buffer()) + await self.protocol._writer.drain() + + # await self.protocol._channels[CC.CONTROL].write(b'ala ma kota') + # await self.protocol._channels[CC.CONTROL].drain() + data = await self.protocol._channels[CC.CONTROL].read(5) + print('x') + #assert self.protocol._channels[CC.CONTROL]._queue.qsize() == 1 + await self.protocol.stop() diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index af467e3cf..d2aa9dfba 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -2,7 +2,7 @@ import socket from attrs import define, field -from inet import TCPSegment,IPPacket +from inet import IPPacket from hardcoded_magic_values import CommunicationChannels as CC @define @@ -10,32 +10,54 @@ class Tecemux: @define class ChannelContext: _name: str - _queue: asyncio.Queue + _reader: asyncio.StreamReader _writer: asyncio.StreamWriter + _queue: asyncio.Queue - _task_reader: asyncio.coroutine = None - _task_writer: asyncio.coroutine = None - + _channel_paused: bool = False + + async def queue_up(self, data): + + if self._channel_paused: + self._queue.put(data) + else: + while not self._queue.empty(): + try: + buf = await self._queue.get() + self._writer.write(buf) + self._queue.task_done() + except asyncio.QueueEmpty: + await self._writer.drain() + break + self._writer.write(data) + + async def read(self, len): + return await self._reader.read(len) - def encode(data): - pass + async def write(self, data): + await self.queue_up(data) - def decode(data): - pass + async def drain(self): + if not self._channel_paused: + return await self._writer.drain() + return True + + def set_pause(self, state): + self._channel_paused = state def get_name(self): return self._name.name - _queue: asyncio.Queue = asyncio.Queue() + _queue: asyncio.Queue = field(default=None) _reader: asyncio.StreamReader = field(default=None) _writer: asyncio.StreamWriter = field(default=None) _incoming_data_forwarder: asyncio.coroutine = field(default=None) _outcoming_data_forwarder: asyncio.coroutine = field(default=None) - _logger = field(default=None) _channels = field(default={}) + _logger = field(default=None) _stop_event = asyncio.Event() async def connect(self, reader, writer): @@ -43,7 +65,6 @@ async def connect(self, reader, writer): self._writer = writer self._stop_event.clear() - @staticmethod async def prepare_tcp_connection(server_host, server_port): return await asyncio.open_connection(server_host, server_port) @@ -55,8 +76,14 @@ async def prepare_socket_connection(): _, writer = await asyncio.open_unix_connection(sock=wsock) return reader,writer + async def prepare(self): - self._channels = {channel : Tecemux.ChannelContext(channel, *await Tecemux.prepare_socket_connection(),asyncio.Queue()) for channel in CC } + self._channels = {channel : Tecemux.ChannelContext(channel, *await Tecemux.prepare_socket_connection(), asyncio.Queue()) for channel in CC } + self._queue = asyncio.Queue() + + + def set_logger(self,logger): + self._logger = logger def get_channel(self, channel): return self._channels[channel] @@ -67,11 +94,14 @@ async def stop(self): await self._writer.wait_closed() self._stop_event.set() + for channel in self._channels.values(): + await channel._writer.drain() + channel._writer.close() + await channel._writer.wait_closed() + + async def wait_until_end(self): await self._stop_event.wait() - - await asyncio.gather(*[ channel._task_reader for channel in self._channels.values()]) - await asyncio.gather(*[ channel._task_writer for channel in self._channels.values()]) await asyncio.gather(*[self._incoming_data_forwarder, self._outcoming_data_forwarder]) self._logger.debug(f'Tecemux/MAIN: [-] Finished') @@ -79,9 +109,6 @@ async def wait_until_end(self): async def loop(self): loop = asyncio.get_event_loop() - for channel in CC: - self._channels[channel]._task_reader = loop.create_task(Tecemux.channel_reader(self._channels[channel], self._stop_event)) - self._channels[channel]._task_writer = loop.create_task(Tecemux.channel_writer(self._channels[channel], self._stop_event)) self._incoming_data_forwarder = loop.create_task(self.incoming_data_forward()) self._outcoming_data_forwarder = loop.create_task(self.outcoming_data_forward()) @@ -99,17 +126,16 @@ async def incoming_data_forward(self): self._logger.debug(f'Tecemux/MAIN: [<] Incomming chunk from Transform Hub was received') - pkt = TCPSegment().from_buffer(buf) - channel = CC(pkt.dst_port) - await self._channels[channel]._queue.put(pkt.data) - self._logger.debug(f'Tecemux/MAIN: [<] Chunk forwarded to {channel} steam') + pkt = IPPacket().from_buffer(buf) + channel = CC(str(pkt.get_segment().dst_port)) + await self._channels[channel].write(pkt.get_segment().data) + #self._logger.debug(f'Tecemux/MAIN: [<] Chunk forwarded to {channel.name} steam') self._logger.debug(f'Tecemux/MAIN: Incomming data forwarder finished') async def outcoming_data_forward(self): while not self._reader.at_eof(): - try: chunk = await self._queue.get_nowait() self._logger.debug(f'Tecemux/MAIN: [>] Outcoming chunk "{chunk}" is waiting to send to Transform Hub') @@ -121,26 +147,3 @@ async def outcoming_data_forward(self): await asyncio.sleep(0) self._logger.debug(f'Tecemux/MAIN: Outcoming data forwarder finished') - - @staticmethod - async def channel_reader(channel_context, stop_event): - - #get_logger().debug(f'Tecemux/{channel_context.get_name()}: Channel reader started.') - - while not stop_event.is_set(): - #get_logger().debug(f'Tecemux/{channel_context.get_name()}: READ!') - await asyncio.sleep(0) - - #get_logger().debug(f'Tecemux/{channel_context.get_name()}: Channel writer finished.') - - @staticmethod - async def channel_writer(channel_context, stop_event): - - #get_logger().debug(f'Tecemux/{channel_context.get_name()}: Channel writer started.') - - while not stop_event.is_set(): - #get_logger().debug(f'Tecemux/{channel_context.get_name()}: WRITE!') - await asyncio.sleep(0) - - #get_logger().debug(f'Tecemux/{channel_context.get_name()}: Channel writer finished.') - diff --git a/packages/python-runner/test/test_inet.py b/packages/python-runner/test/test_inet.py index 3c2bb6ea1..6e3b12df1 100644 --- a/packages/python-runner/test/test_inet.py +++ b/packages/python-runner/test/test_inet.py @@ -51,6 +51,15 @@ def test_checksum_calc(self): assert pkt.checksum == checksum + def test_packet_creation(self): + + pkt = IPPacket(src_addr='172.25.44.3',dst_addr='172.25.44.254',segment=TCPSegment(dst_port=3,flags=['ACK'])) + + assert pkt.src_addr == '172.25.44.3' + assert pkt.dst_addr == '172.25.44.254' + + #data = pkt.build().to_buffer() + class TestTCP: def test_tcp_offset(self): data = b'\x01\xbb\xc0\xd7\xb6\x56\xa8\xb9\xd1\xac\xaa\xb1\x50\x18\x40\x00\x56\xf8\x00\x00' diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index 997bf2d14..ca822efe2 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -1,18 +1,22 @@ import asyncio import pytest from tecemux import Tecemux +from inet import IPPacket,TCPSegment +from hardcoded_magic_values import CommunicationChannels as CC from runner_tecemux import get_logger @pytest.fixture() async def local_socket_connection(): - protocol = Tecemux() + protocol = Tecemux() + protocol.set_logger(get_logger()) await protocol.connect(*await Tecemux.prepare_socket_connection()) return protocol class TestTecemux: def test_default_init(self): protocol = Tecemux() + protocol.set_logger(get_logger()) assert isinstance(protocol, Tecemux) @pytest.mark.asyncio @@ -36,4 +40,24 @@ async def test_tecemux_pipe(self, local_socket_connection): protocol._writer.write(b'dnkrozz') await protocol._writer.drain() - assert await protocol._reader.read(100) == b'dnkrozz' \ No newline at end of file + assert await protocol._reader.read(100) == b'dnkrozz' + + + @pytest.mark.asyncio + async def test_tecemux_specific_channel(self, local_socket_connection): + protocol = local_socket_connection + data_to_send ="{'foo':'bar'}" + destination_channel = CC.CONTROL + await protocol.prepare() + await protocol.loop() + + pkt = IPPacket(src_addr='172.25.44.3',dst_addr='172.25.44.254',segment=TCPSegment(dst_port=int(destination_channel.value),flags=['ACK'],data=data_to_send)) + protocol._writer.write(pkt.to_buffer()) + await protocol._writer.drain() + + assert (await protocol.get_channel(destination_channel).read(len(data_to_send))) + # # await self.protocol._channels[CC.CONTROL].write(b'ala ma kota') + # # await self.protocol._channels[CC.CONTROL].drain() + # data = await self.protocol._channels[CC.CONTROL].read(5) + + From dd426ad5143b67cb2f5dc7c2fcdec0fc7a237484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 19 May 2023 14:25:39 +0200 Subject: [PATCH 055/231] Tests for channel forwarding --- packages/python-runner/runner_tecemux.py | 12 --------- packages/python-runner/tecemux.py | 24 +++++++++++------ packages/python-runner/test/test_tecemux.py | 29 ++++++++++++++++----- 3 files changed, 39 insertions(+), 26 deletions(-) diff --git a/packages/python-runner/runner_tecemux.py b/packages/python-runner/runner_tecemux.py index 14a3a57c7..6c6a1b18b 100644 --- a/packages/python-runner/runner_tecemux.py +++ b/packages/python-runner/runner_tecemux.py @@ -2,7 +2,6 @@ import sys import codecs from tecemux import Tecemux -from inet import TCPSegment, IPPacket from logging_setup import LoggingSetup from hardcoded_magic_values import CommunicationChannels as CC @@ -54,17 +53,6 @@ async def main(self, server_host, server_port): await self.protocol.loop() self.logger.debug("Run sequence") - - pkt = IPPacket(src_addr='172.25.44.3',dst_addr='172.25.44.254',segment=TCPSegment(dst_port=int(CC.CONTROL.value),flags=['ACK'],data="ala ma kota")) - self.protocol._writer.write(pkt.to_buffer()) - await self.protocol._writer.drain() - - # await self.protocol._channels[CC.CONTROL].write(b'ala ma kota') - # await self.protocol._channels[CC.CONTROL].drain() - data = await self.protocol._channels[CC.CONTROL].read(5) - print('x') - #assert self.protocol._channels[CC.CONTROL]._queue.qsize() == 1 - await self.protocol.stop() diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index d2aa9dfba..12ef04546 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -2,7 +2,7 @@ import socket from attrs import define, field -from inet import IPPacket +from inet import IPPacket, TCPSegment from hardcoded_magic_values import CommunicationChannels as CC @define @@ -14,23 +14,30 @@ class ChannelContext: _reader: asyncio.StreamReader _writer: asyncio.StreamWriter _queue: asyncio.Queue + _global_writer: asyncio.StreamWriter _channel_paused: bool = False async def queue_up(self, data): - + + def wrap(channel_enum, buf): + channel_id = int(channel_enum.value) + return IPPacket(src_addr='0.0.0.0',dst_addr='0.0.0.0',segment=TCPSegment(dst_port=channel_id,data=buf)).build().to_buffer() + if self._channel_paused: self._queue.put(data) else: while not self._queue.empty(): try: buf = await self._queue.get() - self._writer.write(buf) + + self._global_writer.write(wrap(self._name,data)) self._queue.task_done() except asyncio.QueueEmpty: - await self._writer.drain() + await self._global_writer.drain() break - self._writer.write(data) + self._global_writer.write(wrap(self._name,data)) + async def read(self, len): return await self._reader.read(len) @@ -40,7 +47,8 @@ async def write(self, data): async def drain(self): if not self._channel_paused: - return await self._writer.drain() + await self._writer.drain() + return await self._global_writer.drain() return True def set_pause(self, state): @@ -78,7 +86,7 @@ async def prepare_socket_connection(): async def prepare(self): - self._channels = {channel : Tecemux.ChannelContext(channel, *await Tecemux.prepare_socket_connection(), asyncio.Queue()) for channel in CC } + self._channels = {channel : Tecemux.ChannelContext(channel, *await Tecemux.prepare_socket_connection(), asyncio.Queue(), self._writer) for channel in CC } self._queue = asyncio.Queue() @@ -129,7 +137,7 @@ async def incoming_data_forward(self): pkt = IPPacket().from_buffer(buf) channel = CC(str(pkt.get_segment().dst_port)) await self._channels[channel].write(pkt.get_segment().data) - #self._logger.debug(f'Tecemux/MAIN: [<] Chunk forwarded to {channel.name} steam') + self._logger.debug(f'Tecemux/MAIN: [<] Chunk forwarded to {channel.name} steam') self._logger.debug(f'Tecemux/MAIN: Incomming data forwarder finished') diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index ca822efe2..8be59d0e1 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -16,7 +16,6 @@ async def local_socket_connection(): class TestTecemux: def test_default_init(self): protocol = Tecemux() - protocol.set_logger(get_logger()) assert isinstance(protocol, Tecemux) @pytest.mark.asyncio @@ -42,9 +41,8 @@ async def test_tecemux_pipe(self, local_socket_connection): assert await protocol._reader.read(100) == b'dnkrozz' - @pytest.mark.asyncio - async def test_tecemux_specific_channel(self, local_socket_connection): + async def test_tecemux_forward_on_channel(self, local_socket_connection): protocol = local_socket_connection data_to_send ="{'foo':'bar'}" destination_channel = CC.CONTROL @@ -56,8 +54,27 @@ async def test_tecemux_specific_channel(self, local_socket_connection): await protocol._writer.drain() assert (await protocol.get_channel(destination_channel).read(len(data_to_send))) - # # await self.protocol._channels[CC.CONTROL].write(b'ala ma kota') - # # await self.protocol._channels[CC.CONTROL].drain() - # data = await self.protocol._channels[CC.CONTROL].read(5) + + @pytest.mark.asyncio + async def test_tecemux_forward_from_channel(self, local_socket_connection): + protocol = local_socket_connection + + data_to_send ="{'foo':'bar'}" + source_channel = CC.CONTROL + + await protocol.prepare() + + channel = protocol.get_channel(source_channel) + + await protocol.loop() + await channel.write(data_to_send) + await channel.drain() + + data = await protocol._reader.read(100) + pkt = IPPacket.from_buffer(data).get_segment() + + assert pkt.data == data_to_send.encode('utf-8') + assert pkt.dst_port == int(source_channel.value) + From 3e390eb353ff1b9edda0c595aec8ca4fac835a0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 19 May 2023 15:58:46 +0200 Subject: [PATCH 056/231] Tecemux communication between endpoints --- packages/python-runner/inet.py | 2 +- packages/python-runner/tecemux.py | 37 ++++----- packages/python-runner/test/test_tecemux.py | 84 +++++++++++---------- 3 files changed, 67 insertions(+), 56 deletions(-) diff --git a/packages/python-runner/inet.py b/packages/python-runner/inet.py index 6a547b36d..ec7227710 100644 --- a/packages/python-runner/inet.py +++ b/packages/python-runner/inet.py @@ -81,7 +81,7 @@ def to_buffer(self): self.flags,\ self.win,\ self.checksum,\ - self.urp) + self.data.encode("utf-8") + self.urp) + (self.data.encode("utf-8") if isinstance(self.data,str) else self.data) def set_flags(self, list_of_flags): self.flags = list_of_flags diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 12ef04546..64c822e98 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -14,41 +14,44 @@ class ChannelContext: _reader: asyncio.StreamReader _writer: asyncio.StreamWriter _queue: asyncio.Queue - _global_writer: asyncio.StreamWriter + _global_queue: asyncio.Queue _channel_paused: bool = False - async def queue_up(self, data): + + async def queue_up_incoming(self,buf): + pkt = IPPacket().from_buffer(buf) + self._writer.write(pkt.get_segment().data) + await self._writer.drain() + + async def queue_up_outcoming(self, data): def wrap(channel_enum, buf): channel_id = int(channel_enum.value) return IPPacket(src_addr='0.0.0.0',dst_addr='0.0.0.0',segment=TCPSegment(dst_port=channel_id,data=buf)).build().to_buffer() if self._channel_paused: - self._queue.put(data) + await self._queue.put(data) else: while not self._queue.empty(): try: buf = await self._queue.get() - - self._global_writer.write(wrap(self._name,data)) - self._queue.task_done() + await self._global_queue.put(wrap(self._name,data)) except asyncio.QueueEmpty: - await self._global_writer.drain() break - self._global_writer.write(wrap(self._name,data)) + await self._global_queue.put(wrap(self._name,data)) + async def read(self, len): return await self._reader.read(len) async def write(self, data): - await self.queue_up(data) + await self.queue_up_outcoming(data) async def drain(self): if not self._channel_paused: await self._writer.drain() - return await self._global_writer.drain() return True def set_pause(self, state): @@ -86,9 +89,9 @@ async def prepare_socket_connection(): async def prepare(self): - self._channels = {channel : Tecemux.ChannelContext(channel, *await Tecemux.prepare_socket_connection(), asyncio.Queue(), self._writer) for channel in CC } self._queue = asyncio.Queue() - + self._channels = {channel : Tecemux.ChannelContext(channel, *await Tecemux.prepare_socket_connection(), asyncio.Queue(), self._queue) for channel in CC } + def set_logger(self,logger): self._logger = logger @@ -122,7 +125,7 @@ async def loop(self): self._outcoming_data_forwarder = loop.create_task(self.outcoming_data_forward()) async def incoming_data_forward(self): - while not self._reader.at_eof(): + while not self._stop_event.is_set(): buf = await self._reader.read(1024) if not buf: @@ -136,20 +139,20 @@ async def incoming_data_forward(self): pkt = IPPacket().from_buffer(buf) channel = CC(str(pkt.get_segment().dst_port)) - await self._channels[channel].write(pkt.get_segment().data) + await self._channels[channel].queue_up_incoming(buf) self._logger.debug(f'Tecemux/MAIN: [<] Chunk forwarded to {channel.name} steam') self._logger.debug(f'Tecemux/MAIN: Incomming data forwarder finished') async def outcoming_data_forward(self): - while not self._reader.at_eof(): + while not self._stop_event.is_set(): try: - chunk = await self._queue.get_nowait() + chunk = self._queue.get_nowait() self._logger.debug(f'Tecemux/MAIN: [>] Outcoming chunk "{chunk}" is waiting to send to Transform Hub') self._writer.write(chunk) await self._writer.drain() - await self._queue.task_done() + self._queue.task_done() self._logger.debug(f'Tecemux/MAIN: [>] Chunk "{chunk}" was sent to Transform Hub') except asyncio.QueueEmpty: await asyncio.sleep(0) diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index 8be59d0e1..61da6ab8e 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -8,10 +8,27 @@ @pytest.fixture() async def local_socket_connection(): - protocol = Tecemux() - protocol.set_logger(get_logger()) - await protocol.connect(*await Tecemux.prepare_socket_connection()) - return protocol + client_a = Tecemux() + client_a.set_logger(get_logger()) + + client_b = Tecemux() + client_b.set_logger(get_logger()) + + + rsock_a, wsock_a = await Tecemux.prepare_socket_connection() + rsock_b, wsock_b = await Tecemux.prepare_socket_connection() + + await client_a.connect(rsock_a,wsock_b) + await client_b.connect(rsock_b,wsock_a) + + await client_a.prepare() + await client_b.prepare() + + await client_a.loop() + await client_b.loop() + + + return client_a, client_b class TestTecemux: def test_default_init(self): @@ -30,51 +47,42 @@ async def test_socket_connection(self): assert isinstance(protocol._reader,asyncio.StreamReader) assert isinstance(protocol._writer,asyncio.StreamWriter) - - assert True - - @pytest.mark.asyncio - async def test_tecemux_pipe(self, local_socket_connection): - protocol = local_socket_connection - protocol._writer.write(b'dnkrozz') - await protocol._writer.drain() - assert await protocol._reader.read(100) == b'dnkrozz' @pytest.mark.asyncio - async def test_tecemux_forward_on_channel(self, local_socket_connection): - protocol = local_socket_connection + async def test_forward_from_main_to_channel(self, local_socket_connection): + client_a, client_b = local_socket_connection + data_to_send ="{'foo':'bar'}" destination_channel = CC.CONTROL - await protocol.prepare() - await protocol.loop() - + pkt = IPPacket(src_addr='172.25.44.3',dst_addr='172.25.44.254',segment=TCPSegment(dst_port=int(destination_channel.value),flags=['ACK'],data=data_to_send)) - protocol._writer.write(pkt.to_buffer()) - await protocol._writer.drain() + + client_a._writer.write(pkt.to_buffer()) + await client_a._writer.drain() - assert (await protocol.get_channel(destination_channel).read(len(data_to_send))) + assert (await client_b.get_channel(destination_channel).read(100)).decode() == data_to_send - @pytest.mark.asyncio - async def test_tecemux_forward_from_channel(self, local_socket_connection): - protocol = local_socket_connection - - data_to_send ="{'foo':'bar'}" - source_channel = CC.CONTROL + await client_a.stop() + await client_b.stop() - await protocol.prepare() - - channel = protocol.get_channel(source_channel) + await client_a.wait_until_end() + await client_a.wait_until_end() - await protocol.loop() - await channel.write(data_to_send) - await channel.drain() + @pytest.mark.asyncio + async def test_forward_channel_between_a_b(self, local_socket_connection): + client_a, client_b = local_socket_connection - data = await protocol._reader.read(100) - pkt = IPPacket.from_buffer(data).get_segment() + data_to_send ="{'foo':'bar'}" + source_channel = CC.CONTROL + + await client_a.get_channel(source_channel).write(data_to_send) + await client_a.get_channel(source_channel).drain() - assert pkt.data == data_to_send.encode('utf-8') - assert pkt.dst_port == int(source_channel.value) - + assert (await client_b.get_channel(source_channel).read(100)).decode() == data_to_send + await client_a.stop() + await client_b.stop() + await client_a.wait_until_end() + await client_a.wait_until_end() From ed81a039e8d3b392c9dcb4824d22462df11fd084 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Mon, 22 May 2023 13:22:29 +0200 Subject: [PATCH 057/231] Fix bug with forward multiple packets in one chunk --- packages/python-runner/inet.py | 9 +++- packages/python-runner/tecemux.py | 58 +++++++++++++++------ packages/python-runner/test/test_inet.py | 54 ++++++++++++------- packages/python-runner/test/test_tecemux.py | 17 +++--- 4 files changed, 94 insertions(+), 44 deletions(-) diff --git a/packages/python-runner/inet.py b/packages/python-runner/inet.py index ec7227710..783922933 100644 --- a/packages/python-runner/inet.py +++ b/packages/python-runner/inet.py @@ -127,7 +127,7 @@ def parse_flags(value): ihl: int = 5 version: int = field( default=4, converter = lambda value: value >> 4) tos: int = field(default=0) - len: int = field(default=0) + len: int = field(default=None) ids: int = field(default=0) flags_offset: int = field(default = 0) flags: int = field(default = 0, init= False, repr = lambda value: IPPacket.Flags.flags_to_str(value), \ @@ -144,6 +144,9 @@ def __attrs_post_init__(self): self.offset = self.flags_offset & 0x1FFF self.flags = self.flags_offset >> 13 + #Cut data buffer to IP packet length + self.get_segment().data=self.get_segment().data[:self.len-(self.ihl*4)-20] + @staticmethod def calc_checksum(pkt: bytes) -> int: #source: https://github.com/secdev/scapy @@ -166,6 +169,8 @@ def is_flag(self,flag): def to_buffer(self): ihl_ver = (int(self.version) << 4) + int(self.ihl) + data = self.get_segment().to_buffer() if self.segment else b'' + self.len = 20 + len(data) return struct.pack('!BBHHHBBH4s4s', \ ihl_ver, \ self.tos, \ @@ -176,7 +181,7 @@ def to_buffer(self): self.protocol, \ self.checksum, \ inet_aton(self.src_addr), \ - inet_aton(self.dst_addr)) + (self.get_segment().to_buffer() if self.segment else b'') + inet_aton(self.dst_addr)) + data def _validate_tcp(self): diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 64c822e98..9478d7fdd 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -69,12 +69,12 @@ def get_name(self): _channels = field(default={}) _logger = field(default=None) - _stop_event = asyncio.Event() + _global_stop_event = asyncio.Event() async def connect(self, reader, writer): self._reader = reader self._writer = writer - self._stop_event.clear() + self._global_stop_event.clear() @staticmethod async def prepare_tcp_connection(server_host, server_port): @@ -103,7 +103,7 @@ async def stop(self): await self._writer.drain() self._writer.close() await self._writer.wait_closed() - self._stop_event.set() + self._global_stop_event.set() for channel in self._channels.values(): await channel._writer.drain() @@ -112,7 +112,7 @@ async def stop(self): async def wait_until_end(self): - await self._stop_event.wait() + await self._global_stop_event.wait() await asyncio.gather(*[self._incoming_data_forwarder, self._outcoming_data_forwarder]) self._logger.debug(f'Tecemux/MAIN: [-] Finished') @@ -125,28 +125,52 @@ async def loop(self): self._outcoming_data_forwarder = loop.create_task(self.outcoming_data_forward()) async def incoming_data_forward(self): - while not self._stop_event.is_set(): - buf = await self._reader.read(1024) + buffer = b'' + + incoming_parser_finish_loop = asyncio.Event() + + MINIMAL_IP_PACKET_LENGTH = 20 + READ_CHUNK_SIZE = 1024 - if not buf: - break + while not self._global_stop_event.is_set(): - if len(buf) < 20: - self._logger.warning(f'Tecemux/MAIN: [<] Too few data from Transform Hub received') - continue + chunk = await self._reader.read(READ_CHUNK_SIZE) - self._logger.debug(f'Tecemux/MAIN: [<] Incomming chunk from Transform Hub was received') + if not chunk: + incoming_parser_finish_loop.set() - pkt = IPPacket().from_buffer(buf) - channel = CC(str(pkt.get_segment().dst_port)) - await self._channels[channel].queue_up_incoming(buf) - self._logger.debug(f'Tecemux/MAIN: [<] Chunk forwarded to {channel.name} steam') + buffer = buffer + chunk + + if incoming_parser_finish_loop.is_set() and len(buffer) < MINIMAL_IP_PACKET_LENGTH: + self._logger.warning(f'Tecemux/MAIN: [<] Too few data is waiting in global buffer but stream finished') + break + + while not (len(buffer) < MINIMAL_IP_PACKET_LENGTH): + + current_packet_size = IPPacket().from_buffer(buffer).len + + if len(buffer) >= current_packet_size: + + self._logger.debug(f'Tecemux/MAIN: [<] Full incomming packet from Transform Hub was received') + + single_packet_buffer = buffer[:current_packet_size] + pkt = IPPacket().from_buffer(single_packet_buffer) + channel = CC(str(pkt.get_segment().dst_port)) + + await self._channels[channel].queue_up_incoming(single_packet_buffer) + + self._logger.debug(f'Tecemux/MAIN: [<] Packet forwarded to {channel.name} steam') + buffer = buffer[current_packet_size:] + else: + self._logger.warning(f'Tecemux/MAIN: [<] Not full packet received. Getting additional data chunk') + break self._logger.debug(f'Tecemux/MAIN: Incomming data forwarder finished') + async def outcoming_data_forward(self): - while not self._stop_event.is_set(): + while not self._global_stop_event.is_set(): try: chunk = self._queue.get_nowait() self._logger.debug(f'Tecemux/MAIN: [>] Outcoming chunk "{chunk}" is waiting to send to Transform Hub') diff --git a/packages/python-runner/test/test_inet.py b/packages/python-runner/test/test_inet.py index 6e3b12df1..de17ddfb8 100644 --- a/packages/python-runner/test/test_inet.py +++ b/packages/python-runner/test/test_inet.py @@ -1,12 +1,14 @@ import pytest from inet import TCPSegment, IPPacket, EthernetFrame -class TestIP: + +class TestIP: def test_mf_df_flags(self): data = b'E\x00\x00\x14\x00\x01`\x00@\x00\x1c\xe7\x7f\x00\x00\x01\x7f\x00\x00\x01' pkt = IPPacket.from_buffer(data) - assert pkt.flags == (IPPacket.Flags.MF | IPPacket.Flags.DF) & ~IPPacket.Flags.RF + assert pkt.flags == (IPPacket.Flags.MF | + IPPacket.Flags.DF) & ~IPPacket.Flags.RF assert pkt.is_flag('MF') == True assert pkt.is_flag('DF') == True assert pkt.is_flag('RF') == False @@ -21,21 +23,22 @@ def test_mf_flags(self): assert pkt.is_flag('RF') == False def test_df_flags(self): - data= (b'\x00\x23\x20\xd4\x2a\x8c\x00\x23\x20\xd4\x2a\x8c\x08\x00\x45\x00\x00\x54\x00\x00\x40\x00' + data = (b'\x00\x23\x20\xd4\x2a\x8c\x00\x23\x20\xd4\x2a\x8c\x08\x00\x45\x00\x00\x54\x00\x00\x40\x00' b'\x40\x01\x25\x8d\x0a\x00\x00\x8f\x0a\x00\x00\x8e\x08\x00\x2e\xa0\x01\xff\x23\x73\x20\x48' b'\x4a\x4d\x00\x00\x00\x00\x78\x85\x02\x00\x00\x00\x00\x00\x10\x11\x12\x13\x14\x15\x16\x17' b'\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d' b'\x2e\x2f\x30\x31\x32\x33\x34\x35\x36\x37') - + pkt = EthernetFrame.from_buffer(data).get_packet() assert pkt.flags == IPPacket.Flags.DF - assert pkt.flags == (IPPacket.Flags.DF & (~IPPacket.Flags.MF & ~IPPacket.Flags.RF)) + assert pkt.flags == (IPPacket.Flags.DF & ( + ~IPPacket.Flags.MF & ~IPPacket.Flags.RF)) assert pkt.is_flag('DF') == True assert pkt.is_flag('MF') == False assert pkt.is_flag('RF') == False - + assert pkt.offset == 0 def test_checksum_calc(self): @@ -52,52 +55,52 @@ def test_checksum_calc(self): assert pkt.checksum == checksum def test_packet_creation(self): - - pkt = IPPacket(src_addr='172.25.44.3',dst_addr='172.25.44.254',segment=TCPSegment(dst_port=3,flags=['ACK'])) + + pkt = IPPacket(src_addr='172.25.44.3', dst_addr='172.25.44.254', + segment=TCPSegment(dst_port=3, flags=['ACK'])) assert pkt.src_addr == '172.25.44.3' assert pkt.dst_addr == '172.25.44.254' - #data = pkt.build().to_buffer() - + # data = pkt.build().to_buffer() + + class TestTCP: def test_tcp_offset(self): data = b'\x01\xbb\xc0\xd7\xb6\x56\xa8\xb9\xd1\xac\xaa\xb1\x50\x18\x40\x00\x56\xf8\x00\x00' segment = TCPSegment.from_buffer(data) assert segment.offset == 5 - def test_prepare_segment_with_flags(self): - segment = TCPSegment(flags=['FIN','SYN','ACK']) + segment = TCPSegment(flags=['FIN', 'SYN', 'ACK']) assert segment.is_flag('FIN') == True assert segment.is_flag('SYN') == True assert segment.is_flag('ACK') == True assert segment.is_flag('PSH') == False - - segment = TCPSegment(flags=['SYN','PSH']) + + segment = TCPSegment(flags=['SYN', 'PSH']) assert segment.is_flag('FIN') == False assert segment.is_flag('SYN') == True assert segment.is_flag('ACK') == False assert segment.is_flag('PSH') == True - segment = TCPSegment().set_flags(['FIN','SYN','ACK']) + segment = TCPSegment().set_flags(['FIN', 'SYN', 'ACK']) assert segment.is_flag('FIN') == True assert segment.is_flag('SYN') == True assert segment.is_flag('ACK') == True assert segment.is_flag('PSH') == False - segment = TCPSegment().set_flags(['SYN','PSH']) + segment = TCPSegment().set_flags(['SYN', 'PSH']) assert segment.is_flag('FIN') == False assert segment.is_flag('SYN') == True assert segment.is_flag('ACK') == False assert segment.is_flag('PSH') == True - def test_basic_details(self): data = b'E\x00\x00(\x00\x01@\x00@\x06,\xbe\x01\x02\x03\x04\x04\x05\x06\x07\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x81m\x00\x00' pkt = IPPacket.from_buffer(data) @@ -106,7 +109,6 @@ def test_basic_details(self): assert pkt.segment.is_flag('SYN') == True assert pkt.segment.is_flag('PSH') == False - def test_checksum(self): data = b'E\x00\x00(\x00\x01@\x00@\x06,\xbe\x01\x02\x03\x04\x04\x05\x06\x07\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x81m\x00\x00' pkt = IPPacket.from_buffer(data) @@ -131,4 +133,18 @@ def test_unpack(self): assert pkt.win == 5840 assert pkt.dst_port == 3372 assert pkt.seq == 290218379 - assert pkt.ack == 951057940 \ No newline at end of file + assert pkt.ack == 951057940 + + def test_parse_only_first_packet(self): + data = (b"\x05\x00\x005\x00\x00\x00\x00\xff\x06\xfb\xc3\x00\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\x00\x00\x00#g\x00\x00{'foo':'bar'}\x05\x00\x006\x00\x00\x00\x00" + b"\xff\x06\xfb\xc2\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05" + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00x\xdc\x00" + b"\x00{'bar':'foo2'}") + + pkt = IPPacket.from_buffer(data) + + assert pkt.len == 53 + assert pkt.get_segment().data == b"{'foo':'bar'}" + diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index 61da6ab8e..69f376960 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -73,14 +73,19 @@ async def test_forward_from_main_to_channel(self, local_socket_connection): async def test_forward_channel_between_a_b(self, local_socket_connection): client_a, client_b = local_socket_connection - data_to_send ="{'foo':'bar'}" - source_channel = CC.CONTROL - - await client_a.get_channel(source_channel).write(data_to_send) - await client_a.get_channel(source_channel).drain() - assert (await client_b.get_channel(source_channel).read(100)).decode() == data_to_send + channel_alpha = CC.CONTROL + channel_beta = CC.IN + + await client_a.get_channel(channel_alpha).write("{'foo':'bar'}") + await client_a._writer.drain() + + await client_a.get_channel(channel_beta).write("{'bar':'foo2'}") + await client_a._writer.drain() + assert (await client_b.get_channel(channel_alpha).read(100)).decode() == "{'foo':'bar'}" + assert (await client_b.get_channel(channel_beta).read(100)).decode() == "{'bar':'foo2'}" + await client_a.stop() await client_b.stop() From 31ee1291e9e95674e400ff5bc3e7665450d9d636 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 20 Feb 2023 11:25:36 +0000 Subject: [PATCH 058/231] Add platform args to STH --- packages/host/src/lib/cpm-connector.ts | 2 +- packages/types/src/sth-configuration.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/host/src/lib/cpm-connector.ts b/packages/host/src/lib/cpm-connector.ts index 1f91ae393..1f19fd8b5 100644 --- a/packages/host/src/lib/cpm-connector.ts +++ b/packages/host/src/lib/cpm-connector.ts @@ -171,8 +171,8 @@ export class CPMConnector extends TypedEmitter { headers: { "x-sth-description": typeof this.config.description !== "undefined" ? this.config.description : "", "x-sth-tags": JSON.stringify(typeof this.config.tags !== "undefined" ? this.config.tags : []), - "x-manager-id": cpmId, "x-sth-id": this.config.id || "", + "x-manager-id": cpmId, ...(orgId ? { "x-org-id": orgId } : {}), ...(sthKey ? { "Authorization": `Digest cnonce="${sthKey}"` } : {}) }, diff --git a/packages/types/src/sth-configuration.ts b/packages/types/src/sth-configuration.ts index 6044c8be5..9de337d82 100644 --- a/packages/types/src/sth-configuration.ts +++ b/packages/types/src/sth-configuration.ts @@ -159,7 +159,7 @@ export type STHConfiguration = { apiVersion: string; api: string; space: string; - hostType: "hub" | "federation" + hostType: "hub" | "federation"; }; /** From 27b891a45e4d9b305ca405634c545945d2f22d67 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Fri, 3 Mar 2023 08:59:55 +0000 Subject: [PATCH 059/231] si space accesskey generate|revoke --- packages/cli/src/lib/commands/space.ts | 6 +++--- packages/types/src/rest-api-manager/common.ts | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/cli/src/lib/commands/space.ts b/packages/cli/src/lib/commands/space.ts index 578116938..23d278ec3 100644 --- a/packages/cli/src/lib/commands/space.ts +++ b/packages/cli/src/lib/commands/space.ts @@ -106,9 +106,9 @@ export const space: CommandDefinition = (program) => { displayObject(await managerClient.disconnectHubs(opts), profileManager.getProfileConfig().format); }); - const accessKeyCmd = spaceCmd - .command("access") - .description("Manages Access Keys for active Space"); + const accessKeyCmd = spaceCmd + .command("access") + .description("Manages Access Keys for active Space"); accessKeyCmd.command("create") .argument("", "Key description") diff --git a/packages/types/src/rest-api-manager/common.ts b/packages/types/src/rest-api-manager/common.ts index 6f170174d..bf3f4daa6 100644 --- a/packages/types/src/rest-api-manager/common.ts +++ b/packages/types/src/rest-api-manager/common.ts @@ -5,13 +5,13 @@ export type ConnectedSTHInfoDetails = { } export type ConnectedSTHInfo = { - id: string, - info: ConnectedSTHInfoDetails, - healthy: boolean, - selfHosted: boolean, - isConnectionActive: boolean, - description?: string, - tags?: Array , + id: string; + info: ConnectedSTHInfoDetails; + healthy: boolean; + selfHosted: boolean; + isConnectionActive: boolean; + description?: string; + tags?: Array; disconnectReason?: string; }; From c3208a80fc792c4a7dbf566c9b67cac67b145428 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 20 Apr 2023 22:01:45 +0000 Subject: [PATCH 060/231] Disable nagle R-H --- packages/runner/src/host-client.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/runner/src/host-client.ts b/packages/runner/src/host-client.ts index 5635b6dd0..4ecdbf338 100644 --- a/packages/runner/src/host-client.ts +++ b/packages/runner/src/host-client.ts @@ -45,6 +45,7 @@ class HostClient implements IHostClient { .map(() => { // Error handling for each connection is process crash for now const connection = net.createConnection(this.instancesServerPort, this.instancesServerHost); + connection.setNoDelay(true); connection.setNoDelay(true); From 830bc27c2882c2f9f73de32bed174bb9fb2b368f Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 10 Jan 2023 10:08:50 +0000 Subject: [PATCH 061/231] Start for bpmux replacement --- packages/verser/package.json | 2 +- packages/verser/src/index.ts | 1 + .../src/lib/tc-verser/codecs/frame-decoder.ts | 53 +++++++++++ .../src/lib/tc-verser/codecs/frame-encoder.ts | 68 +++++++++++++ .../verser/src/lib/tc-verser/codecs/index.ts | 2 + .../verser/src/lib/tc-verser/playground.ts | 95 +++++++++++++++++++ .../verser/src/lib/tc-verser/utils/index.ts | 10 ++ yarn.lock | 5 + 8 files changed, 235 insertions(+), 1 deletion(-) create mode 100644 packages/verser/src/lib/tc-verser/codecs/frame-decoder.ts create mode 100644 packages/verser/src/lib/tc-verser/codecs/frame-encoder.ts create mode 100644 packages/verser/src/lib/tc-verser/codecs/index.ts create mode 100644 packages/verser/src/lib/tc-verser/playground.ts create mode 100644 packages/verser/src/lib/tc-verser/utils/index.ts diff --git a/packages/verser/package.json b/packages/verser/package.json index 5f7875cf8..203fff311 100644 --- a/packages/verser/package.json +++ b/packages/verser/package.json @@ -26,7 +26,7 @@ "ts-node": "^10.9.1", "typedoc": "^0.23.17", "typedoc-plugin-markdown": "^3.13.6", - "typescript": "~4.7.4" + "typescript": "~4.9.4" }, "ava": { "extensions": [ diff --git a/packages/verser/src/index.ts b/packages/verser/src/index.ts index 19bca6084..b962fa556 100644 --- a/packages/verser/src/index.ts +++ b/packages/verser/src/index.ts @@ -1,3 +1,4 @@ export * from "./lib/verser"; export * from "./lib/verser-client"; export * from "./lib/verser-connection"; +export * from "./lib/tc-verser/playground"; diff --git a/packages/verser/src/lib/tc-verser/codecs/frame-decoder.ts b/packages/verser/src/lib/tc-verser/codecs/frame-decoder.ts new file mode 100644 index 000000000..2795da985 --- /dev/null +++ b/packages/verser/src/lib/tc-verser/codecs/frame-decoder.ts @@ -0,0 +1,53 @@ +import { ObjLogger } from "@scramjet/obj-logger"; +import { Duplex, Transform, TransformCallback, TransformOptions } from "stream"; +import { HEADER_LENGTH, toHex } from "../utils"; + +export class FrameDecoder extends Transform { + buff: Buffer; + size = 0; + logger: ObjLogger; + + _streams = new Map(); + + constructor(opts?: TransformOptions, params: { name: string } = { name: "FrameDecoder"}) { + super(opts); + this.buff = Buffer.alloc(64 * 1024, 0, undefined); //@TODO: optimize + this.logger = new ObjLogger(params.name); + + this.on("pipe", () => { + this.logger.debug("onPipe"); + }); + } + + _transform(chunk: Buffer, encoding: BufferEncoding, callback: TransformCallback): void { + this.logger.trace("Decoding frame...", toHex(chunk), "Size:", chunk.length); + + if (!Buffer.isBuffer(chunk)) { + this.push(JSON.stringify({ error: "not a buffer"}), undefined); + callback(); + + return; + } + + chunk.copy(this.buff, this.size, 0, chunk.length); + + this.size += chunk.length; + + if (this.size >= 10 && this.buff.readInt32LE(10) === this.size) { + this.push(JSON.stringify({ + sourceAddress: `${this.buff.readInt8(0).toString()}.${this.buff.readInt8(1).toString()}.${this.buff.readInt8(2).toString()}.${this.buff.readInt8(3).toString()}`, + destinationAddress: `${this.buff.readInt8(4).toString()}.${this.buff.readInt8(5).toString()}.${this.buff.readInt8(6).toString()}.${this.buff.readInt8(7).toString()}`, + chunk: this.buff.subarray(32, this.buff.readInt32LE(10) - HEADER_LENGTH + 32), + dataLength: this.buff.readInt32LE(10) - HEADER_LENGTH, + stringified: this.buff.subarray(32, this.buff.readInt32LE(10) - HEADER_LENGTH + 32).toString() + }, null, 0) + "\n"); + + this.size = 0; + this.buff.fill(0); + } else { + this.logger.error("too few data", this.size, this.buff.readInt32LE(10)); + } + + callback(); + } +} diff --git a/packages/verser/src/lib/tc-verser/codecs/frame-encoder.ts b/packages/verser/src/lib/tc-verser/codecs/frame-encoder.ts new file mode 100644 index 000000000..a353e0d4b --- /dev/null +++ b/packages/verser/src/lib/tc-verser/codecs/frame-encoder.ts @@ -0,0 +1,68 @@ +import { Transform, TransformCallback, TransformOptions } from "stream"; +import { FrameTarget, HEADER_LENGTH, toHex as getHexString } from "../utils"; +import { ObjLogger } from "@scramjet/obj-logger"; + +export class FrameEncoder extends Transform { + sequenceNumber = 0; + logger = new ObjLogger("FrameEncoder",); + + constructor(private frameTarget: FrameTarget, opts?: TransformOptions, params: { name: string } = { name: "FrameEncoder"}) { + super(opts); + this.logger = new ObjLogger(params.name, { id: this.frameTarget.toString() }); + + this.on("pipe", () => { + this.logger.debug("onPipe"); + }); + } + + createChunkFrame(chunk: any) { + const checksum = 255; + + const buffer = Buffer.concat([ + // 0: source address 0 - 3 + new Uint8Array([10, 0, 0, 1]), + // 32: destination address 4 - 7 + new Uint8Array([10, 0, 0, 2]), + + // 64: zeroes (8bit), protocol (8bit), 8 - 9 + new Uint8Array([0, 1]), + // tcp length (16bit) 10 - 11 + new Uint8Array(new Uint16Array([chunk.length + HEADER_LENGTH]).buffer), + + // 96: Source port, destination port 12 - 15 + new Uint8Array(new Uint16Array([0, this.frameTarget]).buffer), + + // 128: sequenceNumber(32 bit, acnowledge number) 16 - 19 + new Uint8Array(new Uint32Array([this.sequenceNumber++]).buffer), + + // 160: Acknowledgement number 20-23 + new Uint8Array([0,0,0,0]), + + // 192: data offset (4bit), reserved (4bit), 24 + new Uint8Array([0b00000000]), + // 224: flags (8bit), 25 + new Uint8Array([0b00000000]), + // window(16bit) 26 - 27 + new Uint8Array(new Uint16Array([0]).buffer), + + // checksum(16bit) 28 - 29 + new Uint8Array(new Uint16Array([checksum]).buffer), + // pointer (16bit) 30 - 31 + new Uint8Array(new Uint16Array([checksum]).buffer), + + // 256: data 32 - + new Uint8Array(chunk) + ]); + + return buffer; + } + + _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void { + const buffer = this.createChunkFrame(chunk); + + this.logger.trace("Encoded frame", getHexString(buffer), "Size: ", buffer.length, "Pushing"); + + this.push(buffer, undefined); + callback(null); + }; +} diff --git a/packages/verser/src/lib/tc-verser/codecs/index.ts b/packages/verser/src/lib/tc-verser/codecs/index.ts new file mode 100644 index 000000000..2a0fcd378 --- /dev/null +++ b/packages/verser/src/lib/tc-verser/codecs/index.ts @@ -0,0 +1,2 @@ +export * from "./frame-decoder"; +export * from "./frame-encoder"; diff --git a/packages/verser/src/lib/tc-verser/playground.ts b/packages/verser/src/lib/tc-verser/playground.ts new file mode 100644 index 000000000..8ba3ea03a --- /dev/null +++ b/packages/verser/src/lib/tc-verser/playground.ts @@ -0,0 +1,95 @@ +/*eslint no-unused-vars: ["error", { "args": "none" }]*/ + +import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; +import { IncomingMessage, ServerResponse, createServer, request } from "http"; +import { DataStream } from "scramjet"; +import { FrameTarget } from "./utils"; +import { FrameDecoder, FrameEncoder } from "./codecs"; + +(async () => { + const logger = new ObjLogger("Sandbox"); + + logger.pipe(new DataStream().map(prettyPrint({ colors: true }))).pipe(process.stdout); + + const PORT = 6660; + const server = createServer(); + + server.setTimeout(0); + server.requestTimeout = 0; + + server.on("request", (req: IncomingMessage, res: ServerResponse) => { + res.socket!.setNoDelay(true); + + res.writeHead(200, { "Content-Type": "application/octet-stream", "Transfer-Encoding" : "chunked" }); + res.flushHeaders(); + + logger.debug("on request"); + + const serverFrameEncoder = new FrameEncoder(FrameTarget.API, { encoding: undefined }, { name: "ServerEncoder"}); + serverFrameEncoder.logger.pipe(logger); + + serverFrameEncoder.pipe(res); + let i = 0; + + req.on("pause", () => { logger.warn("Request paused") }); + + const serverFrameDecoder = new FrameDecoder({ highWaterMark: 60 * 1024, encoding: undefined, emitClose: false }, { name: "ServerDecoder"}); + serverFrameDecoder.logger.pipe(logger); + + req.pipe(serverFrameDecoder).pipe(process.stdout); + + serverFrameDecoder.on("data", (d: any) => { + logger.debug(`Server on request data [${i++}]`, JSON.parse(d).chunk, d.length); + serverFrameEncoder.write(Buffer.from(new Uint8Array([255, 255, 255, 255]))); + }) + }); + + server.listen(PORT); + + const req = request({ + hostname: "0.0.0.0", + port: PORT, + headers: { "Content-Type": "application/octet-stream", "Transfer-Encoding": "chunked" } + }); + + + req.on("response", (response) => { + req.socket!.setNoDelay(true); + + logger.debug("Response"); + + let i = 0; + let m = 0; + + const frameEncoder = new FrameEncoder(FrameTarget.API, { encoding: undefined, emitClose: false }, { name: "RequestEncoder"}); + frameEncoder.logger.pipe(logger); + + frameEncoder.pipe(req); + + const responseFrameDecoder = new FrameDecoder({ highWaterMark: 60 * 1024, encoding: undefined, emitClose: false }, { name: "RequestDecoder"}); + + responseFrameDecoder.logger.pipe(logger); + + response.pipe( + responseFrameDecoder + ).on("data", (d) => { + logger.debug(`Echo from server [${i++}]`, JSON.parse(d).chunk, d.length); + }); + + response.on("data", (d) => console.log("plain response", d)); + + req.on("error", (err) => { + console.error(err); + }); + + setInterval(async () => { + logger.debug(`Writing [${++m}]..`); + + frameEncoder.write(Buffer.from(new Uint8Array([0x61, 0x62, 0x63, 0x64]))); + }, 500); + }); + + req.flushHeaders(); + + await new Promise((_res, _rej) => {}); +})(); diff --git a/packages/verser/src/lib/tc-verser/utils/index.ts b/packages/verser/src/lib/tc-verser/utils/index.ts new file mode 100644 index 000000000..6dacf2285 --- /dev/null +++ b/packages/verser/src/lib/tc-verser/utils/index.ts @@ -0,0 +1,10 @@ +export function toHex(chunk: Buffer) { + return chunk.toString('hex').match(/../g)?.join(' '); +} + +export const HEADER_LENGTH = 32; + +export enum FrameTarget { + API, + INPUT = 1001 +} diff --git a/yarn.lock b/yarn.lock index eb667583a..6c8a47843 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7366,6 +7366,11 @@ typescript@~4.7.4: resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.7.4.tgz#1a88596d1cf47d59507a1bcdfb5b9dfe4d488235" integrity sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ== +typescript@~4.9.4: + version "4.9.4" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.4.tgz#a2a3d2756c079abda241d75f149df9d561091e78" + integrity sha512-Uz+dTXYzxXXbsFpM86Wh3dKCxrQqUcVMxwU54orwlJjOpO3ao8L7j5lH+dWfTwgCwIuM9GQ2kvVotzYJMXTBZg== + uglify-js@^3.1.4: version "3.17.4" resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.17.4.tgz#61678cf5fa3f5b7eb789bb345df29afb8257c22c" From 235c22a9bdfaf364c598efdd7d9b5f30ddebf2d1 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 10 Jan 2023 12:14:59 +0000 Subject: [PATCH 062/231] tc-verser Switch to connect method --- packages/host/src/lib/cpm-connector.ts | 2 +- .../src/lib/tc-verser/codecs/frame-decoder.ts | 4 +- .../src/lib/tc-verser/codecs/frame-encoder.ts | 8 ++-- .../verser/src/lib/tc-verser/playground.ts | 44 ++++++++++--------- .../verser/src/lib/tc-verser/utils/index.ts | 2 +- packages/verser/src/lib/verser-client.ts | 5 +-- packages/verser/src/types/index.ts | 2 +- 7 files changed, 35 insertions(+), 32 deletions(-) diff --git a/packages/host/src/lib/cpm-connector.ts b/packages/host/src/lib/cpm-connector.ts index 1f19fd8b5..5c0b3a889 100644 --- a/packages/host/src/lib/cpm-connector.ts +++ b/packages/host/src/lib/cpm-connector.ts @@ -366,7 +366,7 @@ export class CPMConnector extends TypedEmitter { this.connected = true; this.connectionAttempts = 0; - connection.res.once("error", async (error: any) => { + connection.response.once("error", async (error: any) => { this.logger.error("Request error", error); try { diff --git a/packages/verser/src/lib/tc-verser/codecs/frame-decoder.ts b/packages/verser/src/lib/tc-verser/codecs/frame-decoder.ts index 2795da985..f1fcb1c45 100644 --- a/packages/verser/src/lib/tc-verser/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tc-verser/codecs/frame-decoder.ts @@ -9,7 +9,7 @@ export class FrameDecoder extends Transform { _streams = new Map(); - constructor(opts?: TransformOptions, params: { name: string } = { name: "FrameDecoder"}) { + constructor(opts?: TransformOptions, params: { name: string } = { name: "FrameDecoder" }) { super(opts); this.buff = Buffer.alloc(64 * 1024, 0, undefined); //@TODO: optimize this.logger = new ObjLogger(params.name); @@ -23,7 +23,7 @@ export class FrameDecoder extends Transform { this.logger.trace("Decoding frame...", toHex(chunk), "Size:", chunk.length); if (!Buffer.isBuffer(chunk)) { - this.push(JSON.stringify({ error: "not a buffer"}), undefined); + this.push(JSON.stringify({ error: "not a buffer" }), undefined); callback(); return; diff --git a/packages/verser/src/lib/tc-verser/codecs/frame-encoder.ts b/packages/verser/src/lib/tc-verser/codecs/frame-encoder.ts index a353e0d4b..b1fe54f36 100644 --- a/packages/verser/src/lib/tc-verser/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tc-verser/codecs/frame-encoder.ts @@ -6,9 +6,9 @@ export class FrameEncoder extends Transform { sequenceNumber = 0; logger = new ObjLogger("FrameEncoder",); - constructor(private frameTarget: FrameTarget, opts?: TransformOptions, params: { name: string } = { name: "FrameEncoder"}) { + constructor(private frameTarget: FrameTarget, opts?: TransformOptions, params: { name: string } = { name: "FrameEncoder" }) { super(opts); - this.logger = new ObjLogger(params.name, { id: this.frameTarget.toString() }); + this.logger = new ObjLogger(params.name, { id: this.frameTarget.toString() }); this.on("pipe", () => { this.logger.debug("onPipe"); @@ -36,7 +36,7 @@ export class FrameEncoder extends Transform { new Uint8Array(new Uint32Array([this.sequenceNumber++]).buffer), // 160: Acknowledgement number 20-23 - new Uint8Array([0,0,0,0]), + new Uint8Array([0, 0, 0, 0]), // 192: data offset (4bit), reserved (4bit), 24 new Uint8Array([0b00000000]), @@ -64,5 +64,5 @@ export class FrameEncoder extends Transform { this.push(buffer, undefined); callback(null); - }; + } } diff --git a/packages/verser/src/lib/tc-verser/playground.ts b/packages/verser/src/lib/tc-verser/playground.ts index 8ba3ea03a..7c11e815c 100644 --- a/packages/verser/src/lib/tc-verser/playground.ts +++ b/packages/verser/src/lib/tc-verser/playground.ts @@ -1,10 +1,13 @@ /*eslint no-unused-vars: ["error", { "args": "none" }]*/ +/* eslint-disable @typescript-eslint/no-floating-promises */ +/* eslint-disable no-console */ import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; -import { IncomingMessage, ServerResponse, createServer, request } from "http"; +import { IncomingMessage, createServer, request } from "http"; import { DataStream } from "scramjet"; import { FrameTarget } from "./utils"; import { FrameDecoder, FrameEncoder } from "./codecs"; +import { Socket } from "net"; (async () => { const logger = new ObjLogger("Sandbox"); @@ -17,60 +20,61 @@ import { FrameDecoder, FrameEncoder } from "./codecs"; server.setTimeout(0); server.requestTimeout = 0; - server.on("request", (req: IncomingMessage, res: ServerResponse) => { - res.socket!.setNoDelay(true); + server.on("connect", (req: IncomingMessage, socket: Socket) => { + socket.setNoDelay(true); + socket.write(`HTTP/1.1 ${200} \r\n\r\n`); - res.writeHead(200, { "Content-Type": "application/octet-stream", "Transfer-Encoding" : "chunked" }); - res.flushHeaders(); + logger.debug("on connect"); - logger.debug("on request"); + const serverFrameEncoder = new FrameEncoder(FrameTarget.API, { encoding: undefined }, { name: "ServerEncoder" }); - const serverFrameEncoder = new FrameEncoder(FrameTarget.API, { encoding: undefined }, { name: "ServerEncoder"}); serverFrameEncoder.logger.pipe(logger); - serverFrameEncoder.pipe(res); + serverFrameEncoder.pipe(socket); let i = 0; - req.on("pause", () => { logger.warn("Request paused") }); + req.on("pause", () => { logger.warn("Request paused"); }); + + const serverFrameDecoder = new FrameDecoder({ highWaterMark: 60 * 1024, encoding: undefined, emitClose: false }, { name: "ServerDecoder" }); - const serverFrameDecoder = new FrameDecoder({ highWaterMark: 60 * 1024, encoding: undefined, emitClose: false }, { name: "ServerDecoder"}); serverFrameDecoder.logger.pipe(logger); - req.pipe(serverFrameDecoder).pipe(process.stdout); + socket.pipe(serverFrameDecoder).pipe(process.stdout); serverFrameDecoder.on("data", (d: any) => { logger.debug(`Server on request data [${i++}]`, JSON.parse(d).chunk, d.length); serverFrameEncoder.write(Buffer.from(new Uint8Array([255, 255, 255, 255]))); - }) + }); }); server.listen(PORT); const req = request({ hostname: "0.0.0.0", + method: "connect", port: PORT, headers: { "Content-Type": "application/octet-stream", "Transfer-Encoding": "chunked" } }); + req.on("connect", (response, socket, head) => { + socket.setNoDelay(true); - req.on("response", (response) => { - req.socket!.setNoDelay(true); - - logger.debug("Response"); + logger.debug("Response. Head:", head.toString()); let i = 0; let m = 0; - const frameEncoder = new FrameEncoder(FrameTarget.API, { encoding: undefined, emitClose: false }, { name: "RequestEncoder"}); + const frameEncoder = new FrameEncoder(FrameTarget.API, { encoding: undefined, emitClose: false }, { name: "RequestEncoder" }); + frameEncoder.logger.pipe(logger); - frameEncoder.pipe(req); + frameEncoder.pipe(socket); - const responseFrameDecoder = new FrameDecoder({ highWaterMark: 60 * 1024, encoding: undefined, emitClose: false }, { name: "RequestDecoder"}); + const responseFrameDecoder = new FrameDecoder({ highWaterMark: 60 * 1024, encoding: undefined, emitClose: false }, { name: "RequestDecoder" }); responseFrameDecoder.logger.pipe(logger); - response.pipe( + socket.pipe( responseFrameDecoder ).on("data", (d) => { logger.debug(`Echo from server [${i++}]`, JSON.parse(d).chunk, d.length); diff --git a/packages/verser/src/lib/tc-verser/utils/index.ts b/packages/verser/src/lib/tc-verser/utils/index.ts index 6dacf2285..229a29b8d 100644 --- a/packages/verser/src/lib/tc-verser/utils/index.ts +++ b/packages/verser/src/lib/tc-verser/utils/index.ts @@ -1,5 +1,5 @@ export function toHex(chunk: Buffer) { - return chunk.toString('hex').match(/../g)?.join(' '); + return chunk.toString("hex").match(/../g)?.join(" "); } export const HEADER_LENGTH = 32; diff --git a/packages/verser/src/lib/verser-client.ts b/packages/verser/src/lib/verser-client.ts index 568007edd..1e0724337 100644 --- a/packages/verser/src/lib/verser-client.ts +++ b/packages/verser/src/lib/verser-client.ts @@ -113,12 +113,11 @@ export class VerserClient extends TypedEmitter { reject(err); }); - connectRequest.once("connect", (res, socket, head) => { - this.logger.info("HEAD", head.toString()); + connectRequest.on("connect", (response, socket) => { this.socket = socket; this.mux(); - resolve({ res, socket }); + resolve({ response, socket }); }); connectRequest.flushHeaders(); diff --git a/packages/verser/src/types/index.ts b/packages/verser/src/types/index.ts index d6d8c195f..9c9ecb1c9 100644 --- a/packages/verser/src/types/index.ts +++ b/packages/verser/src/types/index.ts @@ -40,7 +40,7 @@ export type VerserClientConnection = { /** * Connection request object. */ - res: IncomingMessage; + response: IncomingMessage; }; export type VerserRequestResult = { incomingMessage: IncomingMessage; clientRequest: ClientRequest } From cd8b5996b0effbc0be79e31102ef55af5b346c50 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 10 Jan 2023 17:27:04 +0000 Subject: [PATCH 063/231] Rename to tecemux --- packages/verser/src/index.ts | 1 - .../src/lib/{tc-verser => tecemux}/codecs/frame-decoder.ts | 0 .../src/lib/{tc-verser => tecemux}/codecs/frame-encoder.ts | 0 packages/verser/src/lib/{tc-verser => tecemux}/codecs/index.ts | 0 packages/verser/src/lib/{tc-verser => tecemux}/playground.ts | 0 packages/verser/src/lib/{tc-verser => tecemux}/utils/index.ts | 0 6 files changed, 1 deletion(-) rename packages/verser/src/lib/{tc-verser => tecemux}/codecs/frame-decoder.ts (100%) rename packages/verser/src/lib/{tc-verser => tecemux}/codecs/frame-encoder.ts (100%) rename packages/verser/src/lib/{tc-verser => tecemux}/codecs/index.ts (100%) rename packages/verser/src/lib/{tc-verser => tecemux}/playground.ts (100%) rename packages/verser/src/lib/{tc-verser => tecemux}/utils/index.ts (100%) diff --git a/packages/verser/src/index.ts b/packages/verser/src/index.ts index b962fa556..19bca6084 100644 --- a/packages/verser/src/index.ts +++ b/packages/verser/src/index.ts @@ -1,4 +1,3 @@ export * from "./lib/verser"; export * from "./lib/verser-client"; export * from "./lib/verser-connection"; -export * from "./lib/tc-verser/playground"; diff --git a/packages/verser/src/lib/tc-verser/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts similarity index 100% rename from packages/verser/src/lib/tc-verser/codecs/frame-decoder.ts rename to packages/verser/src/lib/tecemux/codecs/frame-decoder.ts diff --git a/packages/verser/src/lib/tc-verser/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts similarity index 100% rename from packages/verser/src/lib/tc-verser/codecs/frame-encoder.ts rename to packages/verser/src/lib/tecemux/codecs/frame-encoder.ts diff --git a/packages/verser/src/lib/tc-verser/codecs/index.ts b/packages/verser/src/lib/tecemux/codecs/index.ts similarity index 100% rename from packages/verser/src/lib/tc-verser/codecs/index.ts rename to packages/verser/src/lib/tecemux/codecs/index.ts diff --git a/packages/verser/src/lib/tc-verser/playground.ts b/packages/verser/src/lib/tecemux/playground.ts similarity index 100% rename from packages/verser/src/lib/tc-verser/playground.ts rename to packages/verser/src/lib/tecemux/playground.ts diff --git a/packages/verser/src/lib/tc-verser/utils/index.ts b/packages/verser/src/lib/tecemux/utils/index.ts similarity index 100% rename from packages/verser/src/lib/tc-verser/utils/index.ts rename to packages/verser/src/lib/tecemux/utils/index.ts From f622e035f9d8caa95359dfaf96506c89f71be0ac Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 11 Jan 2023 11:10:31 +0000 Subject: [PATCH 064/231] DecodedFrame type --- .../src/lib/tecemux/codecs/frame-decoder.ts | 26 ++++++---- .../verser/src/lib/tecemux/codecs/index.ts | 48 +++++++++++++++++++ packages/verser/src/lib/tecemux/playground.ts | 10 ++-- .../verser/src/lib/tecemux/utils/index.ts | 13 +++++ 4 files changed, 85 insertions(+), 12 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index f1fcb1c45..2af72925d 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -9,8 +9,9 @@ export class FrameDecoder extends Transform { _streams = new Map(); - constructor(opts?: TransformOptions, params: { name: string } = { name: "FrameDecoder" }) { - super(opts); + constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { + super(Object.assign({}, { readableObjectMode: true }, opts)); + this.buff = Buffer.alloc(64 * 1024, 0, undefined); //@TODO: optimize this.logger = new ObjLogger(params.name); @@ -34,13 +35,20 @@ export class FrameDecoder extends Transform { this.size += chunk.length; if (this.size >= 10 && this.buff.readInt32LE(10) === this.size) { - this.push(JSON.stringify({ - sourceAddress: `${this.buff.readInt8(0).toString()}.${this.buff.readInt8(1).toString()}.${this.buff.readInt8(2).toString()}.${this.buff.readInt8(3).toString()}`, - destinationAddress: `${this.buff.readInt8(4).toString()}.${this.buff.readInt8(5).toString()}.${this.buff.readInt8(6).toString()}.${this.buff.readInt8(7).toString()}`, - chunk: this.buff.subarray(32, this.buff.readInt32LE(10) - HEADER_LENGTH + 32), - dataLength: this.buff.readInt32LE(10) - HEADER_LENGTH, - stringified: this.buff.subarray(32, this.buff.readInt32LE(10) - HEADER_LENGTH + 32).toString() - }, null, 0) + "\n"); + const frameSize = this.buff.readInt32LE(10); + + const payload = { + sourceAddress: [this.buff.readInt8(0), this.buff.readInt8(1), this.buff.readInt8(2), this.buff.readInt8(3)], + destinationAddress: [this.buff.readInt8(4), this.buff.readInt8(5), this.buff.readInt8(6), this.buff.readInt8(7)], + destinationPort: this.buff.readInt8(9), + chunk: this.buff.subarray(32, this.buff.readInt32LE(10)), + dataLength: frameSize - HEADER_LENGTH, + chunkLength: frameSize, + stringified: this.buff.subarray(32, frameSize).toString() + }; + + this.push(JSON.stringify(payload) + "\n"); + //this.push(payload); this.size = 0; this.buff.fill(0); diff --git a/packages/verser/src/lib/tecemux/codecs/index.ts b/packages/verser/src/lib/tecemux/codecs/index.ts index 2a0fcd378..2e0d50d40 100644 --- a/packages/verser/src/lib/tecemux/codecs/index.ts +++ b/packages/verser/src/lib/tecemux/codecs/index.ts @@ -1,2 +1,50 @@ +import { Socket } from "net"; +import { Duplex } from "stream"; +import { FrameEncoder } from "./frame-encoder"; +import { FrameTarget } from "../utils"; +import { TypedEmitter } from "@scramjet/utility"; +import { FrameDecoder } from "./frame-decoder"; + export * from "./frame-decoder"; export * from "./frame-encoder"; + +type TeceMuxEvents = { + channel(socket: Duplex): void; +} + +export class TeceMux extends TypedEmitter{ + socket: Duplex; + socketCount = 0; + decoder = new FrameDecoder(); + + channels: Duplex[] = []; + + constructor(socket: Socket) { + super(); + this.socket = socket; + + this.decoder.on("data", (chunk) => { + const frame = JSON.parse(chunk); + + const channel = frame.destinationPort; + + if (!this.channels[channel]) { + this.channels[channel] = new Duplex({ }); + this.channels[channel].pipe(new FrameEncoder(channel).pipe(this.socket)); // ? lot of encoders / encoder.writeToChannel ?; + this.emit("channel", this.channels[channel]); + } + + this.channels[channel].write(frame.chunk.data); + }); + + socket.pipe(this.decoder); + } + + multiplex() { + const multiplex = new Socket(); + + multiplex.pipe(new FrameEncoder(FrameTarget.API)).pipe(this.socket); + + return multiplex; + } +} diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/src/lib/tecemux/playground.ts index 7c11e815c..41d7894cc 100644 --- a/packages/verser/src/lib/tecemux/playground.ts +++ b/packages/verser/src/lib/tecemux/playground.ts @@ -5,7 +5,7 @@ import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; import { IncomingMessage, createServer, request } from "http"; import { DataStream } from "scramjet"; -import { FrameTarget } from "./utils"; +import { DecodedFrame, FrameTarget } from "./utils"; import { FrameDecoder, FrameEncoder } from "./codecs"; import { Socket } from "net"; @@ -42,7 +42,9 @@ import { Socket } from "net"; socket.pipe(serverFrameDecoder).pipe(process.stdout); serverFrameDecoder.on("data", (d: any) => { - logger.debug(`Server on request data [${i++}]`, JSON.parse(d).chunk, d.length); + const parsed = JSON.parse(d) as DecodedFrame; + + logger.debug(`Server on request data [${i++}]`, parsed.chunk, parsed.dataLength); serverFrameEncoder.write(Buffer.from(new Uint8Array([255, 255, 255, 255]))); }); }); @@ -77,7 +79,9 @@ import { Socket } from "net"; socket.pipe( responseFrameDecoder ).on("data", (d) => { - logger.debug(`Echo from server [${i++}]`, JSON.parse(d).chunk, d.length); + const parsed = JSON.parse(d) as DecodedFrame; + + logger.debug(`Echo from server [${i++}]`, parsed.chunk, parsed.chunkLength); }); response.on("data", (d) => console.log("plain response", d)); diff --git a/packages/verser/src/lib/tecemux/utils/index.ts b/packages/verser/src/lib/tecemux/utils/index.ts index 229a29b8d..631dba5d0 100644 --- a/packages/verser/src/lib/tecemux/utils/index.ts +++ b/packages/verser/src/lib/tecemux/utils/index.ts @@ -8,3 +8,16 @@ export enum FrameTarget { API, INPUT = 1001 } + +export type DecodedFrame = { + sourceAddress: [number, number, number, number]; + destinationAddress: [number, number, number, number]; + destinationPort: number; + chunk: { + type: string; + data: any; + }; + dataLength: number; + chunkLength: number; + stringified: string; +}; From 534ec8cf7bee4c8228334e50632a80f62efad0f3 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 12 Jan 2023 20:49:34 +0000 Subject: [PATCH 065/231] Flags parsers --- .../src/lib/tecemux/codecs/frame-decoder.ts | 14 +++- .../src/lib/tecemux/codecs/frame-encoder.ts | 64 +++++++++++++++-- .../verser/src/lib/tecemux/codecs/index.ts | 53 ++++---------- packages/verser/src/lib/tecemux/playground.ts | 2 +- packages/verser/src/lib/tecemux/tecemux.ts | 70 +++++++++++++++++++ 5 files changed, 153 insertions(+), 50 deletions(-) create mode 100644 packages/verser/src/lib/tecemux/tecemux.ts diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 2af72925d..8269f0d0a 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -1,6 +1,7 @@ import { ObjLogger } from "@scramjet/obj-logger"; import { Duplex, Transform, TransformCallback, TransformOptions } from "stream"; import { HEADER_LENGTH, toHex } from "../utils"; +import { frameFlags } from "."; export class FrameDecoder extends Transform { buff: Buffer; @@ -9,8 +10,9 @@ export class FrameDecoder extends Transform { _streams = new Map(); + constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { - super(Object.assign({}, { readableObjectMode: true }, opts)); + super(Object.assign({}, { readableObjectMode: true, emitClose: false }, opts)); this.buff = Buffer.alloc(64 * 1024, 0, undefined); //@TODO: optimize this.logger = new ObjLogger(params.name); @@ -20,6 +22,11 @@ export class FrameDecoder extends Transform { }); } + parseFlags(byte: number) { + return frameFlags.filter((_flag, index) => byte >>> index & 1) + .reduce((p, c) => ({ ...p, [c]: true }), {}); + } + _transform(chunk: Buffer, encoding: BufferEncoding, callback: TransformCallback): void { this.logger.trace("Decoding frame...", toHex(chunk), "Size:", chunk.length); @@ -40,15 +47,16 @@ export class FrameDecoder extends Transform { const payload = { sourceAddress: [this.buff.readInt8(0), this.buff.readInt8(1), this.buff.readInt8(2), this.buff.readInt8(3)], destinationAddress: [this.buff.readInt8(4), this.buff.readInt8(5), this.buff.readInt8(6), this.buff.readInt8(7)], - destinationPort: this.buff.readInt8(9), chunk: this.buff.subarray(32, this.buff.readInt32LE(10)), + flags: this.parseFlags(this.buff.readInt8(25)), + sourcePort: this.buff.readInt16LE(12), + destinationPort: this.buff.readInt16LE(14), dataLength: frameSize - HEADER_LENGTH, chunkLength: frameSize, stringified: this.buff.subarray(32, frameSize).toString() }; this.push(JSON.stringify(payload) + "\n"); - //this.push(payload); this.size = 0; this.buff.fill(0); diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index b1fe54f36..b8ec9a96b 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -1,13 +1,16 @@ import { Transform, TransformCallback, TransformOptions } from "stream"; import { FrameTarget, HEADER_LENGTH, toHex as getHexString } from "../utils"; import { ObjLogger } from "@scramjet/obj-logger"; - +import { binaryFlags, frameFlags } from "."; export class FrameEncoder extends Transform { sequenceNumber = 0; logger = new ObjLogger("FrameEncoder",); - constructor(private frameTarget: FrameTarget, opts?: TransformOptions, params: { name: string } = { name: "FrameEncoder" }) { + constructor(private frameTarget: FrameTarget, opts: TransformOptions = {}, params: { name: string } = { name: "FrameEncoder" }) { + opts.emitClose = false; + super(opts); + this.logger = new ObjLogger(params.name, { id: this.frameTarget.toString() }); this.on("pipe", () => { @@ -15,9 +18,15 @@ export class FrameEncoder extends Transform { }); } - createChunkFrame(chunk: any) { - const checksum = 255; + setFlag(flag: keyof typeof binaryFlags, flags: Uint8Array = new Uint8Array([0])) { + this.logger.debug("settingFlag", flag); + flags[0] |= 1 << frameFlags.indexOf(flag); + + return flags; + } + setChannel() { + const checksum = this.getChecksum(); const buffer = Buffer.concat([ // 0: source address 0 - 3 new Uint8Array([10, 0, 0, 1]), @@ -27,7 +36,7 @@ export class FrameEncoder extends Transform { // 64: zeroes (8bit), protocol (8bit), 8 - 9 new Uint8Array([0, 1]), // tcp length (16bit) 10 - 11 - new Uint8Array(new Uint16Array([chunk.length + HEADER_LENGTH]).buffer), + new Uint8Array(new Uint16Array([0 + HEADER_LENGTH]).buffer), // 96: Source port, destination port 12 - 15 new Uint8Array(new Uint16Array([0, this.frameTarget]).buffer), @@ -41,7 +50,52 @@ export class FrameEncoder extends Transform { // 192: data offset (4bit), reserved (4bit), 24 new Uint8Array([0b00000000]), // 224: flags (8bit), 25 + this.setFlag("ECE"), + // window(16bit) 26 - 27 + new Uint8Array(new Uint16Array([0]).buffer), + + // checksum(16bit) 28 - 29 + new Uint8Array(new Uint16Array([checksum]).buffer), + // pointer (16bit) 30 - 31 + new Uint8Array(new Uint16Array([checksum]).buffer), + + // 256: data 32 - + new Uint8Array([]) + ]); + + this.push(buffer, undefined); + } + + getChecksum() { + return 255; + } + + createChunkFrame(chunk: any) { + const checksum = this.getChecksum(); + const buffer = Buffer.concat([ + // 0: source address 0 - 3 + new Uint8Array([10, 0, 0, 1]), + // 32: destination address 4 - 7 + new Uint8Array([10, 0, 0, 2]), + + // 64: zeroes (8bit), protocol (8bit), 8 - 9 + new Uint8Array([0, 1]), + // tcp length (16bit) 10 - 11 + new Uint8Array(new Uint16Array([chunk.length + HEADER_LENGTH]).buffer), + + // 96: Source port, destination port 12 - 15 + new Uint8Array(new Uint16Array([0, this.frameTarget]).buffer), + + // 128: sequenceNumber(32 bit, acnowledge number) 16 - 19 + new Uint8Array(new Uint32Array([this.sequenceNumber++]).buffer), + + // 160: Acknowledgement number 20-23 + new Uint8Array([0, 0, 0, 0]), + + // 192: data offset (4bit), reserved (4bit), 24 new Uint8Array([0b00000000]), + // 224: flags (8bit), 25 + this.setFlag("PSH"), // window(16bit) 26 - 27 new Uint8Array(new Uint16Array([0]).buffer), diff --git a/packages/verser/src/lib/tecemux/codecs/index.ts b/packages/verser/src/lib/tecemux/codecs/index.ts index 2e0d50d40..f2556a8ea 100644 --- a/packages/verser/src/lib/tecemux/codecs/index.ts +++ b/packages/verser/src/lib/tecemux/codecs/index.ts @@ -1,50 +1,21 @@ -import { Socket } from "net"; import { Duplex } from "stream"; -import { FrameEncoder } from "./frame-encoder"; -import { FrameTarget } from "../utils"; -import { TypedEmitter } from "@scramjet/utility"; -import { FrameDecoder } from "./frame-decoder"; export * from "./frame-decoder"; export * from "./frame-encoder"; -type TeceMuxEvents = { +export type TeceMuxEvents = { channel(socket: Duplex): void; + error(error: any): void; } -export class TeceMux extends TypedEmitter{ - socket: Duplex; - socketCount = 0; - decoder = new FrameDecoder(); - - channels: Duplex[] = []; - - constructor(socket: Socket) { - super(); - this.socket = socket; - - this.decoder.on("data", (chunk) => { - const frame = JSON.parse(chunk); - - const channel = frame.destinationPort; - - if (!this.channels[channel]) { - this.channels[channel] = new Duplex({ }); - this.channels[channel].pipe(new FrameEncoder(channel).pipe(this.socket)); // ? lot of encoders / encoder.writeToChannel ?; - this.emit("channel", this.channels[channel]); - } - - this.channels[channel].write(frame.chunk.data); - }); - - socket.pipe(this.decoder); - } - - multiplex() { - const multiplex = new Socket(); - - multiplex.pipe(new FrameEncoder(FrameTarget.API)).pipe(this.socket); - - return multiplex; - } +export const frameFlags = ["FIN", "SYN", "RST", "PSH", "ACK", "URG", "ECE", "CWR"]; +export const binaryFlags = { + FIN: 0b00000001, + SYN: 0b00000010, + RST: 0b00000100, + PSH: 0b00001000, + ACK: 0b00010000, + URG: 0b00100000, + ECE: 0b01000000, + CWR: 0b10000000 } diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/src/lib/tecemux/playground.ts index 41d7894cc..cc7429c8a 100644 --- a/packages/verser/src/lib/tecemux/playground.ts +++ b/packages/verser/src/lib/tecemux/playground.ts @@ -81,7 +81,7 @@ import { Socket } from "net"; ).on("data", (d) => { const parsed = JSON.parse(d) as DecodedFrame; - logger.debug(`Echo from server [${i++}]`, parsed.chunk, parsed.chunkLength); + logger.debug(`Echo from server [${i++}]`, parsed.chunk, parsed.dataLength, parsed.chunkLength); }); response.on("data", (d) => console.log("plain response", d)); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts new file mode 100644 index 000000000..45ff67d10 --- /dev/null +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -0,0 +1,70 @@ +import { TypedEmitter } from "@scramjet/utility"; +import { FrameDecoder, FrameEncoder, TeceMuxEvents } from "./codecs"; +import { Duplex } from "stream"; +import { Socket } from "net"; + +export type TeceMuxChannel = Duplex & { _id: number }; + +export class TeceMux extends TypedEmitter{ + carrierSocket: Duplex; + channelCount = 0; + decoder = new FrameDecoder(); + + channels: Duplex[] = []; + + private createChannel(): TeceMuxChannel { + const channel: TeceMuxChannel = Object.assign(new Duplex({ }), { _id: this.channelCount }); + + const encoder = new FrameEncoder(this.channelCount); + + channel.pipe(encoder).pipe(this.carrierSocket); + channel.on("error", (error) => { + this.emit("error", { error, source: channel }) + }); + + encoder.setChannel(); + + return channel; + } + + constructor(socket: Socket) { + super(); + this.carrierSocket = socket; + + this.main().catch((error) => { + this.emit("error", error); + }); + + socket.pipe(this.decoder); + } + + async main() { + for await (const chunk of this.decoder) { + const frame = JSON.parse(chunk); + const channel = frame.destinationPort; + + if (!this.channels[channel]) { + const channel = this.createChannel(); + + this.addChannel(channel); + } + + this.channels[channel].write(frame.chunk.data); + } + } + + addChannel(channel: TeceMuxChannel) { + this.channels[this.channelCount] = channel; + this.channelCount++; + + this.emit("channel", channel); + } + + multiplex(): TeceMuxChannel { + const channel = this.createChannel(); + + this.addChannel(channel); + + return channel; + } +} From 732461bcf6b55e2d1d17c20f5b0b2056d5b2affe Mon Sep 17 00:00:00 2001 From: patuwwy Date: Fri, 13 Jan 2023 22:54:35 +0000 Subject: [PATCH 066/231] Flags utils --- .../src/lib/tecemux/codecs/frame-decoder.ts | 9 +-- .../src/lib/tecemux/codecs/frame-encoder.ts | 60 ++++++++++++++++--- .../verser/src/lib/tecemux/codecs/index.ts | 16 +++++ packages/verser/src/lib/tecemux/playground.ts | 6 +- packages/verser/src/lib/tecemux/tecemux.ts | 36 ++++++++--- .../verser/src/lib/tecemux/utils/index.ts | 13 ++-- 6 files changed, 110 insertions(+), 30 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 8269f0d0a..0657af2eb 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -1,7 +1,7 @@ import { ObjLogger } from "@scramjet/obj-logger"; import { Duplex, Transform, TransformCallback, TransformOptions } from "stream"; import { HEADER_LENGTH, toHex } from "../utils"; -import { frameFlags } from "."; +import { frameFlags, parseFlags } from "."; export class FrameDecoder extends Transform { buff: Buffer; @@ -22,11 +22,6 @@ export class FrameDecoder extends Transform { }); } - parseFlags(byte: number) { - return frameFlags.filter((_flag, index) => byte >>> index & 1) - .reduce((p, c) => ({ ...p, [c]: true }), {}); - } - _transform(chunk: Buffer, encoding: BufferEncoding, callback: TransformCallback): void { this.logger.trace("Decoding frame...", toHex(chunk), "Size:", chunk.length); @@ -48,7 +43,7 @@ export class FrameDecoder extends Transform { sourceAddress: [this.buff.readInt8(0), this.buff.readInt8(1), this.buff.readInt8(2), this.buff.readInt8(3)], destinationAddress: [this.buff.readInt8(4), this.buff.readInt8(5), this.buff.readInt8(6), this.buff.readInt8(7)], chunk: this.buff.subarray(32, this.buff.readInt32LE(10)), - flags: this.parseFlags(this.buff.readInt8(25)), + flags: parseFlags(this.buff.readInt8(25)), sourcePort: this.buff.readInt16LE(12), destinationPort: this.buff.readInt16LE(14), dataLength: frameSize - HEADER_LENGTH, diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index b8ec9a96b..e196c67b2 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -1,5 +1,5 @@ import { Transform, TransformCallback, TransformOptions } from "stream"; -import { FrameTarget, HEADER_LENGTH, toHex as getHexString } from "../utils"; +import { FrameData, FrameTarget, HEADER_LENGTH, toHex as getHexString } from "../utils"; import { ObjLogger } from "@scramjet/obj-logger"; import { binaryFlags, frameFlags } from "."; export class FrameEncoder extends Transform { @@ -18,9 +18,11 @@ export class FrameEncoder extends Transform { }); } - setFlag(flag: keyof typeof binaryFlags, flags: Uint8Array = new Uint8Array([0])) { - this.logger.debug("settingFlag", flag); - flags[0] |= 1 << frameFlags.indexOf(flag); + setFlags(flag: (keyof typeof binaryFlags)[] = [], flags: Uint8Array = new Uint8Array([0])) { + for (const f in flag) { + this.logger.debug("settingFlag", flag); + flags[0] |= 1 << frameFlags.indexOf(flag[f]); + } return flags; } @@ -50,7 +52,7 @@ export class FrameEncoder extends Transform { // 192: data offset (4bit), reserved (4bit), 24 new Uint8Array([0b00000000]), // 224: flags (8bit), 25 - this.setFlag("ECE"), + this.setFlags(["ECE"]), // window(16bit) 26 - 27 new Uint8Array(new Uint16Array([0]).buffer), @@ -70,6 +72,49 @@ export class FrameEncoder extends Transform { return 255; } + createFrame(chunk: any = new Uint8Array([]), frame: Partial) { + const checksum = this.getChecksum(); + const buffer = Buffer.concat([ + // 0: source address 0 - 3 + new Uint8Array([10, 0, 0, 1]), + // 32: destination address 4 - 7 + new Uint8Array([10, 0, 0, 2]), + + // 64: zeroes (8bit), protocol (8bit), 8 - 9 + new Uint8Array([0, 1]), + + // tcp length (16bit) 10 - 11 + new Uint8Array(new Uint16Array([chunk.length + HEADER_LENGTH]).buffer), + + // 96: Source port, destination port 12 - 15 + new Uint8Array(new Uint16Array([0, frame.destinationPort || this.frameTarget]).buffer), + + // 128: sequenceNumber(32 bit, acnowledge number) 16 - 19 + new Uint8Array(new Uint32Array([frame.sequenceNumber || this.sequenceNumber++]).buffer), + + // 160: Acknowledgement number 20-23 + new Uint8Array(new Uint32Array([frame.acknowledgeNumber || 0]).buffer), + + // 192: data offset (4bit), reserved (4bit), 24 + new Uint8Array([0b00000000]), + + // 224: flags (8bit), 25 + this.setFlags(frame.flagsArray, new Uint8Array([0b00000000])), + // window(16bit) 26 - 27 + new Uint8Array(new Uint16Array([0]).buffer), + + // checksum(16bit) 28 - 29 + new Uint8Array(new Uint16Array([checksum]).buffer), + // pointer (16bit) 30 - 31 + new Uint8Array(new Uint16Array([checksum]).buffer), + + // 256: data 32 - + new Uint8Array(chunk) + ]); + + return buffer; + } + createChunkFrame(chunk: any) { const checksum = this.getChecksum(); const buffer = Buffer.concat([ @@ -95,7 +140,7 @@ export class FrameEncoder extends Transform { // 192: data offset (4bit), reserved (4bit), 24 new Uint8Array([0b00000000]), // 224: flags (8bit), 25 - this.setFlag("PSH"), + this.setFlags(["PSH"]), // window(16bit) 26 - 27 new Uint8Array(new Uint16Array([0]).buffer), @@ -112,7 +157,8 @@ export class FrameEncoder extends Transform { } _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void { - const buffer = this.createChunkFrame(chunk); + //const buffer = this.createChunkFrame(chunk); + const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); this.logger.trace("Encoded frame", getHexString(buffer), "Size: ", buffer.length, "Pushing"); diff --git a/packages/verser/src/lib/tecemux/codecs/index.ts b/packages/verser/src/lib/tecemux/codecs/index.ts index f2556a8ea..de478859d 100644 --- a/packages/verser/src/lib/tecemux/codecs/index.ts +++ b/packages/verser/src/lib/tecemux/codecs/index.ts @@ -19,3 +19,19 @@ export const binaryFlags = { ECE: 0b01000000, CWR: 0b10000000 } + +export type flagsObjectType = Partial<{ + FIN: boolean, + SYN: boolean, + RST: boolean, + PSH: boolean, + ACK: boolean, + URG: boolean, + ECE: boolean, + CWR: boolean +}> + +export const parseFlags = (byte: number): flagsObjectType => { + return frameFlags.filter((_flag, index) => byte >>> index & 1) + .reduce((p, c) => ({ ...p, [c]: true }), {}); +} diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/src/lib/tecemux/playground.ts index cc7429c8a..81ce13e25 100644 --- a/packages/verser/src/lib/tecemux/playground.ts +++ b/packages/verser/src/lib/tecemux/playground.ts @@ -5,7 +5,7 @@ import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; import { IncomingMessage, createServer, request } from "http"; import { DataStream } from "scramjet"; -import { DecodedFrame, FrameTarget } from "./utils"; +import { FrameData, FrameTarget } from "./utils"; import { FrameDecoder, FrameEncoder } from "./codecs"; import { Socket } from "net"; @@ -42,7 +42,7 @@ import { Socket } from "net"; socket.pipe(serverFrameDecoder).pipe(process.stdout); serverFrameDecoder.on("data", (d: any) => { - const parsed = JSON.parse(d) as DecodedFrame; + const parsed = JSON.parse(d) as FrameData; logger.debug(`Server on request data [${i++}]`, parsed.chunk, parsed.dataLength); serverFrameEncoder.write(Buffer.from(new Uint8Array([255, 255, 255, 255]))); @@ -79,7 +79,7 @@ import { Socket } from "net"; socket.pipe( responseFrameDecoder ).on("data", (d) => { - const parsed = JSON.parse(d) as DecodedFrame; + const parsed = JSON.parse(d) as FrameData; logger.debug(`Echo from server [${i++}]`, parsed.chunk, parsed.dataLength, parsed.chunkLength); }); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 45ff67d10..10a04a8dd 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -2,6 +2,8 @@ import { TypedEmitter } from "@scramjet/utility"; import { FrameDecoder, FrameEncoder, TeceMuxEvents } from "./codecs"; import { Duplex } from "stream"; import { Socket } from "net"; +import { ObjLogger } from "@scramjet/obj-logger"; +import { FrameData } from "./utils"; export type TeceMuxChannel = Duplex & { _id: number }; @@ -12,6 +14,9 @@ export class TeceMux extends TypedEmitter{ channels: Duplex[] = []; + logger = new ObjLogger(this); + commonEncoder = new FrameEncoder(0); + private createChannel(): TeceMuxChannel { const channel: TeceMuxChannel = Object.assign(new Duplex({ }), { _id: this.channelCount }); @@ -40,21 +45,36 @@ export class TeceMux extends TypedEmitter{ async main() { for await (const chunk of this.decoder) { - const frame = JSON.parse(chunk); - const channel = frame.destinationPort; - - if (!this.channels[channel]) { - const channel = this.createChannel(); + const frame = JSON.parse(chunk) as FrameData; + const { flags, sequenceNumber, dataLength, destinationPort } = frame; - this.addChannel(channel); + if (flags.ACK) { + this.logger.trace("ACKNOWLEDGE", sequenceNumber); + // acknowledge received (confirm packet) + return; } - this.channels[channel].write(frame.chunk.data); + if (flags.PSH) { + if (!this.channels[destinationPort]) { + const channel = this.createChannel(); + + this.addChannel(channel); + this.commonEncoder.createFrame(undefined, { + flagsArray: ["ACK"], + destinationPort, + acknowledgeNumber: sequenceNumber + }) + } + + if (dataLength) { + this.channels[destinationPort].write(frame.chunk); + } + } } } addChannel(channel: TeceMuxChannel) { - this.channels[this.channelCount] = channel; + this.channels[this.channelCount] = channel; // wait for SYN reply? this.channelCount++; this.emit("channel", channel); diff --git a/packages/verser/src/lib/tecemux/utils/index.ts b/packages/verser/src/lib/tecemux/utils/index.ts index 631dba5d0..08cad4374 100644 --- a/packages/verser/src/lib/tecemux/utils/index.ts +++ b/packages/verser/src/lib/tecemux/utils/index.ts @@ -1,3 +1,5 @@ +import { frameFlags, binaryFlags, flagsObjectType } from "../codecs"; + export function toHex(chunk: Buffer) { return chunk.toString("hex").match(/../g)?.join(" "); } @@ -9,15 +11,16 @@ export enum FrameTarget { INPUT = 1001 } -export type DecodedFrame = { +export type FrameData = { sourceAddress: [number, number, number, number]; destinationAddress: [number, number, number, number]; destinationPort: number; - chunk: { - type: string; - data: any; - }; + sequenceNumber: number; + acknowledgeNumber: number; + chunk: Buffer; dataLength: number; chunkLength: number; stringified: string; + flags: flagsObjectType; + flagsArray: (keyof typeof binaryFlags)[]; }; From f597140d8a596756bd61340edcdac7c6e104ab50 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Sun, 15 Jan 2023 22:38:50 +0000 Subject: [PATCH 067/231] push data to channel --- package.json | 2 +- .../src/lib/tecemux/codecs/frame-decoder.ts | 7 +- .../src/lib/tecemux/codecs/frame-encoder.ts | 60 ++++----- .../src/lib/tecemux/playground-tecemux.ts | 115 ++++++++++++++++++ packages/verser/src/lib/tecemux/tecemux.ts | 68 +++++++++-- yarn.lock | 13 +- 6 files changed, 204 insertions(+), 61 deletions(-) create mode 100644 packages/verser/src/lib/tecemux/playground-tecemux.ts diff --git a/package.json b/package.json index fab95ff1f..f12137d1e 100644 --- a/package.json +++ b/package.json @@ -77,7 +77,7 @@ "devDependencies": { "@istanbuljs/nyc-config-typescript": "^1.0.2", "@npmcli/run-script": "4.2.1", - "@types/node": "15.12.5", + "@types/node": "18.11.18", "@typescript-eslint/eslint-plugin": "^5.41.0", "@typescript-eslint/parser": "^5.41.0", "build-if-changed": "^1.5.5", diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 0657af2eb..2bc15bd23 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -1,7 +1,7 @@ import { ObjLogger } from "@scramjet/obj-logger"; import { Duplex, Transform, TransformCallback, TransformOptions } from "stream"; -import { HEADER_LENGTH, toHex } from "../utils"; -import { frameFlags, parseFlags } from "."; +import { FrameData, HEADER_LENGTH, toHex } from "../utils"; +import { parseFlags } from "."; export class FrameDecoder extends Transform { buff: Buffer; @@ -10,7 +10,6 @@ export class FrameDecoder extends Transform { _streams = new Map(); - constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { super(Object.assign({}, { readableObjectMode: true, emitClose: false }, opts)); @@ -49,7 +48,7 @@ export class FrameDecoder extends Transform { dataLength: frameSize - HEADER_LENGTH, chunkLength: frameSize, stringified: this.buff.subarray(32, frameSize).toString() - }; + } as Partial; this.push(JSON.stringify(payload) + "\n"); diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index e196c67b2..f32e9911c 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -2,6 +2,7 @@ import { Transform, TransformCallback, TransformOptions } from "stream"; import { FrameData, FrameTarget, HEADER_LENGTH, toHex as getHexString } from "../utils"; import { ObjLogger } from "@scramjet/obj-logger"; import { binaryFlags, frameFlags } from "."; + export class FrameEncoder extends Transform { sequenceNumber = 0; logger = new ObjLogger("FrameEncoder",); @@ -16,6 +17,14 @@ export class FrameEncoder extends Transform { this.on("pipe", () => { this.logger.debug("onPipe"); }); + + /*const orgPush = this.push; + this.push = (chunk: any, encoding: BufferEncoding | undefined) => { + this.logger.debug("Pushing", chunk) + //return orgPush.call(this, chunk, encoding); + return true; + } + */ } setFlags(flag: (keyof typeof binaryFlags)[] = [], flags: Uint8Array = new Uint8Array([0])) { @@ -27,45 +36,15 @@ export class FrameEncoder extends Transform { return flags; } - setChannel() { - const checksum = this.getChecksum(); - const buffer = Buffer.concat([ - // 0: source address 0 - 3 - new Uint8Array([10, 0, 0, 1]), - // 32: destination address 4 - 7 - new Uint8Array([10, 0, 0, 2]), - - // 64: zeroes (8bit), protocol (8bit), 8 - 9 - new Uint8Array([0, 1]), - // tcp length (16bit) 10 - 11 - new Uint8Array(new Uint16Array([0 + HEADER_LENGTH]).buffer), + setChannel(channelCount: number) { + this.logger.debug("Set channel command", channelCount); - // 96: Source port, destination port 12 - 15 - new Uint8Array(new Uint16Array([0, this.frameTarget]).buffer), - - // 128: sequenceNumber(32 bit, acnowledge number) 16 - 19 - new Uint8Array(new Uint32Array([this.sequenceNumber++]).buffer), - - // 160: Acknowledgement number 20-23 - new Uint8Array([0, 0, 0, 0]), - - // 192: data offset (4bit), reserved (4bit), 24 - new Uint8Array([0b00000000]), - // 224: flags (8bit), 25 - this.setFlags(["ECE"]), - // window(16bit) 26 - 27 - new Uint8Array(new Uint16Array([0]).buffer), - - // checksum(16bit) 28 - 29 - new Uint8Array(new Uint16Array([checksum]).buffer), - // pointer (16bit) 30 - 31 - new Uint8Array(new Uint16Array([checksum]).buffer), - - // 256: data 32 - - new Uint8Array([]) - ]); + //const checksum = this.getChecksum(); - this.push(buffer, undefined); + this.push(this.createFrame([], { + flagsArray: ["PSH"], + destinationPort: channelCount + })); } getChecksum() { @@ -158,11 +137,14 @@ export class FrameEncoder extends Transform { _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void { //const buffer = this.createChunkFrame(chunk); + this.logger.debug("TRANSFORM IN", chunk, chunk.length); const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); this.logger.trace("Encoded frame", getHexString(buffer), "Size: ", buffer.length, "Pushing"); - this.push(buffer, undefined); - callback(null); + //this.push(buffer, undefined); + this.logger.debug("TRANSFORM OUT", getHexString(buffer), buffer.length); + this.push(buffer); + callback(); } } diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts new file mode 100644 index 000000000..891766b4f --- /dev/null +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -0,0 +1,115 @@ +/*eslint no-unused-vars: ["error", { "args": "none" }]*/ +/* eslint-disable @typescript-eslint/no-floating-promises */ +/* eslint-disable no-console */ + +import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; +import { IncomingMessage, createServer, request } from "http"; +import { DataStream } from "scramjet"; +import { FrameData } from "./utils"; + +import { Socket } from "net"; +import { TeceMux, TeceMuxChannel } from "./tecemux"; + +(async () => { + const logger = new ObjLogger("Sandbox"); + + logger.pipe(new DataStream().map(prettyPrint({ colors: true }))).pipe(process.stdout); + + const PORT = 6660; + const server = createServer(); + + server.setTimeout(0); + server.requestTimeout = 0; + + server.on("connect", async (req: IncomingMessage, socket: Socket) => { + //socket.setNoDelay(true); + //socket.write(`HTTP/1.1 ${200} \r\n\r\n`); + socket.write('HTTP/1.1 200 Connection Established\r\n' + + 'Proxy-agent: Node.js-Proxy\r\n' + + '\r\n'); + logger.debug("on connect"); + socket.write("dddd\r\n"); + //socket.pipe(process.stdout) + + const tcmux = new TeceMux(socket, "Server Side"); + tcmux.on("error", (err) => { + logger.error("TCMUX err", err); + }); + + tcmux.logger.pipe(logger); + + const channel = tcmux.multiplex(); + + let i = 0; + + req.on("pause", () => { logger.warn("Request paused"); }); + + setInterval(() => { + logger.warn("writing to server response") + channel.write("som\n"); + }, 250) + + for await (const chunk of channel) { + const parsed = JSON.parse(chunk) as FrameData; + logger.debug(`Server on request data [${i++}]`, parsed.chunk, parsed.dataLength); + + channel.write(new Uint8Array([1,2,3,4])); + //channel.write(Buffer.from(new Uint8Array([255, 255, 255, 255]))); + }; + }); + + server.listen(PORT); + + const req = request({ + hostname: "0.0.0.0", + method: "connect", + port: PORT, + headers: { "Content-Type": "application/octet-stream", "Transfer-Encoding": "chunked" } + }); + + req.on("connect", (response, socket, head) => { + const reqLogger = new ObjLogger("Req", { id: "Request"}); + reqLogger.pipe(logger); + + + //socket.setNoDelay(true); + + reqLogger.debug("Response. Head:", head.toString()); + + let i = 0; + let m = 0; + response.on("data", (d) => { + console.log("req data in", d); + }).pause(); + + + const tcmux = new TeceMux(socket); + + socket.resume(); + tcmux.logger.updateBaseLog({ id: "Client side"}); + + tcmux.logger.pipe(logger); + + tcmux.on("channel", (channel: TeceMuxChannel) => { + reqLogger.debug("New channel", channel._id); + channel.pipe(process.stdout); + + setInterval(async () => { + reqLogger.debug(`Writing [${++m}]..`); + + channel.write(Buffer.from(new Uint8Array([0x61, 0x62, 0x63, 0x64]))); + }, 500); + }); + /*setInterval(async () => { + reqLogger.debug(`Writing [${++m}]..`); + tcmux.multiplex().write(Buffer.from(new Uint8Array([0x61, 0x62, 0x63, 0x64]))); + }, 250);*/ + req.on("error", (err) => { + console.error(err); + }); + }); + + req.flushHeaders(); + + await new Promise((_res, _rej) => {}); +})(); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 10a04a8dd..e3185c524 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -1,50 +1,87 @@ import { TypedEmitter } from "@scramjet/utility"; import { FrameDecoder, FrameEncoder, TeceMuxEvents } from "./codecs"; -import { Duplex } from "stream"; +import { Duplex, PassThrough, pipeline } from "stream"; import { Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; import { FrameData } from "./utils"; +import { IObjectLogger } from "@scramjet/types"; export type TeceMuxChannel = Duplex & { _id: number }; export class TeceMux extends TypedEmitter{ + id: string; carrierSocket: Duplex; channelCount = 0; decoder = new FrameDecoder(); channels: Duplex[] = []; - logger = new ObjLogger(this); + logger: IObjectLogger; commonEncoder = new FrameEncoder(0); private createChannel(): TeceMuxChannel { - const channel: TeceMuxChannel = Object.assign(new Duplex({ }), { _id: this.channelCount }); + this.logger.debug("Create Channel", this.channelCount); const encoder = new FrameEncoder(this.channelCount); + encoder.logger.updateBaseLog({ id: this.id }); + encoder.logger.pipe(this.logger); + + const w = new PassThrough().on("data", (d) => { this.logger.warn("writeable DATA", d); }).pause(); + //w.pipe(process.stdout).pause() + + w.pipe(encoder); + + const channel: TeceMuxChannel = Object.assign( + Duplex.from({ + readable: new PassThrough().on("data", (d) => { this.logger.warn("readable DATA", d); }).pause(), + writable: w, + } as unknown as Iterable) as TeceMuxChannel, + { _id: this.channelCount } + ); + + + //channel.pipe(encoder).pipe(this.carrierSocket); + //channel.pipe(process.stdout); - channel.pipe(encoder).pipe(this.carrierSocket); channel.on("error", (error) => { this.emit("error", { error, source: channel }) }); - encoder.setChannel(); + encoder.setChannel(this.channelCount); + //channel.pipe(w); + //channel.pipe(this.carrierSocket); + encoder.pipe(this.carrierSocket); return channel; } - constructor(socket: Socket) { + constructor(socket: Socket, id = "") { super(); + this.id = id; + this.logger = new ObjLogger(this, { id: this.id }) this.carrierSocket = socket; - this.main().catch((error) => { - this.emit("error", error); - }); + this.decoder.logger.pipe(this.logger); - socket.pipe(this.decoder); + this.carrierSocket.pipe(this.decoder); + this.commonEncoder.pipe(this.carrierSocket); + + this.carrierSocket.on("data", (d) => { console.warn("carrier socket ", d); }); + this.commonEncoder.on("data", (d) => { this.logger.warn("to socket", d); }); + + this.main();//.catch((error) => { + //this.emit("error", error); + //}); + //this.carrierSocket.pipe(process.stdout); } async main() { + this.commonEncoder.logger.updateBaseLog({ id: "Commn" + this.commonEncoder.logger.baseLog.id }) + this.commonEncoder.logger.pipe(this.logger); + for await (const chunk of this.decoder) { + this.logger.debug("Decoded", JSON.parse(chunk)); + const frame = JSON.parse(chunk) as FrameData; const { flags, sequenceNumber, dataLength, destinationPort } = frame; @@ -67,20 +104,25 @@ export class TeceMux extends TypedEmitter{ } if (dataLength) { - this.channels[destinationPort].write(frame.chunk); + this.channels[destinationPort].write(new Uint8Array((frame.chunk) as any)); } } } } addChannel(channel: TeceMuxChannel) { + this.logger.debug("adding channel", channel._id); this.channels[this.channelCount] = channel; // wait for SYN reply? this.channelCount++; - - this.emit("channel", channel); + this.commonEncoder.createFrame(undefined, { + flagsArray: ["PSH"], + destinationPort: channel._id + }) + //this.emit("channel", channel); } multiplex(): TeceMuxChannel { + this.logger.trace("Multiplex") const channel = this.createChannel(); this.addChannel(channel); diff --git a/yarn.lock b/yarn.lock index 6c8a47843..c96e1bb09 100644 --- a/yarn.lock +++ b/yarn.lock @@ -959,16 +959,21 @@ "@types/node" "*" form-data "^3.0.0" -"@types/node@*", "@types/node@>=13.7.0", "@types/node@^18.11.18": - version "18.14.2" - resolved "https://registry.yarnpkg.com/@types/node/-/node-18.14.2.tgz#c076ed1d7b6095078ad3cf21dfeea951842778b1" - integrity sha512-1uEQxww3DaghA0RxqHx0O0ppVlo43pJhepY51OxuQIKHpjbnYLA7vcdwioNPzIqmC2u3I/dmylcqjlh0e7AyUA== +"@types/node@*", "@types/node@18.11.18", "@types/node@^18.11.18": + version "18.11.18" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f" + integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA== "@types/node@15.12.5": version "15.12.5" resolved "https://registry.yarnpkg.com/@types/node/-/node-15.12.5.tgz#9a78318a45d75c9523d2396131bd3cca54b2d185" integrity sha512-se3yX7UHv5Bscf8f1ERKvQOD6sTyycH3hdaoozvaLxgUiY5lIGEeH37AD0G0Qi9kPqihPn0HOfd2yaIEN9VwEg== +"@types/node@>=13.7.0": + version "18.11.3" + resolved "https://registry.npmjs.org/@types/node/-/node-18.11.3.tgz" + integrity sha512-fNjDQzzOsZeKZu5NATgXUPsaFaTxeRgFXoosrHivTl8RGeV733OLawXsGfEk9a8/tySyZUyiZ6E8LcjPFZ2y1A== + "@types/normalize-package-data@^2.4.0": version "2.4.1" resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301" From 48151d5108462139bd82c6031fd9102ee36c0f22 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 16 Jan 2023 17:08:47 +0000 Subject: [PATCH 068/231] Duplex communication --- .../src/lib/tecemux/codecs/frame-decoder.ts | 3 +- .../src/lib/tecemux/codecs/frame-encoder.ts | 28 ++-- .../src/lib/tecemux/playground-tecemux.ts | 120 +++++++++--------- packages/verser/src/lib/tecemux/tecemux.ts | 71 ++++++----- 4 files changed, 109 insertions(+), 113 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 2bc15bd23..9c60fac1c 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -35,7 +35,7 @@ export class FrameDecoder extends Transform { this.size += chunk.length; - if (this.size >= 10 && this.buff.readInt32LE(10) === this.size) { + if (this.size >= 10 ) {//&& this.buff.readInt32LE(10) === this.size) { const frameSize = this.buff.readInt32LE(10); const payload = { @@ -47,6 +47,7 @@ export class FrameDecoder extends Transform { destinationPort: this.buff.readInt16LE(14), dataLength: frameSize - HEADER_LENGTH, chunkLength: frameSize, + sequenceNumber: this.buff.readInt32LE(16), stringified: this.buff.subarray(32, frameSize).toString() } as Partial; diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index f32e9911c..5aec8f657 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -1,11 +1,14 @@ -import { Transform, TransformCallback, TransformOptions } from "stream"; -import { FrameData, FrameTarget, HEADER_LENGTH, toHex as getHexString } from "../utils"; +import { PassThrough, Transform, TransformCallback, TransformOptions } from "stream"; +import { FrameData, FrameTarget, HEADER_LENGTH, toHex as getHexString, toHex } from "../utils"; import { ObjLogger } from "@scramjet/obj-logger"; import { binaryFlags, frameFlags } from "."; export class FrameEncoder extends Transform { sequenceNumber = 0; logger = new ObjLogger("FrameEncoder",); + out = new PassThrough({ readableObjectMode: true }).on("data", (data) => { + this.logger.trace("outcome", toHex(data), data.length); + }); constructor(private frameTarget: FrameTarget, opts: TransformOptions = {}, params: { name: string } = { name: "FrameEncoder" }) { opts.emitClose = false; @@ -18,21 +21,16 @@ export class FrameEncoder extends Transform { this.logger.debug("onPipe"); }); - /*const orgPush = this.push; - this.push = (chunk: any, encoding: BufferEncoding | undefined) => { - this.logger.debug("Pushing", chunk) - //return orgPush.call(this, chunk, encoding); - return true; - } - */ + this.pipe(this.out); } setFlags(flag: (keyof typeof binaryFlags)[] = [], flags: Uint8Array = new Uint8Array([0])) { for (const f in flag) { - this.logger.debug("settingFlag", flag); flags[0] |= 1 << frameFlags.indexOf(flag[f]); } + this.logger.debug("settingFlag", flag, flags[0].toString(2).padStart(8, "0")); + return flags; } @@ -41,7 +39,7 @@ export class FrameEncoder extends Transform { //const checksum = this.getChecksum(); - this.push(this.createFrame([], { + this.out.write(this.createFrame([], { flagsArray: ["PSH"], destinationPort: channelCount })); @@ -137,14 +135,12 @@ export class FrameEncoder extends Transform { _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void { //const buffer = this.createChunkFrame(chunk); - this.logger.debug("TRANSFORM IN", chunk, chunk.length); + this.logger.debug("TRANSFORM IN", toHex(chunk), chunk.length); const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); - this.logger.trace("Encoded frame", getHexString(buffer), "Size: ", buffer.length, "Pushing"); - //this.push(buffer, undefined); - this.logger.debug("TRANSFORM OUT", getHexString(buffer), buffer.length); - this.push(buffer); + this.logger.debug("TRANSFORM OUT", getHexString(buffer), "Size: ", buffer.length); + this.push(buffer, undefined); callback(); } } diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index 891766b4f..8fcf7f363 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -3,18 +3,22 @@ /* eslint-disable no-console */ import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; -import { IncomingMessage, createServer, request } from "http"; +import { IncomingMessage, createServer } from "http"; import { DataStream } from "scramjet"; -import { FrameData } from "./utils"; -import { Socket } from "net"; +import { Socket, createConnection } from "net"; import { TeceMux, TeceMuxChannel } from "./tecemux"; +import { FrameData } from "./utils"; (async () => { const logger = new ObjLogger("Sandbox"); logger.pipe(new DataStream().map(prettyPrint({ colors: true }))).pipe(process.stdout); + /**********************************************/ + /* SERVER + /**********************************************/ + const PORT = 6660; const server = createServer(); @@ -22,94 +26,88 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; server.requestTimeout = 0; server.on("connect", async (req: IncomingMessage, socket: Socket) => { - //socket.setNoDelay(true); - //socket.write(`HTTP/1.1 ${200} \r\n\r\n`); - socket.write('HTTP/1.1 200 Connection Established\r\n' + - 'Proxy-agent: Node.js-Proxy\r\n' + - '\r\n'); - logger.debug("on connect"); - socket.write("dddd\r\n"); - //socket.pipe(process.stdout) - - const tcmux = new TeceMux(socket, "Server Side"); - tcmux.on("error", (err) => { - logger.error("TCMUX err", err); - }); + socket.setNoDelay(true); + + logger.info("Incoming request", req.method, req.headers); + + const tcmux = new TeceMux(socket, "Server Side") + .on("error", (err) => { + logger.error("TCMUX err", err); + }); tcmux.logger.pipe(logger); const channel = tcmux.multiplex(); - let i = 0; - req.on("pause", () => { logger.warn("Request paused"); }); - setInterval(() => { - logger.warn("writing to server response") - channel.write("som\n"); - }, 250) + logger.warn("writing to server response") + channel.write("som\n"); for await (const chunk of channel) { const parsed = JSON.parse(chunk) as FrameData; - logger.debug(`Server on request data [${i++}]`, parsed.chunk, parsed.dataLength); + logger.debug(`Server on request data [C: ${parsed.sequenceNumber}, SN: ${parsed.sequenceNumber}]`, parsed.chunk, parsed.dataLength); - channel.write(new Uint8Array([1,2,3,4])); + //channel.write(new Uint8Array([1,2,3,4])); //channel.write(Buffer.from(new Uint8Array([255, 255, 255, 255]))); }; + /* + channel.on("data", (d) => { + logger.error("Server on request data", d); + }); + */ + + //channel.pipe(process.stdout); }); - server.listen(PORT); + server.listen(PORT, "0.0.0.0"); - const req = request({ - hostname: "0.0.0.0", - method: "connect", - port: PORT, - headers: { "Content-Type": "application/octet-stream", "Transfer-Encoding": "chunked" } - }); + /**********************************************/ + /* CLIENT + /**********************************************/ - req.on("connect", (response, socket, head) => { - const reqLogger = new ObjLogger("Req", { id: "Request"}); - reqLogger.pipe(logger); + const socket = createConnection({ port: PORT, allowHalfOpen: true, host: "0.0.0.0" }, () => {}); + socket.setNoDelay(true); - //socket.setNoDelay(true); + const reqLogger = new ObjLogger("Req", { id: "Request"}); + reqLogger.pipe(logger); - reqLogger.debug("Response. Head:", head.toString()); - let i = 0; - let m = 0; - response.on("data", (d) => { - console.log("req data in", d); - }).pause(); + socket.write("CONNECT HTTP/1.1\r\n\r\n\r\n"); + socket.on("error", (error) => { + reqLogger.error("ERROR", error); + }); - const tcmux = new TeceMux(socket); + reqLogger.info('connected to server!'); - socket.resume(); - tcmux.logger.updateBaseLog({ id: "Client side"}); + const tcmux = new TeceMux(socket); - tcmux.logger.pipe(logger); + tcmux.logger.updateBaseLog({ id: "Client side"}); - tcmux.on("channel", (channel: TeceMuxChannel) => { - reqLogger.debug("New channel", channel._id); - channel.pipe(process.stdout); + tcmux.logger.pipe(logger); - setInterval(async () => { - reqLogger.debug(`Writing [${++m}]..`); + tcmux.on("channel", async (channel: TeceMuxChannel) => { + reqLogger.debug("New channel", channel._id); - channel.write(Buffer.from(new Uint8Array([0x61, 0x62, 0x63, 0x64]))); - }, 500); - }); - /*setInterval(async () => { - reqLogger.debug(`Writing [${++m}]..`); - tcmux.multiplex().write(Buffer.from(new Uint8Array([0x61, 0x62, 0x63, 0x64]))); - }, 250);*/ - req.on("error", (err) => { - console.error(err); - }); + //channel.write(Buffer.from(new Uint8Array([0x61, 0x62, 0x63, 0x64]))); + channel.pipe(process.stdout); + + for await (const chunk of channel) { + reqLogger.info("request on response data", chunk); + } }); + /*setInterval(async () => { + reqLogger.debug(`Writing [${++m}]..`); + tcmux.multiplex().write(Buffer.from(new Uint8Array([0x61, 0x62, 0x63, 0x64]))); + }, 250);*/ + socket.on("error", (err) => { + console.error(err); + }); + - req.flushHeaders(); + //socket.flushHeaders(); await new Promise((_res, _rej) => {}); })(); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index e3185c524..9a7913dc1 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -1,6 +1,6 @@ import { TypedEmitter } from "@scramjet/utility"; import { FrameDecoder, FrameEncoder, TeceMuxEvents } from "./codecs"; -import { Duplex, PassThrough, pipeline } from "stream"; +import { Duplex, PassThrough } from "stream"; import { Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; import { FrameData } from "./utils"; @@ -11,16 +11,16 @@ export type TeceMuxChannel = Duplex & { _id: number }; export class TeceMux extends TypedEmitter{ id: string; carrierSocket: Duplex; - channelCount = 0; + channelCount = 1; decoder = new FrameDecoder(); - channels: Duplex[] = []; + channels: TeceMuxChannel[] = []; logger: IObjectLogger; commonEncoder = new FrameEncoder(0); - private createChannel(): TeceMuxChannel { - this.logger.debug("Create Channel", this.channelCount); + private createChannel(destinationPort?: number): TeceMuxChannel { + this.logger.debug("Create Channel", destinationPort || this.channelCount); const encoder = new FrameEncoder(this.channelCount); encoder.logger.updateBaseLog({ id: this.id }); @@ -33,13 +33,12 @@ export class TeceMux extends TypedEmitter{ const channel: TeceMuxChannel = Object.assign( Duplex.from({ - readable: new PassThrough().on("data", (d) => { this.logger.warn("readable DATA", d); }).pause(), + readable: new PassThrough().on("data", (d) => { this.logger.warn("readable DATA", d); }), writable: w, } as unknown as Iterable) as TeceMuxChannel, - { _id: this.channelCount } + { _id: destinationPort || this.channelCount } ); - //channel.pipe(encoder).pipe(this.carrierSocket); //channel.pipe(process.stdout); @@ -47,10 +46,10 @@ export class TeceMux extends TypedEmitter{ this.emit("error", { error, source: channel }) }); - encoder.setChannel(this.channelCount); + encoder.setChannel(destinationPort || this.channelCount); //channel.pipe(w); //channel.pipe(this.carrierSocket); - encoder.pipe(this.carrierSocket); + encoder.out.pipe(this.carrierSocket); return channel; } @@ -64,15 +63,12 @@ export class TeceMux extends TypedEmitter{ this.decoder.logger.pipe(this.logger); this.carrierSocket.pipe(this.decoder); - this.commonEncoder.pipe(this.carrierSocket); + this.commonEncoder.out.pipe(this.carrierSocket); - this.carrierSocket.on("data", (d) => { console.warn("carrier socket ", d); }); - this.commonEncoder.on("data", (d) => { this.logger.warn("to socket", d); }); - this.main();//.catch((error) => { - //this.emit("error", error); - //}); - //this.carrierSocket.pipe(process.stdout); + this.main().catch((error) => { + this.emit("error", error); + }); } async main() { @@ -92,41 +88,46 @@ export class TeceMux extends TypedEmitter{ } if (flags.PSH) { - if (!this.channels[destinationPort]) { - const channel = this.createChannel(); + this.logger.trace(`Received PSH command [C: ${destinationPort}]`, dataLength, frame.chunk, !!this.channels[destinationPort]); + let channel = this.channels[destinationPort] + + if (!channel) { + this.logger.warn("NEW CHANNEL"); + channel = this.createChannel(destinationPort); this.addChannel(channel); - this.commonEncoder.createFrame(undefined, { - flagsArray: ["ACK"], - destinationPort, - acknowledgeNumber: sequenceNumber - }) } if (dataLength) { - this.channels[destinationPort].write(new Uint8Array((frame.chunk) as any)); + this.logger.warn("writing DATA LENGHT REC", dataLength); + channel.push(new Uint8Array(((frame.chunk as any).data) as any)); } } + + this.commonEncoder.out.write( + this.commonEncoder.createFrame(undefined, { + flagsArray: ["ACK"], + sequenceNumber, + destinationPort + }) + ); } } addChannel(channel: TeceMuxChannel) { - this.logger.debug("adding channel", channel._id); - this.channels[this.channelCount] = channel; // wait for SYN reply? - this.channelCount++; - this.commonEncoder.createFrame(undefined, { - flagsArray: ["PSH"], - destinationPort: channel._id - }) - //this.emit("channel", channel); + this.logger.debug("adding channel", ); + this.channels[channel._id] = channel; // wait for SYN reply? + + this.emit("channel", channel); } multiplex(): TeceMuxChannel { - this.logger.trace("Multiplex") - const channel = this.createChannel(); + const channel = this.createChannel(this.channelCount); + this.logger.trace("Multiplex", channel._id); this.addChannel(channel); + this.channelCount++; return channel; } } From 2c4b6c929e24b9f615e1336b71d3752b74896d43 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 16 Jan 2023 18:12:36 +0000 Subject: [PATCH 069/231] Send stdin over frames --- .../src/lib/tecemux/playground-tecemux.ts | 33 +++++-------------- packages/verser/src/lib/tecemux/tecemux.ts | 7 ++-- 2 files changed, 14 insertions(+), 26 deletions(-) diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index 8fcf7f363..fc3635f8e 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -42,22 +42,16 @@ import { FrameData } from "./utils"; req.on("pause", () => { logger.warn("Request paused"); }); logger.warn("writing to server response") - channel.write("som\n"); + //channel.write("som\n"); + + + process.stdout.pipe(channel); + for await (const chunk of channel) { const parsed = JSON.parse(chunk) as FrameData; logger.debug(`Server on request data [C: ${parsed.sequenceNumber}, SN: ${parsed.sequenceNumber}]`, parsed.chunk, parsed.dataLength); - - //channel.write(new Uint8Array([1,2,3,4])); - //channel.write(Buffer.from(new Uint8Array([255, 255, 255, 255]))); }; - /* - channel.on("data", (d) => { - logger.error("Server on request data", d); - }); - */ - - //channel.pipe(process.stdout); }); server.listen(PORT, "0.0.0.0"); @@ -70,7 +64,7 @@ import { FrameData } from "./utils"; socket.setNoDelay(true); - const reqLogger = new ObjLogger("Req", { id: "Request"}); + const reqLogger = new ObjLogger("Req", { id: "Client Side"}); reqLogger.pipe(logger); @@ -82,7 +76,7 @@ import { FrameData } from "./utils"; reqLogger.info('connected to server!'); - const tcmux = new TeceMux(socket); + const tcmux = new TeceMux(socket, "Request"); tcmux.logger.updateBaseLog({ id: "Client side"}); @@ -91,23 +85,14 @@ import { FrameData } from "./utils"; tcmux.on("channel", async (channel: TeceMuxChannel) => { reqLogger.debug("New channel", channel._id); - //channel.write(Buffer.from(new Uint8Array([0x61, 0x62, 0x63, 0x64]))); - channel.pipe(process.stdout); - for await (const chunk of channel) { - reqLogger.info("request on response data", chunk); + reqLogger.info("Data from server", chunk.toString()); } }); - /*setInterval(async () => { - reqLogger.debug(`Writing [${++m}]..`); - tcmux.multiplex().write(Buffer.from(new Uint8Array([0x61, 0x62, 0x63, 0x64]))); - }, 250);*/ + socket.on("error", (err) => { console.error(err); }); - - //socket.flushHeaders(); - await new Promise((_res, _rej) => {}); })(); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 9a7913dc1..4e15bb36e 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -60,6 +60,7 @@ export class TeceMux extends TypedEmitter{ this.logger = new ObjLogger(this, { id: this.id }) this.carrierSocket = socket; + this.decoder.logger.updateBaseLog({ id: this.id }); this.decoder.logger.pipe(this.logger); this.carrierSocket.pipe(this.decoder); @@ -92,7 +93,7 @@ export class TeceMux extends TypedEmitter{ let channel = this.channels[destinationPort] if (!channel) { - this.logger.warn("NEW CHANNEL"); + this.logger.warn("Unknown channel"); channel = this.createChannel(destinationPort); this.addChannel(channel); @@ -122,11 +123,13 @@ export class TeceMux extends TypedEmitter{ } multiplex(): TeceMuxChannel { + this.logger.trace("Multiplex"); + const channel = this.createChannel(this.channelCount); - this.logger.trace("Multiplex", channel._id); this.addChannel(channel); + this.logger.trace("Multiplex ready", channel._id); this.channelCount++; return channel; } From 3e429fc86de7be514bd8c6edadb91785219a4a00 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 16 Jan 2023 22:31:47 +0000 Subject: [PATCH 070/231] Issue, stream paused --- .../src/lib/tecemux/codecs/frame-decoder.ts | 2 +- .../src/lib/tecemux/codecs/frame-encoder.ts | 16 ++++-- .../src/lib/tecemux/playground-tecemux.ts | 54 +++++++++++++++---- packages/verser/src/lib/tecemux/tecemux.ts | 49 ++++++++++------- 4 files changed, 87 insertions(+), 34 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 9c60fac1c..f6a62efe4 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -11,7 +11,7 @@ export class FrameDecoder extends Transform { _streams = new Map(); constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { - super(Object.assign({}, { readableObjectMode: true, emitClose: false }, opts)); + super(Object.assign({}, { writableObjectMode: true, readableObjectMode: true, emitClose: false }, opts)); this.buff = Buffer.alloc(64 * 1024, 0, undefined); //@TODO: optimize this.logger = new ObjLogger(params.name); diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index 5aec8f657..9d3acf6ee 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -6,9 +6,19 @@ import { binaryFlags, frameFlags } from "."; export class FrameEncoder extends Transform { sequenceNumber = 0; logger = new ObjLogger("FrameEncoder",); - out = new PassThrough({ readableObjectMode: true }).on("data", (data) => { - this.logger.trace("outcome", toHex(data), data.length); - }); + out = new PassThrough({ readableObjectMode: true }) + .on("data", (data) => { + this.logger.trace("output to socket:", toHex(data), data.length, this.readableFlowing); + }) + .on("pause", () => { + this.logger.trace("output to socket paused"); + }) + .on("end", () => { + this.logger.trace("output to socket ended"); + }) + .on("resume", () => { + this.logger.trace("output to socket resumed"); + }) constructor(private frameTarget: FrameTarget, opts: TransformOptions = {}, params: { name: string } = { name: "FrameEncoder" }) { opts.emitClose = false; diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index fc3635f8e..f08adab9d 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -30,23 +30,38 @@ import { FrameData } from "./utils"; logger.info("Incoming request", req.method, req.headers); - const tcmux = new TeceMux(socket, "Server Side") + socket + .on("data", (data) => { + console.log("SERVER socket ondata", data); + console.log("SOCKET TX RX", socket.bytesWritten, socket.bytesRead) + }) + .on("pause", () => { + logger.info("Socket paused"); + }) + .on("resume", () => { + logger.info("Socket resumed"); + }) + + const tcmux = new TeceMux(socket, "Server") .on("error", (err) => { logger.error("TCMUX err", err); }); + + tcmux.logger.pipe(logger); const channel = tcmux.multiplex(); req.on("pause", () => { logger.warn("Request paused"); }); - logger.warn("writing to server response") - //channel.write("som\n"); - + logger.warn("Waiting for stdin..."); - process.stdout.pipe(channel); + process.stdin.pipe(channel); + const somePayload = "smth\n"; + logger.info("writing some payload to channel", somePayload); + channel.write(somePayload); for await (const chunk of channel) { const parsed = JSON.parse(chunk) as FrameData; @@ -64,10 +79,9 @@ import { FrameData } from "./utils"; socket.setNoDelay(true); - const reqLogger = new ObjLogger("Req", { id: "Client Side"}); + const reqLogger = new ObjLogger("Req", { id: "Client"}); reqLogger.pipe(logger); - socket.write("CONNECT HTTP/1.1\r\n\r\n\r\n"); socket.on("error", (error) => { @@ -78,16 +92,34 @@ import { FrameData } from "./utils"; const tcmux = new TeceMux(socket, "Request"); - tcmux.logger.updateBaseLog({ id: "Client side"}); + tcmux.logger.updateBaseLog({ id: reqLogger.baseLog.id }); tcmux.logger.pipe(logger); tcmux.on("channel", async (channel: TeceMuxChannel) => { reqLogger.debug("New channel", channel._id); - for await (const chunk of channel) { - reqLogger.info("Data from server", chunk.toString()); - } + // for await (const chunk of channel) { + // reqLogger.info("Data from server", chunk.toString()); + + // await new Promise((resolve, reject) => { + // setTimeout(() => { + // //channel.encoder.write("Echo\n"); + // resolve(); + // }, 2000); + // }); + // } + + channel.on("data", async (chunk) => { + reqLogger.info("SERVER->CLIENT->CHANNEL", chunk.toString()); + + await new Promise((resolve, reject) => { + setTimeout(() => { + //channel.encoder.write("Echo\n"); + resolve(); + }, 2000); + }); + }) }); socket.on("error", (err) => { diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 4e15bb36e..92d024a49 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -6,37 +6,41 @@ import { ObjLogger } from "@scramjet/obj-logger"; import { FrameData } from "./utils"; import { IObjectLogger } from "@scramjet/types"; -export type TeceMuxChannel = Duplex & { _id: number }; +export type TeceMuxChannel = Duplex & { _id: number, encoder: FrameEncoder }; export class TeceMux extends TypedEmitter{ id: string; carrierSocket: Duplex; channelCount = 1; - decoder = new FrameDecoder(); + decoder = new FrameDecoder().resume(); channels: TeceMuxChannel[] = []; logger: IObjectLogger; - commonEncoder = new FrameEncoder(0); + commonEncoder = new FrameEncoder(0).resume(); - private createChannel(destinationPort?: number): TeceMuxChannel { + private createChannel(destinationPort?: number, emit?: boolean): TeceMuxChannel { this.logger.debug("Create Channel", destinationPort || this.channelCount); - const encoder = new FrameEncoder(this.channelCount); + const encoder = new FrameEncoder(this.channelCount, { encoding: undefined }); + encoder.logger.updateBaseLog({ id: this.id }); encoder.logger.pipe(this.logger); - const w = new PassThrough().on("data", (d) => { this.logger.warn("writeable DATA", d); }).pause(); + const w = new PassThrough({ encoding: undefined }).on("data", (d) => { this.logger.warn("channel writeable on DATA", d); }); //w.pipe(process.stdout).pause() w.pipe(encoder); const channel: TeceMuxChannel = Object.assign( Duplex.from({ - readable: new PassThrough().on("data", (d) => { this.logger.warn("readable DATA", d); }), + readable: new PassThrough({ encoding: undefined }).on("data", (d) => { this.logger.warn("channel readable on DATA", d); }), writable: w, } as unknown as Iterable) as TeceMuxChannel, - { _id: destinationPort || this.channelCount } + { + _id: destinationPort || this.channelCount, + encoder + } ); //channel.pipe(encoder).pipe(this.carrierSocket); @@ -46,7 +50,10 @@ export class TeceMux extends TypedEmitter{ this.emit("error", { error, source: channel }) }); - encoder.setChannel(destinationPort || this.channelCount); + if (emit) { + encoder.setChannel(destinationPort || this.channelCount); + } + //channel.pipe(w); //channel.pipe(this.carrierSocket); encoder.out.pipe(this.carrierSocket); @@ -66,7 +73,6 @@ export class TeceMux extends TypedEmitter{ this.carrierSocket.pipe(this.decoder); this.commonEncoder.out.pipe(this.carrierSocket); - this.main().catch((error) => { this.emit("error", error); }); @@ -83,7 +89,7 @@ export class TeceMux extends TypedEmitter{ const { flags, sequenceNumber, dataLength, destinationPort } = frame; if (flags.ACK) { - this.logger.trace("ACKNOWLEDGE", sequenceNumber); + this.logger.trace("ACKNOWLEDGE frame received for sequenceNumber", sequenceNumber); // acknowledge received (confirm packet) return; } @@ -94,17 +100,18 @@ export class TeceMux extends TypedEmitter{ if (!channel) { this.logger.warn("Unknown channel"); - channel = this.createChannel(destinationPort); + channel = this.createChannel(destinationPort, false); - this.addChannel(channel); + this.addChannel(channel, true); } if (dataLength) { - this.logger.warn("writing DATA LENGHT REC", dataLength); + this.logger.warn("writing to channel [channel, length]", channel._id, dataLength); channel.push(new Uint8Array(((frame.chunk as any).data) as any)); } } + this.logger.debug("Write acknowledge frame for sequenceNumber", sequenceNumber); this.commonEncoder.out.write( this.commonEncoder.createFrame(undefined, { flagsArray: ["ACK"], @@ -115,19 +122,23 @@ export class TeceMux extends TypedEmitter{ } } - addChannel(channel: TeceMuxChannel) { - this.logger.debug("adding channel", ); + addChannel(channel: TeceMuxChannel, emit: boolean) { + this.logger.debug("adding channel", channel._id); this.channels[channel._id] = channel; // wait for SYN reply? - this.emit("channel", channel); + if (emit) { + this.emit("channel", channel); + this.logger.debug("channel event emitted", channel._id); + this.channelCount++; + } } multiplex(): TeceMuxChannel { this.logger.trace("Multiplex"); - const channel = this.createChannel(this.channelCount); + const channel = this.createChannel(this.channelCount, true); - this.addChannel(channel); + this.addChannel(channel, false); this.logger.trace("Multiplex ready", channel._id); this.channelCount++; From 0fa423c13af85367e1fb352f9a54a42160e6cda4 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 17 Jan 2023 21:37:41 +0000 Subject: [PATCH 071/231] Debug AbortError --- .../src/lib/tecemux/codecs/frame-decoder.ts | 70 +++++++++++-------- .../src/lib/tecemux/codecs/frame-encoder.ts | 49 ++----------- .../src/lib/tecemux/playground-tecemux.ts | 45 +++++++++--- packages/verser/src/lib/tecemux/tecemux.ts | 37 ++++++---- 4 files changed, 100 insertions(+), 101 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index f6a62efe4..803d27c57 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -11,54 +11,62 @@ export class FrameDecoder extends Transform { _streams = new Map(); constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { - super(Object.assign({}, { writableObjectMode: true, readableObjectMode: true, emitClose: false }, opts)); + super(Object.assign({}, opts, { writableObjectMode: true, readableObjectMode: true, emitClose: false })); this.buff = Buffer.alloc(64 * 1024, 0, undefined); //@TODO: optimize this.logger = new ObjLogger(params.name); this.on("pipe", () => { this.logger.debug("onPipe"); - }); + }).on("close", () => { + this.logger.debug("onClose"); + }) } _transform(chunk: Buffer, encoding: BufferEncoding, callback: TransformCallback): void { - this.logger.trace("Decoding frame...", toHex(chunk), "Size:", chunk.length); + try { + this.logger.trace("Decoding frame...", toHex(chunk), "Size:", chunk.length); - if (!Buffer.isBuffer(chunk)) { - this.push(JSON.stringify({ error: "not a buffer" }), undefined); - callback(); + if (!Buffer.isBuffer(chunk)) { + this.push(JSON.stringify({ error: "not a buffer" }), undefined); + callback(); - return; - } + return; + } - chunk.copy(this.buff, this.size, 0, chunk.length); + if (Buffer.isBuffer(chunk)) { + chunk.copy(this.buff, this.size, 0, chunk.length); + } - this.size += chunk.length; + this.size += chunk.length; - if (this.size >= 10 ) {//&& this.buff.readInt32LE(10) === this.size) { - const frameSize = this.buff.readInt32LE(10); + if (this.size >= 10 ) {//&& this.buff.readInt32LE(10) === this.size) { + const frameSize = this.buff.readInt32LE(10); - const payload = { - sourceAddress: [this.buff.readInt8(0), this.buff.readInt8(1), this.buff.readInt8(2), this.buff.readInt8(3)], - destinationAddress: [this.buff.readInt8(4), this.buff.readInt8(5), this.buff.readInt8(6), this.buff.readInt8(7)], - chunk: this.buff.subarray(32, this.buff.readInt32LE(10)), - flags: parseFlags(this.buff.readInt8(25)), - sourcePort: this.buff.readInt16LE(12), - destinationPort: this.buff.readInt16LE(14), - dataLength: frameSize - HEADER_LENGTH, - chunkLength: frameSize, - sequenceNumber: this.buff.readInt32LE(16), - stringified: this.buff.subarray(32, frameSize).toString() - } as Partial; + const payload = { + sourceAddress: [this.buff.readInt8(0), this.buff.readInt8(1), this.buff.readInt8(2), this.buff.readInt8(3)], + destinationAddress: [this.buff.readInt8(4), this.buff.readInt8(5), this.buff.readInt8(6), this.buff.readInt8(7)], + chunk: this.buff.subarray(32, this.buff.readInt32LE(10)), + flags: parseFlags(this.buff.readInt8(25)), + sourcePort: this.buff.readInt16LE(12), + destinationPort: this.buff.readInt16LE(14), + dataLength: frameSize - HEADER_LENGTH, + chunkLength: frameSize, + sequenceNumber: this.buff.readInt32LE(16), + stringified: this.buff.subarray(32, frameSize).toString() + } as Partial; - this.push(JSON.stringify(payload) + "\n"); + this.push(JSON.stringify(payload) + "\n"); - this.size = 0; - this.buff.fill(0); - } else { - this.logger.error("too few data", this.size, this.buff.readInt32LE(10)); - } + this.size = 0; + this.buff.fill(0); + } else { + this.logger.error("too few data", this.size, this.buff.readInt32LE(10)); + } - callback(); + callback(); + } catch(err) { + this.logger.error("ERROR", err) + } } } diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index 9d3acf6ee..c3c0dd571 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -8,7 +8,7 @@ export class FrameEncoder extends Transform { logger = new ObjLogger("FrameEncoder",); out = new PassThrough({ readableObjectMode: true }) .on("data", (data) => { - this.logger.trace("output to socket:", toHex(data), data.length, this.readableFlowing); + this.logger.trace("output to socket: " + (data.length == HEADER_LENGTH ? "HEADER ONLY" : ""), toHex(data), data.length, this.readableFlowing); }) .on("pause", () => { this.logger.trace("output to socket paused"); @@ -19,9 +19,12 @@ export class FrameEncoder extends Transform { .on("resume", () => { this.logger.trace("output to socket resumed"); }) + .on("error", (error) => { + this.logger.error("output to socket paused", error); + }) constructor(private frameTarget: FrameTarget, opts: TransformOptions = {}, params: { name: string } = { name: "FrameEncoder" }) { - opts.emitClose = false; + //opts.emitClose = false; super(opts); @@ -102,53 +105,11 @@ export class FrameEncoder extends Transform { return buffer; } - createChunkFrame(chunk: any) { - const checksum = this.getChecksum(); - const buffer = Buffer.concat([ - // 0: source address 0 - 3 - new Uint8Array([10, 0, 0, 1]), - // 32: destination address 4 - 7 - new Uint8Array([10, 0, 0, 2]), - - // 64: zeroes (8bit), protocol (8bit), 8 - 9 - new Uint8Array([0, 1]), - // tcp length (16bit) 10 - 11 - new Uint8Array(new Uint16Array([chunk.length + HEADER_LENGTH]).buffer), - - // 96: Source port, destination port 12 - 15 - new Uint8Array(new Uint16Array([0, this.frameTarget]).buffer), - - // 128: sequenceNumber(32 bit, acnowledge number) 16 - 19 - new Uint8Array(new Uint32Array([this.sequenceNumber++]).buffer), - - // 160: Acknowledgement number 20-23 - new Uint8Array([0, 0, 0, 0]), - - // 192: data offset (4bit), reserved (4bit), 24 - new Uint8Array([0b00000000]), - // 224: flags (8bit), 25 - this.setFlags(["PSH"]), - // window(16bit) 26 - 27 - new Uint8Array(new Uint16Array([0]).buffer), - - // checksum(16bit) 28 - 29 - new Uint8Array(new Uint16Array([checksum]).buffer), - // pointer (16bit) 30 - 31 - new Uint8Array(new Uint16Array([checksum]).buffer), - - // 256: data 32 - - new Uint8Array(chunk) - ]); - - return buffer; - } - _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void { //const buffer = this.createChunkFrame(chunk); this.logger.debug("TRANSFORM IN", toHex(chunk), chunk.length); const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); - //this.push(buffer, undefined); this.logger.debug("TRANSFORM OUT", getHexString(buffer), "Size: ", buffer.length); this.push(buffer, undefined); callback(); diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index f08adab9d..59e157221 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -22,7 +22,9 @@ import { FrameData } from "./utils"; const PORT = 6660; const server = createServer(); - server.setTimeout(0); + server.on("timeout", (socket) => { + logger.warn("Server on timeout"); + }); server.requestTimeout = 0; server.on("connect", async (req: IncomingMessage, socket: Socket) => { @@ -32,14 +34,31 @@ import { FrameData } from "./utils"; socket .on("data", (data) => { - console.log("SERVER socket ondata", data); - console.log("SOCKET TX RX", socket.bytesWritten, socket.bytesRead) + logger.info("SERVER Carrier socket ondata", data); + logger.info("SOCKET Carrier TX RX", socket.bytesWritten, socket.bytesRead) + }) + .on("pipe", () => { + logger.info("Carrier Socket piped"); + }) + .on("unpipe", () => { + logger.info("Carrier Socket unpiped"); }) .on("pause", () => { - logger.info("Socket paused"); + //socket.resume(); + logger.fatal("Carrier Socket paused"); + //debugger; }) .on("resume", () => { - logger.info("Socket resumed"); + logger.info("Carrier Socket resumed"); + }) + .on("error", (error) => { + logger.error("Carrier Socket error", error); + }) + .on("close", () => { + logger.info("Carrier Socket closed"); + }) + .on("timeout", () => { + logger.info("Carrier Socket timeout"); }) const tcmux = new TeceMux(socket, "Server") @@ -47,8 +66,6 @@ import { FrameData } from "./utils"; logger.error("TCMUX err", err); }); - - tcmux.logger.pipe(logger); const channel = tcmux.multiplex(); @@ -59,9 +76,9 @@ import { FrameData } from "./utils"; process.stdin.pipe(channel); - const somePayload = "smth\n"; - logger.info("writing some payload to channel", somePayload); - channel.write(somePayload); + // const somePayload = "smth\n"; + // logger.info("writing some payload to channel", somePayload); + // channel.write(somePayload); for await (const chunk of channel) { const parsed = JSON.parse(chunk) as FrameData; @@ -77,6 +94,12 @@ import { FrameData } from "./utils"; const socket = createConnection({ port: PORT, allowHalfOpen: true, host: "0.0.0.0" }, () => {}); + await new Promise((resolve, reject) => { + socket + .on("connect", resolve) + .on("error", reject); + }); + socket.setNoDelay(true); const reqLogger = new ObjLogger("Req", { id: "Client"}); @@ -111,7 +134,7 @@ import { FrameData } from "./utils"; // } channel.on("data", async (chunk) => { - reqLogger.info("SERVER->CLIENT->CHANNEL", chunk.toString()); + reqLogger.info("SERVER->CLIENT->CHANNEL", channel._id, chunk.toString()); await new Promise((resolve, reject) => { setTimeout(() => { diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 92d024a49..6632e4d02 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -12,7 +12,20 @@ export class TeceMux extends TypedEmitter{ id: string; carrierSocket: Duplex; channelCount = 1; - decoder = new FrameDecoder().resume(); + decoder = new FrameDecoder({ emitClose: false }) + .on("pause", () => { + this.logger.warn("Decoder paused"); + }) + .on("close", () => { + this.logger.warn("Decoder closed"); + }) + .on("end", () => { + this.logger.warn("Decoder ended"); + }) + .on("error", (error) => { + this.logger.error("Decoder error", error); + //debugger; + }) channels: TeceMuxChannel[] = []; @@ -27,10 +40,9 @@ export class TeceMux extends TypedEmitter{ encoder.logger.updateBaseLog({ id: this.id }); encoder.logger.pipe(this.logger); - const w = new PassThrough({ encoding: undefined }).on("data", (d) => { this.logger.warn("channel writeable on DATA", d); }); - //w.pipe(process.stdout).pause() + const w = new PassThrough({ encoding: undefined, readableObjectMode: true }).on("data", (d) => { this.logger.warn("channel writeable on DATA", d); }); - w.pipe(encoder); + w.pipe(encoder).out.pipe(this.carrierSocket); const channel: TeceMuxChannel = Object.assign( Duplex.from({ @@ -43,9 +55,6 @@ export class TeceMux extends TypedEmitter{ } ); - //channel.pipe(encoder).pipe(this.carrierSocket); - //channel.pipe(process.stdout); - channel.on("error", (error) => { this.emit("error", { error, source: channel }) }); @@ -54,10 +63,6 @@ export class TeceMux extends TypedEmitter{ encoder.setChannel(destinationPort || this.channelCount); } - //channel.pipe(w); - //channel.pipe(this.carrierSocket); - encoder.out.pipe(this.carrierSocket); - return channel; } @@ -70,8 +75,9 @@ export class TeceMux extends TypedEmitter{ this.decoder.logger.updateBaseLog({ id: this.id }); this.decoder.logger.pipe(this.logger); - this.carrierSocket.pipe(this.decoder); - this.commonEncoder.out.pipe(this.carrierSocket); + this.carrierSocket.pipe(this.decoder, { end: false }); + this.carrierSocket.on("data", (data) => { this.logger.info("CARRIER DATA", data); }) + this.commonEncoder.out.pipe(this.carrierSocket, { end: false }); this.main().catch((error) => { this.emit("error", error); @@ -79,7 +85,7 @@ export class TeceMux extends TypedEmitter{ } async main() { - this.commonEncoder.logger.updateBaseLog({ id: "Commn" + this.commonEncoder.logger.baseLog.id }) + this.commonEncoder.logger.updateBaseLog({ id: "Comm" + this.commonEncoder.logger.baseLog.id }) this.commonEncoder.logger.pipe(this.logger); for await (const chunk of this.decoder) { @@ -129,8 +135,9 @@ export class TeceMux extends TypedEmitter{ if (emit) { this.emit("channel", channel); this.logger.debug("channel event emitted", channel._id); - this.channelCount++; } + + this.channelCount++; } multiplex(): TeceMuxChannel { From 34139379ff18c9320883760e40c73177e605cd0b Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 18 Jan 2023 13:41:26 +0000 Subject: [PATCH 072/231] Add handlers --- .../src/lib/tecemux/codecs/frame-decoder.ts | 2 +- .../src/lib/tecemux/playground-tecemux.ts | 17 +-- packages/verser/src/lib/tecemux/tecemux.ts | 103 ++++++++++++------ 3 files changed, 81 insertions(+), 41 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 803d27c57..c49f6cb42 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -11,7 +11,7 @@ export class FrameDecoder extends Transform { _streams = new Map(); constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { - super(Object.assign({}, opts, { writableObjectMode: true, readableObjectMode: true, emitClose: false })); + super(Object.assign({}, opts, { writableObjectMode: false, readableObjectMode: false })); this.buff = Buffer.alloc(64 * 1024, 0, undefined); //@TODO: optimize this.logger = new ObjLogger(params.name); diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index 59e157221..b38480821 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -20,7 +20,7 @@ import { FrameData } from "./utils"; /**********************************************/ const PORT = 6660; - const server = createServer(); + const server = createServer({}); server.on("timeout", (socket) => { logger.warn("Server on timeout"); @@ -33,10 +33,10 @@ import { FrameData } from "./utils"; logger.info("Incoming request", req.method, req.headers); socket - .on("data", (data) => { - logger.info("SERVER Carrier socket ondata", data); - logger.info("SOCKET Carrier TX RX", socket.bytesWritten, socket.bytesRead) - }) + // .on("data", (data) => { + // logger.info("SERVER Carrier socket ondata", data); + // logger.info("SOCKET Carrier TX RX", socket.bytesWritten, socket.bytesRead) + // }) .on("pipe", () => { logger.info("Carrier Socket piped"); }) @@ -60,6 +60,7 @@ import { FrameData } from "./utils"; .on("timeout", () => { logger.info("Carrier Socket timeout"); }) + .pause() const tcmux = new TeceMux(socket, "Server") .on("error", (err) => { @@ -81,8 +82,7 @@ import { FrameData } from "./utils"; // channel.write(somePayload); for await (const chunk of channel) { - const parsed = JSON.parse(chunk) as FrameData; - logger.debug(`Server on request data [C: ${parsed.sequenceNumber}, SN: ${parsed.sequenceNumber}]`, parsed.chunk, parsed.dataLength); + logger.debug(`reading CHANNEL chunk`, chunk.toString());// [C: ${parsed.sequenceNumber}, SN: ${parsed.sequenceNumber}]`, parsed.chunk, parsed.dataLength); }; }); @@ -133,12 +133,13 @@ import { FrameData } from "./utils"; // }); // } + //for await (const chunk of channel) { channel.on("data", async (chunk) => { reqLogger.info("SERVER->CLIENT->CHANNEL", channel._id, chunk.toString()); await new Promise((resolve, reject) => { setTimeout(() => { - //channel.encoder.write("Echo\n"); + channel.write("XEcho\n"); resolve(); }, 2000); }); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 6632e4d02..cc43ebe25 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -4,7 +4,6 @@ import { Duplex, PassThrough } from "stream"; import { Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; import { FrameData } from "./utils"; -import { IObjectLogger } from "@scramjet/types"; export type TeceMuxChannel = Duplex & { _id: number, encoder: FrameEncoder }; @@ -12,25 +11,12 @@ export class TeceMux extends TypedEmitter{ id: string; carrierSocket: Duplex; channelCount = 1; - decoder = new FrameDecoder({ emitClose: false }) - .on("pause", () => { - this.logger.warn("Decoder paused"); - }) - .on("close", () => { - this.logger.warn("Decoder closed"); - }) - .on("end", () => { - this.logger.warn("Decoder ended"); - }) - .on("error", (error) => { - this.logger.error("Decoder error", error); - //debugger; - }) + decoder: FrameDecoder; channels: TeceMuxChannel[] = []; - logger: IObjectLogger; - commonEncoder = new FrameEncoder(0).resume(); + logger: ObjLogger; + commonEncoder = new FrameEncoder(0); private createChannel(destinationPort?: number, emit?: boolean): TeceMuxChannel { this.logger.debug("Create Channel", destinationPort || this.channelCount); @@ -40,15 +26,31 @@ export class TeceMux extends TypedEmitter{ encoder.logger.updateBaseLog({ id: this.id }); encoder.logger.pipe(this.logger); - const w = new PassThrough({ encoding: undefined, readableObjectMode: true }).on("data", (d) => { this.logger.warn("channel writeable on DATA", d); }); + const w = new PassThrough({ encoding: undefined }) + + process.nextTick(() => { + w.on("data", (d) => { this.logger.warn("channel writeable on DATA", d); }); + }); w.pipe(encoder).out.pipe(this.carrierSocket); + const duplex = new Duplex({ + write: (chunk, encoding, next) => { + this.logger.trace("WRITE channel", channel._id, chunk ); + w.write(chunk); + next(); + }, + read: (size) => { + this.logger.trace("READ channel", channel._id ); + //setTimeout(() => (channel as unknown as Duplex).write("a"), 1000); + } + }); const channel: TeceMuxChannel = Object.assign( - Duplex.from({ - readable: new PassThrough({ encoding: undefined }).on("data", (d) => { this.logger.warn("channel readable on DATA", d); }), - writable: w, - } as unknown as Iterable) as TeceMuxChannel, + // Duplex.from({ + // readable: new PassThrough({ encoding: undefined }).on("data", (d) => { this.logger.warn("channel readable on DATA", d); }), + // writable: w, + // } as unknown as Iterable) as TeceMuxChannel, + duplex, { _id: destinationPort || this.channelCount, encoder @@ -56,8 +58,14 @@ export class TeceMux extends TypedEmitter{ ); channel.on("error", (error) => { - this.emit("error", { error, source: channel }) - }); + this.logger.error("CHANNEL ERROR", error) + //this.emit("error", { error, source: channel }) + }).on("destroy", () => { + this.logger.trace("channel on DESTROY ", channel._id ); + }) + .on("abort", () => { + this.logger.trace("channel on DESTROY ", channel._id ); + }) if (emit) { encoder.setChannel(destinationPort || this.channelCount); @@ -72,20 +80,48 @@ export class TeceMux extends TypedEmitter{ this.logger = new ObjLogger(this, { id: this.id }) this.carrierSocket = socket; + this.decoder = new FrameDecoder({ emitClose: false }) + .on("pause", () => { + this.logger.warn("Decoder paused"); + }) + .on("close", () => { + this.logger.warn("Decoder closed"); + }) + .on("end", () => { + this.logger.warn("Decoder ended"); + }) + .on("error", (error) => { + this.logger.error("Decoder error", error); + //debugger; + }) + .on("abort", (error) => { + this.logger.error("Decoder abort", error); + //debugger; + }) + .on("destroy", (error) => { + this.logger.error("Decoder destroy", error); + //debugger; + }); + this.decoder.logger.updateBaseLog({ id: this.id }); this.decoder.logger.pipe(this.logger); this.carrierSocket.pipe(this.decoder, { end: false }); - this.carrierSocket.on("data", (data) => { this.logger.info("CARRIER DATA", data); }) + + + process.nextTick(() => { + //this.carrierSocket.on("data", (data) => { this.logger.info("CARRIER DATA", data); }) + }); + this.commonEncoder.out.pipe(this.carrierSocket, { end: false }); - this.main().catch((error) => { + this.main().then(() => 1);/*.catch((error) => { this.emit("error", error); - }); + });*/ } async main() { - this.commonEncoder.logger.updateBaseLog({ id: "Comm" + this.commonEncoder.logger.baseLog.id }) + this.commonEncoder.logger.updateBaseLog({ id: "CMN " + this.logger.baseLog.id }) this.commonEncoder.logger.pipe(this.logger); for await (const chunk of this.decoder) { @@ -113,18 +149,21 @@ export class TeceMux extends TypedEmitter{ if (dataLength) { this.logger.warn("writing to channel [channel, length]", channel._id, dataLength); - channel.push(new Uint8Array(((frame.chunk as any).data) as any)); + + const written = channel.push(new Uint8Array(((frame.chunk as any).data) as any)); + + this.logger.info("Bytes written to channel [writeResult, channel, length]", written, destinationPort, dataLength); } } - this.logger.debug("Write acknowledge frame for sequenceNumber", sequenceNumber); - this.commonEncoder.out.write( + this.logger.debug("OFF Write acknowledge frame for sequenceNumber", sequenceNumber); + /*this.commonEncoder.push( this.commonEncoder.createFrame(undefined, { flagsArray: ["ACK"], sequenceNumber, destinationPort }) - ); + );*/ } } From 8b133ad65ca37dddff58829727dd2f34732be864 Mon Sep 17 00:00:00 2001 From: Budleigh Salterton Date: Wed, 18 Jan 2023 20:27:11 +0100 Subject: [PATCH 073/231] Ack on, alternate stdin to cli/srv --- .../verser/src/lib/tecemux/playground-tecemux.ts | 16 +++++++++------- packages/verser/src/lib/tecemux/tecemux.ts | 4 ++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index b38480821..fb2e026d1 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -75,7 +75,7 @@ import { FrameData } from "./utils"; logger.warn("Waiting for stdin..."); - process.stdin.pipe(channel); + DataStream.from(process.stdin).filter((x: Buffer) => (x[0] % 2 !== 0)).pipe(channel); // const somePayload = "smth\n"; // logger.info("writing some payload to channel", somePayload); @@ -133,16 +133,18 @@ import { FrameData } from "./utils"; // }); // } + DataStream.from(process.stdin).filter((x: Buffer) => (x[0] % 2 !== 1)).pipe(channel); + //for await (const chunk of channel) { channel.on("data", async (chunk) => { reqLogger.info("SERVER->CLIENT->CHANNEL", channel._id, chunk.toString()); - await new Promise((resolve, reject) => { - setTimeout(() => { - channel.write("XEcho\n"); - resolve(); - }, 2000); - }); + // await new Promise((resolve, reject) => { + // setTimeout(() => { + // channel.write("abcde\n"); + // resolve(); + // }, 2000); + // }); }) }); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index cc43ebe25..7b4eacb47 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -157,13 +157,13 @@ export class TeceMux extends TypedEmitter{ } this.logger.debug("OFF Write acknowledge frame for sequenceNumber", sequenceNumber); - /*this.commonEncoder.push( + this.commonEncoder.push( this.commonEncoder.createFrame(undefined, { flagsArray: ["ACK"], sequenceNumber, destinationPort }) - );*/ + ); } } From e6818eb86667ae7831505c788bec6843d090ed03 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 18 Jan 2023 22:05:00 +0000 Subject: [PATCH 074/231] Issue. Decoder receives more than 1 frame --- .../src/lib/tecemux/codecs/frame-decoder.ts | 48 +++++++++---------- .../src/lib/tecemux/codecs/frame-encoder.ts | 19 ++++++-- .../src/lib/tecemux/playground-tecemux.ts | 40 +++++++++------- packages/verser/src/lib/tecemux/tecemux.ts | 14 ++---- 4 files changed, 68 insertions(+), 53 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index c49f6cb42..124b6630d 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -11,7 +11,7 @@ export class FrameDecoder extends Transform { _streams = new Map(); constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { - super(Object.assign({}, opts, { writableObjectMode: false, readableObjectMode: false })); + super(Object.assign({}, opts, { writableObjectMode: true, readableObjectMode: true, readableHighWaterMark: 2 })); this.buff = Buffer.alloc(64 * 1024, 0, undefined); //@TODO: optimize this.logger = new ObjLogger(params.name); @@ -36,33 +36,33 @@ export class FrameDecoder extends Transform { if (Buffer.isBuffer(chunk)) { chunk.copy(this.buff, this.size, 0, chunk.length); + } else { + this.emit("error", "Chunk is not a buffer"); + callback(); } this.size += chunk.length; - if (this.size >= 10 ) {//&& this.buff.readInt32LE(10) === this.size) { - const frameSize = this.buff.readInt32LE(10); - - const payload = { - sourceAddress: [this.buff.readInt8(0), this.buff.readInt8(1), this.buff.readInt8(2), this.buff.readInt8(3)], - destinationAddress: [this.buff.readInt8(4), this.buff.readInt8(5), this.buff.readInt8(6), this.buff.readInt8(7)], - chunk: this.buff.subarray(32, this.buff.readInt32LE(10)), - flags: parseFlags(this.buff.readInt8(25)), - sourcePort: this.buff.readInt16LE(12), - destinationPort: this.buff.readInt16LE(14), - dataLength: frameSize - HEADER_LENGTH, - chunkLength: frameSize, - sequenceNumber: this.buff.readInt32LE(16), - stringified: this.buff.subarray(32, frameSize).toString() - } as Partial; - - this.push(JSON.stringify(payload) + "\n"); - - this.size = 0; - this.buff.fill(0); - } else { - this.logger.error("too few data", this.size, this.buff.readInt32LE(10)); - } + const frameSize = this.buff.readInt32LE(10); + + const payload = { + sourceAddress: [this.buff.readInt8(0), this.buff.readInt8(1), this.buff.readInt8(2), this.buff.readInt8(3)], + destinationAddress: [this.buff.readInt8(4), this.buff.readInt8(5), this.buff.readInt8(6), this.buff.readInt8(7)], + chunk: this.buff.subarray(32, this.buff.readInt32LE(10)), + flags: parseFlags(this.buff.readInt8(25)), + sourcePort: this.buff.readInt16LE(12), + destinationPort: this.buff.readInt16LE(14), + dataLength: frameSize - HEADER_LENGTH, + chunkLength: frameSize, + sequenceNumber: this.buff.readInt32LE(16), + stringified: this.buff.subarray(32, frameSize).toString() + } as Partial; + + this.push(JSON.stringify(payload) + "\n"); + + this.size = 0; + this.buff.fill(0); + callback(); } catch(err) { diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index c3c0dd571..1a5f8c63c 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -25,8 +25,12 @@ export class FrameEncoder extends Transform { constructor(private frameTarget: FrameTarget, opts: TransformOptions = {}, params: { name: string } = { name: "FrameEncoder" }) { //opts.emitClose = false; - - super(opts); + //opts.readableObjectMode = true; + super(Object.assign(opts, { + writableObjectMode: true, + readableObjectMode: true, + readableHighWaterMark: 0 + })); this.logger = new ObjLogger(params.name, { id: this.frameTarget.toString() }); @@ -111,7 +115,16 @@ export class FrameEncoder extends Transform { const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); this.logger.debug("TRANSFORM OUT", getHexString(buffer), "Size: ", buffer.length); - this.push(buffer, undefined); + + if (!this.push(buffer, undefined)) { + this.once("drain", () => { + this.push(buffer, undefined); + }); + }; + this.read(0); + callback(); } + + } diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index fb2e026d1..1c8b939c3 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -69,21 +69,30 @@ import { FrameData } from "./utils"; tcmux.logger.pipe(logger); - const channel = tcmux.multiplex(); + const channel1 = tcmux.multiplex(); + const channel2 = tcmux.multiplex(); req.on("pause", () => { logger.warn("Request paused"); }); logger.warn("Waiting for stdin..."); - DataStream.from(process.stdin).filter((x: Buffer) => (x[0] % 2 !== 0)).pipe(channel); - + process.stdin.pipe(channel1); + process.stdin.pipe(channel2); // const somePayload = "smth\n"; // logger.info("writing some payload to channel", somePayload); // channel.write(somePayload); - for await (const chunk of channel) { - logger.debug(`reading CHANNEL chunk`, chunk.toString());// [C: ${parsed.sequenceNumber}, SN: ${parsed.sequenceNumber}]`, parsed.chunk, parsed.dataLength); - }; + (async () => { + for await (const chunk of channel1) { + logger.debug(`reading CHANNEL1 chunk`, chunk.toString());// [C: ${parsed.sequenceNumber}, SN: ${parsed.sequenceNumber}]`, parsed.chunk, parsed.dataLength); + }; + })(); + + (async () => { + for await (const chunk of channel2) { + logger.debug(`reading CHANNEL2 chunk`, chunk.toString());// [C: ${parsed.sequenceNumber}, SN: ${parsed.sequenceNumber}]`, parsed.chunk, parsed.dataLength); + }; + })(); }); server.listen(PORT, "0.0.0.0"); @@ -133,19 +142,18 @@ import { FrameData } from "./utils"; // }); // } - DataStream.from(process.stdin).filter((x: Buffer) => (x[0] % 2 !== 1)).pipe(channel); + //DataStream.from(process.stdin).filter((x: Buffer) => (x[0] % 2 !== 1)).pipe(channel); - //for await (const chunk of channel) { - channel.on("data", async (chunk) => { + for await (const chunk of channel) { reqLogger.info("SERVER->CLIENT->CHANNEL", channel._id, chunk.toString()); - // await new Promise((resolve, reject) => { - // setTimeout(() => { - // channel.write("abcde\n"); - // resolve(); - // }, 2000); - // }); - }) + await new Promise((resolve, reject) => { + setTimeout(() => { + channel.write("abcde\n"); + resolve(); + }, 2000); + }); + } }); socket.on("error", (err) => { diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 7b4eacb47..2c17754de 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -108,16 +108,11 @@ export class TeceMux extends TypedEmitter{ this.carrierSocket.pipe(this.decoder, { end: false }); - - process.nextTick(() => { - //this.carrierSocket.on("data", (data) => { this.logger.info("CARRIER DATA", data); }) - }); - this.commonEncoder.out.pipe(this.carrierSocket, { end: false }); - this.main().then(() => 1);/*.catch((error) => { + this.main().catch((error) => { this.emit("error", error); - });*/ + }); } async main() { @@ -133,7 +128,7 @@ export class TeceMux extends TypedEmitter{ if (flags.ACK) { this.logger.trace("ACKNOWLEDGE frame received for sequenceNumber", sequenceNumber); // acknowledge received (confirm packet) - return; + continue; } if (flags.PSH) { @@ -156,7 +151,7 @@ export class TeceMux extends TypedEmitter{ } } - this.logger.debug("OFF Write acknowledge frame for sequenceNumber", sequenceNumber); + this.logger.debug("Write acknowledge frame for sequenceNumber", sequenceNumber); this.commonEncoder.push( this.commonEncoder.createFrame(undefined, { flagsArray: ["ACK"], @@ -187,7 +182,6 @@ export class TeceMux extends TypedEmitter{ this.addChannel(channel, false); this.logger.trace("Multiplex ready", channel._id); - this.channelCount++; return channel; } } From 97f0900ae27904738c0add547630d35cbe8d6931 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 18 Jan 2023 22:53:34 +0000 Subject: [PATCH 075/231] Read buffer of frames to decode --- .../src/lib/tecemux/codecs/frame-decoder.ts | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 124b6630d..bc4c3a936 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -13,7 +13,7 @@ export class FrameDecoder extends Transform { constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { super(Object.assign({}, opts, { writableObjectMode: true, readableObjectMode: true, readableHighWaterMark: 2 })); - this.buff = Buffer.alloc(64 * 1024, 0, undefined); //@TODO: optimize + this.buff = Buffer.alloc(0);// Buffer.alloc(64 * 1024, 0, undefined); //@TODO: optimize this.logger = new ObjLogger(params.name); this.on("pipe", () => { @@ -25,7 +25,6 @@ export class FrameDecoder extends Transform { _transform(chunk: Buffer, encoding: BufferEncoding, callback: TransformCallback): void { try { - this.logger.trace("Decoding frame...", toHex(chunk), "Size:", chunk.length); if (!Buffer.isBuffer(chunk)) { this.push(JSON.stringify({ error: "not a buffer" }), undefined); @@ -35,15 +34,24 @@ export class FrameDecoder extends Transform { } if (Buffer.isBuffer(chunk)) { - chunk.copy(this.buff, this.size, 0, chunk.length); + this.buff = Buffer.concat([this.buff, chunk]); + //chunk.copy(this.buff, this.size, 0, chunk.length); } else { + this.logger.error("Decoding buffer...", chunk); this.emit("error", "Chunk is not a buffer"); callback(); } - this.size += chunk.length; + this.logger.trace("Decoding buffer...", toHex(this.buff), "Size:", this.buff.length); - const frameSize = this.buff.readInt32LE(10); + let frameSize = 0; + + if (this.buff.length >= HEADER_LENGTH) { + frameSize = this.buff.readInt32LE(10); + } else { + this.logger.trace("To few data"); + callback(); + } const payload = { sourceAddress: [this.buff.readInt8(0), this.buff.readInt8(1), this.buff.readInt8(2), this.buff.readInt8(3)], @@ -60,11 +68,16 @@ export class FrameDecoder extends Transform { this.push(JSON.stringify(payload) + "\n"); - this.size = 0; - this.buff.fill(0); + this.buff = this.buff.subarray(frameSize); + if (this.buff.length === 0) { + this.logger.info("No remaining data!") + callback(); + return; + } - callback(); + this.logger.trace("More than one frame in chunk. processing", this.buff.length); + this._transform(Buffer.alloc(0), encoding, callback); } catch(err) { this.logger.error("ERROR", err) } From 6556e1a5cb45e52182bb535316b775e4030573b9 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 19 Jan 2023 13:07:03 +0000 Subject: [PATCH 076/231] Cleanup, common sequenceNumber for TeceMux instance --- .../src/lib/tecemux/codecs/frame-decoder.ts | 42 ++++--- .../src/lib/tecemux/codecs/frame-encoder.ts | 10 +- .../verser/src/lib/tecemux/codecs/index.ts | 3 +- .../src/lib/tecemux/playground-tecemux.ts | 29 +---- packages/verser/src/lib/tecemux/playground.ts | 103 ------------------ packages/verser/src/lib/tecemux/tecemux.ts | 17 +-- 6 files changed, 36 insertions(+), 168 deletions(-) delete mode 100644 packages/verser/src/lib/tecemux/playground.ts diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index bc4c3a936..42bcc12a5 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -4,8 +4,7 @@ import { FrameData, HEADER_LENGTH, toHex } from "../utils"; import { parseFlags } from "."; export class FrameDecoder extends Transform { - buff: Buffer; - size = 0; + buffer: Buffer; logger: ObjLogger; _streams = new Map(); @@ -13,7 +12,7 @@ export class FrameDecoder extends Transform { constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { super(Object.assign({}, opts, { writableObjectMode: true, readableObjectMode: true, readableHighWaterMark: 2 })); - this.buff = Buffer.alloc(0);// Buffer.alloc(64 * 1024, 0, undefined); //@TODO: optimize + this.buffer = Buffer.alloc(0); this.logger = new ObjLogger(params.name); this.on("pipe", () => { @@ -25,7 +24,6 @@ export class FrameDecoder extends Transform { _transform(chunk: Buffer, encoding: BufferEncoding, callback: TransformCallback): void { try { - if (!Buffer.isBuffer(chunk)) { this.push(JSON.stringify({ error: "not a buffer" }), undefined); callback(); @@ -34,52 +32,52 @@ export class FrameDecoder extends Transform { } if (Buffer.isBuffer(chunk)) { - this.buff = Buffer.concat([this.buff, chunk]); - //chunk.copy(this.buff, this.size, 0, chunk.length); + this.buffer = Buffer.concat([this.buffer, chunk]); } else { this.logger.error("Decoding buffer...", chunk); this.emit("error", "Chunk is not a buffer"); callback(); } - this.logger.trace("Decoding buffer...", toHex(this.buff), "Size:", this.buff.length); + this.logger.trace("Decoding buffer...", toHex(this.buffer), "Size:", this.buffer.length); let frameSize = 0; - if (this.buff.length >= HEADER_LENGTH) { - frameSize = this.buff.readInt32LE(10); + if (this.buffer.length >= HEADER_LENGTH) { + frameSize = this.buffer.readInt32LE(10); } else { this.logger.trace("To few data"); callback(); } const payload = { - sourceAddress: [this.buff.readInt8(0), this.buff.readInt8(1), this.buff.readInt8(2), this.buff.readInt8(3)], - destinationAddress: [this.buff.readInt8(4), this.buff.readInt8(5), this.buff.readInt8(6), this.buff.readInt8(7)], - chunk: this.buff.subarray(32, this.buff.readInt32LE(10)), - flags: parseFlags(this.buff.readInt8(25)), - sourcePort: this.buff.readInt16LE(12), - destinationPort: this.buff.readInt16LE(14), + sourceAddress: [this.buffer.readInt8(0), this.buffer.readInt8(1), this.buffer.readInt8(2), this.buffer.readInt8(3)], + destinationAddress: [this.buffer.readInt8(4), this.buffer.readInt8(5), this.buffer.readInt8(6), this.buffer.readInt8(7)], + chunk: this.buffer.subarray(32, this.buffer.readInt32LE(10)), + flags: parseFlags(this.buffer.readInt8(25)), + sourcePort: this.buffer.readInt16LE(12), + destinationPort: this.buffer.readInt16LE(14), dataLength: frameSize - HEADER_LENGTH, chunkLength: frameSize, - sequenceNumber: this.buff.readInt32LE(16), - stringified: this.buff.subarray(32, frameSize).toString() + sequenceNumber: this.buffer.readInt32LE(16), + stringified: this.buffer.subarray(32, frameSize).toString() } as Partial; this.push(JSON.stringify(payload) + "\n"); - this.buff = this.buff.subarray(frameSize); + this.buffer = this.buffer.subarray(frameSize); - if (this.buff.length === 0) { + if (this.buffer.length === 0) { this.logger.info("No remaining data!") callback(); return; } - this.logger.trace("More than one frame in chunk. processing", this.buff.length); + this.logger.trace("More than one frame in chunk. processing", this.buffer.length); this._transform(Buffer.alloc(0), encoding, callback); - } catch(err) { - this.logger.error("ERROR", err) + } catch(error) { + this.logger.error("ERROR", error); + this.emit("error", error); } } } diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index 1a5f8c63c..f4b7483fd 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -2,9 +2,10 @@ import { PassThrough, Transform, TransformCallback, TransformOptions } from "str import { FrameData, FrameTarget, HEADER_LENGTH, toHex as getHexString, toHex } from "../utils"; import { ObjLogger } from "@scramjet/obj-logger"; import { binaryFlags, frameFlags } from "."; +import { TeceMux } from "../tecemux"; export class FrameEncoder extends Transform { - sequenceNumber = 0; + tecemux: TeceMux; logger = new ObjLogger("FrameEncoder",); out = new PassThrough({ readableObjectMode: true }) .on("data", (data) => { @@ -21,9 +22,9 @@ export class FrameEncoder extends Transform { }) .on("error", (error) => { this.logger.error("output to socket paused", error); - }) + }).pause(); - constructor(private frameTarget: FrameTarget, opts: TransformOptions = {}, params: { name: string } = { name: "FrameEncoder" }) { + constructor(private frameTarget: FrameTarget, tecemux: TeceMux, opts: TransformOptions = {}, params: { name: string } = { name: "FrameEncoder" }) { //opts.emitClose = false; //opts.readableObjectMode = true; super(Object.assign(opts, { @@ -32,6 +33,7 @@ export class FrameEncoder extends Transform { readableHighWaterMark: 0 })); + this.tecemux = tecemux; this.logger = new ObjLogger(params.name, { id: this.frameTarget.toString() }); this.on("pipe", () => { @@ -84,7 +86,7 @@ export class FrameEncoder extends Transform { new Uint8Array(new Uint16Array([0, frame.destinationPort || this.frameTarget]).buffer), // 128: sequenceNumber(32 bit, acnowledge number) 16 - 19 - new Uint8Array(new Uint32Array([frame.sequenceNumber || this.sequenceNumber++]).buffer), + new Uint8Array(new Uint32Array([this.tecemux.sequenceNumber++]).buffer), // 160: Acknowledgement number 20-23 new Uint8Array(new Uint32Array([frame.acknowledgeNumber || 0]).buffer), diff --git a/packages/verser/src/lib/tecemux/codecs/index.ts b/packages/verser/src/lib/tecemux/codecs/index.ts index de478859d..245e3e9fa 100644 --- a/packages/verser/src/lib/tecemux/codecs/index.ts +++ b/packages/verser/src/lib/tecemux/codecs/index.ts @@ -8,7 +8,6 @@ export type TeceMuxEvents = { error(error: any): void; } -export const frameFlags = ["FIN", "SYN", "RST", "PSH", "ACK", "URG", "ECE", "CWR"]; export const binaryFlags = { FIN: 0b00000001, SYN: 0b00000010, @@ -20,6 +19,8 @@ export const binaryFlags = { CWR: 0b10000000 } +export const frameFlags = Object.keys(binaryFlags); + export type flagsObjectType = Partial<{ FIN: boolean, SYN: boolean, diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index 1c8b939c3..fe3ec0cf1 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -8,7 +8,6 @@ import { DataStream } from "scramjet"; import { Socket, createConnection } from "net"; import { TeceMux, TeceMuxChannel } from "./tecemux"; -import { FrameData } from "./utils"; (async () => { const logger = new ObjLogger("Sandbox"); @@ -33,10 +32,6 @@ import { FrameData } from "./utils"; logger.info("Incoming request", req.method, req.headers); socket - // .on("data", (data) => { - // logger.info("SERVER Carrier socket ondata", data); - // logger.info("SOCKET Carrier TX RX", socket.bytesWritten, socket.bytesRead) - // }) .on("pipe", () => { logger.info("Carrier Socket piped"); }) @@ -76,21 +71,18 @@ import { FrameData } from "./utils"; logger.warn("Waiting for stdin..."); - process.stdin.pipe(channel1); - process.stdin.pipe(channel2); - // const somePayload = "smth\n"; - // logger.info("writing some payload to channel", somePayload); - // channel.write(somePayload); + DataStream.from(process.stdin).filter((x: Buffer) => !(parseInt(x[0].toString()) % 2)).pipe(channel1); + DataStream.from(process.stdin).filter((x: Buffer) => !!(parseInt(x[0].toString()) % 2)).pipe(channel2); (async () => { for await (const chunk of channel1) { - logger.debug(`reading CHANNEL1 chunk`, chunk.toString());// [C: ${parsed.sequenceNumber}, SN: ${parsed.sequenceNumber}]`, parsed.chunk, parsed.dataLength); + logger.debug(`reading CHANNEL1 chunk`, chunk.toString()); }; })(); (async () => { for await (const chunk of channel2) { - logger.debug(`reading CHANNEL2 chunk`, chunk.toString());// [C: ${parsed.sequenceNumber}, SN: ${parsed.sequenceNumber}]`, parsed.chunk, parsed.dataLength); + logger.debug(`reading CHANNEL2 chunk`, chunk.toString()); }; })(); }); @@ -131,19 +123,6 @@ import { FrameData } from "./utils"; tcmux.on("channel", async (channel: TeceMuxChannel) => { reqLogger.debug("New channel", channel._id); - // for await (const chunk of channel) { - // reqLogger.info("Data from server", chunk.toString()); - - // await new Promise((resolve, reject) => { - // setTimeout(() => { - // //channel.encoder.write("Echo\n"); - // resolve(); - // }, 2000); - // }); - // } - - //DataStream.from(process.stdin).filter((x: Buffer) => (x[0] % 2 !== 1)).pipe(channel); - for await (const chunk of channel) { reqLogger.info("SERVER->CLIENT->CHANNEL", channel._id, chunk.toString()); diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/src/lib/tecemux/playground.ts deleted file mode 100644 index 81ce13e25..000000000 --- a/packages/verser/src/lib/tecemux/playground.ts +++ /dev/null @@ -1,103 +0,0 @@ -/*eslint no-unused-vars: ["error", { "args": "none" }]*/ -/* eslint-disable @typescript-eslint/no-floating-promises */ -/* eslint-disable no-console */ - -import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; -import { IncomingMessage, createServer, request } from "http"; -import { DataStream } from "scramjet"; -import { FrameData, FrameTarget } from "./utils"; -import { FrameDecoder, FrameEncoder } from "./codecs"; -import { Socket } from "net"; - -(async () => { - const logger = new ObjLogger("Sandbox"); - - logger.pipe(new DataStream().map(prettyPrint({ colors: true }))).pipe(process.stdout); - - const PORT = 6660; - const server = createServer(); - - server.setTimeout(0); - server.requestTimeout = 0; - - server.on("connect", (req: IncomingMessage, socket: Socket) => { - socket.setNoDelay(true); - socket.write(`HTTP/1.1 ${200} \r\n\r\n`); - - logger.debug("on connect"); - - const serverFrameEncoder = new FrameEncoder(FrameTarget.API, { encoding: undefined }, { name: "ServerEncoder" }); - - serverFrameEncoder.logger.pipe(logger); - - serverFrameEncoder.pipe(socket); - let i = 0; - - req.on("pause", () => { logger.warn("Request paused"); }); - - const serverFrameDecoder = new FrameDecoder({ highWaterMark: 60 * 1024, encoding: undefined, emitClose: false }, { name: "ServerDecoder" }); - - serverFrameDecoder.logger.pipe(logger); - - socket.pipe(serverFrameDecoder).pipe(process.stdout); - - serverFrameDecoder.on("data", (d: any) => { - const parsed = JSON.parse(d) as FrameData; - - logger.debug(`Server on request data [${i++}]`, parsed.chunk, parsed.dataLength); - serverFrameEncoder.write(Buffer.from(new Uint8Array([255, 255, 255, 255]))); - }); - }); - - server.listen(PORT); - - const req = request({ - hostname: "0.0.0.0", - method: "connect", - port: PORT, - headers: { "Content-Type": "application/octet-stream", "Transfer-Encoding": "chunked" } - }); - - req.on("connect", (response, socket, head) => { - socket.setNoDelay(true); - - logger.debug("Response. Head:", head.toString()); - - let i = 0; - let m = 0; - - const frameEncoder = new FrameEncoder(FrameTarget.API, { encoding: undefined, emitClose: false }, { name: "RequestEncoder" }); - - frameEncoder.logger.pipe(logger); - - frameEncoder.pipe(socket); - - const responseFrameDecoder = new FrameDecoder({ highWaterMark: 60 * 1024, encoding: undefined, emitClose: false }, { name: "RequestDecoder" }); - - responseFrameDecoder.logger.pipe(logger); - - socket.pipe( - responseFrameDecoder - ).on("data", (d) => { - const parsed = JSON.parse(d) as FrameData; - - logger.debug(`Echo from server [${i++}]`, parsed.chunk, parsed.dataLength, parsed.chunkLength); - }); - - response.on("data", (d) => console.log("plain response", d)); - - req.on("error", (err) => { - console.error(err); - }); - - setInterval(async () => { - logger.debug(`Writing [${++m}]..`); - - frameEncoder.write(Buffer.from(new Uint8Array([0x61, 0x62, 0x63, 0x64]))); - }, 500); - }); - - req.flushHeaders(); - - await new Promise((_res, _rej) => {}); -})(); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 2c17754de..c159ba7e5 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -12,16 +12,16 @@ export class TeceMux extends TypedEmitter{ carrierSocket: Duplex; channelCount = 1; decoder: FrameDecoder; - + sequenceNumber = 0; channels: TeceMuxChannel[] = []; logger: ObjLogger; - commonEncoder = new FrameEncoder(0); + commonEncoder = new FrameEncoder(0, this); private createChannel(destinationPort?: number, emit?: boolean): TeceMuxChannel { this.logger.debug("Create Channel", destinationPort || this.channelCount); - const encoder = new FrameEncoder(this.channelCount, { encoding: undefined }); + const encoder = new FrameEncoder(this.channelCount, this, { encoding: undefined }); encoder.logger.updateBaseLog({ id: this.id }); encoder.logger.pipe(this.logger); @@ -42,14 +42,9 @@ export class TeceMux extends TypedEmitter{ }, read: (size) => { this.logger.trace("READ channel", channel._id ); - //setTimeout(() => (channel as unknown as Duplex).write("a"), 1000); } }); const channel: TeceMuxChannel = Object.assign( - // Duplex.from({ - // readable: new PassThrough({ encoding: undefined }).on("data", (d) => { this.logger.warn("channel readable on DATA", d); }), - // writable: w, - // } as unknown as Iterable) as TeceMuxChannel, duplex, { _id: destinationPort || this.channelCount, @@ -59,7 +54,7 @@ export class TeceMux extends TypedEmitter{ channel.on("error", (error) => { this.logger.error("CHANNEL ERROR", error) - //this.emit("error", { error, source: channel }) + this.emit("error", { error, source: channel }) }).on("destroy", () => { this.logger.trace("channel on DESTROY ", channel._id ); }) @@ -92,15 +87,12 @@ export class TeceMux extends TypedEmitter{ }) .on("error", (error) => { this.logger.error("Decoder error", error); - //debugger; }) .on("abort", (error) => { this.logger.error("Decoder abort", error); - //debugger; }) .on("destroy", (error) => { this.logger.error("Decoder destroy", error); - //debugger; }); this.decoder.logger.updateBaseLog({ id: this.id }); @@ -127,7 +119,6 @@ export class TeceMux extends TypedEmitter{ if (flags.ACK) { this.logger.trace("ACKNOWLEDGE frame received for sequenceNumber", sequenceNumber); - // acknowledge received (confirm packet) continue; } From 7a6512e1beb4ed9c5a0e02a88f10065b827d6cdd Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 19 Jan 2023 19:57:07 +0000 Subject: [PATCH 077/231] Send FIN --- .../src/lib/tecemux/codecs/frame-encoder.ts | 33 ++++-- .../src/lib/tecemux/playground-tecemux.ts | 64 +++++++++-- packages/verser/src/lib/tecemux/tecemux.ts | 105 +++++++++++++----- 3 files changed, 154 insertions(+), 48 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index f4b7483fd..fc7bc9643 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -7,22 +7,26 @@ import { TeceMux } from "../tecemux"; export class FrameEncoder extends Transform { tecemux: TeceMux; logger = new ObjLogger("FrameEncoder",); - out = new PassThrough({ readableObjectMode: true }) + out = Object.assign(new PassThrough({ readableObjectMode: true }) .on("data", (data) => { this.logger.trace("output to socket: " + (data.length == HEADER_LENGTH ? "HEADER ONLY" : ""), toHex(data), data.length, this.readableFlowing); }) .on("pause", () => { - this.logger.trace("output to socket paused"); + this.logger.trace("output to socket paused!"); }) .on("end", () => { - this.logger.trace("output to socket ended"); + this.logger.trace("output to socket ended!", this.frameTarget); + //this.tecemux.sendFIN(this.frameTarget); }) .on("resume", () => { this.logger.trace("output to socket resumed"); }) .on("error", (error) => { - this.logger.error("output to socket paused", error); - }).pause(); + this.logger.error("output to socket error", error); + }).pause() + ,{ + _id: this.frameTarget + }); constructor(private frameTarget: FrameTarget, tecemux: TeceMux, opts: TransformOptions = {}, params: { name: string } = { name: "FrameEncoder" }) { //opts.emitClose = false; @@ -36,9 +40,14 @@ export class FrameEncoder extends Transform { this.tecemux = tecemux; this.logger = new ObjLogger(params.name, { id: this.frameTarget.toString() }); - this.on("pipe", () => { - this.logger.debug("onPipe"); - }); + this + .on("pipe", () => { + this.logger.debug("onPipe"); + }) + .on("end", () => { + this.logger.debug("onEnd"); + }) + this.pipe(this.out); } @@ -64,6 +73,14 @@ export class FrameEncoder extends Transform { })); } + onChannelEnd(channelId: number) { + this.logger.debug("sending FIN for channel", channelId); + this.out.write(this.createFrame([], { + flagsArray: ["FIN"], + destinationPort: channelId + })); + } + getChecksum() { return 255; } diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index fe3ec0cf1..8054f1ce9 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -12,7 +12,18 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; (async () => { const logger = new ObjLogger("Sandbox"); - logger.pipe(new DataStream().map(prettyPrint({ colors: true }))).pipe(process.stdout); + logger + .pipe( + new DataStream() + .map(prettyPrint({ colors: true })) + .map((chunk: string) => ( + chunk.replace( + /(:?FIN|SYN|RST|PSH|ACK|URG|ECE|CWR)|^$]/, + "\x1b[41m" + "$&" + "\x1b[0m" + ) + )) + ) + .pipe(process.stdout); /**********************************************/ /* SERVER @@ -35,8 +46,8 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; .on("pipe", () => { logger.info("Carrier Socket piped"); }) - .on("unpipe", () => { - logger.info("Carrier Socket unpiped"); + .on("unpipe", (c: any) => { + logger.info("Carrier Socket unpiped", c); }) .on("pause", () => { //socket.resume(); @@ -75,16 +86,37 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; DataStream.from(process.stdin).filter((x: Buffer) => !!(parseInt(x[0].toString()) % 2)).pipe(channel2); (async () => { - for await (const chunk of channel1) { - logger.debug(`reading CHANNEL1 chunk`, chunk.toString()); - }; + try { + for await (const chunk of channel1) { + console.log("CHHUNK", chunk); + logger.debug(`reading CHANNEL1 chunk`, chunk.toString()); + }; + } catch (error) { + logger.error(`reading CHANNEL1 ERROR`, error); + } + + logger.debug(`reading CHANNEL1 END`); })(); (async () => { - for await (const chunk of channel2) { - logger.debug(`reading CHANNEL2 chunk`, chunk.toString()); - }; + try { + for await (const chunk of channel2) { + logger.debug(`reading CHANNEL2 chunk`, chunk.toString()); + }; + } catch (error) { + logger.error(`reading CHANNEL2 ERROR`, error); + } + + logger.debug(`reading CHANNEL2 END`); })(); + + + setTimeout(() => { + console.log("\n\n\n\n"); + logger.trace("Ending channels"); + channel1.push(null); + //channel2.end(); + }, 4000); }); server.listen(PORT, "0.0.0.0"); @@ -123,12 +155,24 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; tcmux.on("channel", async (channel: TeceMuxChannel) => { reqLogger.debug("New channel", channel._id); + channel + .on("finish", () => { + tcmux.logger.info("Channel finish", channel._id) + }) + .on("end", () => { + tcmux.logger.info("Channel end", channel._id) + }); + for await (const chunk of channel) { + console.log(chunk); + if (chunk === null) { + break; + } reqLogger.info("SERVER->CLIENT->CHANNEL", channel._id, chunk.toString()); await new Promise((resolve, reject) => { setTimeout(() => { - channel.write("abcde\n"); + //channel.write("abcde\n"); resolve(); }, 2000); }); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index c159ba7e5..aa9d24a05 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -5,7 +5,11 @@ import { Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; import { FrameData } from "./utils"; -export type TeceMuxChannel = Duplex & { _id: number, encoder: FrameEncoder }; +export type TeceMuxChannel = Duplex & { + _id: number, + encoder: FrameEncoder, + closedByFIN: boolean +}; export class TeceMux extends TypedEmitter{ id: string; @@ -26,41 +30,55 @@ export class TeceMux extends TypedEmitter{ encoder.logger.updateBaseLog({ id: this.id }); encoder.logger.pipe(this.logger); - const w = new PassThrough({ encoding: undefined }) + const w = new PassThrough({ encoding: undefined, allowHalfOpen: true }); - process.nextTick(() => { - w.on("data", (d) => { this.logger.warn("channel writeable on DATA", d); }); - }); - - w.pipe(encoder).out.pipe(this.carrierSocket); + w.pipe(encoder, { end: false }).out.pipe(this.carrierSocket, { end: false }); const duplex = new Duplex({ write: (chunk, encoding, next) => { - this.logger.trace("WRITE channel", channel._id, chunk ); - w.write(chunk); - next(); + this.logger.trace("WRITE channel", channel._id, chunk, chunk.toString() ); + + if (!w.push(chunk)) { + w.once("drain", next); + } else { + next(); + } }, read: (size) => { this.logger.trace("READ channel", channel._id ); - } + }, + allowHalfOpen: true }); const channel: TeceMuxChannel = Object.assign( duplex, { _id: destinationPort || this.channelCount, - encoder + encoder, + closedByFIN: false } ); - channel.on("error", (error) => { - this.logger.error("CHANNEL ERROR", error) - this.emit("error", { error, source: channel }) - }).on("destroy", () => { + process.nextTick(() => { + w.on("data", (d) => { this.logger.warn("channel writeable on DATA", d); }); + }); + + channel + .on("error", (error) => { + this.logger.error("CHANNEL ERROR", error) + //this.emit("error", { error, source: channel }) + }) + .on("destroy", () => { this.logger.trace("channel on DESTROY ", channel._id ); }) .on("abort", () => { - this.logger.trace("channel on DESTROY ", channel._id ); + this.logger.trace("channel on ABORT ", channel._id ); }) + .on("end", () => { + this.logger.info("CHANNEL end", channel._id); + if (!channel.closedByFIN) { + this.sendFIN(channel._id); + } + }); if (emit) { encoder.setChannel(destinationPort || this.channelCount); @@ -102,29 +120,42 @@ export class TeceMux extends TypedEmitter{ this.commonEncoder.out.pipe(this.carrierSocket, { end: false }); + this.commonEncoder.logger.updateBaseLog({ id }) + this.commonEncoder.logger.pipe(this.logger); + this.main().catch((error) => { this.emit("error", error); }); } async main() { - this.commonEncoder.logger.updateBaseLog({ id: "CMN " + this.logger.baseLog.id }) - this.commonEncoder.logger.pipe(this.logger); - for await (const chunk of this.decoder) { this.logger.debug("Decoded", JSON.parse(chunk)); const frame = JSON.parse(chunk) as FrameData; const { flags, sequenceNumber, dataLength, destinationPort } = frame; + let channel = this.channels[destinationPort] + if (flags.ACK) { - this.logger.trace("ACKNOWLEDGE frame received for sequenceNumber", sequenceNumber); + this.logger.trace("ACK frame received for sequenceNumber", sequenceNumber); continue; } + if (flags.FIN) { + this.logger.trace(`Received FIN command [C: ${destinationPort}]`, dataLength, frame.chunk, !!this.channels[destinationPort]); + + if (channel) { + //channel.end(); + channel.closedByFIN = true; + channel.push(null) + } else { + this.logger.error("FIN for unknown channel"); + } + } + if (flags.PSH) { this.logger.trace(`Received PSH command [C: ${destinationPort}]`, dataLength, frame.chunk, !!this.channels[destinationPort]); - let channel = this.channels[destinationPort] if (!channel) { this.logger.warn("Unknown channel"); @@ -142,17 +173,21 @@ export class TeceMux extends TypedEmitter{ } } - this.logger.debug("Write acknowledge frame for sequenceNumber", sequenceNumber); - this.commonEncoder.push( - this.commonEncoder.createFrame(undefined, { - flagsArray: ["ACK"], - sequenceNumber, - destinationPort - }) - ); + this.sendACK(sequenceNumber, destinationPort); } } + sendACK(sequenceNumber: number, channel: number) { + this.logger.debug("Write acknowledge frame for sequenceNumber", sequenceNumber); + this.commonEncoder.push( + this.commonEncoder.createFrame(undefined, { + flagsArray: ["ACK"], + sequenceNumber, + destinationPort: channel + }) + ); + } + addChannel(channel: TeceMuxChannel, emit: boolean) { this.logger.debug("adding channel", channel._id); this.channels[channel._id] = channel; // wait for SYN reply? @@ -165,6 +200,16 @@ export class TeceMux extends TypedEmitter{ this.channelCount++; } + sendFIN(channel: number) { + this.logger.debug("Write acknowledge frame for sequenceNumber"); + this.commonEncoder.push( + this.commonEncoder.createFrame(undefined, { + flagsArray: ["FIN"], + destinationPort: channel + }) + ); + } + multiplex(): TeceMuxChannel { this.logger.trace("Multiplex"); From 472f0cb818b59ba42efc974919f22d8bac852ea6 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Fri, 20 Jan 2023 12:40:22 +0000 Subject: [PATCH 078/231] Replace BPMux with TeCeMux --- packages/verser/package.json | 3 +- packages/verser/src/lib/tecemux/tecemux.ts | 4 +-- .../verser/src/lib/tecemux/utils/index.ts | 2 +- packages/verser/src/lib/verser-client.ts | 13 +++++---- packages/verser/src/lib/verser-connection.ts | 29 ++++++++++--------- 5 files changed, 26 insertions(+), 25 deletions(-) diff --git a/packages/verser/package.json b/packages/verser/package.json index 203fff311..4c29af24c 100644 --- a/packages/verser/package.json +++ b/packages/verser/package.json @@ -15,8 +15,7 @@ "license": "AGPL-3.0", "dependencies": { "@scramjet/obj-logger": "^0.33.5", - "@scramjet/utility": "^0.33.5", - "bpmux": "^8.2.1" + "@scramjet/utility": "^0.33.5" }, "devDependencies": { "@scramjet/api-server": "^0.33.5", diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index aa9d24a05..8b3a9d2cd 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -210,10 +210,10 @@ export class TeceMux extends TypedEmitter{ ); } - multiplex(): TeceMuxChannel { + multiplex(opts: { channel?: number } = {} ): TeceMuxChannel { this.logger.trace("Multiplex"); - const channel = this.createChannel(this.channelCount, true); + const channel = this.createChannel(opts.channel || this.channelCount, true); this.addChannel(channel, false); diff --git a/packages/verser/src/lib/tecemux/utils/index.ts b/packages/verser/src/lib/tecemux/utils/index.ts index 08cad4374..8528167e9 100644 --- a/packages/verser/src/lib/tecemux/utils/index.ts +++ b/packages/verser/src/lib/tecemux/utils/index.ts @@ -1,4 +1,4 @@ -import { frameFlags, binaryFlags, flagsObjectType } from "../codecs"; +import { binaryFlags, flagsObjectType } from "../codecs"; export function toHex(chunk: Buffer) { return chunk.toString("hex").match(/../g)?.join(" "); diff --git a/packages/verser/src/lib/verser-client.ts b/packages/verser/src/lib/verser-client.ts index 1e0724337..92d293d5f 100644 --- a/packages/verser/src/lib/verser-client.ts +++ b/packages/verser/src/lib/verser-client.ts @@ -8,8 +8,7 @@ import { createConnection, Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; import { defaultVerserClientOptions } from "./verser-client-default-config"; import { URL } from "url"; - -const BPMux = require("bpmux").BPMux; +import { TeceMux } from "./tecemux/tecemux"; type Events = { error: (err: Error) => void; @@ -25,7 +24,7 @@ export class VerserClient extends TypedEmitter { /** * BPMux instance. */ - private bpmux: any; + private teceMux: any; /** * VerserClient options. @@ -131,8 +130,8 @@ export class VerserClient extends TypedEmitter { * otherwise stream will be passed to the server. */ private mux() { - this.bpmux = new BPMux(this.socket) - .on("peer_multiplex", async (mSocket: Duplex & { _chan: number }) => { + this.teceMux = new TeceMux(this.socket!) + .on("channel", async (mSocket: Duplex & { _chan: number }) => { const registeredChannelCallback = this.registeredChannels.get(mSocket._chan); if (registeredChannelCallback) { @@ -145,13 +144,15 @@ export class VerserClient extends TypedEmitter { this.emit("error", err); }); + this.teceMux.logger.pipe(this.logger); + this._verserAgent = new HttpAgent() as HttpAgent & { createConnection: typeof createConnection }; // lack of types? this._verserAgent.createConnection = () => { try { - const socket = this.bpmux!.multiplex() as Socket; + const socket = this.teceMux!.multiplex() as Socket; socket.on("error", () => { this.logger.error("Muxed stream error"); diff --git a/packages/verser/src/lib/verser-connection.ts b/packages/verser/src/lib/verser-connection.ts index 2e5b9045a..08146d012 100644 --- a/packages/verser/src/lib/verser-connection.ts +++ b/packages/verser/src/lib/verser-connection.ts @@ -10,8 +10,7 @@ import { import { ObjLogger } from "@scramjet/obj-logger"; import { createConnection, Socket } from "net"; import { VerserRequestResult } from "../types"; - -const BPMux = require("bpmux").BPMux; +import { TeceMux } from "./tecemux/tecemux"; /** * VerserConnection class. @@ -22,21 +21,21 @@ export class VerserConnection { logger = new ObjLogger(this); private request: IncomingMessage; - private bpmux?: { [key: string]: any }; + private teceMux?: TeceMux; - private _socket: Duplex; + private _socket: Socket; private agent?: Agent & { createConnection: typeof createConnection }; private channelListeners: ((socket: Duplex, data?: any) => any)[] = []; get connected() { - return !(this._socket.destroyed && this.bpmux); + return !(this._socket.destroyed && this.teceMux); } - get socket(): Duplex { + get socket(): Socket { return this._socket; } - constructor(request: IncomingMessage, socket: Duplex) { + constructor(request: IncomingMessage, socket: Socket) { this.request = request; this._socket = socket; @@ -102,7 +101,7 @@ export class VerserConnection { async forward(req: IncomingMessage, res: ServerResponse) { if (!this.connected) throw new Error("BPMux not connected"); - const channel = this.bpmux?.multiplex() as Duplex; + const channel = this.teceMux?.multiplex() as Duplex; channel .on("error", (error: Error) => { @@ -193,22 +192,24 @@ export class VerserConnection { * @returns Duplex stream. */ createChannel(id: number): Duplex { - if (!this.bpmux) throw new Error("BPMux not connected"); + if (!this.teceMux) throw new Error("TeCeMux not connected"); - return this.bpmux.multiplex({ channel: id }); + return this.teceMux.multiplex({ channel: id }); } reconnect() { this.logger.debug("Reconnecting..."); - this.bpmux = this.bpmux = new BPMux(this.socket).on("error", (error: Error) => { + this.teceMux = new TeceMux(this.socket).on("error", (error: Error) => { this.logger.error("BPMux Error", error.message); // TODO: Error handling? }); + this.teceMux.logger.pipe(this.logger); + this.agent = new Agent() as Agent & { createConnection: typeof createConnection }; // lack of types? this.agent.createConnection = () => { try { - const socket = this.bpmux!.multiplex() as Socket; + const socket = this.teceMux!.multiplex() as unknown as Socket; socket.on("error", () => { this.logger.error("Muxed stream error"); @@ -229,9 +230,9 @@ export class VerserConnection { } }; - this.bpmux!.on("peer_multiplex", (socket: Duplex, data: any) => { + this.teceMux!.on("channel", (socket: Duplex) => { this.channelListeners.forEach((listener) => { - listener(socket, data); + listener(socket); }); }); } From 6d157166ad0373a7d3a8dc5ee5c581ba5dd27039 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Fri, 20 Jan 2023 14:06:57 +0000 Subject: [PATCH 079/231] Cleanup --- package.json | 2 +- packages/obj-logger/src/obj-logger.ts | 4 +- .../src/lib/tecemux/codecs/frame-decoder.ts | 26 +++++--- .../src/lib/tecemux/codecs/frame-encoder.ts | 26 ++++---- .../verser/src/lib/tecemux/codecs/index.ts | 36 ----------- packages/verser/src/lib/tecemux/constants.ts | 19 ++++++ .../src/lib/tecemux/playground-tecemux.ts | 47 +++++++------- packages/verser/src/lib/tecemux/tecemux.ts | 64 +++++++++---------- packages/verser/src/lib/tecemux/types.ts | 31 +++++++++ .../verser/src/lib/tecemux/utils/index.ts | 15 ++--- packages/verser/src/lib/verser-client.ts | 13 ++-- packages/verser/src/lib/verser-connection.ts | 4 +- yarn.lock | 30 +++++++-- 13 files changed, 175 insertions(+), 142 deletions(-) create mode 100644 packages/verser/src/lib/tecemux/constants.ts create mode 100644 packages/verser/src/lib/tecemux/types.ts diff --git a/package.json b/package.json index f12137d1e..fab95ff1f 100644 --- a/package.json +++ b/package.json @@ -77,7 +77,7 @@ "devDependencies": { "@istanbuljs/nyc-config-typescript": "^1.0.2", "@npmcli/run-script": "4.2.1", - "@types/node": "18.11.18", + "@types/node": "15.12.5", "@typescript-eslint/eslint-plugin": "^5.41.0", "@typescript-eslint/parser": "^5.41.0", "build-if-changed": "^1.5.5", diff --git a/packages/obj-logger/src/obj-logger.ts b/packages/obj-logger/src/obj-logger.ts index ef2f77b76..5222d3998 100644 --- a/packages/obj-logger/src/obj-logger.ts +++ b/packages/obj-logger/src/obj-logger.ts @@ -165,9 +165,11 @@ export class ObjLogger implements IObjectLogger { private _stringifiedOutput?: StringStream; get stringifiedOutput(): StringStream { - if (!this._stringifiedOutput) + if (!this._stringifiedOutput) { // eslint-disable-next-line no-console this._stringifiedOutput = this.output.JSONStringify().catch((e: any) => { console.error(e); }); + } + return this._stringifiedOutput; } diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 42bcc12a5..3382646c8 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -1,7 +1,7 @@ import { ObjLogger } from "@scramjet/obj-logger"; import { Duplex, Transform, TransformCallback, TransformOptions } from "stream"; -import { FrameData, HEADER_LENGTH, toHex } from "../utils"; -import { parseFlags } from "."; +import { FrameData, parseFlags, toHex } from "../utils"; +import { HEADER_LENGTH } from "../constants"; export class FrameDecoder extends Transform { buffer: Buffer; @@ -10,7 +10,11 @@ export class FrameDecoder extends Transform { _streams = new Map(); constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { - super(Object.assign({}, opts, { writableObjectMode: true, readableObjectMode: true, readableHighWaterMark: 2 })); + super(Object.assign({}, opts, { + writableObjectMode: true, + readableObjectMode: true, + readableHighWaterMark: 2 + })); this.buffer = Buffer.alloc(0); this.logger = new ObjLogger(params.name); @@ -19,7 +23,7 @@ export class FrameDecoder extends Transform { this.logger.debug("onPipe"); }).on("close", () => { this.logger.debug("onClose"); - }) + }); } _transform(chunk: Buffer, encoding: BufferEncoding, callback: TransformCallback): void { @@ -51,8 +55,12 @@ export class FrameDecoder extends Transform { } const payload = { - sourceAddress: [this.buffer.readInt8(0), this.buffer.readInt8(1), this.buffer.readInt8(2), this.buffer.readInt8(3)], - destinationAddress: [this.buffer.readInt8(4), this.buffer.readInt8(5), this.buffer.readInt8(6), this.buffer.readInt8(7)], + sourceAddress: [ + this.buffer.readInt8(0), this.buffer.readInt8(1), this.buffer.readInt8(2), this.buffer.readInt8(3) + ], + destinationAddress: [ + this.buffer.readInt8(4), this.buffer.readInt8(5), this.buffer.readInt8(6), this.buffer.readInt8(7) + ], chunk: this.buffer.subarray(32, this.buffer.readInt32LE(10)), flags: parseFlags(this.buffer.readInt8(25)), sourcePort: this.buffer.readInt16LE(12), @@ -67,15 +75,15 @@ export class FrameDecoder extends Transform { this.buffer = this.buffer.subarray(frameSize); - if (this.buffer.length === 0) { - this.logger.info("No remaining data!") + if (this.buffer.length === 0) { + this.logger.info("No remaining data!"); callback(); return; } this.logger.trace("More than one frame in chunk. processing", this.buffer.length); this._transform(Buffer.alloc(0), encoding, callback); - } catch(error) { + } catch (error) { this.logger.error("ERROR", error); this.emit("error", error); } diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index fc7bc9643..5c43d4d92 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -1,15 +1,16 @@ import { PassThrough, Transform, TransformCallback, TransformOptions } from "stream"; -import { FrameData, FrameTarget, HEADER_LENGTH, toHex as getHexString, toHex } from "../utils"; +import { FrameData, toHex as getHexString, toHex } from "../utils"; import { ObjLogger } from "@scramjet/obj-logger"; -import { binaryFlags, frameFlags } from "."; -import { TeceMux } from "../tecemux"; + +import { FrameTarget, HEADER_LENGTH, binaryFlags, frameFlags } from "../constants"; +import { ITeCeMux } from "../types"; export class FrameEncoder extends Transform { - tecemux: TeceMux; + tecemux: ITeCeMux; logger = new ObjLogger("FrameEncoder",); out = Object.assign(new PassThrough({ readableObjectMode: true }) .on("data", (data) => { - this.logger.trace("output to socket: " + (data.length == HEADER_LENGTH ? "HEADER ONLY" : ""), toHex(data), data.length, this.readableFlowing); + this.logger.trace("output to socket: " + (data.length === HEADER_LENGTH ? "HEADER ONLY" : ""), toHex(data), data.length, this.readableFlowing); }) .on("pause", () => { this.logger.trace("output to socket paused!"); @@ -24,11 +25,11 @@ export class FrameEncoder extends Transform { .on("error", (error) => { this.logger.error("output to socket error", error); }).pause() - ,{ + , { _id: this.frameTarget }); - constructor(private frameTarget: FrameTarget, tecemux: TeceMux, opts: TransformOptions = {}, params: { name: string } = { name: "FrameEncoder" }) { + constructor(private frameTarget: FrameTarget, tecemux: ITeCeMux, opts: TransformOptions = {}, params: { name: string } = { name: "FrameEncoder" }) { //opts.emitClose = false; //opts.readableObjectMode = true; super(Object.assign(opts, { @@ -46,8 +47,7 @@ export class FrameEncoder extends Transform { }) .on("end", () => { this.logger.debug("onEnd"); - }) - + }); this.pipe(this.out); } @@ -96,10 +96,10 @@ export class FrameEncoder extends Transform { // 64: zeroes (8bit), protocol (8bit), 8 - 9 new Uint8Array([0, 1]), - // tcp length (16bit) 10 - 11 + // tcp length (16bit) 10 - 11 new Uint8Array(new Uint16Array([chunk.length + HEADER_LENGTH]).buffer), - // 96: Source port, destination port 12 - 15 + // 96: Source port, destination port 12 - 15 new Uint8Array(new Uint16Array([0, frame.destinationPort || this.frameTarget]).buffer), // 128: sequenceNumber(32 bit, acnowledge number) 16 - 19 @@ -139,11 +139,9 @@ export class FrameEncoder extends Transform { this.once("drain", () => { this.push(buffer, undefined); }); - }; + } this.read(0); callback(); } - - } diff --git a/packages/verser/src/lib/tecemux/codecs/index.ts b/packages/verser/src/lib/tecemux/codecs/index.ts index 245e3e9fa..2a0fcd378 100644 --- a/packages/verser/src/lib/tecemux/codecs/index.ts +++ b/packages/verser/src/lib/tecemux/codecs/index.ts @@ -1,38 +1,2 @@ -import { Duplex } from "stream"; - export * from "./frame-decoder"; export * from "./frame-encoder"; - -export type TeceMuxEvents = { - channel(socket: Duplex): void; - error(error: any): void; -} - -export const binaryFlags = { - FIN: 0b00000001, - SYN: 0b00000010, - RST: 0b00000100, - PSH: 0b00001000, - ACK: 0b00010000, - URG: 0b00100000, - ECE: 0b01000000, - CWR: 0b10000000 -} - -export const frameFlags = Object.keys(binaryFlags); - -export type flagsObjectType = Partial<{ - FIN: boolean, - SYN: boolean, - RST: boolean, - PSH: boolean, - ACK: boolean, - URG: boolean, - ECE: boolean, - CWR: boolean -}> - -export const parseFlags = (byte: number): flagsObjectType => { - return frameFlags.filter((_flag, index) => byte >>> index & 1) - .reduce((p, c) => ({ ...p, [c]: true }), {}); -} diff --git a/packages/verser/src/lib/tecemux/constants.ts b/packages/verser/src/lib/tecemux/constants.ts new file mode 100644 index 000000000..4e4aa9f38 --- /dev/null +++ b/packages/verser/src/lib/tecemux/constants.ts @@ -0,0 +1,19 @@ +export const binaryFlags = { + FIN: 0b00000001, + SYN: 0b00000010, + RST: 0b00000100, + PSH: 0b00001000, + ACK: 0b00010000, + URG: 0b00100000, + ECE: 0b01000000, + CWR: 0b10000000 +}; + +export const frameFlags = Object.keys(binaryFlags); + +export const HEADER_LENGTH = 32; + +export enum FrameTarget { + API, + INPUT = 1001 +} diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index 8054f1ce9..7cca2a4c3 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -7,7 +7,8 @@ import { IncomingMessage, createServer } from "http"; import { DataStream } from "scramjet"; import { Socket, createConnection } from "net"; -import { TeceMux, TeceMuxChannel } from "./tecemux"; +import { TeceMux } from "./tecemux"; +import { TeceMuxChannel } from "./types"; (async () => { const logger = new ObjLogger("Sandbox"); @@ -16,12 +17,12 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; .pipe( new DataStream() .map(prettyPrint({ colors: true })) - .map((chunk: string) => ( + .map((chunk: string) => chunk.replace( /(:?FIN|SYN|RST|PSH|ACK|URG|ECE|CWR)|^$]/, - "\x1b[41m" + "$&" + "\x1b[0m" + "\x1b[41m\$&\x1b[0m" ) - )) + ) ) .pipe(process.stdout); @@ -32,7 +33,7 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; const PORT = 6660; const server = createServer({}); - server.on("timeout", (socket) => { + server.on("timeout", (_socket) => { logger.warn("Server on timeout"); }); server.requestTimeout = 0; @@ -66,7 +67,7 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; .on("timeout", () => { logger.info("Carrier Socket timeout"); }) - .pause() + .pause(); const tcmux = new TeceMux(socket, "Server") .on("error", (err) => { @@ -82,35 +83,34 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; logger.warn("Waiting for stdin..."); - DataStream.from(process.stdin).filter((x: Buffer) => !(parseInt(x[0].toString()) % 2)).pipe(channel1); - DataStream.from(process.stdin).filter((x: Buffer) => !!(parseInt(x[0].toString()) % 2)).pipe(channel2); + //DataStream.from(process.stdin).filter((x: Buffer) => !(parseInt(x[0].toString(), 10) % 2)).pipe(channel1); + //DataStream.from(process.stdin).filter((x: Buffer) => !!(parseInt(x[0].toString(), 10) % 2)).pipe(channel2); (async () => { try { for await (const chunk of channel1) { console.log("CHHUNK", chunk); - logger.debug(`reading CHANNEL1 chunk`, chunk.toString()); - }; + logger.debug("reading CHANNEL1 chunk", chunk.toString()); + } } catch (error) { - logger.error(`reading CHANNEL1 ERROR`, error); + logger.error("reading CHANNEL1 ERROR", error); } - logger.debug(`reading CHANNEL1 END`); + logger.debug("reading CHANNEL1 END"); })(); (async () => { try { for await (const chunk of channel2) { - logger.debug(`reading CHANNEL2 chunk`, chunk.toString()); - }; + logger.debug("reading CHANNEL2 chunk", chunk.toString()); + } } catch (error) { - logger.error(`reading CHANNEL2 ERROR`, error); + logger.error("reading CHANNEL2 ERROR", error); } - logger.debug(`reading CHANNEL2 END`); + logger.debug("reading CHANNEL2 END"); })(); - setTimeout(() => { console.log("\n\n\n\n"); logger.trace("Ending channels"); @@ -135,7 +135,8 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; socket.setNoDelay(true); - const reqLogger = new ObjLogger("Req", { id: "Client"}); + const reqLogger = new ObjLogger("Req", { id: "Client" }); + reqLogger.pipe(logger); socket.write("CONNECT HTTP/1.1\r\n\r\n\r\n"); @@ -144,7 +145,7 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; reqLogger.error("ERROR", error); }); - reqLogger.info('connected to server!'); + reqLogger.info("connected to server!"); const tcmux = new TeceMux(socket, "Request"); @@ -157,10 +158,10 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; channel .on("finish", () => { - tcmux.logger.info("Channel finish", channel._id) + tcmux.logger.info("Channel finish", channel._id); }) .on("end", () => { - tcmux.logger.info("Channel end", channel._id) + tcmux.logger.info("Channel end", channel._id); }); for await (const chunk of channel) { @@ -170,12 +171,12 @@ import { TeceMux, TeceMuxChannel } from "./tecemux"; } reqLogger.info("SERVER->CLIENT->CHANNEL", channel._id, chunk.toString()); - await new Promise((resolve, reject) => { + /*await new Promise((resolve, _reject) => { setTimeout(() => { //channel.write("abcde\n"); resolve(); }, 2000); - }); + });*/ } }); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 8b3a9d2cd..aaaf122da 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -1,17 +1,12 @@ import { TypedEmitter } from "@scramjet/utility"; -import { FrameDecoder, FrameEncoder, TeceMuxEvents } from "./codecs"; +import { FrameDecoder, FrameEncoder } from "./codecs"; import { Duplex, PassThrough } from "stream"; import { Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; import { FrameData } from "./utils"; +import { TeceMuxChannel, TeceMuxEvents } from "./types"; -export type TeceMuxChannel = Duplex & { - _id: number, - encoder: FrameEncoder, - closedByFIN: boolean -}; - -export class TeceMux extends TypedEmitter{ +export class TeceMux extends TypedEmitter { id: string; carrierSocket: Duplex; channelCount = 1; @@ -34,23 +29,22 @@ export class TeceMux extends TypedEmitter{ w.pipe(encoder, { end: false }).out.pipe(this.carrierSocket, { end: false }); - const duplex = new Duplex({ - write: (chunk, encoding, next) => { - this.logger.trace("WRITE channel", channel._id, chunk, chunk.toString() ); - - if (!w.push(chunk)) { - w.once("drain", next); - } else { - next(); - } - }, - read: (size) => { - this.logger.trace("READ channel", channel._id ); - }, - allowHalfOpen: true - }); const channel: TeceMuxChannel = Object.assign( - duplex, + new Duplex({ + write: (chunk, encoding, next) => { + this.logger.trace("WRITE channel", channel._id, chunk, chunk.toString()); + + if (!w.push(chunk)) { + w.once("drain", next); + } else { + next(); + } + }, + read: (_size) => { + this.logger.trace("READ channel", channel._id); + }, + allowHalfOpen: true + }), { _id: destinationPort || this.channelCount, encoder, @@ -64,14 +58,14 @@ export class TeceMux extends TypedEmitter{ channel .on("error", (error) => { - this.logger.error("CHANNEL ERROR", error) + this.logger.error("CHANNEL ERROR", error); //this.emit("error", { error, source: channel }) }) .on("destroy", () => { - this.logger.trace("channel on DESTROY ", channel._id ); + this.logger.trace("channel on DESTROY ", channel._id); }) .on("abort", () => { - this.logger.trace("channel on ABORT ", channel._id ); + this.logger.trace("channel on ABORT ", channel._id); }) .on("end", () => { this.logger.info("CHANNEL end", channel._id); @@ -90,7 +84,7 @@ export class TeceMux extends TypedEmitter{ constructor(socket: Socket, id = "") { super(); this.id = id; - this.logger = new ObjLogger(this, { id: this.id }) + this.logger = new ObjLogger(this, { id: this.id }); this.carrierSocket = socket; this.decoder = new FrameDecoder({ emitClose: false }) @@ -120,7 +114,7 @@ export class TeceMux extends TypedEmitter{ this.commonEncoder.out.pipe(this.carrierSocket, { end: false }); - this.commonEncoder.logger.updateBaseLog({ id }) + this.commonEncoder.logger.updateBaseLog({ id }); this.commonEncoder.logger.pipe(this.logger); this.main().catch((error) => { @@ -135,7 +129,7 @@ export class TeceMux extends TypedEmitter{ const frame = JSON.parse(chunk) as FrameData; const { flags, sequenceNumber, dataLength, destinationPort } = frame; - let channel = this.channels[destinationPort] + let channel = this.channels[destinationPort]; if (flags.ACK) { this.logger.trace("ACK frame received for sequenceNumber", sequenceNumber); @@ -148,14 +142,14 @@ export class TeceMux extends TypedEmitter{ if (channel) { //channel.end(); channel.closedByFIN = true; - channel.push(null) + channel.push(null); } else { this.logger.error("FIN for unknown channel"); } } if (flags.PSH) { - this.logger.trace(`Received PSH command [C: ${destinationPort}]`, dataLength, frame.chunk, !!this.channels[destinationPort]); + this.logger.trace(`Received PSH command [C: ${destinationPort}]`, dataLength); if (!channel) { this.logger.warn("Unknown channel"); @@ -201,7 +195,7 @@ export class TeceMux extends TypedEmitter{ } sendFIN(channel: number) { - this.logger.debug("Write acknowledge frame for sequenceNumber"); + this.logger.debug("Write FIN frame for channel", channel); this.commonEncoder.push( this.commonEncoder.createFrame(undefined, { flagsArray: ["FIN"], @@ -210,7 +204,7 @@ export class TeceMux extends TypedEmitter{ ); } - multiplex(opts: { channel?: number } = {} ): TeceMuxChannel { + multiplex(opts: { channel?: number } = {}): TeceMuxChannel { this.logger.trace("Multiplex"); const channel = this.createChannel(opts.channel || this.channelCount, true); @@ -221,3 +215,5 @@ export class TeceMux extends TypedEmitter{ return channel; } } +export { TeceMuxChannel }; + diff --git a/packages/verser/src/lib/tecemux/types.ts b/packages/verser/src/lib/tecemux/types.ts new file mode 100644 index 000000000..7d8c041f8 --- /dev/null +++ b/packages/verser/src/lib/tecemux/types.ts @@ -0,0 +1,31 @@ +import { Duplex } from "stream"; + +export type TeceMuxEvents = { + channel(socket: Duplex): void; + error(error: any): void; +} + +export type flagsObjectType = Partial<{ + FIN: boolean, + SYN: boolean, + RST: boolean, + PSH: boolean, + ACK: boolean, + URG: boolean, + ECE: boolean, + CWR: boolean +}> + +export interface ITeCeMux { + sequenceNumber: number; +} + +export interface IFrameEncoder { + +} + +export type TeceMuxChannel = Duplex & { + _id: number, + encoder: IFrameEncoder, + closedByFIN: boolean +}; diff --git a/packages/verser/src/lib/tecemux/utils/index.ts b/packages/verser/src/lib/tecemux/utils/index.ts index 8528167e9..dde25f260 100644 --- a/packages/verser/src/lib/tecemux/utils/index.ts +++ b/packages/verser/src/lib/tecemux/utils/index.ts @@ -1,16 +1,10 @@ -import { binaryFlags, flagsObjectType } from "../codecs"; +import { binaryFlags, frameFlags } from "../constants"; +import { flagsObjectType } from "../types"; export function toHex(chunk: Buffer) { return chunk.toString("hex").match(/../g)?.join(" "); } -export const HEADER_LENGTH = 32; - -export enum FrameTarget { - API, - INPUT = 1001 -} - export type FrameData = { sourceAddress: [number, number, number, number]; destinationAddress: [number, number, number, number]; @@ -24,3 +18,8 @@ export type FrameData = { flags: flagsObjectType; flagsArray: (keyof typeof binaryFlags)[]; }; + +export const parseFlags = (byte: number): flagsObjectType => { + return frameFlags.filter((_flag, index) => byte >>> index & 1) + .reduce((p, c) => ({ ...p, [c]: true }), {}); +}; diff --git a/packages/verser/src/lib/verser-client.ts b/packages/verser/src/lib/verser-client.ts index 92d293d5f..db8d0c50e 100644 --- a/packages/verser/src/lib/verser-client.ts +++ b/packages/verser/src/lib/verser-client.ts @@ -3,12 +3,11 @@ import { Agent as HttpsAgent, request } from "https"; import { merge, TypedEmitter } from "@scramjet/utility"; import { IObjectLogger } from "@scramjet/types"; import { VerserClientOptions, VerserClientConnection, RegisteredChannels, RegisteredChannelCallback } from "../types"; -import { Duplex } from "stream"; import { createConnection, Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; import { defaultVerserClientOptions } from "./verser-client-default-config"; import { URL } from "url"; -import { TeceMux } from "./tecemux/tecemux"; +import { TeceMux, TeceMuxChannel } from "./tecemux/tecemux"; type Events = { error: (err: Error) => void; @@ -130,14 +129,14 @@ export class VerserClient extends TypedEmitter { * otherwise stream will be passed to the server. */ private mux() { - this.teceMux = new TeceMux(this.socket!) - .on("channel", async (mSocket: Duplex & { _chan: number }) => { - const registeredChannelCallback = this.registeredChannels.get(mSocket._chan); + this.teceMux = new TeceMux(this.socket!, "client") + .on("channel", async (channel: TeceMuxChannel) => { + const registeredChannelCallback = this.registeredChannels.get(channel._id); if (registeredChannelCallback) { - await registeredChannelCallback(mSocket); + await registeredChannelCallback(channel); } else { - this.opts.server?.emit("connection", mSocket); + this.opts.server?.emit("connection", channel); } }) .on("error", (err: Error) => { diff --git a/packages/verser/src/lib/verser-connection.ts b/packages/verser/src/lib/verser-connection.ts index 08146d012..4928b4531 100644 --- a/packages/verser/src/lib/verser-connection.ts +++ b/packages/verser/src/lib/verser-connection.ts @@ -199,8 +199,8 @@ export class VerserConnection { reconnect() { this.logger.debug("Reconnecting..."); - this.teceMux = new TeceMux(this.socket).on("error", (error: Error) => { - this.logger.error("BPMux Error", error.message); + this.teceMux = new TeceMux(this.socket, "client").on("error", (error: Error) => { + this.logger.error("TeCeMux Error", error.message); // TODO: Error handling? }); diff --git a/yarn.lock b/yarn.lock index c96e1bb09..8e08e5d7f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -814,6 +814,22 @@ resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== +"@scramjet/obj-logger@^0.31.4": + version "0.31.4" + resolved "https://registry.yarnpkg.com/@scramjet/obj-logger/-/obj-logger-0.31.4.tgz#9c755403afa544afa31cc30dbca58913b95c8388" + integrity sha512-6sfS45IWkRMmdiG+2egr7FTyjpNW5Dc+n2WvzWbk6M/37/B30lRUtVkYGKTpIBbw/arp6lUuH0w/QjlC+QAiDQ== + dependencies: + "@scramjet/utility" "^0.31.4" + scramjet "^4.36.9" + +"@scramjet/utility@^0.31.4": + version "0.31.4" + resolved "https://registry.yarnpkg.com/@scramjet/utility/-/utility-0.31.4.tgz#9e13f30ef66636e77a7a065d1496f35e1dfc7017" + integrity sha512-BFwoYu+u5SbuJX5QgzrSgb+6Vak+cCAu141C5zasoAMplM4AmvqPWQAZ7CQ6Du7KQA7HsJqgD0xCHHky+mgoEA== + dependencies: + normalize-url "^5.3.1" + yaml "^2.1.3" + "@sindresorhus/is@^0.14.0": version "0.14.0" resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea" @@ -959,14 +975,9 @@ "@types/node" "*" form-data "^3.0.0" -"@types/node@*", "@types/node@18.11.18", "@types/node@^18.11.18": - version "18.11.18" - resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f" - integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA== - -"@types/node@15.12.5": +"@types/node@*", "@types/node@15.12.5": version "15.12.5" - resolved "https://registry.yarnpkg.com/@types/node/-/node-15.12.5.tgz#9a78318a45d75c9523d2396131bd3cca54b2d185" + resolved "https://registry.npmjs.org/@types/node/-/node-15.12.5.tgz" integrity sha512-se3yX7UHv5Bscf8f1ERKvQOD6sTyycH3hdaoozvaLxgUiY5lIGEeH37AD0G0Qi9kPqihPn0HOfd2yaIEN9VwEg== "@types/node@>=13.7.0": @@ -974,6 +985,11 @@ resolved "https://registry.npmjs.org/@types/node/-/node-18.11.3.tgz" integrity sha512-fNjDQzzOsZeKZu5NATgXUPsaFaTxeRgFXoosrHivTl8RGeV733OLawXsGfEk9a8/tySyZUyiZ6E8LcjPFZ2y1A== +"@types/node@^18.11.18": + version "18.11.18" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f" + integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA== + "@types/normalize-package-data@^2.4.0": version "2.4.1" resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301" From 5a92353d93a2220bfa0cc5326b4f5353c65aa7bf Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 25 Jan 2023 13:02:13 +0000 Subject: [PATCH 080/231] Playground pass file --- .../src/lib/tecemux/codecs/frame-decoder.ts | 42 ++++++--- .../src/lib/tecemux/codecs/frame-encoder.ts | 73 ++++++++++----- .../verser/src/lib/tecemux/frames-keeper.ts | 36 +++++++ .../src/lib/tecemux/playground-tecemux.ts | 78 ++++++++++------ packages/verser/src/lib/tecemux/playground.ts | 93 +++++++++++++++++++ packages/verser/src/lib/tecemux/tecemux.ts | 82 +++++++++++----- yarn.lock | 5 + 7 files changed, 323 insertions(+), 86 deletions(-) create mode 100644 packages/verser/src/lib/tecemux/frames-keeper.ts create mode 100644 packages/verser/src/lib/tecemux/playground.ts diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 3382646c8..5d7da89a0 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -10,11 +10,7 @@ export class FrameDecoder extends Transform { _streams = new Map(); constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { - super(Object.assign({}, opts, { - writableObjectMode: true, - readableObjectMode: true, - readableHighWaterMark: 2 - })); + super(Object.assign({}, opts, { readableObjectMode: true})); this.buffer = Buffer.alloc(0); this.logger = new ObjLogger(params.name); @@ -29,9 +25,9 @@ export class FrameDecoder extends Transform { _transform(chunk: Buffer, encoding: BufferEncoding, callback: TransformCallback): void { try { if (!Buffer.isBuffer(chunk)) { - this.push(JSON.stringify({ error: "not a buffer" }), undefined); - callback(); + this.push(JSON.stringify({ error: "not a buffer" }), undefined); + callback(); return; } @@ -40,18 +36,32 @@ export class FrameDecoder extends Transform { } else { this.logger.error("Decoding buffer...", chunk); this.emit("error", "Chunk is not a buffer"); + callback(); + return; } - this.logger.trace("Decoding buffer...", toHex(this.buffer), "Size:", this.buffer.length); - let frameSize = 0; + this.logger.trace("Decoding buffer...", /* toHex(this.buffer) */ "Size:", this.buffer.length); + + if (this.buffer.length === 0) { + callback(null); + return; + } + if (this.buffer.length >= HEADER_LENGTH) { frameSize = this.buffer.readInt32LE(10); } else { this.logger.trace("To few data"); - callback(); + callback(null); + return; + } + + if (this.buffer.length < frameSize) { + this.logger.trace("To few data"); + callback(null); + return; } const payload = { @@ -68,13 +78,21 @@ export class FrameDecoder extends Transform { dataLength: frameSize - HEADER_LENGTH, chunkLength: frameSize, sequenceNumber: this.buffer.readInt32LE(16), - stringified: this.buffer.subarray(32, frameSize).toString() + acknowledgeNumber: this.buffer.readInt32LE(20), + //stringified: this.buffer.subarray(32, frameSize).toString() } as Partial; - this.push(JSON.stringify(payload) + "\n"); + if (payload.dataLength && payload.dataLength < 0) { + this.emit("error", "Data length incorrect"); + return; + } + + this.push(JSON.stringify(payload), "utf-8") this.buffer = this.buffer.subarray(frameSize); + this.logger.trace("Decoded", { ...payload, stringified: "--not-displayed--" }); + if (this.buffer.length === 0) { this.logger.info("No remaining data!"); callback(); diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index 5c43d4d92..d2e4a481c 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -7,35 +7,38 @@ import { ITeCeMux } from "../types"; export class FrameEncoder extends Transform { tecemux: ITeCeMux; - logger = new ObjLogger("FrameEncoder",); + total = 0; + logger = new ObjLogger("FrameEncoder"); out = Object.assign(new PassThrough({ readableObjectMode: true }) - .on("data", (data) => { + /*.on("data", (data) => { this.logger.trace("output to socket: " + (data.length === HEADER_LENGTH ? "HEADER ONLY" : ""), toHex(data), data.length, this.readableFlowing); - }) + })*/ .on("pause", () => { - this.logger.trace("output to socket paused!"); + this.logger.trace("OUT paused!"); }) .on("end", () => { - this.logger.trace("output to socket ended!", this.frameTarget); + this.logger.trace("OUT ended!", this.frameTarget); //this.tecemux.sendFIN(this.frameTarget); }) .on("resume", () => { - this.logger.trace("output to socket resumed"); + this.logger.trace("OUT resumed"); }) .on("error", (error) => { - this.logger.error("output to socket error", error); - }).pause() + this.logger.error("OUT error", error); + })//.pause() , { _id: this.frameTarget }); - constructor(private frameTarget: FrameTarget, tecemux: ITeCeMux, opts: TransformOptions = {}, params: { name: string } = { name: "FrameEncoder" }) { - //opts.emitClose = false; - //opts.readableObjectMode = true; + constructor( + private frameTarget: FrameTarget, + tecemux: ITeCeMux, + opts: TransformOptions = {}, + params: { name: string } = { name: "FrameEncoder" + }) { super(Object.assign(opts, { - writableObjectMode: true, readableObjectMode: true, - readableHighWaterMark: 0 + writableObjectMode: true })); this.tecemux = tecemux; @@ -47,6 +50,12 @@ export class FrameEncoder extends Transform { }) .on("end", () => { this.logger.debug("onEnd"); + }) + .on("pause", () => { + this.logger.debug("onPause"); + }) + .on("resume", () => { + this.logger.debug("onResume"); }); this.pipe(this.out); @@ -103,7 +112,7 @@ export class FrameEncoder extends Transform { new Uint8Array(new Uint16Array([0, frame.destinationPort || this.frameTarget]).buffer), // 128: sequenceNumber(32 bit, acnowledge number) 16 - 19 - new Uint8Array(new Uint32Array([this.tecemux.sequenceNumber++]).buffer), + new Uint8Array(new Uint32Array([this.tecemux.sequenceNumber]).buffer), // 160: Acknowledgement number 20-23 new Uint8Array(new Uint32Array([frame.acknowledgeNumber || 0]).buffer), @@ -129,19 +138,35 @@ export class FrameEncoder extends Transform { } _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void { - //const buffer = this.createChunkFrame(chunk); - this.logger.debug("TRANSFORM IN", toHex(chunk), chunk.length); - const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); + const MAX_CHUNK_SIZE = 10 * 1024 - HEADER_LENGTH; - this.logger.debug("TRANSFORM OUT", getHexString(buffer), "Size: ", buffer.length); + if (chunk) { - if (!this.push(buffer, undefined)) { - this.once("drain", () => { - this.push(buffer, undefined); - }); } - this.read(0); - callback(); + this.total += chunk.length; + + this.logger.debug("TRANSFORM IN", /* toHex(chunk), */ chunk.length, this.total); + + let remaining = Buffer.alloc(0); + + if (chunk.length > MAX_CHUNK_SIZE) { + this.logger.debug("TRANSFORM big chunk, splitting", chunk.length); + remaining = (chunk as Buffer).subarray(MAX_CHUNK_SIZE); + chunk = chunk.subarray(0, MAX_CHUNK_SIZE); + this.logger.debug("TRANSFORM processing part/remaining", chunk.length, remaining.length); + } + + const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); + this.tecemux.sequenceNumber++; + + this.logger.debug("TRANSFORM OUT", /*getHexString(buffer), */ "Size: ", buffer.length); + + if (remaining.length) { + this.push(buffer); + this._transform(remaining, encoding, callback); + } else { + callback(null, buffer); + } } } diff --git a/packages/verser/src/lib/tecemux/frames-keeper.ts b/packages/verser/src/lib/tecemux/frames-keeper.ts new file mode 100644 index 000000000..1aece983e --- /dev/null +++ b/packages/verser/src/lib/tecemux/frames-keeper.ts @@ -0,0 +1,36 @@ +import { ObjLogger } from "@scramjet/obj-logger"; +import { PassThrough } from "stream"; + +export class FramesKeeper extends PassThrough { + logger = new ObjLogger(this); + archive = new Map(); + lastSequenceNumberSent: number = -1; + + _write(chunk: any, encoding: BufferEncoding, cb: ((error: Error | null | undefined) => void) | undefined) { + const sequenceNumber = chunk.readInt32LE(16); + this.archive.set(sequenceNumber, chunk); + this.logger.debug(`sequenceNumber ${sequenceNumber} stored, size: ${chunk.length}`); + + this.push(chunk, encoding); + if (cb) cb(undefined); + + this.lastSequenceNumberSent = sequenceNumber; + + // await new Promise((resolve, reject) => { + // while (this.archive.get(sequenceNumber)) { + // this.logger.debug("wait for ack for", sequenceNumber) + // } + // resolve(); + // }); + + return true + } + + _read(size: number) { + } + + onACK(sequenceNumber: number) { + this.logger.debug("onACK", sequenceNumber); + this.archive.delete(sequenceNumber); + } +} diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index 7cca2a4c3..2c5686597 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -9,6 +9,9 @@ import { DataStream } from "scramjet"; import { Socket, createConnection } from "net"; import { TeceMux } from "./tecemux"; import { TeceMuxChannel } from "./types"; +import { createReadStream, createWriteStream } from "fs"; +import path from "path"; +import { Readable } from "stream"; (async () => { const logger = new ObjLogger("Sandbox"); @@ -67,7 +70,7 @@ import { TeceMuxChannel } from "./types"; .on("timeout", () => { logger.info("Carrier Socket timeout"); }) - .pause(); + const tcmux = new TeceMux(socket, "Server") .on("error", (err) => { @@ -77,19 +80,22 @@ import { TeceMuxChannel } from "./types"; tcmux.logger.pipe(logger); const channel1 = tcmux.multiplex(); - const channel2 = tcmux.multiplex(); + //const channel2 = tcmux.multiplex(); + + createReadStream(path.join(__dirname, "../../../../forever.tar.gz")).pipe(channel1); + //Readable.from(Buffer.alloc(1024 * 100)).pipe(channel1, { end: false }); req.on("pause", () => { logger.warn("Request paused"); }); - logger.warn("Waiting for stdin..."); + //logger.warn("Waiting for stdin..."); //DataStream.from(process.stdin).filter((x: Buffer) => !(parseInt(x[0].toString(), 10) % 2)).pipe(channel1); //DataStream.from(process.stdin).filter((x: Buffer) => !!(parseInt(x[0].toString(), 10) % 2)).pipe(channel2); - + /* (async () => { try { for await (const chunk of channel1) { - console.log("CHHUNK", chunk); + console.log("CHUNK", chunk); logger.debug("reading CHANNEL1 chunk", chunk.toString()); } } catch (error) { @@ -98,25 +104,27 @@ import { TeceMuxChannel } from "./types"; logger.debug("reading CHANNEL1 END"); })(); - - (async () => { - try { - for await (const chunk of channel2) { - logger.debug("reading CHANNEL2 chunk", chunk.toString()); - } - } catch (error) { - logger.error("reading CHANNEL2 ERROR", error); - } - - logger.debug("reading CHANNEL2 END"); - })(); - +*/ + // (async () => { + // try { + // for await (const chunk of channel2) { + // logger.debug("reading CHANNEL2 chunk", chunk.toString()); + // } + // } catch (error) { + // logger.error("reading CHANNEL2 ERROR", error); + // } + + // logger.debug("reading CHANNEL2 END"); + // })(); + + /* setTimeout(() => { console.log("\n\n\n\n"); logger.trace("Ending channels"); channel1.push(null); //channel2.end(); }, 4000); + */ }); server.listen(PORT, "0.0.0.0"); @@ -164,20 +172,38 @@ import { TeceMuxChannel } from "./types"; tcmux.logger.info("Channel end", channel._id); }); - for await (const chunk of channel) { + channel + //.pipe(process.stdout); + .pipe(createWriteStream(path.join(__dirname, "output.tar.gz"))); + + let total = 0; + channel + .on("data", (d) => { + total += d.length; + tcmux.logger.info("-------------------- data", channel._id, d.length, total); + + }) + .on("pause", () => { + tcmux.logger.info("-------------------- paused", channel._id); + }) + .on("resume", () => { + tcmux.logger.info("-------------------- resumed", channel._id); + }) + /*for await (const chunk of channel) { console.log(chunk); if (chunk === null) { break; } reqLogger.info("SERVER->CLIENT->CHANNEL", channel._id, chunk.toString()); - /*await new Promise((resolve, _reject) => { - setTimeout(() => { - //channel.write("abcde\n"); - resolve(); - }, 2000); - });*/ - } + // await new Promise((resolve, _reject) => { + // setTimeout(() => { + // //channel.write("abcde\n"); + // resolve(); + // }, 2000); + // }); + }*/ + }); socket.on("error", (err) => { diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/src/lib/tecemux/playground.ts new file mode 100644 index 000000000..33e3adf3a --- /dev/null +++ b/packages/verser/src/lib/tecemux/playground.ts @@ -0,0 +1,93 @@ +import { DataStream } from "scramjet"; +import { FrameEncoder } from "./codecs"; +import { FrameDecoder } from "./codecs"; +import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; +import { createReadStream, createWriteStream } from "fs"; +import path from "path"; +import { Duplex, PassThrough, Transform } from "stream"; +import { wrap } from "module"; + +const tcm = { + sequenceNumber: 0 +}; + +const logger = new ObjLogger("Sandbox"); + +logger.pipe( + new DataStream() + .map(prettyPrint({ colors: true })) + .map((chunk: string) => + chunk.replace( + /(:?FIN|SYN|RST|PSH|ACK|URG|ECE|CWR)|^$]/, + "\x1b[41m\$&\x1b[0m" + ) + ) +) +.pipe(process.stdout); + +const encoder = new FrameEncoder(0, tcm); +const decoder = new FrameDecoder(); + +encoder.logger.pipe(logger); +decoder.logger.pipe(logger); + +/* +process.stdin + .pipe(encoder).out + .pipe(decoder) + //.pipe(process.stdout); +*/ +let i = 0; +let t = 0; + +const ws = createWriteStream(path.join(__dirname, "output.tar.gz")) + +const delayedPassThrough = () => new PassThrough({ + async transform(chunk, encoding, callback) { + await new Promise((res,_rej)=> { + setTimeout(res, 1000); + }); + this.push(chunk, encoding) + callback(null) + }, +}); + +const dh = new Transform({ writableObjectMode: true, transform: (chunk, encoding, callback) => { + try { + console.log(i++, t); + + if (chunk && chunk.length) { + t += chunk.length; + + console.log({ ...JSON.parse(chunk.toString()).chunk, }); + if (!ws.write(new Uint8Array(JSON.parse(chunk.toString()).chunk.data))) { + dh.pause(); + ws.once("drain", () => { + callback(); + dh.resume(); + }) + } else { + callback(null); + } + } + + //callback(null); + } catch (e) { + debugger; + } + +}}) + +createReadStream(path.join(__dirname, "../../../../forever.tar.gz"), { encoding: undefined }) + .pipe(encoder, { end: false}).out + + .pipe(delayedPassThrough()) + .pipe(decoder) + .pipe(delayedPassThrough()) + .pipe(dh) + + // .pipe( + // ws + // ) + + diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index aaaf122da..13b06e479 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -1,16 +1,18 @@ import { TypedEmitter } from "@scramjet/utility"; import { FrameDecoder, FrameEncoder } from "./codecs"; -import { Duplex, PassThrough } from "stream"; +import { Duplex, PassThrough, Readable, ReadableOptions } from "stream"; import { Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; import { FrameData } from "./utils"; import { TeceMuxChannel, TeceMuxEvents } from "./types"; +import { FramesKeeper } from "./frames-keeper"; export class TeceMux extends TypedEmitter { id: string; carrierSocket: Duplex; channelCount = 1; decoder: FrameDecoder; + framesKeeper = new FramesKeeper(); sequenceNumber = 0; channels: TeceMuxChannel[] = []; @@ -25,20 +27,16 @@ export class TeceMux extends TypedEmitter { encoder.logger.updateBaseLog({ id: this.id }); encoder.logger.pipe(this.logger); - const w = new PassThrough({ encoding: undefined, allowHalfOpen: true }); + //const w = new PassThrough().on("error", (error) => { this.logger.error("W error", error)}); - w.pipe(encoder, { end: false }).out.pipe(this.carrierSocket, { end: false }); + //w.pipe(encoder) const channel: TeceMuxChannel = Object.assign( new Duplex({ write: (chunk, encoding, next) => { - this.logger.trace("WRITE channel", channel._id, chunk, chunk.toString()); + this.logger.trace("WRITE channel", channel._id, chunk); - if (!w.push(chunk)) { - w.once("drain", next); - } else { - next(); - } + return encoder.write(chunk, encoding, next); }, read: (_size) => { this.logger.trace("READ channel", channel._id); @@ -52,13 +50,14 @@ export class TeceMux extends TypedEmitter { } ); - process.nextTick(() => { - w.on("data", (d) => { this.logger.warn("channel writeable on DATA", d); }); - }); + encoder.out + .pipe(this.carrierSocket, { end: false }); channel .on("error", (error) => { this.logger.error("CHANNEL ERROR", error); + + debugger; //this.emit("error", { error, source: channel }) }) .on("destroy", () => { @@ -97,9 +96,9 @@ export class TeceMux extends TypedEmitter { .on("end", () => { this.logger.warn("Decoder ended"); }) - .on("error", (error) => { + /*.on("error", (error) => { this.logger.error("Decoder error", error); - }) + })*/ .on("abort", (error) => { this.logger.error("Decoder abort", error); }) @@ -117,22 +116,36 @@ export class TeceMux extends TypedEmitter { this.commonEncoder.logger.updateBaseLog({ id }); this.commonEncoder.logger.pipe(this.logger); + this.framesKeeper.logger.pipe(this.logger); + this.main().catch((error) => { this.emit("error", error); }); } async main() { + let t = 0; for await (const chunk of this.decoder) { - this.logger.debug("Decoded", JSON.parse(chunk)); + //console.log(chunk); + let frame: FrameData; + + try { + frame = JSON.parse(chunk) as FrameData; + } catch (error) { + console.debug( chunk.toString()) + this.logger.error("error Parsing data from decoder", error, chunk, chunk.length, chunk.toString()); + continue; + } + + //this.logger.debug("Decoded", { ...frame, stringified: "--not-displayed--" }); - const frame = JSON.parse(chunk) as FrameData; - const { flags, sequenceNumber, dataLength, destinationPort } = frame; + const { flags, sequenceNumber, dataLength, destinationPort, acknowledgeNumber } = frame; let channel = this.channels[destinationPort]; if (flags.ACK) { - this.logger.trace("ACK frame received for sequenceNumber", sequenceNumber); + this.logger.trace("ACK frame received for sequenceNumber", acknowledgeNumber); + //this.framesKeeper.onACK(sequenceNumber); continue; } @@ -146,10 +159,13 @@ export class TeceMux extends TypedEmitter { } else { this.logger.error("FIN for unknown channel"); } + + this.sendACK(sequenceNumber, destinationPort); + continue; } if (flags.PSH) { - this.logger.trace(`Received PSH command [C: ${destinationPort}]`, dataLength); + this.logger.trace(`Received PSH command [C: ${destinationPort}, SIZE: ${dataLength}]`); if (!channel) { this.logger.warn("Unknown channel"); @@ -160,14 +176,32 @@ export class TeceMux extends TypedEmitter { if (dataLength) { this.logger.warn("writing to channel [channel, length]", channel._id, dataLength); + this.logger.warn("writing to channel [flowing, isPaused]", channel.readableFlowing, channel.isPaused()); + + //Readable.from(chunk).pipe(channel, { end: false });//channel.write(new Uint8Array(((frame.chunk as any).data) as any)); + //channel.push(frame.chunk, undefined); + channel.push(new Uint8Array((frame.chunk as any).data), undefined); + + t += (frame.chunk as any).data.length; + this.logger.info("Writen", t) - const written = channel.push(new Uint8Array(((frame.chunk as any).data) as any)); - this.logger.info("Bytes written to channel [writeResult, channel, length]", written, destinationPort, dataLength); + /* + await new Promise((resolve, reject) => { + this.logger.debug("waiting for drain!") + channel.on("drain", () => { + //channel.push(new Uint8Array(((frame.chunk as any).data) as any)); + this.logger.debug("Drained!") + resolve(); + }); + }); + }*/ + + //this.logger.info("Bytes written to channel [writeResult, channel, length]", written, destinationPort, dataLength); } - } - this.sendACK(sequenceNumber, destinationPort); + this.sendACK(sequenceNumber, destinationPort); + } } } @@ -176,7 +210,7 @@ export class TeceMux extends TypedEmitter { this.commonEncoder.push( this.commonEncoder.createFrame(undefined, { flagsArray: ["ACK"], - sequenceNumber, + acknowledgeNumber: sequenceNumber, destinationPort: channel }) ); diff --git a/yarn.lock b/yarn.lock index 8e08e5d7f..d6a6a59e9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -980,6 +980,11 @@ resolved "https://registry.npmjs.org/@types/node/-/node-15.12.5.tgz" integrity sha512-se3yX7UHv5Bscf8f1ERKvQOD6sTyycH3hdaoozvaLxgUiY5lIGEeH37AD0G0Qi9kPqihPn0HOfd2yaIEN9VwEg== +"@types/node@18.11.18": + version "18.11.18" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f" + integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA== + "@types/node@>=13.7.0": version "18.11.3" resolved "https://registry.npmjs.org/@types/node/-/node-18.11.3.tgz" From 8ad0709f42ea94988913a2ce35e67eaca10d7ca0 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 25 Jan 2023 18:47:39 +0000 Subject: [PATCH 081/231] Pass big file --- packages/verser/package.json | 2 +- .../src/lib/tecemux/codecs/frame-decoder.ts | 6 +- .../src/lib/tecemux/codecs/frame-encoder.ts | 9 ++- .../verser/src/lib/tecemux/frames-keeper.ts | 5 +- .../src/lib/tecemux/playground-tecemux.ts | 24 +------ packages/verser/src/lib/tecemux/playground.ts | 69 ++++++++----------- packages/verser/src/lib/tecemux/tecemux.ts | 29 ++------ packages/verser/src/lib/verser-client.ts | 2 +- packages/verser/src/lib/verser-connection.ts | 2 +- yarn.lock | 10 --- 10 files changed, 50 insertions(+), 108 deletions(-) diff --git a/packages/verser/package.json b/packages/verser/package.json index 4c29af24c..f2c426a17 100644 --- a/packages/verser/package.json +++ b/packages/verser/package.json @@ -25,7 +25,7 @@ "ts-node": "^10.9.1", "typedoc": "^0.23.17", "typedoc-plugin-markdown": "^3.13.6", - "typescript": "~4.9.4" + "typescript": "~4.7.4" }, "ava": { "extensions": [ diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 5d7da89a0..a4452891b 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -1,6 +1,6 @@ import { ObjLogger } from "@scramjet/obj-logger"; import { Duplex, Transform, TransformCallback, TransformOptions } from "stream"; -import { FrameData, parseFlags, toHex } from "../utils"; +import { FrameData, parseFlags } from "../utils"; import { HEADER_LENGTH } from "../constants"; export class FrameDecoder extends Transform { @@ -10,7 +10,7 @@ export class FrameDecoder extends Transform { _streams = new Map(); constructor(opts: TransformOptions = {}, params: { name: string } = { name: "FrameDecoder" }) { - super(Object.assign({}, opts, { readableObjectMode: true})); + super(Object.assign({}, opts, { readableObjectMode: true })); this.buffer = Buffer.alloc(0); this.logger = new ObjLogger(params.name); @@ -87,7 +87,7 @@ export class FrameDecoder extends Transform { return; } - this.push(JSON.stringify(payload), "utf-8") + this.push(JSON.stringify(payload), "utf-8"); this.buffer = this.buffer.subarray(frameSize); diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index d2e4a481c..d31cf0138 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -1,5 +1,5 @@ import { PassThrough, Transform, TransformCallback, TransformOptions } from "stream"; -import { FrameData, toHex as getHexString, toHex } from "../utils"; +import { FrameData } from "../utils"; import { ObjLogger } from "@scramjet/obj-logger"; import { FrameTarget, HEADER_LENGTH, binaryFlags, frameFlags } from "../constants"; @@ -140,10 +140,6 @@ export class FrameEncoder extends Transform { _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void { const MAX_CHUNK_SIZE = 10 * 1024 - HEADER_LENGTH; - if (chunk) { - - } - this.total += chunk.length; this.logger.debug("TRANSFORM IN", /* toHex(chunk), */ chunk.length, this.total); @@ -152,12 +148,15 @@ export class FrameEncoder extends Transform { if (chunk.length > MAX_CHUNK_SIZE) { this.logger.debug("TRANSFORM big chunk, splitting", chunk.length); + remaining = (chunk as Buffer).subarray(MAX_CHUNK_SIZE); chunk = chunk.subarray(0, MAX_CHUNK_SIZE); + this.logger.debug("TRANSFORM processing part/remaining", chunk.length, remaining.length); } const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); + this.tecemux.sequenceNumber++; this.logger.debug("TRANSFORM OUT", /*getHexString(buffer), */ "Size: ", buffer.length); diff --git a/packages/verser/src/lib/tecemux/frames-keeper.ts b/packages/verser/src/lib/tecemux/frames-keeper.ts index 1aece983e..dc402e5db 100644 --- a/packages/verser/src/lib/tecemux/frames-keeper.ts +++ b/packages/verser/src/lib/tecemux/frames-keeper.ts @@ -8,6 +8,7 @@ export class FramesKeeper extends PassThrough { _write(chunk: any, encoding: BufferEncoding, cb: ((error: Error | null | undefined) => void) | undefined) { const sequenceNumber = chunk.readInt32LE(16); + this.archive.set(sequenceNumber, chunk); this.logger.debug(`sequenceNumber ${sequenceNumber} stored, size: ${chunk.length}`); @@ -23,10 +24,10 @@ export class FramesKeeper extends PassThrough { // resolve(); // }); - return true + return true; } - _read(size: number) { + _read(_size: number) { } onACK(sequenceNumber: number) { diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index 2c5686597..f66cbf171 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -11,7 +11,6 @@ import { TeceMux } from "./tecemux"; import { TeceMuxChannel } from "./types"; import { createReadStream, createWriteStream } from "fs"; import path from "path"; -import { Readable } from "stream"; (async () => { const logger = new ObjLogger("Sandbox"); @@ -69,8 +68,7 @@ import { Readable } from "stream"; }) .on("timeout", () => { logger.info("Carrier Socket timeout"); - }) - + }); const tcmux = new TeceMux(socket, "Server") .on("error", (err) => { @@ -173,37 +171,21 @@ import { Readable } from "stream"; }); channel - //.pipe(process.stdout); .pipe(createWriteStream(path.join(__dirname, "output.tar.gz"))); let total = 0; + channel .on("data", (d) => { total += d.length; tcmux.logger.info("-------------------- data", channel._id, d.length, total); - }) .on("pause", () => { tcmux.logger.info("-------------------- paused", channel._id); }) .on("resume", () => { tcmux.logger.info("-------------------- resumed", channel._id); - }) - /*for await (const chunk of channel) { - console.log(chunk); - if (chunk === null) { - break; - } - reqLogger.info("SERVER->CLIENT->CHANNEL", channel._id, chunk.toString()); - - // await new Promise((resolve, _reject) => { - // setTimeout(() => { - // //channel.write("abcde\n"); - // resolve(); - // }, 2000); - // }); - }*/ - + }); }); socket.on("error", (err) => { diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/src/lib/tecemux/playground.ts index 33e3adf3a..badd45084 100644 --- a/packages/verser/src/lib/tecemux/playground.ts +++ b/packages/verser/src/lib/tecemux/playground.ts @@ -4,8 +4,7 @@ import { FrameDecoder } from "./codecs"; import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; import { createReadStream, createWriteStream } from "fs"; import path from "path"; -import { Duplex, PassThrough, Transform } from "stream"; -import { wrap } from "module"; +import { PassThrough, Transform } from "stream"; const tcm = { sequenceNumber: 0 @@ -37,57 +36,49 @@ process.stdin .pipe(decoder) //.pipe(process.stdout); */ -let i = 0; -let t = 0; - -const ws = createWriteStream(path.join(__dirname, "output.tar.gz")) +const ws = createWriteStream(path.join(__dirname, "output.tar.gz")); const delayedPassThrough = () => new PassThrough({ async transform(chunk, encoding, callback) { - await new Promise((res,_rej)=> { + await new Promise((res,_rej) => { setTimeout(res, 1000); }); - this.push(chunk, encoding) - callback(null) + + this.push(chunk, encoding); + callback(null); }, }); -const dh = new Transform({ writableObjectMode: true, transform: (chunk, encoding, callback) => { - try { - console.log(i++, t); - - if (chunk && chunk.length) { - t += chunk.length; +let t = 0; - console.log({ ...JSON.parse(chunk.toString()).chunk, }); - if (!ws.write(new Uint8Array(JSON.parse(chunk.toString()).chunk.data))) { - dh.pause(); - ws.once("drain", () => { - callback(); - dh.resume(); - }) - } else { - callback(null); +const dh = new Transform({ + writableObjectMode: true, + transform: (chunk, encoding, callback) => { + try { + if (chunk && chunk.length) { + t = t + chunk.length; + + if (!ws.write(new Uint8Array(JSON.parse(chunk.toString()).chunk.data))) { + dh.pause(); + ws.once("drain", () => { + callback(); + dh.resume(); + }); + } else { + callback(null); + } } - } - //callback(null); - } catch (e) { - debugger; + //callback(null); + } catch (e) { + logger.error("dh error", e); + } } - -}}) +}); createReadStream(path.join(__dirname, "../../../../forever.tar.gz"), { encoding: undefined }) - .pipe(encoder, { end: false}).out - + .pipe(encoder, { end: false }).out .pipe(delayedPassThrough()) .pipe(decoder) .pipe(delayedPassThrough()) - .pipe(dh) - - // .pipe( - // ws - // ) - - + .pipe(dh); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 13b06e479..588ab7468 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -1,6 +1,6 @@ import { TypedEmitter } from "@scramjet/utility"; import { FrameDecoder, FrameEncoder } from "./codecs"; -import { Duplex, PassThrough, Readable, ReadableOptions } from "stream"; +import { Duplex } from "stream"; import { Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; import { FrameData } from "./utils"; @@ -56,9 +56,7 @@ export class TeceMux extends TypedEmitter { channel .on("error", (error) => { this.logger.error("CHANNEL ERROR", error); - - debugger; - //this.emit("error", { error, source: channel }) + this.emit("error", { error, source: channel }); }) .on("destroy", () => { this.logger.trace("channel on DESTROY ", channel._id); @@ -125,20 +123,17 @@ export class TeceMux extends TypedEmitter { async main() { let t = 0; + for await (const chunk of this.decoder) { - //console.log(chunk); let frame: FrameData; try { frame = JSON.parse(chunk) as FrameData; } catch (error) { - console.debug( chunk.toString()) this.logger.error("error Parsing data from decoder", error, chunk, chunk.length, chunk.toString()); continue; } - //this.logger.debug("Decoded", { ...frame, stringified: "--not-displayed--" }); - const { flags, sequenceNumber, dataLength, destinationPort, acknowledgeNumber } = frame; let channel = this.channels[destinationPort]; @@ -178,26 +173,10 @@ export class TeceMux extends TypedEmitter { this.logger.warn("writing to channel [channel, length]", channel._id, dataLength); this.logger.warn("writing to channel [flowing, isPaused]", channel.readableFlowing, channel.isPaused()); - //Readable.from(chunk).pipe(channel, { end: false });//channel.write(new Uint8Array(((frame.chunk as any).data) as any)); - //channel.push(frame.chunk, undefined); channel.push(new Uint8Array((frame.chunk as any).data), undefined); t += (frame.chunk as any).data.length; - this.logger.info("Writen", t) - - - /* - await new Promise((resolve, reject) => { - this.logger.debug("waiting for drain!") - channel.on("drain", () => { - //channel.push(new Uint8Array(((frame.chunk as any).data) as any)); - this.logger.debug("Drained!") - resolve(); - }); - }); - }*/ - - //this.logger.info("Bytes written to channel [writeResult, channel, length]", written, destinationPort, dataLength); + this.logger.info("Writen", t); } this.sendACK(sequenceNumber, destinationPort); diff --git a/packages/verser/src/lib/verser-client.ts b/packages/verser/src/lib/verser-client.ts index db8d0c50e..d28af8378 100644 --- a/packages/verser/src/lib/verser-client.ts +++ b/packages/verser/src/lib/verser-client.ts @@ -143,7 +143,7 @@ export class VerserClient extends TypedEmitter { this.emit("error", err); }); - this.teceMux.logger.pipe(this.logger); + //this.teceMux.logger.pipe(this.logger); this._verserAgent = new HttpAgent() as HttpAgent & { createConnection: typeof createConnection diff --git a/packages/verser/src/lib/verser-connection.ts b/packages/verser/src/lib/verser-connection.ts index 4928b4531..c8c10e470 100644 --- a/packages/verser/src/lib/verser-connection.ts +++ b/packages/verser/src/lib/verser-connection.ts @@ -204,7 +204,7 @@ export class VerserConnection { // TODO: Error handling? }); - this.teceMux.logger.pipe(this.logger); + //this.teceMux.logger.pipe(this.logger); this.agent = new Agent() as Agent & { createConnection: typeof createConnection }; // lack of types? this.agent.createConnection = () => { diff --git a/yarn.lock b/yarn.lock index d6a6a59e9..1bbc97518 100644 --- a/yarn.lock +++ b/yarn.lock @@ -980,11 +980,6 @@ resolved "https://registry.npmjs.org/@types/node/-/node-15.12.5.tgz" integrity sha512-se3yX7UHv5Bscf8f1ERKvQOD6sTyycH3hdaoozvaLxgUiY5lIGEeH37AD0G0Qi9kPqihPn0HOfd2yaIEN9VwEg== -"@types/node@18.11.18": - version "18.11.18" - resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f" - integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA== - "@types/node@>=13.7.0": version "18.11.3" resolved "https://registry.npmjs.org/@types/node/-/node-18.11.3.tgz" @@ -7392,11 +7387,6 @@ typescript@~4.7.4: resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.7.4.tgz#1a88596d1cf47d59507a1bcdfb5b9dfe4d488235" integrity sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ== -typescript@~4.9.4: - version "4.9.4" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.4.tgz#a2a3d2756c079abda241d75f149df9d561091e78" - integrity sha512-Uz+dTXYzxXXbsFpM86Wh3dKCxrQqUcVMxwU54orwlJjOpO3ao8L7j5lH+dWfTwgCwIuM9GQ2kvVotzYJMXTBZg== - uglify-js@^3.1.4: version "3.17.4" resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.17.4.tgz#61678cf5fa3f5b7eb789bb345df29afb8257c22c" From dd7b3e1a4f9e297058f5262a3f46c50de6943f42 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Sat, 28 Jan 2023 15:25:28 +0000 Subject: [PATCH 082/231] Calc/check checksum --- .../src/lib/tecemux/codecs/frame-decoder.ts | 15 ++++++++ .../src/lib/tecemux/codecs/frame-encoder.ts | 16 +++++---- .../verser/src/lib/tecemux/codecs/utils.ts | 25 +++++++++++++ packages/verser/src/lib/tecemux/tecemux.ts | 11 ++++-- .../verser/src/lib/tecemux/utils/index.ts | 1 + yarn.lock | 36 +++++++++++++++++++ 6 files changed, 95 insertions(+), 9 deletions(-) create mode 100644 packages/verser/src/lib/tecemux/codecs/utils.ts diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index a4452891b..1aa7df77b 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -2,6 +2,7 @@ import { ObjLogger } from "@scramjet/obj-logger"; import { Duplex, Transform, TransformCallback, TransformOptions } from "stream"; import { FrameData, parseFlags } from "../utils"; import { HEADER_LENGTH } from "../constants"; +import { calculateChecksum, getChecksum } from "./utils"; export class FrameDecoder extends Transform { buffer: Buffer; @@ -64,6 +65,10 @@ export class FrameDecoder extends Transform { return; } + const checksum = getChecksum(this.buffer.subarray(0, frameSize)); + + this.logger.debug("getChecksum", checksum.toString()); + const payload = { sourceAddress: [ this.buffer.readInt8(0), this.buffer.readInt8(1), this.buffer.readInt8(2), this.buffer.readInt8(3) @@ -79,9 +84,19 @@ export class FrameDecoder extends Transform { chunkLength: frameSize, sequenceNumber: this.buffer.readInt32LE(16), acknowledgeNumber: this.buffer.readInt32LE(20), + checksum //stringified: this.buffer.subarray(32, frameSize).toString() } as Partial; + const expectedChecksum = calculateChecksum(this.buffer.subarray(0, frameSize)); + + if (checksum !== expectedChecksum) { + this.emit("error", { code: "INVALID_CHECKSUM", payload, expectedChecksum }); + payload.error = "checksum"; + } else { + this.logger.info("Checksum match!", expectedChecksum); + } + if (payload.dataLength && payload.dataLength < 0) { this.emit("error", "Data length incorrect"); return; diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index d31cf0138..6f16e9875 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -4,6 +4,7 @@ import { ObjLogger } from "@scramjet/obj-logger"; import { FrameTarget, HEADER_LENGTH, binaryFlags, frameFlags } from "../constants"; import { ITeCeMux } from "../types"; +import { calculateChecksum } from "./utils"; export class FrameEncoder extends Transform { tecemux: ITeCeMux; @@ -34,8 +35,8 @@ export class FrameEncoder extends Transform { private frameTarget: FrameTarget, tecemux: ITeCeMux, opts: TransformOptions = {}, - params: { name: string } = { name: "FrameEncoder" - }) { + params: { name: string } = { name: "FrameEncoder" } + ) { super(Object.assign(opts, { readableObjectMode: true, writableObjectMode: true @@ -95,7 +96,6 @@ export class FrameEncoder extends Transform { } createFrame(chunk: any = new Uint8Array([]), frame: Partial) { - const checksum = this.getChecksum(); const buffer = Buffer.concat([ // 0: source address 0 - 3 new Uint8Array([10, 0, 0, 1]), @@ -122,18 +122,22 @@ export class FrameEncoder extends Transform { // 224: flags (8bit), 25 this.setFlags(frame.flagsArray, new Uint8Array([0b00000000])), - // window(16bit) 26 - 27 + // window(16bit) 26 - 27, ZEROes before calculation new Uint8Array(new Uint16Array([0]).buffer), // checksum(16bit) 28 - 29 - new Uint8Array(new Uint16Array([checksum]).buffer), + new Uint8Array(new Uint16Array([0]).buffer), // pointer (16bit) 30 - 31 - new Uint8Array(new Uint16Array([checksum]).buffer), + new Uint8Array(new Uint16Array([0]).buffer), // 256: data 32 - new Uint8Array(chunk) ]); + buffer.writeUInt16LE( + calculateChecksum(buffer), 28 + ); + return buffer; } diff --git a/packages/verser/src/lib/tecemux/codecs/utils.ts b/packages/verser/src/lib/tecemux/codecs/utils.ts new file mode 100644 index 000000000..98db2e4bf --- /dev/null +++ b/packages/verser/src/lib/tecemux/codecs/utils.ts @@ -0,0 +1,25 @@ +export const calculateChecksum = (buffer: Buffer) => { + let tempFrame = Buffer.concat([buffer, Buffer.alloc(0)]); + + if (buffer.length % 1) { + tempFrame = Buffer.concat([buffer, Buffer.alloc(1, 0)]); + } + + let checksum = 0; + let i = 0; + + while (i <= tempFrame.length - 2) { + if (i !== 28) { + checksum += tempFrame.readUInt16LE(i); + } + + i += 2; + } + + return checksum % 0x10000; +}; + +export const getChecksum = (buffer: Buffer) => { + return buffer.readUInt16LE(28); +}; + diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 588ab7468..c7c4ea150 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -129,12 +129,17 @@ export class TeceMux extends TypedEmitter { try { frame = JSON.parse(chunk) as FrameData; - } catch (error) { - this.logger.error("error Parsing data from decoder", error, chunk, chunk.length, chunk.toString()); + } catch (err) { + this.logger.error("error Parsing data from decoder", err, chunk, chunk.length, chunk.toString()); continue; } - const { flags, sequenceNumber, dataLength, destinationPort, acknowledgeNumber } = frame; + const { flags, sequenceNumber, dataLength, destinationPort, acknowledgeNumber, error } = frame; + + if (error) { + this.emit("error", frame); + continue; + } let channel = this.channels[destinationPort]; diff --git a/packages/verser/src/lib/tecemux/utils/index.ts b/packages/verser/src/lib/tecemux/utils/index.ts index dde25f260..91dc00170 100644 --- a/packages/verser/src/lib/tecemux/utils/index.ts +++ b/packages/verser/src/lib/tecemux/utils/index.ts @@ -17,6 +17,7 @@ export type FrameData = { stringified: string; flags: flagsObjectType; flagsArray: (keyof typeof binaryFlags)[]; + error?: "checksum" }; export const parseFlags = (byte: number): flagsObjectType => { diff --git a/yarn.lock b/yarn.lock index 1bbc97518..df5227dda 100644 --- a/yarn.lock +++ b/yarn.lock @@ -814,6 +814,29 @@ resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== +"@scramjet/api-server@^0.31.4": + version "0.31.4" + resolved "https://registry.yarnpkg.com/@scramjet/api-server/-/api-server-0.31.4.tgz#a2855de120c1093d6cfd96b50447babd62116823" + integrity sha512-ghhKbK6omrTJIT4xgRhT5bLUE/Hk+ShmM3/J9XwcVwxGgltqtEC10k1G6Dm42vqYnUxLgW3iIsmW9OMCNh2jfw== + dependencies: + "0http" "^3.4.1" + "@scramjet/model" "^0.31.4" + "@scramjet/obj-logger" "^0.31.4" + "@scramjet/symbols" "^0.31.4" + "@scramjet/utility" "^0.31.4" + http-status-codes "^2.2.0" + scramjet "^4.36.9" + +"@scramjet/model@^0.31.4": + version "0.31.4" + resolved "https://registry.yarnpkg.com/@scramjet/model/-/model-0.31.4.tgz#2cc4e9a9855abeee0e450c7a9d66525bfdb03301" + integrity sha512-NkM/6DhVpN0tsE/1aCY9v2EOSxq3IecBtwtnaz9EYEAb0XTt78hkseALevzvOLDvc4G+F0BqmZKeqHWxbAJFgg== + dependencies: + "@scramjet/obj-logger" "^0.31.4" + "@scramjet/symbols" "^0.31.4" + scramjet "^4.36.9" + uuid "^8.3.2" + "@scramjet/obj-logger@^0.31.4": version "0.31.4" resolved "https://registry.yarnpkg.com/@scramjet/obj-logger/-/obj-logger-0.31.4.tgz#9c755403afa544afa31cc30dbca58913b95c8388" @@ -822,6 +845,19 @@ "@scramjet/utility" "^0.31.4" scramjet "^4.36.9" +"@scramjet/symbols@^0.31.4": + version "0.31.4" + resolved "https://registry.yarnpkg.com/@scramjet/symbols/-/symbols-0.31.4.tgz#f1ccaeee59b26530caf47fe021bf45727f6da4f0" + integrity sha512-dJWBd7VgMU627SlXVg27n3F/aBzvUHTEGMCTq5N46W91OeU/YhjRT+dO5NyJhAxpyTHdlMFCHKAFC9FrGzdoMw== + +"@scramjet/types@^0.31.4": + version "0.31.4" + resolved "https://registry.yarnpkg.com/@scramjet/types/-/types-0.31.4.tgz#73e298bbcf01abf7a378b26fa0da9a9093d45341" + integrity sha512-RxJzbRD+q4w5RrlaT5cg6GdlMzlXnDrqBSKaAeDdzu/2OgApHJOBE4roP1hqtwQkgtQvXMNeF8uUJM0IFCsjSQ== + dependencies: + "@scramjet/symbols" "^0.31.4" + http-status-codes "^2.2.0" + "@scramjet/utility@^0.31.4": version "0.31.4" resolved "https://registry.yarnpkg.com/@scramjet/utility/-/utility-0.31.4.tgz#9e13f30ef66636e77a7a065d1496f35e1dfc7017" From 3a20fc0c974c48ae1a1c3816204cedc932e5a32d Mon Sep 17 00:00:00 2001 From: patuwwy Date: Sun, 29 Jan 2023 21:33:55 +0000 Subject: [PATCH 083/231] FramesKeeper --- .../src/lib/tecemux/codecs/frame-encoder.ts | 30 +++++---- .../verser/src/lib/tecemux/frames-keeper.ts | 63 +++++++++++-------- .../src/lib/tecemux/playground-tecemux.ts | 2 +- packages/verser/src/lib/tecemux/playground.ts | 7 +-- packages/verser/src/lib/tecemux/tecemux.ts | 23 +++---- .../verser/src/lib/tecemux/utils/index.ts | 3 +- 6 files changed, 72 insertions(+), 56 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index 6f16e9875..98cadb502 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -12,7 +12,9 @@ export class FrameEncoder extends Transform { logger = new ObjLogger("FrameEncoder"); out = Object.assign(new PassThrough({ readableObjectMode: true }) /*.on("data", (data) => { - this.logger.trace("output to socket: " + (data.length === HEADER_LENGTH ? "HEADER ONLY" : ""), toHex(data), data.length, this.readableFlowing); + this.logger.trace( + "output to socket: " + (data.length === HEADER_LENGTH ? "HEADER ONLY" : ""), + toHex(data), data.length, this.readableFlowing); })*/ .on("pause", () => { this.logger.trace("OUT paused!"); @@ -64,7 +66,9 @@ export class FrameEncoder extends Transform { setFlags(flag: (keyof typeof binaryFlags)[] = [], flags: Uint8Array = new Uint8Array([0])) { for (const f in flag) { - flags[0] |= 1 << frameFlags.indexOf(flag[f]); + if (Object.prototype.hasOwnProperty.call(flag, f)) { + flags[0] |= 1 << frameFlags.indexOf(flag[f]); + } } this.logger.debug("settingFlag", flag, flags[0].toString(2).padStart(8, "0")); @@ -75,20 +79,22 @@ export class FrameEncoder extends Transform { setChannel(channelCount: number) { this.logger.debug("Set channel command", channelCount); - //const checksum = this.getChecksum(); - - this.out.write(this.createFrame([], { - flagsArray: ["PSH"], - destinationPort: channelCount - })); + this.out.write( + this.createFrame([], { + flagsArray: ["PSH"], + destinationPort: channelCount + }) + ); } onChannelEnd(channelId: number) { this.logger.debug("sending FIN for channel", channelId); - this.out.write(this.createFrame([], { - flagsArray: ["FIN"], - destinationPort: channelId - })); + this.out.write( + this.createFrame([], { + flagsArray: ["FIN"], + destinationPort: channelId + }) + ); } getChecksum() { diff --git a/packages/verser/src/lib/tecemux/frames-keeper.ts b/packages/verser/src/lib/tecemux/frames-keeper.ts index dc402e5db..40dae5918 100644 --- a/packages/verser/src/lib/tecemux/frames-keeper.ts +++ b/packages/verser/src/lib/tecemux/frames-keeper.ts @@ -1,37 +1,46 @@ import { ObjLogger } from "@scramjet/obj-logger"; -import { PassThrough } from "stream"; - -export class FramesKeeper extends PassThrough { - logger = new ObjLogger(this); - archive = new Map(); - lastSequenceNumberSent: number = -1; - - _write(chunk: any, encoding: BufferEncoding, cb: ((error: Error | null | undefined) => void) | undefined) { - const sequenceNumber = chunk.readInt32LE(16); - - this.archive.set(sequenceNumber, chunk); - this.logger.debug(`sequenceNumber ${sequenceNumber} stored, size: ${chunk.length}`); +import { TransformOptions, Writable } from "stream"; +import { IObjectLogger } from "@scramjet/types"; + +export class FramesKeeper extends Writable { + logger: IObjectLogger; + framesSent = new Map(); + lastSequenceSent: number = -1; + lastSequenceReceived: number = -1; + + constructor( + opts: TransformOptions = {}, + params: { name: string } = { name: "Keeper" } + ) { + super(Object.assign(opts, { + readableObjectMode: true, + writableObjectMode: true + })); + + this.logger = new ObjLogger(params.name); + } - this.push(chunk, encoding); - if (cb) cb(undefined); + _write(chunk: any, encoding: BufferEncoding, cb: ((error: Error | null | undefined) => void)) { + if (Buffer.isBuffer(chunk)) { + this.logger.info("transform buffer"); + const sequenceNumber = chunk.readInt32LE(16); - this.lastSequenceNumberSent = sequenceNumber; + this.framesSent.set(sequenceNumber, { buffer: chunk, received: false, sequenceNumber }); - // await new Promise((resolve, reject) => { - // while (this.archive.get(sequenceNumber)) { - // this.logger.debug("wait for ack for", sequenceNumber) - // } - // resolve(); - // }); + this.lastSequenceSent = sequenceNumber; + this.logger.debug(`lastSequenceSent ${sequenceNumber}, size: ${chunk.length}`); + } - return true; + cb(undefined); } - _read(_size: number) { - } + onACK(acknowledgeNumber: number,) { + this.logger.debug("onACK", acknowledgeNumber); + + const storedFrame = this.framesSent.get(acknowledgeNumber); - onACK(sequenceNumber: number) { - this.logger.debug("onACK", sequenceNumber); - this.archive.delete(sequenceNumber); + if (storedFrame) { + this.framesSent.set(acknowledgeNumber, { ...storedFrame, received: true }); + } } } diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index f66cbf171..7b3586db3 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -89,7 +89,7 @@ import path from "path"; //DataStream.from(process.stdin).filter((x: Buffer) => !(parseInt(x[0].toString(), 10) % 2)).pipe(channel1); //DataStream.from(process.stdin).filter((x: Buffer) => !!(parseInt(x[0].toString(), 10) % 2)).pipe(channel2); - /* + /* (async () => { try { for await (const chunk of channel1) { diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/src/lib/tecemux/playground.ts index badd45084..304316f47 100644 --- a/packages/verser/src/lib/tecemux/playground.ts +++ b/packages/verser/src/lib/tecemux/playground.ts @@ -1,6 +1,5 @@ import { DataStream } from "scramjet"; -import { FrameEncoder } from "./codecs"; -import { FrameDecoder } from "./codecs"; +import { FrameDecoder, FrameEncoder } from "./codecs"; import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; import { createReadStream, createWriteStream } from "fs"; import path from "path"; @@ -22,7 +21,7 @@ logger.pipe( ) ) ) -.pipe(process.stdout); + .pipe(process.stdout); const encoder = new FrameEncoder(0, tcm); const decoder = new FrameDecoder(); @@ -40,7 +39,7 @@ const ws = createWriteStream(path.join(__dirname, "output.tar.gz")); const delayedPassThrough = () => new PassThrough({ async transform(chunk, encoding, callback) { - await new Promise((res,_rej) => { + await new Promise((res, _rej) => { setTimeout(res, 1000); }); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index c7c4ea150..dd60a0b4d 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -11,7 +11,7 @@ export class TeceMux extends TypedEmitter { id: string; carrierSocket: Duplex; channelCount = 1; - decoder: FrameDecoder; + carrierDecoder: FrameDecoder; framesKeeper = new FramesKeeper(); sequenceNumber = 0; channels: TeceMuxChannel[] = []; @@ -27,10 +27,6 @@ export class TeceMux extends TypedEmitter { encoder.logger.updateBaseLog({ id: this.id }); encoder.logger.pipe(this.logger); - //const w = new PassThrough().on("error", (error) => { this.logger.error("W error", error)}); - - //w.pipe(encoder) - const channel: TeceMuxChannel = Object.assign( new Duplex({ write: (chunk, encoding, next) => { @@ -50,6 +46,9 @@ export class TeceMux extends TypedEmitter { } ); + encoder.out + .pipe(this.framesKeeper); + encoder.out .pipe(this.carrierSocket, { end: false }); @@ -84,7 +83,7 @@ export class TeceMux extends TypedEmitter { this.logger = new ObjLogger(this, { id: this.id }); this.carrierSocket = socket; - this.decoder = new FrameDecoder({ emitClose: false }) + this.carrierDecoder = new FrameDecoder({ emitClose: false }) .on("pause", () => { this.logger.warn("Decoder paused"); }) @@ -104,10 +103,10 @@ export class TeceMux extends TypedEmitter { this.logger.error("Decoder destroy", error); }); - this.decoder.logger.updateBaseLog({ id: this.id }); - this.decoder.logger.pipe(this.logger); + this.carrierDecoder.logger.updateBaseLog({ id: this.id }); + this.carrierDecoder.logger.pipe(this.logger); - this.carrierSocket.pipe(this.decoder, { end: false }); + this.carrierSocket.pipe(this.carrierDecoder, { end: false }); this.commonEncoder.out.pipe(this.carrierSocket, { end: false }); @@ -124,7 +123,9 @@ export class TeceMux extends TypedEmitter { async main() { let t = 0; - for await (const chunk of this.decoder) { + //this.carrierDecoder.pipe(this.framesKeeper); + + for await (const chunk of this.carrierDecoder) { let frame: FrameData; try { @@ -145,7 +146,7 @@ export class TeceMux extends TypedEmitter { if (flags.ACK) { this.logger.trace("ACK frame received for sequenceNumber", acknowledgeNumber); - //this.framesKeeper.onACK(sequenceNumber); + this.framesKeeper.onACK(acknowledgeNumber); continue; } diff --git a/packages/verser/src/lib/tecemux/utils/index.ts b/packages/verser/src/lib/tecemux/utils/index.ts index 91dc00170..b90056eb7 100644 --- a/packages/verser/src/lib/tecemux/utils/index.ts +++ b/packages/verser/src/lib/tecemux/utils/index.ts @@ -17,7 +17,8 @@ export type FrameData = { stringified: string; flags: flagsObjectType; flagsArray: (keyof typeof binaryFlags)[]; - error?: "checksum" + error?: "checksum"; + received: boolean; }; export const parseFlags = (byte: number): flagsObjectType => { From c3332cb271d4ba54e318315e3181f2c01c85c46e Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 30 Jan 2023 13:19:32 +0000 Subject: [PATCH 084/231] Check for ACK of last frames send --- .../src/lib/tecemux/codecs/frame-encoder.ts | 26 +++++++++++++--- packages/verser/src/lib/tecemux/constants.ts | 2 ++ .../verser/src/lib/tecemux/frames-keeper.ts | 31 ++++++++++++++++--- packages/verser/src/lib/tecemux/playground.ts | 13 +++----- packages/verser/src/lib/tecemux/types.ts | 15 +++++++++ 5 files changed, 71 insertions(+), 16 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index 98cadb502..d11f9d32d 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -2,7 +2,7 @@ import { PassThrough, Transform, TransformCallback, TransformOptions } from "str import { FrameData } from "../utils"; import { ObjLogger } from "@scramjet/obj-logger"; -import { FrameTarget, HEADER_LENGTH, binaryFlags, frameFlags } from "../constants"; +import { ACK_FRAME_DELTA, FrameTarget, HEADER_LENGTH, binaryFlags, frameFlags } from "../constants"; import { ITeCeMux } from "../types"; import { calculateChecksum } from "./utils"; @@ -147,7 +147,7 @@ export class FrameEncoder extends Transform { return buffer; } - _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void { + async _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): Promise { const MAX_CHUNK_SIZE = 10 * 1024 - HEADER_LENGTH; this.total += chunk.length; @@ -167,13 +167,31 @@ export class FrameEncoder extends Transform { const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); - this.tecemux.sequenceNumber++; + if (this.tecemux.sequenceNumber > -ACK_FRAME_DELTA) { + // eslint-disable-next-line no-loop-func + while (!await new Promise((res, _rej) => { + setImmediate(() => { + const frame = this.tecemux.framesKeeper.getFrame(this.tecemux.sequenceNumber + ACK_FRAME_DELTA); + + if (frame) { + const rec = frame?.received || frame?.flags.ACK; + + this.logger.info(`Sending ${this.tecemux.sequenceNumber}, ${frame?.sequenceNumber} ${frame?.received}`); + res(!!rec); + } else { + _rej("frame not exists"); + } + }); + })); + } + + this.tecemux.sequenceNumber++; this.logger.debug("TRANSFORM OUT", /*getHexString(buffer), */ "Size: ", buffer.length); if (remaining.length) { this.push(buffer); - this._transform(remaining, encoding, callback); + await this._transform(remaining, encoding, callback); } else { callback(null, buffer); } diff --git a/packages/verser/src/lib/tecemux/constants.ts b/packages/verser/src/lib/tecemux/constants.ts index 4e4aa9f38..17e8362a3 100644 --- a/packages/verser/src/lib/tecemux/constants.ts +++ b/packages/verser/src/lib/tecemux/constants.ts @@ -17,3 +17,5 @@ export enum FrameTarget { API, INPUT = 1001 } + +export const ACK_FRAME_DELTA = -5; diff --git a/packages/verser/src/lib/tecemux/frames-keeper.ts b/packages/verser/src/lib/tecemux/frames-keeper.ts index 40dae5918..27146824e 100644 --- a/packages/verser/src/lib/tecemux/frames-keeper.ts +++ b/packages/verser/src/lib/tecemux/frames-keeper.ts @@ -1,10 +1,13 @@ import { ObjLogger } from "@scramjet/obj-logger"; -import { TransformOptions, Writable } from "stream"; import { IObjectLogger } from "@scramjet/types"; -export class FramesKeeper extends Writable { +import { TransformOptions, Writable } from "stream"; +import { parseFlags } from "./utils"; +import { IFramesKeeper, FramesKeeperFrame } from "./types"; + +export class FramesKeeper extends Writable implements IFramesKeeper { logger: IObjectLogger; - framesSent = new Map(); + framesSent = new Map(); lastSequenceSent: number = -1; lastSequenceReceived: number = -1; @@ -24,8 +27,13 @@ export class FramesKeeper extends Writable { if (Buffer.isBuffer(chunk)) { this.logger.info("transform buffer"); const sequenceNumber = chunk.readInt32LE(16); + const destinationPort = chunk.readInt16LE(14); + const flags = parseFlags(chunk.readInt8(25)); - this.framesSent.set(sequenceNumber, { buffer: chunk, received: false, sequenceNumber }); + this.framesSent.set( + sequenceNumber, { + buffer: chunk, received: false, sequenceNumber, destinationPort, flags + }); this.lastSequenceSent = sequenceNumber; this.logger.debug(`lastSequenceSent ${sequenceNumber}, size: ${chunk.length}`); @@ -43,4 +51,19 @@ export class FramesKeeper extends Writable { this.framesSent.set(acknowledgeNumber, { ...storedFrame, received: true }); } } + + isReceived(sequenceNumber: number) { + const frame = this.framesSent.get(sequenceNumber); + + // received or not stored + return frame === undefined || !!this.framesSent.get(sequenceNumber)?.received; + } + + getFrame(sequenceNumber: number) { + return this.framesSent.get(sequenceNumber); + } + + getDestinationPort(sequenceNumber: number) { + return this.framesSent.get(sequenceNumber)?.received; + } } diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/src/lib/tecemux/playground.ts index 304316f47..f51d43ecd 100644 --- a/packages/verser/src/lib/tecemux/playground.ts +++ b/packages/verser/src/lib/tecemux/playground.ts @@ -4,10 +4,13 @@ import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; import { createReadStream, createWriteStream } from "fs"; import path from "path"; import { PassThrough, Transform } from "stream"; +import { FramesKeeper } from "./frames-keeper"; +import { ITeCeMux } from "./types"; const tcm = { - sequenceNumber: 0 -}; + sequenceNumber: 0, + framesKeeper: new FramesKeeper() +} as ITeCeMux; const logger = new ObjLogger("Sandbox"); @@ -29,12 +32,6 @@ const decoder = new FrameDecoder(); encoder.logger.pipe(logger); decoder.logger.pipe(logger); -/* -process.stdin - .pipe(encoder).out - .pipe(decoder) - //.pipe(process.stdout); -*/ const ws = createWriteStream(path.join(__dirname, "output.tar.gz")); const delayedPassThrough = () => new PassThrough({ diff --git a/packages/verser/src/lib/tecemux/types.ts b/packages/verser/src/lib/tecemux/types.ts index 7d8c041f8..9b04efc2f 100644 --- a/packages/verser/src/lib/tecemux/types.ts +++ b/packages/verser/src/lib/tecemux/types.ts @@ -16,8 +16,23 @@ export type flagsObjectType = Partial<{ CWR: boolean }> +export type FramesKeeperFrame = { + buffer: Buffer; + received: boolean; + sequenceNumber: number; + destinationPort: number; + flags: any +}; + +export interface IFramesKeeper { + onACK(acknowledgeNumber: number): void; + isReceived(sequenceNumber: number): boolean; + getFrame(sequenceNumber: number): FramesKeeperFrame | undefined +} + export interface ITeCeMux { sequenceNumber: number; + framesKeeper: IFramesKeeper; } export interface IFrameEncoder { From d160314cc0078444d3de57e6b0438833d03ab25c Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 31 Jan 2023 11:43:53 +0000 Subject: [PATCH 085/231] Fix send FIN --- .../src/lib/tecemux/playground-tecemux.ts | 5 ++++- packages/verser/src/lib/tecemux/tecemux.ts | 18 +++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index 7b3586db3..69199e99b 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -80,7 +80,10 @@ import path from "path"; const channel1 = tcmux.multiplex(); //const channel2 = tcmux.multiplex(); - createReadStream(path.join(__dirname, "../../../../forever.tar.gz")).pipe(channel1); + createReadStream(path.join(__dirname, "../../../../forever.tar.gz")).on("end", () => { + logger.info("FILE END"); + }).pipe(channel1); + //Readable.from(Buffer.alloc(1024 * 100)).pipe(channel1, { end: false }); req.on("pause", () => { logger.warn("Request paused"); }); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index dd60a0b4d..6a0dc4eaa 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -32,11 +32,20 @@ export class TeceMux extends TypedEmitter { write: (chunk, encoding, next) => { this.logger.trace("WRITE channel", channel._id, chunk); + if (chunk === null) { + this.logger.info("NULL ON CHANNEL"); + channel.end(); + return false; + } + return encoder.write(chunk, encoding, next); }, read: (_size) => { this.logger.trace("READ channel", channel._id); }, + final() { + //channel.emit("end"); + }, allowHalfOpen: true }), { @@ -65,6 +74,7 @@ export class TeceMux extends TypedEmitter { }) .on("end", () => { this.logger.info("CHANNEL end", channel._id); + if (!channel.closedByFIN) { this.sendFIN(channel._id); } @@ -154,8 +164,14 @@ export class TeceMux extends TypedEmitter { this.logger.trace(`Received FIN command [C: ${destinationPort}]`, dataLength, frame.chunk, !!this.channels[destinationPort]); if (channel) { - //channel.end(); channel.closedByFIN = true; + + if (channel.readableEnded) { + channel.once("end", () => { + this.logger.info("channel --------- ENDED"); + }); + } + channel.push(null); } else { this.logger.error("FIN for unknown channel"); From a41e00daca0aced2dcffe42ebf07cf30830b3aea Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 31 Jan 2023 22:19:43 +0000 Subject: [PATCH 086/231] Send file both directions --- .../src/lib/tecemux/codecs/frame-encoder.ts | 3 +- .../verser/src/lib/tecemux/frames-keeper.ts | 2 +- .../src/lib/tecemux/playground-tecemux.ts | 16 +++--- packages/verser/src/lib/tecemux/playground.ts | 3 +- packages/verser/src/lib/tecemux/tecemux.ts | 27 +++++----- packages/verser/src/lib/tecemux/types.ts | 1 + yarn.lock | 52 ------------------- 7 files changed, 29 insertions(+), 75 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index d11f9d32d..f9370dc1f 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -167,7 +167,7 @@ export class FrameEncoder extends Transform { const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); - if (this.tecemux.sequenceNumber > -ACK_FRAME_DELTA) { + if (this.tecemux.framesSent > -ACK_FRAME_DELTA) { // eslint-disable-next-line no-loop-func while (!await new Promise((res, _rej) => { setImmediate(() => { @@ -186,6 +186,7 @@ export class FrameEncoder extends Transform { })); } + this.tecemux.framesSent++; this.tecemux.sequenceNumber++; this.logger.debug("TRANSFORM OUT", /*getHexString(buffer), */ "Size: ", buffer.length); diff --git a/packages/verser/src/lib/tecemux/frames-keeper.ts b/packages/verser/src/lib/tecemux/frames-keeper.ts index 27146824e..d0e9bf8a2 100644 --- a/packages/verser/src/lib/tecemux/frames-keeper.ts +++ b/packages/verser/src/lib/tecemux/frames-keeper.ts @@ -36,7 +36,7 @@ export class FramesKeeper extends Writable implements IFramesKeeper { }); this.lastSequenceSent = sequenceNumber; - this.logger.debug(`lastSequenceSent ${sequenceNumber}, size: ${chunk.length}`); + this.logger.debug(`lastSequenceSent ${sequenceNumber}, size: ${chunk.length} total frames sent ${this.framesSent.size}`,); } cb(undefined); diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index 69199e99b..b75bf78d7 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -80,9 +80,13 @@ import path from "path"; const channel1 = tcmux.multiplex(); //const channel2 = tcmux.multiplex(); - createReadStream(path.join(__dirname, "../../../../forever.tar.gz")).on("end", () => { - logger.info("FILE END"); - }).pipe(channel1); + channel1.pipe(createWriteStream(path.join(__dirname, "output-server.tar.gz"))); + + createReadStream(path.join(__dirname, "../../../../forever.tar.gz")) + .on("end", () => { + logger.info("FILE END"); + }) + .pipe(channel1); //Readable.from(Buffer.alloc(1024 * 100)).pipe(channel1, { end: false }); @@ -170,11 +174,11 @@ import path from "path"; tcmux.logger.info("Channel finish", channel._id); }) .on("end", () => { - tcmux.logger.info("Channel end", channel._id); + tcmux.logger.info("Channel readable end", channel._id, channel.readableEnded, channel.writableEnded); }); - channel - .pipe(createWriteStream(path.join(__dirname, "output.tar.gz"))); + createReadStream(path.join(__dirname, "../../../../forever.tar.gz")).pipe(channel, { end: false }); + channel.pipe(createWriteStream(path.join(__dirname, "output-client.tar.gz"))); let total = 0; diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/src/lib/tecemux/playground.ts index f51d43ecd..694ad1a45 100644 --- a/packages/verser/src/lib/tecemux/playground.ts +++ b/packages/verser/src/lib/tecemux/playground.ts @@ -9,7 +9,8 @@ import { ITeCeMux } from "./types"; const tcm = { sequenceNumber: 0, - framesKeeper: new FramesKeeper() + framesKeeper: new FramesKeeper(), + framesSent: 0 } as ITeCeMux; const logger = new ObjLogger("Sandbox"); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 6a0dc4eaa..e37595768 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -11,9 +11,10 @@ export class TeceMux extends TypedEmitter { id: string; carrierSocket: Duplex; channelCount = 1; + framesSent = 0; carrierDecoder: FrameDecoder; framesKeeper = new FramesKeeper(); - sequenceNumber = 0; + sequenceNumber = Math.abs((Math.random() * (2 ** 32)) | 0); channels: TeceMuxChannel[] = []; logger: ObjLogger; @@ -34,6 +35,7 @@ export class TeceMux extends TypedEmitter { if (chunk === null) { this.logger.info("NULL ON CHANNEL"); + channel.end(); return false; } @@ -74,10 +76,7 @@ export class TeceMux extends TypedEmitter { }) .on("end", () => { this.logger.info("CHANNEL end", channel._id); - - if (!channel.closedByFIN) { - this.sendFIN(channel._id); - } + this.sendFIN(channel._id); }); if (emit) { @@ -149,30 +148,30 @@ export class TeceMux extends TypedEmitter { if (error) { this.emit("error", frame); - continue; + break; } let channel = this.channels[destinationPort]; if (flags.ACK) { - this.logger.trace("ACK frame received for sequenceNumber", acknowledgeNumber); + this.logger.trace("Received ACK flag for sequenceNumber", acknowledgeNumber); this.framesKeeper.onACK(acknowledgeNumber); continue; } if (flags.FIN) { - this.logger.trace(`Received FIN command [C: ${destinationPort}]`, dataLength, frame.chunk, !!this.channels[destinationPort]); + this.logger.trace(`Received FIN flag [C: ${destinationPort}]`, dataLength, frame.chunk, !!this.channels[destinationPort], channel._id); if (channel) { channel.closedByFIN = true; - - if (channel.readableEnded) { - channel.once("end", () => { - this.logger.info("channel --------- ENDED"); - }); + if (!channel.writableEnded) { + channel.push(null); } - channel.push(null); + if (channel.writableEnded && channel.readableEnded) { + channel.destroy(); + this.logger.info("Channel destroy"); + } } else { this.logger.error("FIN for unknown channel"); } diff --git a/packages/verser/src/lib/tecemux/types.ts b/packages/verser/src/lib/tecemux/types.ts index 9b04efc2f..67a5a6443 100644 --- a/packages/verser/src/lib/tecemux/types.ts +++ b/packages/verser/src/lib/tecemux/types.ts @@ -32,6 +32,7 @@ export interface IFramesKeeper { export interface ITeCeMux { sequenceNumber: number; + framesSent: number; framesKeeper: IFramesKeeper; } diff --git a/yarn.lock b/yarn.lock index df5227dda..07dea5cdb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -814,58 +814,6 @@ resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== -"@scramjet/api-server@^0.31.4": - version "0.31.4" - resolved "https://registry.yarnpkg.com/@scramjet/api-server/-/api-server-0.31.4.tgz#a2855de120c1093d6cfd96b50447babd62116823" - integrity sha512-ghhKbK6omrTJIT4xgRhT5bLUE/Hk+ShmM3/J9XwcVwxGgltqtEC10k1G6Dm42vqYnUxLgW3iIsmW9OMCNh2jfw== - dependencies: - "0http" "^3.4.1" - "@scramjet/model" "^0.31.4" - "@scramjet/obj-logger" "^0.31.4" - "@scramjet/symbols" "^0.31.4" - "@scramjet/utility" "^0.31.4" - http-status-codes "^2.2.0" - scramjet "^4.36.9" - -"@scramjet/model@^0.31.4": - version "0.31.4" - resolved "https://registry.yarnpkg.com/@scramjet/model/-/model-0.31.4.tgz#2cc4e9a9855abeee0e450c7a9d66525bfdb03301" - integrity sha512-NkM/6DhVpN0tsE/1aCY9v2EOSxq3IecBtwtnaz9EYEAb0XTt78hkseALevzvOLDvc4G+F0BqmZKeqHWxbAJFgg== - dependencies: - "@scramjet/obj-logger" "^0.31.4" - "@scramjet/symbols" "^0.31.4" - scramjet "^4.36.9" - uuid "^8.3.2" - -"@scramjet/obj-logger@^0.31.4": - version "0.31.4" - resolved "https://registry.yarnpkg.com/@scramjet/obj-logger/-/obj-logger-0.31.4.tgz#9c755403afa544afa31cc30dbca58913b95c8388" - integrity sha512-6sfS45IWkRMmdiG+2egr7FTyjpNW5Dc+n2WvzWbk6M/37/B30lRUtVkYGKTpIBbw/arp6lUuH0w/QjlC+QAiDQ== - dependencies: - "@scramjet/utility" "^0.31.4" - scramjet "^4.36.9" - -"@scramjet/symbols@^0.31.4": - version "0.31.4" - resolved "https://registry.yarnpkg.com/@scramjet/symbols/-/symbols-0.31.4.tgz#f1ccaeee59b26530caf47fe021bf45727f6da4f0" - integrity sha512-dJWBd7VgMU627SlXVg27n3F/aBzvUHTEGMCTq5N46W91OeU/YhjRT+dO5NyJhAxpyTHdlMFCHKAFC9FrGzdoMw== - -"@scramjet/types@^0.31.4": - version "0.31.4" - resolved "https://registry.yarnpkg.com/@scramjet/types/-/types-0.31.4.tgz#73e298bbcf01abf7a378b26fa0da9a9093d45341" - integrity sha512-RxJzbRD+q4w5RrlaT5cg6GdlMzlXnDrqBSKaAeDdzu/2OgApHJOBE4roP1hqtwQkgtQvXMNeF8uUJM0IFCsjSQ== - dependencies: - "@scramjet/symbols" "^0.31.4" - http-status-codes "^2.2.0" - -"@scramjet/utility@^0.31.4": - version "0.31.4" - resolved "https://registry.yarnpkg.com/@scramjet/utility/-/utility-0.31.4.tgz#9e13f30ef66636e77a7a065d1496f35e1dfc7017" - integrity sha512-BFwoYu+u5SbuJX5QgzrSgb+6Vak+cCAu141C5zasoAMplM4AmvqPWQAZ7CQ6Du7KQA7HsJqgD0xCHHky+mgoEA== - dependencies: - normalize-url "^5.3.1" - yaml "^2.1.3" - "@sindresorhus/is@^0.14.0": version "0.14.0" resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea" From 96b19ebd820bf98b6ee6ac447723d44069ece381 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 2 Feb 2023 23:27:35 +0000 Subject: [PATCH 087/231] Debug w/o FrameKeeper --- packages/verser/src/lib/tecemux/codecs/frame-decoder.ts | 2 +- packages/verser/src/lib/tecemux/codecs/frame-encoder.ts | 9 +++++---- packages/verser/src/lib/tecemux/frames-keeper.ts | 5 +++-- packages/verser/src/lib/tecemux/tecemux.ts | 5 +++-- packages/verser/src/lib/verser.ts | 4 ++++ 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 1aa7df77b..5c81b7378 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -106,7 +106,7 @@ export class FrameDecoder extends Transform { this.buffer = this.buffer.subarray(frameSize); - this.logger.trace("Decoded", { ...payload, stringified: "--not-displayed--" }); + this.logger.trace("Decoded", { ...payload, stringified: payload.chunk?.toString() }); if (this.buffer.length === 0) { this.logger.info("No remaining data!"); diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index f9370dc1f..a7f1c1de8 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -2,7 +2,7 @@ import { PassThrough, Transform, TransformCallback, TransformOptions } from "str import { FrameData } from "../utils"; import { ObjLogger } from "@scramjet/obj-logger"; -import { ACK_FRAME_DELTA, FrameTarget, HEADER_LENGTH, binaryFlags, frameFlags } from "../constants"; +import { FrameTarget, HEADER_LENGTH, binaryFlags, frameFlags } from "../constants"; import { ITeCeMux } from "../types"; import { calculateChecksum } from "./utils"; @@ -167,7 +167,7 @@ export class FrameEncoder extends Transform { const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); - if (this.tecemux.framesSent > -ACK_FRAME_DELTA) { + /*if (this.tecemux.framesSent > -ACK_FRAME_DELTA) { // eslint-disable-next-line no-loop-func while (!await new Promise((res, _rej) => { setImmediate(() => { @@ -180,11 +180,12 @@ export class FrameEncoder extends Transform { res(!!rec); } else { - _rej("frame not exists"); + console.log((this.tecemux.framesKeeper as any)["framesSent"]) + _rej("frame not exists" + (this.tecemux.sequenceNumber + ACK_FRAME_DELTA)); } }); })); - } + }*/ this.tecemux.framesSent++; this.tecemux.sequenceNumber++; diff --git a/packages/verser/src/lib/tecemux/frames-keeper.ts b/packages/verser/src/lib/tecemux/frames-keeper.ts index d0e9bf8a2..418ed1661 100644 --- a/packages/verser/src/lib/tecemux/frames-keeper.ts +++ b/packages/verser/src/lib/tecemux/frames-keeper.ts @@ -17,7 +17,8 @@ export class FramesKeeper extends Writable implements IFramesKeeper { ) { super(Object.assign(opts, { readableObjectMode: true, - writableObjectMode: true + writableObjectMode: true, + writableHighWaterMark: 0 })); this.logger = new ObjLogger(params.name); @@ -25,11 +26,11 @@ export class FramesKeeper extends Writable implements IFramesKeeper { _write(chunk: any, encoding: BufferEncoding, cb: ((error: Error | null | undefined) => void)) { if (Buffer.isBuffer(chunk)) { - this.logger.info("transform buffer"); const sequenceNumber = chunk.readInt32LE(16); const destinationPort = chunk.readInt16LE(14); const flags = parseFlags(chunk.readInt8(25)); + this.logger.info("transform buffer", sequenceNumber); this.framesSent.set( sequenceNumber, { buffer: chunk, received: false, sequenceNumber, destinationPort, flags diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index e37595768..18badd578 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -117,6 +117,7 @@ export class TeceMux extends TypedEmitter { this.carrierSocket.pipe(this.carrierDecoder, { end: false }); + this.commonEncoder.out.pipe(this.framesKeeper); this.commonEncoder.out.pipe(this.carrierSocket, { end: false }); this.commonEncoder.logger.updateBaseLog({ id }); @@ -204,9 +205,9 @@ export class TeceMux extends TypedEmitter { } } } - + i = 0; sendACK(sequenceNumber: number, channel: number) { - this.logger.debug("Write acknowledge frame for sequenceNumber", sequenceNumber); + this.logger.debug("Write acknowledge frame for sequenceNumber", sequenceNumber, this.i++); this.commonEncoder.push( this.commonEncoder.createFrame(undefined, { flagsArray: ["ACK"], diff --git a/packages/verser/src/lib/verser.ts b/packages/verser/src/lib/verser.ts index 9f2fab0da..35f3cfc82 100644 --- a/packages/verser/src/lib/verser.ts +++ b/packages/verser/src/lib/verser.ts @@ -24,11 +24,15 @@ export class Verser extends TypedEmitter { super(); this.server = server; + this.server.timeout = 0; + this.server.on("connect", (req, socket: Socket) => { this.logger.info("New connection:", req.url); const connection = new VerserConnection(req, socket); + connection.logger.pipe(this.logger); + this.connections.push(connection); this.logger.info("Total connections:", this.connections.length); From 4cfedde4545ed1535c1d0bae54dbec69a9bd6100 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 9 Feb 2023 21:35:24 +0000 Subject: [PATCH 088/231] FramesKeeper generator --- .../src/lib/tecemux/codecs/frame-decoder.ts | 2 +- .../src/lib/tecemux/codecs/frame-encoder.ts | 24 ++---- .../verser/src/lib/tecemux/frames-keeper.ts | 80 +++++++++---------- packages/verser/src/lib/tecemux/playground.ts | 2 +- packages/verser/src/lib/tecemux/tecemux.ts | 21 +++-- packages/verser/src/lib/tecemux/types.ts | 16 ++-- 6 files changed, 61 insertions(+), 84 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 5c81b7378..65ef3f5ab 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -106,7 +106,7 @@ export class FrameDecoder extends Transform { this.buffer = this.buffer.subarray(frameSize); - this.logger.trace("Decoded", { ...payload, stringified: payload.chunk?.toString() }); + this.logger.trace("Decoded", { ...payload, stringified: "N/A" });//payload.chunk?.toString() }); if (this.buffer.length === 0) { this.logger.info("No remaining data!"); diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index a7f1c1de8..6f6a56108 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -167,25 +167,11 @@ export class FrameEncoder extends Transform { const buffer = this.createFrame(chunk, { destinationPort: this.frameTarget, flagsArray: ["PSH"] }); - /*if (this.tecemux.framesSent > -ACK_FRAME_DELTA) { - // eslint-disable-next-line no-loop-func - while (!await new Promise((res, _rej) => { - setImmediate(() => { - const frame = this.tecemux.framesKeeper.getFrame(this.tecemux.sequenceNumber + ACK_FRAME_DELTA); - - if (frame) { - const rec = frame?.received || frame?.flags.ACK; - - this.logger.info(`Sending ${this.tecemux.sequenceNumber}, ${frame?.sequenceNumber} ${frame?.received}`); - - res(!!rec); - } else { - console.log((this.tecemux.framesKeeper as any)["framesSent"]) - _rej("frame not exists" + (this.tecemux.sequenceNumber + ACK_FRAME_DELTA)); - } - }); - })); - }*/ + this.logger.debug("Awaiting keeper"); + await this.tecemux.framesKeeper.generator.next(this.tecemux.sequenceNumber); + this.logger.debug("Awaiting keeper DONE"); + + this.tecemux.framesKeeper.handlePSH(this.tecemux.sequenceNumber); this.tecemux.framesSent++; this.tecemux.sequenceNumber++; diff --git a/packages/verser/src/lib/tecemux/frames-keeper.ts b/packages/verser/src/lib/tecemux/frames-keeper.ts index 418ed1661..9ed00cf10 100644 --- a/packages/verser/src/lib/tecemux/frames-keeper.ts +++ b/packages/verser/src/lib/tecemux/frames-keeper.ts @@ -1,70 +1,62 @@ import { ObjLogger } from "@scramjet/obj-logger"; -import { IObjectLogger } from "@scramjet/types"; +import { EventEmitter } from "stream"; +import { FramesKeeperFrame, IFramesKeeper } from "./types"; -import { TransformOptions, Writable } from "stream"; -import { parseFlags } from "./utils"; -import { IFramesKeeper, FramesKeeperFrame } from "./types"; +export class FramesKeeper extends EventEmitter implements IFramesKeeper { + #MAX_FRAMES_DIFFERENCE = 5; + + #framesSent = new Map(); -export class FramesKeeper extends Writable implements IFramesKeeper { - logger: IObjectLogger; - framesSent = new Map(); lastSequenceSent: number = -1; lastSequenceReceived: number = -1; - constructor( - opts: TransformOptions = {}, - params: { name: string } = { name: "Keeper" } - ) { - super(Object.assign(opts, { - readableObjectMode: true, - writableObjectMode: true, - writableHighWaterMark: 0 - })); - - this.logger = new ObjLogger(params.name); - } - - _write(chunk: any, encoding: BufferEncoding, cb: ((error: Error | null | undefined) => void)) { - if (Buffer.isBuffer(chunk)) { - const sequenceNumber = chunk.readInt32LE(16); - const destinationPort = chunk.readInt16LE(14); - const flags = parseFlags(chunk.readInt8(25)); - - this.logger.info("transform buffer", sequenceNumber); - this.framesSent.set( - sequenceNumber, { - buffer: chunk, received: false, sequenceNumber, destinationPort, flags + logger = new ObjLogger(this); + + generator: AsyncGenerator = (async function* (this: FramesKeeper) { + while (true) { + if (this.lastSequenceSent - this.lastSequenceReceived < this.#MAX_FRAMES_DIFFERENCE) { + this.logger.info("Write allowed"); + yield Promise.resolve(this.lastSequenceReceived); + continue; + } + + this.logger.warn("Write NOT allowed"); + yield new Promise((res) => { + this.logger.info("waiting for ACK..."); + this.once("ack", (acknowledgeNumber: number) => { + this.logger.info("ACK processed"); + res(acknowledgeNumber); }); - - this.lastSequenceSent = sequenceNumber; - this.logger.debug(`lastSequenceSent ${sequenceNumber}, size: ${chunk.length} total frames sent ${this.framesSent.size}`,); + }); } + }).apply(this); - cb(undefined); + handlePSH(sequenceNumber: number) { + this.lastSequenceSent = sequenceNumber; } - onACK(acknowledgeNumber: number,) { + handleACK(acknowledgeNumber: number) { this.logger.debug("onACK", acknowledgeNumber); - const storedFrame = this.framesSent.get(acknowledgeNumber); + const storedFrame = this.#framesSent.get(acknowledgeNumber); if (storedFrame) { - this.framesSent.set(acknowledgeNumber, { ...storedFrame, received: true }); + this.#framesSent.set(acknowledgeNumber, { ...storedFrame, received: true }); } + + this.lastSequenceReceived = acknowledgeNumber; + this.emit("ack", acknowledgeNumber); } isReceived(sequenceNumber: number) { - const frame = this.framesSent.get(sequenceNumber); + const frame = this.#framesSent.get(sequenceNumber); // received or not stored - return frame === undefined || !!this.framesSent.get(sequenceNumber)?.received; + return frame === undefined || !!this.#framesSent.get(sequenceNumber)?.received; } getFrame(sequenceNumber: number) { - return this.framesSent.get(sequenceNumber); - } - - getDestinationPort(sequenceNumber: number) { - return this.framesSent.get(sequenceNumber)?.received; + return this.#framesSent.get(sequenceNumber); } } + diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/src/lib/tecemux/playground.ts index 694ad1a45..1218fceda 100644 --- a/packages/verser/src/lib/tecemux/playground.ts +++ b/packages/verser/src/lib/tecemux/playground.ts @@ -4,8 +4,8 @@ import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; import { createReadStream, createWriteStream } from "fs"; import path from "path"; import { PassThrough, Transform } from "stream"; -import { FramesKeeper } from "./frames-keeper"; import { ITeCeMux } from "./types"; +import { FramesKeeper } from "./frames-keeper"; const tcm = { sequenceNumber: 0, diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 18badd578..8705a4558 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -15,7 +15,7 @@ export class TeceMux extends TypedEmitter { carrierDecoder: FrameDecoder; framesKeeper = new FramesKeeper(); sequenceNumber = Math.abs((Math.random() * (2 ** 32)) | 0); - channels: TeceMuxChannel[] = []; + channels = new Map(); logger: ObjLogger; commonEncoder = new FrameEncoder(0, this); @@ -57,9 +57,6 @@ export class TeceMux extends TypedEmitter { } ); - encoder.out - .pipe(this.framesKeeper); - encoder.out .pipe(this.carrierSocket, { end: false }); @@ -117,7 +114,6 @@ export class TeceMux extends TypedEmitter { this.carrierSocket.pipe(this.carrierDecoder, { end: false }); - this.commonEncoder.out.pipe(this.framesKeeper); this.commonEncoder.out.pipe(this.carrierSocket, { end: false }); this.commonEncoder.logger.updateBaseLog({ id }); @@ -152,16 +148,16 @@ export class TeceMux extends TypedEmitter { break; } - let channel = this.channels[destinationPort]; + let channel = this.channels.get(destinationPort); if (flags.ACK) { this.logger.trace("Received ACK flag for sequenceNumber", acknowledgeNumber); - this.framesKeeper.onACK(acknowledgeNumber); + this.framesKeeper.handleACK(acknowledgeNumber); continue; } if (flags.FIN) { - this.logger.trace(`Received FIN flag [C: ${destinationPort}]`, dataLength, frame.chunk, !!this.channels[destinationPort], channel._id); + this.logger.trace(`Received FIN flag [C: ${destinationPort}]`, dataLength, frame.chunk, !!channel, channel?._id); if (channel) { channel.closedByFIN = true; @@ -205,10 +201,10 @@ export class TeceMux extends TypedEmitter { } } } - i = 0; + sendACK(sequenceNumber: number, channel: number) { - this.logger.debug("Write acknowledge frame for sequenceNumber", sequenceNumber, this.i++); - this.commonEncoder.push( + this.logger.debug("Write acknowledge frame for sequenceNumber", sequenceNumber); + this.channels.get(channel)?.encoder?.push( this.commonEncoder.createFrame(undefined, { flagsArray: ["ACK"], acknowledgeNumber: sequenceNumber, @@ -219,7 +215,7 @@ export class TeceMux extends TypedEmitter { addChannel(channel: TeceMuxChannel, emit: boolean) { this.logger.debug("adding channel", channel._id); - this.channels[channel._id] = channel; // wait for SYN reply? + this.channels.set(channel._id, channel); // wait for SYN reply? if (emit) { this.emit("channel", channel); @@ -231,6 +227,7 @@ export class TeceMux extends TypedEmitter { sendFIN(channel: number) { this.logger.debug("Write FIN frame for channel", channel); + this.commonEncoder.push( this.commonEncoder.createFrame(undefined, { flagsArray: ["FIN"], diff --git a/packages/verser/src/lib/tecemux/types.ts b/packages/verser/src/lib/tecemux/types.ts index 67a5a6443..40174699f 100644 --- a/packages/verser/src/lib/tecemux/types.ts +++ b/packages/verser/src/lib/tecemux/types.ts @@ -1,4 +1,4 @@ -import { Duplex } from "stream"; +import { Duplex, Transform } from "stream"; export type TeceMuxEvents = { channel(socket: Duplex): void; @@ -25,9 +25,11 @@ export type FramesKeeperFrame = { }; export interface IFramesKeeper { - onACK(acknowledgeNumber: number): void; + handleACK(acknowledgeNumber: number): void; isReceived(sequenceNumber: number): boolean; - getFrame(sequenceNumber: number): FramesKeeperFrame | undefined + handlePSH(sequenceNumber: number): void; + getFrame(sequenceNumber: number): FramesKeeperFrame | undefined; + generator: AsyncGenerator; } export interface ITeCeMux { @@ -36,12 +38,12 @@ export interface ITeCeMux { framesKeeper: IFramesKeeper; } -export interface IFrameEncoder { +export interface IFrameEncoder extends Transform { } export type TeceMuxChannel = Duplex & { - _id: number, - encoder: IFrameEncoder, - closedByFIN: boolean + _id: number; + encoder: IFrameEncoder; + closedByFIN: boolean; }; From b05ba1a5db072df35a067e0b3e66a1442bac1a88 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Fri, 10 Feb 2023 12:34:51 +0000 Subject: [PATCH 089/231] Cleanup --- .../src/lib/tecemux/codecs/frame-encoder.ts | 10 +-- .../verser/src/lib/tecemux/frames-keeper.ts | 7 +- .../src/lib/tecemux/playground-tecemux.ts | 65 +++---------------- packages/verser/src/lib/tecemux/playground.ts | 2 - packages/verser/src/lib/tecemux/tecemux.ts | 21 ++++-- packages/verser/src/lib/tecemux/types.ts | 4 ++ packages/verser/test/http-connection.spec.ts | 7 +- 7 files changed, 41 insertions(+), 75 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index 6f6a56108..6198fda38 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -7,6 +7,8 @@ import { ITeCeMux } from "../types"; import { calculateChecksum } from "./utils"; export class FrameEncoder extends Transform { + MAX_CHUNK_SIZE = 10 * 1024 - HEADER_LENGTH; + tecemux: ITeCeMux; total = 0; logger = new ObjLogger("FrameEncoder"); @@ -148,19 +150,17 @@ export class FrameEncoder extends Transform { } async _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): Promise { - const MAX_CHUNK_SIZE = 10 * 1024 - HEADER_LENGTH; - this.total += chunk.length; this.logger.debug("TRANSFORM IN", /* toHex(chunk), */ chunk.length, this.total); let remaining = Buffer.alloc(0); - if (chunk.length > MAX_CHUNK_SIZE) { + if (chunk.length > this.MAX_CHUNK_SIZE) { this.logger.debug("TRANSFORM big chunk, splitting", chunk.length); - remaining = (chunk as Buffer).subarray(MAX_CHUNK_SIZE); - chunk = chunk.subarray(0, MAX_CHUNK_SIZE); + remaining = (chunk as Buffer).subarray(this.MAX_CHUNK_SIZE); + chunk = chunk.subarray(0, this.MAX_CHUNK_SIZE); this.logger.debug("TRANSFORM processing part/remaining", chunk.length, remaining.length); } diff --git a/packages/verser/src/lib/tecemux/frames-keeper.ts b/packages/verser/src/lib/tecemux/frames-keeper.ts index 9ed00cf10..48126f77f 100644 --- a/packages/verser/src/lib/tecemux/frames-keeper.ts +++ b/packages/verser/src/lib/tecemux/frames-keeper.ts @@ -1,8 +1,8 @@ import { ObjLogger } from "@scramjet/obj-logger"; -import { EventEmitter } from "stream"; -import { FramesKeeperFrame, IFramesKeeper } from "./types"; +import { TypedEmitter } from "@scramjet/utility"; +import { FramesKeeperEvents, FramesKeeperFrame, IFramesKeeper } from "./types"; -export class FramesKeeper extends EventEmitter implements IFramesKeeper { +export class FramesKeeper extends TypedEmitter implements IFramesKeeper { #MAX_FRAMES_DIFFERENCE = 5; #framesSent = new Map(); @@ -23,6 +23,7 @@ export class FramesKeeper extends EventEmitter implements IFramesKeeper { this.logger.warn("Write NOT allowed"); yield new Promise((res) => { this.logger.info("waiting for ACK..."); + this.once("ack", (acknowledgeNumber: number) => { this.logger.info("ACK processed"); res(acknowledgeNumber); diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/src/lib/tecemux/playground-tecemux.ts index b75bf78d7..7dfb9f827 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/src/lib/tecemux/playground-tecemux.ts @@ -53,9 +53,7 @@ import path from "path"; logger.info("Carrier Socket unpiped", c); }) .on("pause", () => { - //socket.resume(); logger.fatal("Carrier Socket paused"); - //debugger; }) .on("resume", () => { logger.info("Carrier Socket resumed"); @@ -78,7 +76,6 @@ import path from "path"; tcmux.logger.pipe(logger); const channel1 = tcmux.multiplex(); - //const channel2 = tcmux.multiplex(); channel1.pipe(createWriteStream(path.join(__dirname, "output-server.tar.gz"))); @@ -87,49 +84,6 @@ import path from "path"; logger.info("FILE END"); }) .pipe(channel1); - - //Readable.from(Buffer.alloc(1024 * 100)).pipe(channel1, { end: false }); - - req.on("pause", () => { logger.warn("Request paused"); }); - - //logger.warn("Waiting for stdin..."); - - //DataStream.from(process.stdin).filter((x: Buffer) => !(parseInt(x[0].toString(), 10) % 2)).pipe(channel1); - //DataStream.from(process.stdin).filter((x: Buffer) => !!(parseInt(x[0].toString(), 10) % 2)).pipe(channel2); - /* - (async () => { - try { - for await (const chunk of channel1) { - console.log("CHUNK", chunk); - logger.debug("reading CHANNEL1 chunk", chunk.toString()); - } - } catch (error) { - logger.error("reading CHANNEL1 ERROR", error); - } - - logger.debug("reading CHANNEL1 END"); - })(); -*/ - // (async () => { - // try { - // for await (const chunk of channel2) { - // logger.debug("reading CHANNEL2 chunk", chunk.toString()); - // } - // } catch (error) { - // logger.error("reading CHANNEL2 ERROR", error); - // } - - // logger.debug("reading CHANNEL2 END"); - // })(); - - /* - setTimeout(() => { - console.log("\n\n\n\n"); - logger.trace("Ending channels"); - channel1.push(null); - //channel2.end(); - }, 4000); - */ }); server.listen(PORT, "0.0.0.0"); @@ -169,20 +123,15 @@ import path from "path"; tcmux.on("channel", async (channel: TeceMuxChannel) => { reqLogger.debug("New channel", channel._id); + let total = 0; + channel .on("finish", () => { tcmux.logger.info("Channel finish", channel._id); }) .on("end", () => { - tcmux.logger.info("Channel readable end", channel._id, channel.readableEnded, channel.writableEnded); - }); - - createReadStream(path.join(__dirname, "../../../../forever.tar.gz")).pipe(channel, { end: false }); - channel.pipe(createWriteStream(path.join(__dirname, "output-client.tar.gz"))); - - let total = 0; - - channel + tcmux.logger.info("Channel readable end [id, readableEnded, writableEnded]", channel._id, channel.readableEnded, channel.writableEnded); + }) .on("data", (d) => { total += d.length; tcmux.logger.info("-------------------- data", channel._id, d.length, total); @@ -192,7 +141,11 @@ import path from "path"; }) .on("resume", () => { tcmux.logger.info("-------------------- resumed", channel._id); - }); + }) + .pause(); + + createReadStream(path.join(__dirname, "../../../../forever.tar.gz")).pipe(channel); + channel.pipe(createWriteStream(path.join(__dirname, "output-client.tar.gz"))); }); socket.on("error", (err) => { diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/src/lib/tecemux/playground.ts index 1218fceda..557dd6ed6 100644 --- a/packages/verser/src/lib/tecemux/playground.ts +++ b/packages/verser/src/lib/tecemux/playground.ts @@ -65,8 +65,6 @@ const dh = new Transform({ callback(null); } } - - //callback(null); } catch (e) { logger.error("dh error", e); } diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 8705a4558..c1187bee1 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -45,9 +45,6 @@ export class TeceMux extends TypedEmitter { read: (_size) => { this.logger.trace("READ channel", channel._id); }, - final() { - //channel.emit("end"); - }, allowHalfOpen: true }), { @@ -71,10 +68,22 @@ export class TeceMux extends TypedEmitter { .on("abort", () => { this.logger.trace("channel on ABORT ", channel._id); }) + .on("close", () => { + this.logger.info("CHANNEL close", channel._id); + this.sendFIN(channel._id); + }) .on("end", () => { this.logger.info("CHANNEL end", channel._id); + }) + .on("finish", () => { + this.logger.info("CHANNEL finish", channel._id); this.sendFIN(channel._id); - }); + }) + .on("data", (d) => { + if (d === null) { + this.logger.info("CHANNEL end", channel._id); + } + });//.pause(); if (emit) { encoder.setChannel(destinationPort || this.channelCount); @@ -99,9 +108,9 @@ export class TeceMux extends TypedEmitter { .on("end", () => { this.logger.warn("Decoder ended"); }) - /*.on("error", (error) => { + .on("error", (error) => { this.logger.error("Decoder error", error); - })*/ + }) .on("abort", (error) => { this.logger.error("Decoder abort", error); }) diff --git a/packages/verser/src/lib/tecemux/types.ts b/packages/verser/src/lib/tecemux/types.ts index 40174699f..81d38c7b3 100644 --- a/packages/verser/src/lib/tecemux/types.ts +++ b/packages/verser/src/lib/tecemux/types.ts @@ -5,6 +5,10 @@ export type TeceMuxEvents = { error(error: any): void; } +export type FramesKeeperEvents = { + ack: (acknowledgeNumber: number) => void; +} + export type flagsObjectType = Partial<{ FIN: boolean, SYN: boolean, diff --git a/packages/verser/test/http-connection.spec.ts b/packages/verser/test/http-connection.spec.ts index ba90dc211..c0c4b0a71 100644 --- a/packages/verser/test/http-connection.spec.ts +++ b/packages/verser/test/http-connection.spec.ts @@ -45,7 +45,7 @@ function getJSONResponseFromRequest(request: http.ClientRequest): Promise { test("Connect VerserClient A to Verser B and send HTTP GET Request to VerserClient A", async (t) => { const SERVER_B_PORT = 1999; - const apiB = createServer(); + const apiB = createServer({ }); apiB.server.listen(SERVER_B_PORT); @@ -56,7 +56,7 @@ test("Connect VerserClient A to Verser B and send HTTP GET Request to VerserClie const verserClientA = new VerserClient({ headers: { city: "Valencia" }, - verserUrl: `http://127.0.0.1:${SERVER_B_PORT}`, + verserUrl: `http://0.0.0.0:${SERVER_B_PORT}`, server: apiA.server }); @@ -123,7 +123,7 @@ test("Connect VerserClient A to Verser B over SSL and send HTTPS GET Request to ca: [readFileSync(path.join(certDir, "myCA.pem"))] }); - requestToAThroughB.end(); + requestToAThroughB.flushHeaders(); const responseFromAToB = await getJSONResponseFromRequest(requestToAThroughB); @@ -132,3 +132,4 @@ test("Connect VerserClient A to Verser B over SSL and send HTTPS GET Request to spawnSync("./cleanup-localhost-cert.sh", { cwd: certDir }); }); + From eb7b66b008b77e98d9956f6c738af2d2729eea2a Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 14 Feb 2023 12:18:31 +0000 Subject: [PATCH 090/231] Tests --- .../src/lib/tecemux/codecs/frame-decoder.ts | 3 +- .../src/lib/tecemux/codecs/frame-encoder.ts | 4 +- packages/verser/src/lib/tecemux/tecemux.ts | 7 +- packages/verser/src/lib/tecemux/types.ts | 18 +++++ .../verser/src/lib/tecemux/utils/index.ts | 18 +---- .../playgrounds}/playground-tecemux.ts | 8 +- .../playgrounds}/playground.ts | 6 +- packages/verser/test/tecemux-transfer.spec.ts | 80 +++++++++++++++++++ packages/verser/tsconfig.json | 2 +- 9 files changed, 113 insertions(+), 33 deletions(-) rename packages/verser/{src/lib/tecemux => test/playgrounds}/playground-tecemux.ts (94%) rename packages/verser/{src/lib/tecemux => test/playgrounds}/playground.ts (91%) create mode 100644 packages/verser/test/tecemux-transfer.spec.ts diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts index 65ef3f5ab..5801390dc 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts @@ -1,8 +1,9 @@ import { ObjLogger } from "@scramjet/obj-logger"; import { Duplex, Transform, TransformCallback, TransformOptions } from "stream"; -import { FrameData, parseFlags } from "../utils"; +import { parseFlags } from "../utils"; import { HEADER_LENGTH } from "../constants"; import { calculateChecksum, getChecksum } from "./utils"; +import { FrameData } from "../types"; export class FrameDecoder extends Transform { buffer: Buffer; diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index 6198fda38..7aa386263 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -1,9 +1,8 @@ import { PassThrough, Transform, TransformCallback, TransformOptions } from "stream"; -import { FrameData } from "../utils"; import { ObjLogger } from "@scramjet/obj-logger"; import { FrameTarget, HEADER_LENGTH, binaryFlags, frameFlags } from "../constants"; -import { ITeCeMux } from "../types"; +import { FrameData, ITeCeMux } from "../types"; import { calculateChecksum } from "./utils"; export class FrameEncoder extends Transform { @@ -23,7 +22,6 @@ export class FrameEncoder extends Transform { }) .on("end", () => { this.logger.trace("OUT ended!", this.frameTarget); - //this.tecemux.sendFIN(this.frameTarget); }) .on("resume", () => { this.logger.trace("OUT resumed"); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index c1187bee1..96bb6ccd9 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -3,8 +3,7 @@ import { FrameDecoder, FrameEncoder } from "./codecs"; import { Duplex } from "stream"; import { Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; -import { FrameData } from "./utils"; -import { TeceMuxChannel, TeceMuxEvents } from "./types"; +import { FrameData, TeceMuxChannel, TeceMuxEvents } from "./types"; import { FramesKeeper } from "./frames-keeper"; export class TeceMux extends TypedEmitter { @@ -83,7 +82,7 @@ export class TeceMux extends TypedEmitter { if (d === null) { this.logger.info("CHANNEL end", channel._id); } - });//.pause(); + }); if (emit) { encoder.setChannel(destinationPort || this.channelCount); @@ -144,7 +143,7 @@ export class TeceMux extends TypedEmitter { let frame: FrameData; try { - frame = JSON.parse(chunk) as FrameData; + frame = JSON.parse(chunk); } catch (err) { this.logger.error("error Parsing data from decoder", err, chunk, chunk.length, chunk.toString()); continue; diff --git a/packages/verser/src/lib/tecemux/types.ts b/packages/verser/src/lib/tecemux/types.ts index 81d38c7b3..2cf55c247 100644 --- a/packages/verser/src/lib/tecemux/types.ts +++ b/packages/verser/src/lib/tecemux/types.ts @@ -9,6 +9,8 @@ export type FramesKeeperEvents = { ack: (acknowledgeNumber: number) => void; } +export type BinaryFlags = "FIN" | "SYN" | "RST" | "PSH" | "ACK" | "URG" | "ECE" | "CWR"; + export type flagsObjectType = Partial<{ FIN: boolean, SYN: boolean, @@ -51,3 +53,19 @@ export type TeceMuxChannel = Duplex & { encoder: IFrameEncoder; closedByFIN: boolean; }; + +export type FrameData = { + sourceAddress: [number, number, number, number]; + destinationAddress: [number, number, number, number]; + destinationPort: number; + sequenceNumber: number; + acknowledgeNumber: number; + chunk: Buffer; + dataLength: number; + chunkLength: number; + stringified: string; + flags: flagsObjectType; + flagsArray: BinaryFlags[]; + error?: "checksum"; + received: boolean; +}; diff --git a/packages/verser/src/lib/tecemux/utils/index.ts b/packages/verser/src/lib/tecemux/utils/index.ts index b90056eb7..86fa4a9cf 100644 --- a/packages/verser/src/lib/tecemux/utils/index.ts +++ b/packages/verser/src/lib/tecemux/utils/index.ts @@ -1,26 +1,10 @@ -import { binaryFlags, frameFlags } from "../constants"; +import { frameFlags } from "../constants"; import { flagsObjectType } from "../types"; export function toHex(chunk: Buffer) { return chunk.toString("hex").match(/../g)?.join(" "); } -export type FrameData = { - sourceAddress: [number, number, number, number]; - destinationAddress: [number, number, number, number]; - destinationPort: number; - sequenceNumber: number; - acknowledgeNumber: number; - chunk: Buffer; - dataLength: number; - chunkLength: number; - stringified: string; - flags: flagsObjectType; - flagsArray: (keyof typeof binaryFlags)[]; - error?: "checksum"; - received: boolean; -}; - export const parseFlags = (byte: number): flagsObjectType => { return frameFlags.filter((_flag, index) => byte >>> index & 1) .reduce((p, c) => ({ ...p, [c]: true }), {}); diff --git a/packages/verser/src/lib/tecemux/playground-tecemux.ts b/packages/verser/test/playgrounds/playground-tecemux.ts similarity index 94% rename from packages/verser/src/lib/tecemux/playground-tecemux.ts rename to packages/verser/test/playgrounds/playground-tecemux.ts index 7dfb9f827..fa84cf427 100644 --- a/packages/verser/src/lib/tecemux/playground-tecemux.ts +++ b/packages/verser/test/playgrounds/playground-tecemux.ts @@ -7,8 +7,8 @@ import { IncomingMessage, createServer } from "http"; import { DataStream } from "scramjet"; import { Socket, createConnection } from "net"; -import { TeceMux } from "./tecemux"; -import { TeceMuxChannel } from "./types"; +import { TeceMux } from "../../src/lib/tecemux/tecemux"; +import { TeceMuxChannel } from "../../src/lib/tecemux/types"; import { createReadStream, createWriteStream } from "fs"; import path from "path"; @@ -79,7 +79,7 @@ import path from "path"; channel1.pipe(createWriteStream(path.join(__dirname, "output-server.tar.gz"))); - createReadStream(path.join(__dirname, "../../../../forever.tar.gz")) + createReadStream(path.join(__dirname, "../../../forever.tar.gz")) .on("end", () => { logger.info("FILE END"); }) @@ -144,7 +144,7 @@ import path from "path"; }) .pause(); - createReadStream(path.join(__dirname, "../../../../forever.tar.gz")).pipe(channel); + createReadStream(path.join(__dirname, "../../../forever.tar.gz")).pipe(channel); channel.pipe(createWriteStream(path.join(__dirname, "output-client.tar.gz"))); }); diff --git a/packages/verser/src/lib/tecemux/playground.ts b/packages/verser/test/playgrounds/playground.ts similarity index 91% rename from packages/verser/src/lib/tecemux/playground.ts rename to packages/verser/test/playgrounds/playground.ts index 557dd6ed6..d52e5f452 100644 --- a/packages/verser/src/lib/tecemux/playground.ts +++ b/packages/verser/test/playgrounds/playground.ts @@ -1,11 +1,11 @@ import { DataStream } from "scramjet"; -import { FrameDecoder, FrameEncoder } from "./codecs"; +import { FrameDecoder, FrameEncoder } from "../../src/lib/tecemux/codecs"; import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; import { createReadStream, createWriteStream } from "fs"; import path from "path"; import { PassThrough, Transform } from "stream"; -import { ITeCeMux } from "./types"; -import { FramesKeeper } from "./frames-keeper"; +import { ITeCeMux } from "../../src/lib/tecemux/types"; +import { FramesKeeper } from "../../src/lib/tecemux/frames-keeper"; const tcm = { sequenceNumber: 0, diff --git a/packages/verser/test/tecemux-transfer.spec.ts b/packages/verser/test/tecemux-transfer.spec.ts new file mode 100644 index 000000000..54c9c97d7 --- /dev/null +++ b/packages/verser/test/tecemux-transfer.spec.ts @@ -0,0 +1,80 @@ +/* eslint-disable no-console */ +import test from "ava"; +import { IncomingMessage, Server, createServer } from "http"; +import { TeceMux, TeceMuxChannel } from "../src/lib/tecemux/tecemux"; +import * as crypto from "crypto"; +import { Socket, createConnection } from "net"; +import { Readable } from "stream"; + +let serverTeceMux: TeceMux; + +const PORT = 6660; + +async function startServer() { + const server = createServer({}); + + return new Promise((resolve) => { + server.listen(PORT, () => { resolve(server); }); + }); +} + +test("Protocol send file over http connection", async (t) => { + const server = await startServer(); + + const hashReceived = crypto.createHash("md5"); + const hashSent = crypto.createHash("md5"); + + const serverSideChannelPromise = new Promise((resolve) => { + server.on("connect", async (req: IncomingMessage, socket: Socket) => { + socket.setNoDelay(true); + + serverTeceMux = new TeceMux(socket, "Server") + .on("error", (error) => { + console.error("TeceMux error", error); + }); + + resolve(serverTeceMux.multiplex()); + }); + }); + + const socket = createConnection({ port: PORT, allowHalfOpen: true, host: "0.0.0.0" }, () => {}); + + await new Promise((resolve, reject) => { + socket + .on("connect", resolve) + .on("error", reject); + }); + + socket.write("CONNECT HTTP/1.1\r\n\r\n\r\n"); + + const clientTeceMux = new TeceMux(socket, "Request"); + + await new Promise(resolve => { + clientTeceMux.on("channel", async (clientSideChannel: TeceMuxChannel) => { + function* gen() { + for (let i = 0; i < 1e3; i++) { + const str = crypto.randomBytes(1024).toString("hex"); + + hashSent.update(str); + yield str; + } + } + + Readable.from(gen()).pipe(clientSideChannel); + resolve(clientSideChannel); + }); + }); + + const serverSideChannel = await serverSideChannelPromise; + + for await (const d of serverSideChannel) { + hashReceived.update(d); + } + + console.dir({ + hashSent, + hashReceived + }); + + t.assert(hashReceived.digest("hex") === hashSent.digest("hex"), "Unequal hashes"); +}); diff --git a/packages/verser/tsconfig.json b/packages/verser/tsconfig.json index 6027e8072..55c8006d1 100644 --- a/packages/verser/tsconfig.json +++ b/packages/verser/tsconfig.json @@ -5,7 +5,7 @@ "extends": "../../tsconfig.base.json", "include": [ "./src/**/*.ts", - "./test/**/*.ts", + "./test/**/*.ts", "src/types" ], "exclude": [ From d311f1c8618f5afc882f0b3c8b751106b3167aa6 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Fri, 20 Jan 2023 12:40:22 +0000 Subject: [PATCH 091/231] Replace BPMux with TeCeMux --- packages/verser/src/lib/verser-client.ts | 2 -- packages/verser/src/lib/verser-connection.ts | 2 -- 2 files changed, 4 deletions(-) diff --git a/packages/verser/src/lib/verser-client.ts b/packages/verser/src/lib/verser-client.ts index d28af8378..b010fe07e 100644 --- a/packages/verser/src/lib/verser-client.ts +++ b/packages/verser/src/lib/verser-client.ts @@ -143,8 +143,6 @@ export class VerserClient extends TypedEmitter { this.emit("error", err); }); - //this.teceMux.logger.pipe(this.logger); - this._verserAgent = new HttpAgent() as HttpAgent & { createConnection: typeof createConnection }; // lack of types? diff --git a/packages/verser/src/lib/verser-connection.ts b/packages/verser/src/lib/verser-connection.ts index c8c10e470..87edc0344 100644 --- a/packages/verser/src/lib/verser-connection.ts +++ b/packages/verser/src/lib/verser-connection.ts @@ -204,8 +204,6 @@ export class VerserConnection { // TODO: Error handling? }); - //this.teceMux.logger.pipe(this.logger); - this.agent = new Agent() as Agent & { createConnection: typeof createConnection }; // lack of types? this.agent.createConnection = () => { try { From ad3a5a5661a663da9ca967731e70d520e24c3c88 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 23 Mar 2023 21:05:17 +0000 Subject: [PATCH 092/231] R->STH. WIP --- packages/host/src/lib/csi-controller.ts | 1 + packages/host/src/lib/socket-server.ts | 79 +++++++++++++--------- packages/runner/package.json | 1 + packages/runner/src/host-client.ts | 47 ++++++------- packages/runner/src/runner-app-context.ts | 1 - packages/verser/src/index.ts | 1 + packages/verser/src/lib/tecemux/index.ts | 1 + packages/verser/src/lib/tecemux/tecemux.ts | 13 ++-- packages/verser/src/lib/tecemux/types.ts | 1 + 9 files changed, 84 insertions(+), 61 deletions(-) create mode 100644 packages/verser/src/lib/tecemux/index.ts diff --git a/packages/host/src/lib/csi-controller.ts b/packages/host/src/lib/csi-controller.ts index b92bfc095..2bb7e616c 100644 --- a/packages/host/src/lib/csi-controller.ts +++ b/packages/host/src/lib/csi-controller.ts @@ -480,6 +480,7 @@ export class CSIController extends TypedEmitter { }); this.communicationHandler.addMonitoringHandler(RunnerMessageCode.PANG, async (message) => { + this.logger.info("PANG"); const pangData = message[1]; this.provides ||= this.outputTopic || pangData.provides; diff --git a/packages/host/src/lib/socket-server.ts b/packages/host/src/lib/socket-server.ts index ac48cc069..06dbeb7d0 100644 --- a/packages/host/src/lib/socket-server.ts +++ b/packages/host/src/lib/socket-server.ts @@ -1,21 +1,24 @@ -import { IComponent, DownstreamStreamsConfig, IObjectLogger } from "@scramjet/types"; +import { IComponent, IObjectLogger } from "@scramjet/types"; -import net from "net"; +import net, { Socket } from "net"; import { isDefined, TypedEmitter } from "@scramjet/utility"; import { ObjLogger } from "@scramjet/obj-logger"; +import { TeceMux, TeceMuxChannel } from "@scramjet/verser"; + +type MaybeChannel = TeceMuxChannel | Socket | null; + +type RunnerChannels = [ + TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel +]; -type MaybeSocket = net.Socket | null type RunnerConnectionsInProgress = [ - MaybeSocket, MaybeSocket, MaybeSocket, MaybeSocket, MaybeSocket, MaybeSocket, MaybeSocket, MaybeSocket, MaybeSocket -] -type RunnerOpenConnections = [ - net.Socket, net.Socket, net.Socket, net.Socket, net.Socket, net.Socket, net.Socket, net.Socket, net.Socket -] + MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel +]; type Events = { - connect: (id: string, streams: DownstreamStreamsConfig) => void -} + connect: (id: string, streams: RunnerChannels) => void +}; /** * Server for incoming connections from Runners @@ -37,37 +40,48 @@ export class SocketServer extends TypedEmitter implements IComponent { async start(): Promise { this.server = net.createServer(); - this.server - .on("connection", async (connection) => { - connection.setNoDelay(true); - connection.on("error", (err) => { - this.logger.error("Error on connection from runner", err); - }); + let protocol: TeceMux; - const id = await new Promise((resolve) => { - connection.once("readable", () => { - resolve(connection.read(36).toString()); - }); - }); + this.server.on("connection", async (connection: net.Socket) => { + connection.setNoDelay(true); + connection.on("error", (err) => { + this.logger.error("Error on connection from runner", err); + }); - const channel = await new Promise((resolve) => { - connection.once("readable", () => { - resolve(parseInt(connection.read(1).toString(), 10)); + protocol = new TeceMux(connection); + + protocol.on("channel", async (channel: TeceMuxChannel) => { + const { instanceId, channelId } = + await new Promise<{ instanceId: string, channelId: number }>((resolve) => { + channel.pause(); + channel.once("readable", () => { + const payload = channel.read(37).toString(); + const instId = payload.substring(0, 36); + const chanId = parseInt(payload.substring(36, 37), 10); + + resolve({ + instanceId: instId, + channelId: chanId + }); + }); }); - }); - connection - .on("error", (err) => this.logger.error("Error on Instance in stream", id, channel, err)) - .on("end", () => this.logger.debug(`Channel [${id}:${channel}] ended`)); + + channel + .on("error", (err: any) => this.logger.error("Error on Instance in stream", instanceId, channelId, err)) + .on("end", () => this.logger.debug(`Channel [${instanceId}:${channelId}] ended`)); + try { - await this.handleConnection(id, channel, connection); + await this.handleConnection(instanceId, channelId, connection as Socket); } catch (err: any) { connection.destroy(); } - }); + }) - return new Promise((res, rej) => { + }); + + return new Promise((res, rej) => { this.server! .listen(this.port, this.hostname, () => { this.logger.info("SocketServer on", this.server?.address()); @@ -92,9 +106,10 @@ export class SocketServer extends TypedEmitter implements IComponent { } if (runner.every(isDefined)) { this.runnerConnectionsInProgress.delete(id); - this.emit("connect", id, runner as RunnerOpenConnections); + this.emit("connect", id, runner as RunnerChannels); } } + close() { this.server?.close((err: any) => { if (err) { diff --git a/packages/runner/package.json b/packages/runner/package.json index 9ec7c62ed..e796061b5 100644 --- a/packages/runner/package.json +++ b/packages/runner/package.json @@ -22,6 +22,7 @@ "@scramjet/obj-logger": "^0.33.5", "@scramjet/symbols": "^0.33.5", "@scramjet/utility": "^0.33.5", + "@scramjet/verser": "^0.33.5", "bpmux": "^8.2.1", "scramjet": "^4.36.9" }, diff --git a/packages/runner/src/host-client.ts b/packages/runner/src/host-client.ts index 4ecdbf338..33a880c24 100644 --- a/packages/runner/src/host-client.ts +++ b/packages/runner/src/host-client.ts @@ -4,9 +4,10 @@ import { CommunicationChannel as CC } from "@scramjet/symbols"; import net, { createConnection, Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; import { Agent } from "http"; +import { TeceMux, TeceMuxChannel } from "@scramjet/verser"; type HostOpenConnections = [ - net.Socket, net.Socket, net.Socket, net.Socket, net.Socket, net.Socket, net.Socket, net.Socket, net.Socket + TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel ] const BPMux = require("bpmux").BPMux; @@ -40,32 +41,32 @@ class HostClient implements IHostClient { } async init(id: string): Promise { + const tunnel = net.createConnection(this.instancesServerPort, this.instancesServerHost); + const protocol = new TeceMux(tunnel); + const openConnections = await Promise.all( Array.from(Array(9)) - .map(() => { - // Error handling for each connection is process crash for now - const connection = net.createConnection(this.instancesServerPort, this.instancesServerHost); - connection.setNoDelay(true); - - connection.setNoDelay(true); - - return new Promise(res => { - connection.on("connect", () => res(connection)); - }); + .map((_c, index) => protocol.multiplex({ channel: index })) + ).then(async res => { + return Promise.all( + res.map(async (channel, index) => { + // Assuming id is exactly 36 bytes + channel.write(id + "" + index); + // eslint-disable-next-line no-console + console.log(channel._id); + // Assuming number is from 0-8, sending 1 byte + //channel.write(index.toString()); + + //await defer(500); + + return channel; }) - .map((connPromised, index) => { - return connPromised.then((connection) => { - // Assuming id is exactly 36 bytes - connection.write(id); - // Assuming number is from 0-7, sending 1 byte - connection.write(index.toString()); - - return connection; - }); - }) - ); + ); + }); + + //await defer(500); - this._streams = openConnections as HostOpenConnections; + this._streams = await openConnections as HostOpenConnections; try { this.bpmux = new BPMux(this._streams[CC.PACKAGE]); diff --git a/packages/runner/src/runner-app-context.ts b/packages/runner/src/runner-app-context.ts index 6ccc35f3e..80249a14a 100644 --- a/packages/runner/src/runner-app-context.ts +++ b/packages/runner/src/runner-app-context.ts @@ -44,7 +44,6 @@ implements AppContext { this.hub = hostClient; this.instanceId = id; } - private handleSave(_state: any): void { throw new Error("Method not implemented."); } diff --git a/packages/verser/src/index.ts b/packages/verser/src/index.ts index 19bca6084..c56cde24d 100644 --- a/packages/verser/src/index.ts +++ b/packages/verser/src/index.ts @@ -1,3 +1,4 @@ export * from "./lib/verser"; export * from "./lib/verser-client"; export * from "./lib/verser-connection"; +export * from "./lib/tecemux"; diff --git a/packages/verser/src/lib/tecemux/index.ts b/packages/verser/src/lib/tecemux/index.ts new file mode 100644 index 000000000..802a3f900 --- /dev/null +++ b/packages/verser/src/lib/tecemux/index.ts @@ -0,0 +1 @@ +export * from "./tecemux"; diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 96bb6ccd9..b6752f033 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -20,9 +20,11 @@ export class TeceMux extends TypedEmitter { commonEncoder = new FrameEncoder(0, this); private createChannel(destinationPort?: number, emit?: boolean): TeceMuxChannel { - this.logger.debug("Create Channel", destinationPort || this.channelCount); + const port = destinationPort !== undefined ? destinationPort : this.channelCount; - const encoder = new FrameEncoder(this.channelCount, this, { encoding: undefined }); + this.logger.debug("Create Channel", port); + + const encoder = new FrameEncoder(port, this, { encoding: undefined }); encoder.logger.updateBaseLog({ id: this.id }); encoder.logger.pipe(this.logger); @@ -47,7 +49,7 @@ export class TeceMux extends TypedEmitter { allowHalfOpen: true }), { - _id: destinationPort || this.channelCount, + _id: port || this.channelCount, encoder, closedByFIN: false } @@ -193,6 +195,7 @@ export class TeceMux extends TypedEmitter { channel = this.createChannel(destinationPort, false); this.addChannel(channel, true); + //this.emit("peer", { channelId: destinationPort}) } if (dataLength) { @@ -247,7 +250,7 @@ export class TeceMux extends TypedEmitter { multiplex(opts: { channel?: number } = {}): TeceMuxChannel { this.logger.trace("Multiplex"); - const channel = this.createChannel(opts.channel || this.channelCount, true); + const channel = this.createChannel(opts.channel !== undefined ? opts.channel : this.channelCount, true); this.addChannel(channel, false); @@ -255,5 +258,5 @@ export class TeceMux extends TypedEmitter { return channel; } } -export { TeceMuxChannel }; +export { TeceMuxChannel }; diff --git a/packages/verser/src/lib/tecemux/types.ts b/packages/verser/src/lib/tecemux/types.ts index 2cf55c247..17ac39a18 100644 --- a/packages/verser/src/lib/tecemux/types.ts +++ b/packages/verser/src/lib/tecemux/types.ts @@ -3,6 +3,7 @@ import { Duplex, Transform } from "stream"; export type TeceMuxEvents = { channel(socket: Duplex): void; error(error: any): void; + peer(payload: { channelId: number }): void; } export type FramesKeeperEvents = { From fbdcb0de4c233a76545c4f5d7bbe981893669259 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 28 Mar 2023 14:06:15 +0000 Subject: [PATCH 093/231] R<->STH --- packages/runner/Dockerfile | 1 + packages/runner/src/host-client.ts | 6 ----- packages/runner/src/runner.ts | 2 ++ .../src/lib/tecemux/codecs/frame-encoder.ts | 27 ++++++++++++++----- packages/verser/src/lib/tecemux/tecemux.ts | 10 +++---- packages/verser/src/lib/tecemux/types.ts | 2 ++ packages/verser/src/lib/verser-connection.ts | 6 ++--- 7 files changed, 33 insertions(+), 21 deletions(-) diff --git a/packages/runner/Dockerfile b/packages/runner/Dockerfile index 3d659fea6..7c3076035 100644 --- a/packages/runner/Dockerfile +++ b/packages/runner/Dockerfile @@ -26,6 +26,7 @@ COPY ./dist/symbols ./dist/symbols COPY ./dist/obj-logger ./dist/obj-logger COPY ./dist/model ./dist/model COPY ./dist/runner ./dist/runner +COPY ./dist/verser ./dist/verser COPY ./dist/package.json ./dist/package.json FROM target diff --git a/packages/runner/src/host-client.ts b/packages/runner/src/host-client.ts index 33a880c24..bb58d609b 100644 --- a/packages/runner/src/host-client.ts +++ b/packages/runner/src/host-client.ts @@ -52,12 +52,6 @@ class HostClient implements IHostClient { res.map(async (channel, index) => { // Assuming id is exactly 36 bytes channel.write(id + "" + index); - // eslint-disable-next-line no-console - console.log(channel._id); - // Assuming number is from 0-8, sending 1 byte - //channel.write(index.toString()); - - //await defer(500); return channel; }) diff --git a/packages/runner/src/runner.ts b/packages/runner/src/runner.ts index 880b07c1e..323e764f7 100644 --- a/packages/runner/src/runner.ts +++ b/packages/runner/src/runner.ts @@ -300,6 +300,8 @@ export class Runner implements IComponent { this.logger.debug("Streams initialized"); + //await defer(15); + this.sendHandshakeMessage(); const { appConfig, args } = await this.waitForHandshakeResponse(); diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index 7aa386263..ae3b20c79 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -76,15 +76,28 @@ export class FrameEncoder extends Transform { return flags; } - setChannel(channelCount: number) { + async setChannel(channelCount: number) { this.logger.debug("Set channel command", channelCount); - this.out.write( - this.createFrame([], { - flagsArray: ["PSH"], - destinationPort: channelCount - }) - ); + const frame = this.createFrame([], { + flagsArray: ["PSH"], + destinationPort: channelCount + }); + + this.out.write(frame); + + const sn = +this.tecemux.sequenceNumber; + + return await new Promise((resolve, _reject) => { + const waiter = (sequenceNumber: number) => { + if (sequenceNumber === sn) { + this.tecemux.framesKeeper.off("ack", waiter); + resolve(); + } + }; + + this.tecemux.framesKeeper.on("ack", waiter); + }); } onChannelEnd(channelId: number) { diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index b6752f033..63610f400 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -19,7 +19,7 @@ export class TeceMux extends TypedEmitter { logger: ObjLogger; commonEncoder = new FrameEncoder(0, this); - private createChannel(destinationPort?: number, emit?: boolean): TeceMuxChannel { + private async createChannel(destinationPort?: number, emit?: boolean): Promise { const port = destinationPort !== undefined ? destinationPort : this.channelCount; this.logger.debug("Create Channel", port); @@ -87,7 +87,7 @@ export class TeceMux extends TypedEmitter { }); if (emit) { - encoder.setChannel(destinationPort || this.channelCount); + await encoder.setChannel(destinationPort || this.channelCount); } return channel; @@ -192,7 +192,7 @@ export class TeceMux extends TypedEmitter { if (!channel) { this.logger.warn("Unknown channel"); - channel = this.createChannel(destinationPort, false); + channel = await this.createChannel(destinationPort, false); this.addChannel(channel, true); //this.emit("peer", { channelId: destinationPort}) @@ -247,10 +247,10 @@ export class TeceMux extends TypedEmitter { ); } - multiplex(opts: { channel?: number } = {}): TeceMuxChannel { + async multiplex(opts: { channel?: number } = {}): Promise { this.logger.trace("Multiplex"); - const channel = this.createChannel(opts.channel !== undefined ? opts.channel : this.channelCount, true); + const channel = await this.createChannel(opts.channel !== undefined ? opts.channel : this.channelCount, true); this.addChannel(channel, false); diff --git a/packages/verser/src/lib/tecemux/types.ts b/packages/verser/src/lib/tecemux/types.ts index 17ac39a18..94964849f 100644 --- a/packages/verser/src/lib/tecemux/types.ts +++ b/packages/verser/src/lib/tecemux/types.ts @@ -37,6 +37,8 @@ export interface IFramesKeeper { handlePSH(sequenceNumber: number): void; getFrame(sequenceNumber: number): FramesKeeperFrame | undefined; generator: AsyncGenerator; + on(event: string, handler: (sequenceNumber: number) => void): void; + off(event: string, handler: (sequenceNumber: number) => void): void; } export interface ITeCeMux { diff --git a/packages/verser/src/lib/verser-connection.ts b/packages/verser/src/lib/verser-connection.ts index 87edc0344..16685aaeb 100644 --- a/packages/verser/src/lib/verser-connection.ts +++ b/packages/verser/src/lib/verser-connection.ts @@ -101,7 +101,7 @@ export class VerserConnection { async forward(req: IncomingMessage, res: ServerResponse) { if (!this.connected) throw new Error("BPMux not connected"); - const channel = this.teceMux?.multiplex() as Duplex; + const channel = await this.teceMux?.multiplex() as Duplex; channel .on("error", (error: Error) => { @@ -191,10 +191,10 @@ export class VerserConnection { * @param id {number} Channel id. * @returns Duplex stream. */ - createChannel(id: number): Duplex { + async createChannel(id: number): Promise { if (!this.teceMux) throw new Error("TeCeMux not connected"); - return this.teceMux.multiplex({ channel: id }); + return await this.teceMux.multiplex({ channel: id }); } reconnect() { From f068c5b082bb16d4cbca538524e084d74090ad11 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 3 Apr 2023 21:12:12 +0000 Subject: [PATCH 094/231] TeCeMux wip --- packages/host/src/lib/socket-server.ts | 1 - .../src/lib/tecemux/codecs/frame-encoder.ts | 16 +++++++++++----- packages/verser/src/lib/tecemux/tecemux.ts | 14 ++++++++++---- packages/verser/src/lib/tecemux/types.ts | 2 +- packages/verser/src/lib/verser-connection.ts | 2 +- 5 files changed, 23 insertions(+), 12 deletions(-) diff --git a/packages/host/src/lib/socket-server.ts b/packages/host/src/lib/socket-server.ts index 06dbeb7d0..f2a6f65db 100644 --- a/packages/host/src/lib/socket-server.ts +++ b/packages/host/src/lib/socket-server.ts @@ -53,7 +53,6 @@ export class SocketServer extends TypedEmitter implements IComponent { protocol.on("channel", async (channel: TeceMuxChannel) => { const { instanceId, channelId } = await new Promise<{ instanceId: string, channelId: number }>((resolve) => { - channel.pause(); channel.once("readable", () => { const payload = channel.read(37).toString(); const instId = payload.substring(0, 36); diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index ae3b20c79..80f86768b 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -1,3 +1,5 @@ +/* eslint-disable no-console */ + import { PassThrough, Transform, TransformCallback, TransformOptions } from "stream"; import { ObjLogger } from "@scramjet/obj-logger"; @@ -76,9 +78,11 @@ export class FrameEncoder extends Transform { return flags; } - async setChannel(channelCount: number) { + async establishChannel(channelCount: number) { this.logger.debug("Set channel command", channelCount); + console.log("Set channel command", channelCount); + const frame = this.createFrame([], { flagsArray: ["PSH"], destinationPort: channelCount @@ -88,15 +92,17 @@ export class FrameEncoder extends Transform { const sn = +this.tecemux.sequenceNumber; - return await new Promise((resolve, _reject) => { - const waiter = (sequenceNumber: number) => { + return new Promise((resolve, _reject) => { + const ackTempHandler = (sequenceNumber: number) => { + console.log("ACK RECEIVED", sequenceNumber, sn); + if (sequenceNumber === sn) { - this.tecemux.framesKeeper.off("ack", waiter); + this.tecemux.framesKeeper.off("ack", ackTempHandler); resolve(); } }; - this.tecemux.framesKeeper.on("ack", waiter); + this.tecemux.framesKeeper.on("ack", ackTempHandler); }); } diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 63610f400..c6bd6bb0c 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -1,3 +1,5 @@ +/* eslint-disable no-console */ + import { TypedEmitter } from "@scramjet/utility"; import { FrameDecoder, FrameEncoder } from "./codecs"; import { Duplex } from "stream"; @@ -13,7 +15,7 @@ export class TeceMux extends TypedEmitter { framesSent = 0; carrierDecoder: FrameDecoder; framesKeeper = new FramesKeeper(); - sequenceNumber = Math.abs((Math.random() * (2 ** 32)) | 0); + sequenceNumber = Math.abs((Math.random() * (2 ** 32)) / 2 | 0); channels = new Map(); logger: ObjLogger; @@ -87,7 +89,7 @@ export class TeceMux extends TypedEmitter { }); if (emit) { - await encoder.setChannel(destinationPort || this.channelCount); + await encoder.establishChannel(port || this.channelCount); } return channel; @@ -249,10 +251,14 @@ export class TeceMux extends TypedEmitter { async multiplex(opts: { channel?: number } = {}): Promise { this.logger.trace("Multiplex"); + console.log("Multiplex", opts.channel); + + const id = opts.channel !== undefined ? opts.channel : this.channelCount; + const channel = await this.createChannel(id, true); - const channel = await this.createChannel(opts.channel !== undefined ? opts.channel : this.channelCount, true); + await channel.encoder.establishChannel(id); - this.addChannel(channel, false); + this.addChannel(channel, true); this.logger.trace("Multiplex ready", channel._id); return channel; diff --git a/packages/verser/src/lib/tecemux/types.ts b/packages/verser/src/lib/tecemux/types.ts index 94964849f..f1860077e 100644 --- a/packages/verser/src/lib/tecemux/types.ts +++ b/packages/verser/src/lib/tecemux/types.ts @@ -48,7 +48,7 @@ export interface ITeCeMux { } export interface IFrameEncoder extends Transform { - + establishChannel(id: number): Promise; } export type TeceMuxChannel = Duplex & { diff --git a/packages/verser/src/lib/verser-connection.ts b/packages/verser/src/lib/verser-connection.ts index 16685aaeb..79c1bd826 100644 --- a/packages/verser/src/lib/verser-connection.ts +++ b/packages/verser/src/lib/verser-connection.ts @@ -99,7 +99,7 @@ export class VerserConnection { * @param res {ServerResponse} Response object. */ async forward(req: IncomingMessage, res: ServerResponse) { - if (!this.connected) throw new Error("BPMux not connected"); + if (!this.connected) throw new Error("Not connected"); const channel = await this.teceMux?.multiplex() as Duplex; From d96995bb56c3b3da05b0a20f95acda4f67567a72 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 6 Apr 2023 19:25:21 +0000 Subject: [PATCH 095/231] fix channels _id --- packages/host/src/lib/socket-server.ts | 45 +++++++++++++++++++ packages/runner/src/host-client.ts | 2 + .../verser/src/lib/tecemux/frames-keeper.ts | 4 +- packages/verser/src/lib/tecemux/tecemux.ts | 12 +++-- 4 files changed, 54 insertions(+), 9 deletions(-) diff --git a/packages/host/src/lib/socket-server.ts b/packages/host/src/lib/socket-server.ts index f2a6f65db..3326466ac 100644 --- a/packages/host/src/lib/socket-server.ts +++ b/packages/host/src/lib/socket-server.ts @@ -49,6 +49,7 @@ export class SocketServer extends TypedEmitter implements IComponent { }); protocol = new TeceMux(connection); + //protocol.logger.pipe(this.logger); protocol.on("channel", async (channel: TeceMuxChannel) => { const { instanceId, channelId } = @@ -65,16 +66,60 @@ export class SocketServer extends TypedEmitter implements IComponent { }); }); +<<<<<<< HEAD +||||||| constructed merge base + // const channelId = await new Promise((resolve) => { + // channel.once("readable", () => { + // resolve(parseInt(channel.read(1).toString(), 10)); + // }); + // }); + + this.logger.info("new channel", instanceId, channelId); + + let runner = this.runnerConnectionsInProgress.get(instanceId); + + if (!runner) { + runner = [null, null, null, null, null, null, null, null, null]; + this.runnerConnectionsInProgress.set(instanceId, runner); + } +======= + // const channelId = await new Promise((resolve) => { + // channel.once("readable", () => { + // resolve(parseInt(channel.read(1).toString(), 10)); + // }); + // }); + + this.logger.info("new channel", instanceId, channelId, channel._id); + + let runner = this.runnerConnectionsInProgress.get(instanceId); + + if (!runner) { + runner = [null, null, null, null, null, null, null, null, null]; + this.runnerConnectionsInProgress.set(instanceId, runner); + } +>>>>>>> fix channels _id channel .on("error", (err: any) => this.logger.error("Error on Instance in stream", instanceId, channelId, err)) .on("end", () => this.logger.debug(`Channel [${instanceId}:${channelId}] ended`)); +<<<<<<< HEAD try { await this.handleConnection(instanceId, channelId, connection as Socket); } catch (err: any) { connection.destroy(); +||||||| constructed merge base + if (runner.every(isDefined)) { + this.runnerConnectionsInProgress.delete(instanceId); + this.emit("connect", instanceId, runner as RunnerChannels); +======= + if (runner.every(isDefined)) { + // eslint-disable-next-line no-console + console.log(runner.map(r => r!._id)); + this.runnerConnectionsInProgress.delete(instanceId); + this.emit("connect", instanceId, runner as RunnerChannels); +>>>>>>> fix channels _id } }) diff --git a/packages/runner/src/host-client.ts b/packages/runner/src/host-client.ts index bb58d609b..f43da8928 100644 --- a/packages/runner/src/host-client.ts +++ b/packages/runner/src/host-client.ts @@ -44,6 +44,8 @@ class HostClient implements IHostClient { const tunnel = net.createConnection(this.instancesServerPort, this.instancesServerHost); const protocol = new TeceMux(tunnel); + //protocol.logger.pipe(this.logger); + const openConnections = await Promise.all( Array.from(Array(9)) .map((_c, index) => protocol.multiplex({ channel: index })) diff --git a/packages/verser/src/lib/tecemux/frames-keeper.ts b/packages/verser/src/lib/tecemux/frames-keeper.ts index 48126f77f..cc2fbd3b9 100644 --- a/packages/verser/src/lib/tecemux/frames-keeper.ts +++ b/packages/verser/src/lib/tecemux/frames-keeper.ts @@ -15,7 +15,7 @@ export class FramesKeeper extends TypedEmitter implements IF generator: AsyncGenerator = (async function* (this: FramesKeeper) { while (true) { if (this.lastSequenceSent - this.lastSequenceReceived < this.#MAX_FRAMES_DIFFERENCE) { - this.logger.info("Write allowed"); + this.logger.debug("Write allowed"); yield Promise.resolve(this.lastSequenceReceived); continue; } @@ -53,7 +53,7 @@ export class FramesKeeper extends TypedEmitter implements IF const frame = this.#framesSent.get(sequenceNumber); // received or not stored - return frame === undefined || !!this.#framesSent.get(sequenceNumber)?.received; + return frame === undefined || !!frame.received; } getFrame(sequenceNumber: number) { diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index c6bd6bb0c..6608e0dc5 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -34,7 +34,7 @@ export class TeceMux extends TypedEmitter { const channel: TeceMuxChannel = Object.assign( new Duplex({ write: (chunk, encoding, next) => { - this.logger.trace("WRITE channel", channel._id, chunk); + this.logger.debug("WRITE channel", channel._id, chunk); if (chunk === null) { this.logger.info("NULL ON CHANNEL"); @@ -46,12 +46,12 @@ export class TeceMux extends TypedEmitter { return encoder.write(chunk, encoding, next); }, read: (_size) => { - this.logger.trace("READ channel", channel._id); + this.logger.debug("READ channel", channel._id); }, allowHalfOpen: true }), { - _id: port || this.channelCount, + _id: port, encoder, closedByFIN: false } @@ -89,7 +89,7 @@ export class TeceMux extends TypedEmitter { }); if (emit) { - await encoder.establishChannel(port || this.channelCount); + await encoder.establishChannel(channel._id); } return channel; @@ -141,8 +141,6 @@ export class TeceMux extends TypedEmitter { async main() { let t = 0; - //this.carrierDecoder.pipe(this.framesKeeper); - for await (const chunk of this.carrierDecoder) { let frame: FrameData; @@ -228,7 +226,7 @@ export class TeceMux extends TypedEmitter { addChannel(channel: TeceMuxChannel, emit: boolean) { this.logger.debug("adding channel", channel._id); - this.channels.set(channel._id, channel); // wait for SYN reply? + this.channels.set(channel._id, channel); if (emit) { this.emit("channel", channel); From 377f1b715d48837658b403d23451c6f09eab8a51 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 6 Apr 2023 21:47:02 +0000 Subject: [PATCH 096/231] Test defer --- packages/runner/src/runner.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/runner/src/runner.ts b/packages/runner/src/runner.ts index 323e764f7..29bd8639e 100644 --- a/packages/runner/src/runner.ts +++ b/packages/runner/src/runner.ts @@ -302,7 +302,7 @@ export class Runner implements IComponent { //await defer(15); - this.sendHandshakeMessage(); + await this.sendHandshakeMessage(); const { appConfig, args } = await this.waitForHandshakeResponse(); @@ -451,7 +451,8 @@ export class Runner implements IComponent { // TODO: what if it fails? } - sendHandshakeMessage() { + async sendHandshakeMessage() { + await defer(1000); MessageUtils.writeMessageOnStream([RunnerMessageCode.PING, {}], this.hostClient.monitorStream); this.logger.trace("Handshake sent"); From 83183efcc576b3ea6060b94806319851c4e86284 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 11 Apr 2023 09:41:12 +0000 Subject: [PATCH 097/231] CI test --- packages/host/src/lib/csi-controller.ts | 5 +- packages/host/src/lib/socket-server.ts | 50 +------------------ .../src/lib/tecemux/codecs/frame-encoder.ts | 11 ++-- packages/verser/src/lib/tecemux/tecemux.ts | 7 ++- packages/verser/test/tecemux-transfer.spec.ts | 22 +++++--- 5 files changed, 26 insertions(+), 69 deletions(-) diff --git a/packages/host/src/lib/csi-controller.ts b/packages/host/src/lib/csi-controller.ts index 2bb7e616c..cad7ce566 100644 --- a/packages/host/src/lib/csi-controller.ts +++ b/packages/host/src/lib/csi-controller.ts @@ -539,8 +539,9 @@ export class CSIController extends TypedEmitter { const pongMsg: HandshakeAcknowledgeMessage = { msgCode: RunnerMessageCode.PONG, appConfig: this.appConfig, - args: this.args - }; + args: this.args, + b: Buffer.from(new Uint8Array(1024 * 1024).fill(2)).toString("hex") + } as HandshakeAcknowledgeMessage; await this.controlDataStream.whenWrote(MessageUtilities.serializeMessage(pongMsg)); } else { diff --git a/packages/host/src/lib/socket-server.ts b/packages/host/src/lib/socket-server.ts index 3326466ac..6d02462ec 100644 --- a/packages/host/src/lib/socket-server.ts +++ b/packages/host/src/lib/socket-server.ts @@ -49,7 +49,6 @@ export class SocketServer extends TypedEmitter implements IComponent { }); protocol = new TeceMux(connection); - //protocol.logger.pipe(this.logger); protocol.on("channel", async (channel: TeceMuxChannel) => { const { instanceId, channelId } = @@ -66,63 +65,16 @@ export class SocketServer extends TypedEmitter implements IComponent { }); }); -<<<<<<< HEAD -||||||| constructed merge base - // const channelId = await new Promise((resolve) => { - // channel.once("readable", () => { - // resolve(parseInt(channel.read(1).toString(), 10)); - // }); - // }); - - this.logger.info("new channel", instanceId, channelId); - - let runner = this.runnerConnectionsInProgress.get(instanceId); - - if (!runner) { - runner = [null, null, null, null, null, null, null, null, null]; - this.runnerConnectionsInProgress.set(instanceId, runner); - } -======= - // const channelId = await new Promise((resolve) => { - // channel.once("readable", () => { - // resolve(parseInt(channel.read(1).toString(), 10)); - // }); - // }); - - this.logger.info("new channel", instanceId, channelId, channel._id); - - let runner = this.runnerConnectionsInProgress.get(instanceId); - - if (!runner) { - runner = [null, null, null, null, null, null, null, null, null]; - this.runnerConnectionsInProgress.set(instanceId, runner); - } ->>>>>>> fix channels _id - channel .on("error", (err: any) => this.logger.error("Error on Instance in stream", instanceId, channelId, err)) .on("end", () => this.logger.debug(`Channel [${instanceId}:${channelId}] ended`)); - -<<<<<<< HEAD try { await this.handleConnection(instanceId, channelId, connection as Socket); } catch (err: any) { connection.destroy(); -||||||| constructed merge base - if (runner.every(isDefined)) { - this.runnerConnectionsInProgress.delete(instanceId); - this.emit("connect", instanceId, runner as RunnerChannels); -======= - if (runner.every(isDefined)) { - // eslint-disable-next-line no-console - console.log(runner.map(r => r!._id)); - this.runnerConnectionsInProgress.delete(instanceId); - this.emit("connect", instanceId, runner as RunnerChannels); ->>>>>>> fix channels _id } - }) - + }); }); return new Promise((res, rej) => { diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index 80f86768b..9c6f012e2 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -78,14 +78,12 @@ export class FrameEncoder extends Transform { return flags; } - async establishChannel(channelCount: number) { - this.logger.debug("Set channel command", channelCount); - - console.log("Set channel command", channelCount); + async establishChannel(channelId: number) { + this.logger.debug("Establishing channel", channelId); const frame = this.createFrame([], { flagsArray: ["PSH"], - destinationPort: channelCount + destinationPort: channelId }); this.out.write(frame); @@ -94,10 +92,11 @@ export class FrameEncoder extends Transform { return new Promise((resolve, _reject) => { const ackTempHandler = (sequenceNumber: number) => { - console.log("ACK RECEIVED", sequenceNumber, sn); + this.logger.debug("ACK RECEIVED", sequenceNumber, sn); if (sequenceNumber === sn) { this.tecemux.framesKeeper.off("ack", ackTempHandler); + this.logger.debug("channel established", channelId); resolve(); } }; diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 6608e0dc5..5f99fa3b5 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -248,11 +248,10 @@ export class TeceMux extends TypedEmitter { } async multiplex(opts: { channel?: number } = {}): Promise { - this.logger.trace("Multiplex"); - console.log("Multiplex", opts.channel); - const id = opts.channel !== undefined ? opts.channel : this.channelCount; - const channel = await this.createChannel(id, true); + const channel = await this.createChannel(id); + + this.logger.trace("Multiplex", id); await channel.encoder.establishChannel(id); diff --git a/packages/verser/test/tecemux-transfer.spec.ts b/packages/verser/test/tecemux-transfer.spec.ts index 54c9c97d7..326818ed7 100644 --- a/packages/verser/test/tecemux-transfer.spec.ts +++ b/packages/verser/test/tecemux-transfer.spec.ts @@ -18,7 +18,7 @@ async function startServer() { }); } -test("Protocol send file over http connection", async (t) => { +test.serial("Protocol send file over http connection", async (t) => { const server = await startServer(); const hashReceived = crypto.createHash("md5"); @@ -26,14 +26,15 @@ test("Protocol send file over http connection", async (t) => { const serverSideChannelPromise = new Promise((resolve) => { server.on("connect", async (req: IncomingMessage, socket: Socket) => { + console.log("server on connect!"); socket.setNoDelay(true); serverTeceMux = new TeceMux(socket, "Server") .on("error", (error) => { - console.error("TeceMux error", error); + console.error("TeceMux error", error.code); }); - resolve(serverTeceMux.multiplex()); + resolve(serverTeceMux.multiplex({ channel: 1 })); }); }); @@ -51,6 +52,7 @@ test("Protocol send file over http connection", async (t) => { await new Promise(resolve => { clientTeceMux.on("channel", async (clientSideChannel: TeceMuxChannel) => { + console.log("TEST: on channel", clientSideChannel._id); function* gen() { for (let i = 0; i < 1e3; i++) { const str = crypto.randomBytes(1024).toString("hex"); @@ -67,14 +69,18 @@ test("Protocol send file over http connection", async (t) => { const serverSideChannel = await serverSideChannelPromise; + console.log("Serverside channel", serverSideChannel._id); + for await (const d of serverSideChannel) { hashReceived.update(d); } - console.dir({ - hashSent, - hashReceived - }); + const res = { + txHash: hashSent.digest("hex"), + rxHash: hashReceived.digest("hex") + }; + + console.dir(res); - t.assert(hashReceived.digest("hex") === hashSent.digest("hex"), "Unequal hashes"); + t.assert(res.txHash === res.rxHash, "Unequal hashes"); }); From f8bc943f354e858e3fc029b3d2f9833003d515e5 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 11 Apr 2023 13:51:40 +0000 Subject: [PATCH 098/231] Remove debug listeners --- packages/host/src/lib/csi-controller.ts | 3 +- packages/runner/src/host-client.ts | 4 +- packages/verser/src/lib/tecemux/tecemux.ts | 58 +++++++++++----------- 3 files changed, 31 insertions(+), 34 deletions(-) diff --git a/packages/host/src/lib/csi-controller.ts b/packages/host/src/lib/csi-controller.ts index cad7ce566..aaa2e8550 100644 --- a/packages/host/src/lib/csi-controller.ts +++ b/packages/host/src/lib/csi-controller.ts @@ -539,8 +539,7 @@ export class CSIController extends TypedEmitter { const pongMsg: HandshakeAcknowledgeMessage = { msgCode: RunnerMessageCode.PONG, appConfig: this.appConfig, - args: this.args, - b: Buffer.from(new Uint8Array(1024 * 1024).fill(2)).toString("hex") + args: this.args } as HandshakeAcknowledgeMessage; await this.controlDataStream.whenWrote(MessageUtilities.serializeMessage(pongMsg)); diff --git a/packages/runner/src/host-client.ts b/packages/runner/src/host-client.ts index f43da8928..d22ef850e 100644 --- a/packages/runner/src/host-client.ts +++ b/packages/runner/src/host-client.ts @@ -52,7 +52,7 @@ class HostClient implements IHostClient { ).then(async res => { return Promise.all( res.map(async (channel, index) => { - // Assuming id is exactly 36 bytes + // Assuming id is exactly 36 bytes + Assuming number is from 0-8, sending 1 byte channel.write(id + "" + index); return channel; @@ -60,8 +60,6 @@ class HostClient implements IHostClient { ); }); - //await defer(500); - this._streams = await openConnections as HostOpenConnections; try { diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 5f99fa3b5..004e206ec 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -65,28 +65,28 @@ export class TeceMux extends TypedEmitter { this.logger.error("CHANNEL ERROR", error); this.emit("error", { error, source: channel }); }) - .on("destroy", () => { - this.logger.trace("channel on DESTROY ", channel._id); - }) - .on("abort", () => { - this.logger.trace("channel on ABORT ", channel._id); - }) + // .on("destroy", () => { + // this.logger.trace("channel on DESTROY ", channel._id); + // }) + // .on("abort", () => { + // this.logger.trace("channel on ABORT ", channel._id); + // }) .on("close", () => { this.logger.info("CHANNEL close", channel._id); this.sendFIN(channel._id); }) - .on("end", () => { - this.logger.info("CHANNEL end", channel._id); - }) + // .on("end", () => { + // this.logger.info("CHANNEL end", channel._id); + // }) .on("finish", () => { this.logger.info("CHANNEL finish", channel._id); this.sendFIN(channel._id); - }) - .on("data", (d) => { - if (d === null) { - this.logger.info("CHANNEL end", channel._id); - } }); + // .on("data", (d) => { + // if (d === null) { + // this.logger.info("CHANNEL end", channel._id); + // } + // }); if (emit) { await encoder.establishChannel(channel._id); @@ -102,24 +102,24 @@ export class TeceMux extends TypedEmitter { this.carrierSocket = socket; this.carrierDecoder = new FrameDecoder({ emitClose: false }) - .on("pause", () => { - this.logger.warn("Decoder paused"); - }) - .on("close", () => { - this.logger.warn("Decoder closed"); - }) - .on("end", () => { - this.logger.warn("Decoder ended"); - }) .on("error", (error) => { this.logger.error("Decoder error", error); - }) - .on("abort", (error) => { - this.logger.error("Decoder abort", error); - }) - .on("destroy", (error) => { - this.logger.error("Decoder destroy", error); }); + // .on("pause", () => { + // this.logger.warn("Decoder paused"); + // }) + // .on("close", () => { + // this.logger.warn("Decoder closed"); + // }) + // .on("end", () => { + // this.logger.warn("Decoder ended"); + // }) + // .on("abort", (error) => { + // this.logger.error("Decoder abort", error); + // }) + // .on("destroy", (error) => { + // this.logger.error("Decoder destroy", error); + // }); this.carrierDecoder.logger.updateBaseLog({ id: this.id }); this.carrierDecoder.logger.pipe(this.logger); From 79d202e4eb5b1f4ee760a91dcd3f4dcf49eed233 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 12 Apr 2023 09:22:52 +0000 Subject: [PATCH 099/231] Runner host-client destroy --- packages/runner/src/host-client.ts | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/packages/runner/src/host-client.ts b/packages/runner/src/host-client.ts index d22ef850e..6975e1049 100644 --- a/packages/runner/src/host-client.ts +++ b/packages/runner/src/host-client.ts @@ -60,7 +60,7 @@ class HostClient implements IHostClient { ); }); - this._streams = await openConnections as HostOpenConnections; + this._streams = openConnections as HostOpenConnections; try { this.bpmux = new BPMux(this._streams[CC.PACKAGE]); @@ -103,22 +103,12 @@ class HostClient implements IHostClient { async disconnect() { this.logger.trace("Disconnecting from host"); - const streamsExitedPromised: Promise[] = this.streams.map((stream, i) => + const streamsExitedPromised: Promise[] = this.streams.map((stream, _i) => new Promise( (res) => { - if ("writable" in stream!) { - stream - .on("error", (e) => { - console.error("Error on stream", i, e.stack); - }) - .on("close", () => { - res(); - }) - .end(); - } else { - stream!.destroy(); - res(); - } + // now we have readable and writable always + stream!.destroy(); + res(); } )); From 426ad3b21e9513572d7eec3488493a2f94dcc5c3 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 12 Apr 2023 15:19:14 +0000 Subject: [PATCH 100/231] nextTick before sequence call --- packages/host/src/lib/socket-server.ts | 6 ++++-- packages/runner/src/host-client.ts | 2 +- packages/runner/src/runner.ts | 6 +++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/host/src/lib/socket-server.ts b/packages/host/src/lib/socket-server.ts index 6d02462ec..1684d9b13 100644 --- a/packages/host/src/lib/socket-server.ts +++ b/packages/host/src/lib/socket-server.ts @@ -9,11 +9,13 @@ import { TeceMux, TeceMuxChannel } from "@scramjet/verser"; type MaybeChannel = TeceMuxChannel | Socket | null; type RunnerChannels = [ - TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel + TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, + TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel ]; type RunnerConnectionsInProgress = [ - MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel + MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel, + MaybeChannel, MaybeChannel, MaybeChannel, MaybeChannel ]; type Events = { diff --git a/packages/runner/src/host-client.ts b/packages/runner/src/host-client.ts index 6975e1049..8c938c1a1 100644 --- a/packages/runner/src/host-client.ts +++ b/packages/runner/src/host-client.ts @@ -60,7 +60,7 @@ class HostClient implements IHostClient { ); }); - this._streams = openConnections as HostOpenConnections; + this._streams = await openConnections as HostOpenConnections; try { this.bpmux = new BPMux(this._streams[CC.PACKAGE]); diff --git a/packages/runner/src/runner.ts b/packages/runner/src/runner.ts index 29bd8639e..dbe4035ab 100644 --- a/packages/runner/src/runner.ts +++ b/packages/runner/src/runner.ts @@ -504,12 +504,16 @@ export class Runner implements IComponent { try { this.logger.debug("Processing function on index", sequence.length - itemsLeftInSequence - 1); + // eslint-disable-next-line no-loop-func + await new Promise((res) => { + process.nextTick(res); + }); + out = func.call( this.context, stream, ...args ); - this.logger.debug("Function called", sequence.length - itemsLeftInSequence - 1); } catch (error: any) { this.logger.error("Function errored", sequence.length - itemsLeftInSequence, error.stack); From fe211e7d0dcae740eed38f1449482eb710c4a67b Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 13 Apr 2023 00:11:47 +0000 Subject: [PATCH 101/231] TeceMuxChannel class --- .../src/lib/tecemux/codecs/frame-encoder.ts | 2 +- .../verser/src/lib/tecemux/tecemux-channel.ts | 60 +++++ packages/verser/src/lib/tecemux/tecemux.ts | 245 +++++++++--------- packages/verser/src/lib/tecemux/types.ts | 6 +- 4 files changed, 189 insertions(+), 124 deletions(-) create mode 100644 packages/verser/src/lib/tecemux/tecemux-channel.ts diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts index 9c6f012e2..042f0bfbe 100644 --- a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts +++ b/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts @@ -8,7 +8,7 @@ import { FrameData, ITeCeMux } from "../types"; import { calculateChecksum } from "./utils"; export class FrameEncoder extends Transform { - MAX_CHUNK_SIZE = 10 * 1024 - HEADER_LENGTH; + MAX_CHUNK_SIZE = 64 * 1024 - HEADER_LENGTH; tecemux: ITeCeMux; total = 0; diff --git a/packages/verser/src/lib/tecemux/tecemux-channel.ts b/packages/verser/src/lib/tecemux/tecemux-channel.ts new file mode 100644 index 000000000..36460e5b1 --- /dev/null +++ b/packages/verser/src/lib/tecemux/tecemux-channel.ts @@ -0,0 +1,60 @@ +import { ObjLogger } from "@scramjet/obj-logger"; +import { Duplex } from "stream"; +import { IFrameEncoder, ITeCeMux } from "./types"; +import { FrameEncoder } from "./codecs"; + +export class TeceMuxChannel extends Duplex { + _id: number; + logger: ObjLogger; + allowHalfOpen: boolean; + encoder: IFrameEncoder; + closedByFIN = false; + + constructor(duplexOptions: ConstructorParameters[0], id: number, teceMux: ITeCeMux) { + super(duplexOptions); + this.logger = new ObjLogger(this); + this.allowHalfOpen = true; + this._id = id; + this.encoder = new FrameEncoder(id, teceMux, { encoding: undefined }); + } + + _write(chunk: any, encoding: BufferEncoding, next: (error?: Error | null | undefined) => void) { + this.logger.debug("WRITE channel", this._id, chunk); + + if (chunk === null) { + this.logger.info("NULL ON CHANNEL"); + + this.end(); + return false; + } + + return this.encoder.write(chunk, encoding, next); + } + + _read(_size?: number) { + this.logger.debug("READ channel", this._id); + } + + sendACK(sequenceNumber: number) { + this.encoder.push( + this.encoder.createFrame(undefined, { + flagsArray: ["ACK"], + acknowledgeNumber: sequenceNumber, + destinationPort: this._id + }) + ); + } + + handlerFIN() { + this.closedByFIN = true; + + if (!this.writableEnded) { + this.push(null); + } + + if (this.writableEnded && this.readableEnded) { + this.destroy(); + this.logger.info("Channel destroy"); + } + } +} diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 004e206ec..9cedacf8f 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -1,12 +1,13 @@ /* eslint-disable no-console */ import { TypedEmitter } from "@scramjet/utility"; -import { FrameDecoder, FrameEncoder } from "./codecs"; +import { FrameDecoder } from "./codecs"; import { Duplex } from "stream"; import { Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; -import { FrameData, TeceMuxChannel, TeceMuxEvents } from "./types"; +import { FrameData, TeceMuxEvents } from "./types"; import { FramesKeeper } from "./frames-keeper"; +import { TeceMuxChannel } from "./tecemux-channel"; export class TeceMux extends TypedEmitter { id: string; @@ -19,45 +20,66 @@ export class TeceMux extends TypedEmitter { channels = new Map(); logger: ObjLogger; - commonEncoder = new FrameEncoder(0, this); - private async createChannel(destinationPort?: number, emit?: boolean): Promise { + private async createChannel(destinationPort?: number, establish?: boolean): Promise { const port = destinationPort !== undefined ? destinationPort : this.channelCount; this.logger.debug("Create Channel", port); - const encoder = new FrameEncoder(port, this, { encoding: undefined }); - - encoder.logger.updateBaseLog({ id: this.id }); - encoder.logger.pipe(this.logger); - - const channel: TeceMuxChannel = Object.assign( - new Duplex({ - write: (chunk, encoding, next) => { - this.logger.debug("WRITE channel", channel._id, chunk); - - if (chunk === null) { - this.logger.info("NULL ON CHANNEL"); - - channel.end(); - return false; - } - - return encoder.write(chunk, encoding, next); - }, - read: (_size) => { - this.logger.debug("READ channel", channel._id); - }, - allowHalfOpen: true - }), - { - _id: port, - encoder, - closedByFIN: false - } - ); - - encoder.out + //const encoder = new FrameEncoder(port, this, { encoding: undefined }); + + //encoder.logger.updateBaseLog({ id: this.id }); + //encoder.logger.pipe(this.logger); + + const channel = new TeceMuxChannel({ allowHalfOpen: true }, port, this); + // const channel: TeceMuxChannel = Object.assign( + // new Duplex({ + // write: (chunk, encoding, next) => { + // this.logger.debug("WRITE channel", channel._id, chunk); + + // if (chunk === null) { + // this.logger.info("NULL ON CHANNEL"); + + // channel.end(); + // return false; + // } + + // return encoder.write(chunk, encoding, next); + // }, + // read: (_size) => { + // this.logger.debug("READ channel", channel._id); + // }, + // allowHalfOpen: true + // }), + // { + // _id: port, + // encoder, + // closedByFIN: false, + // sendACK: (sequenceNumber: number) => { + // channel.encoder.push( + // channel.encoder.createFrame(undefined, { + // flagsArray: ["ACK"], + // acknowledgeNumber: sequenceNumber, + // destinationPort: port + // }) + // ); + // }, + // handlerFIN: () => { + // channel.closedByFIN = true; + + // if (!channel.writableEnded) { + // channel.push(null); + // } + + // if (channel.writableEnded && channel.readableEnded) { + // channel.destroy(); + // this.logger.info("Channel destroy"); + // } + // } + // } + // ); + + channel.encoder.out .pipe(this.carrierSocket, { end: false }); channel @@ -65,31 +87,17 @@ export class TeceMux extends TypedEmitter { this.logger.error("CHANNEL ERROR", error); this.emit("error", { error, source: channel }); }) - // .on("destroy", () => { - // this.logger.trace("channel on DESTROY ", channel._id); - // }) - // .on("abort", () => { - // this.logger.trace("channel on ABORT ", channel._id); - // }) .on("close", () => { this.logger.info("CHANNEL close", channel._id); this.sendFIN(channel._id); }) - // .on("end", () => { - // this.logger.info("CHANNEL end", channel._id); - // }) .on("finish", () => { this.logger.info("CHANNEL finish", channel._id); this.sendFIN(channel._id); }); - // .on("data", (d) => { - // if (d === null) { - // this.logger.info("CHANNEL end", channel._id); - // } - // }); - if (emit) { - await encoder.establishChannel(channel._id); + if (establish) { + await channel.encoder.establishChannel(channel._id); } return channel; @@ -126,11 +134,6 @@ export class TeceMux extends TypedEmitter { this.carrierSocket.pipe(this.carrierDecoder, { end: false }); - this.commonEncoder.out.pipe(this.carrierSocket, { end: false }); - - this.commonEncoder.logger.updateBaseLog({ id }); - this.commonEncoder.logger.pipe(this.logger); - this.framesKeeper.logger.pipe(this.logger); this.main().catch((error) => { @@ -139,87 +142,83 @@ export class TeceMux extends TypedEmitter { } async main() { - let t = 0; - for await (const chunk of this.carrierDecoder) { - let frame: FrameData; + if (await this.handleDecodedFrame(chunk)) break; + } + } - try { - frame = JSON.parse(chunk); - } catch (err) { - this.logger.error("error Parsing data from decoder", err, chunk, chunk.length, chunk.toString()); - continue; - } + async handleDecodedFrame(chunk: any) { + let frame: FrameData; - const { flags, sequenceNumber, dataLength, destinationPort, acknowledgeNumber, error } = frame; + try { + frame = JSON.parse(chunk); + } catch (err) { + this.logger.error("error Parsing data from decoder", err, chunk, chunk.length, chunk.toString()); + this.emit("error", { chunk }); + return 1; + } - if (error) { - this.emit("error", frame); - break; - } + const { flags, sequenceNumber, dataLength, destinationPort, acknowledgeNumber, error } = frame; - let channel = this.channels.get(destinationPort); + if (error) { + this.emit("error", { frame, chunk }); + return 2; + } - if (flags.ACK) { - this.logger.trace("Received ACK flag for sequenceNumber", acknowledgeNumber); - this.framesKeeper.handleACK(acknowledgeNumber); - continue; - } + if (flags.ACK) { + this.logger.trace("Received ACK flag for sequenceNumber", acknowledgeNumber); + this.framesKeeper.handleACK(acknowledgeNumber); - if (flags.FIN) { - this.logger.trace(`Received FIN flag [C: ${destinationPort}]`, dataLength, frame.chunk, !!channel, channel?._id); - - if (channel) { - channel.closedByFIN = true; - if (!channel.writableEnded) { - channel.push(null); - } - - if (channel.writableEnded && channel.readableEnded) { - channel.destroy(); - this.logger.info("Channel destroy"); - } - } else { - this.logger.error("FIN for unknown channel"); - } - - this.sendACK(sequenceNumber, destinationPort); - continue; - } + return 0; + } - if (flags.PSH) { - this.logger.trace(`Received PSH command [C: ${destinationPort}, SIZE: ${dataLength}]`); + let channel = this.channels.get(destinationPort); - if (!channel) { - this.logger.warn("Unknown channel"); - channel = await this.createChannel(destinationPort, false); + if (flags.FIN) { + this.logger.trace(`Received FIN flag [C: ${destinationPort}]`, dataLength, frame.chunk, !!channel, channel?._id); - this.addChannel(channel, true); - //this.emit("peer", { channelId: destinationPort}) - } + if (channel) { + channel.handlerFIN(); + channel.sendACK(sequenceNumber); + } else { + this.logger.error("FIN for unknown channel"); + } - if (dataLength) { - this.logger.warn("writing to channel [channel, length]", channel._id, dataLength); - this.logger.warn("writing to channel [flowing, isPaused]", channel.readableFlowing, channel.isPaused()); + return 0; + } - channel.push(new Uint8Array((frame.chunk as any).data), undefined); + if (flags.PSH) { + this.logger.trace(`Received PSH command [C: ${destinationPort}, SIZE: ${dataLength}]`); - t += (frame.chunk as any).data.length; - this.logger.info("Writen", t); - } + if (!channel) { + this.logger.warn("Unknown channel"); + channel = await this.createChannel(destinationPort, false); - this.sendACK(sequenceNumber, destinationPort); + this.addChannel(channel, true); } + + if (dataLength) { + this.logger.warn("writing to channel [channel, length]", channel._id, dataLength); + this.logger.warn("writing to channel [flowing, isPaused]", channel.readableFlowing, channel.isPaused()); + + channel.push(new Uint8Array((frame.chunk as any).data), undefined); + } + + channel.sendACK(sequenceNumber); } + + return 0; } - sendACK(sequenceNumber: number, channel: number) { + sendACK(sequenceNumber: number, channelId: number) { this.logger.debug("Write acknowledge frame for sequenceNumber", sequenceNumber); - this.channels.get(channel)?.encoder?.push( - this.commonEncoder.createFrame(undefined, { + const channel = this.channels.get(channelId); + + channel?.encoder.push( + channel.encoder.createFrame(undefined, { flagsArray: ["ACK"], acknowledgeNumber: sequenceNumber, - destinationPort: channel + destinationPort: channelId }) ); } @@ -236,13 +235,15 @@ export class TeceMux extends TypedEmitter { this.channelCount++; } - sendFIN(channel: number) { - this.logger.debug("Write FIN frame for channel", channel); + sendFIN(channelId: number) { + this.logger.debug("Write FIN frame for channel", channelId); + + const channel = this.channels.get(channelId)!; - this.commonEncoder.push( - this.commonEncoder.createFrame(undefined, { + channel.encoder.push( + channel.encoder.createFrame(undefined, { flagsArray: ["FIN"], - destinationPort: channel + destinationPort: channelId }) ); } diff --git a/packages/verser/src/lib/tecemux/types.ts b/packages/verser/src/lib/tecemux/types.ts index f1860077e..43321ec75 100644 --- a/packages/verser/src/lib/tecemux/types.ts +++ b/packages/verser/src/lib/tecemux/types.ts @@ -1,4 +1,4 @@ -import { Duplex, Transform } from "stream"; +import { Duplex, Readable, Transform } from "stream"; export type TeceMuxEvents = { channel(socket: Duplex): void; @@ -49,12 +49,16 @@ export interface ITeCeMux { export interface IFrameEncoder extends Transform { establishChannel(id: number): Promise; + createFrame(chunk: any, frame: Partial): Buffer; + out: Readable; } export type TeceMuxChannel = Duplex & { _id: number; encoder: IFrameEncoder; closedByFIN: boolean; + sendACK(sequenceNumber: number): void; + handlerFIN(): void; }; export type FrameData = { From ee0fe1e0f8101f9a68d32e5670259cb2abcf45c8 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 13 Apr 2023 20:56:19 +0000 Subject: [PATCH 102/231] Disable nagle in R-H --- packages/host/src/lib/socket-server.ts | 12 ++++++------ packages/runner/src/host-client.ts | 5 +++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/host/src/lib/socket-server.ts b/packages/host/src/lib/socket-server.ts index 1684d9b13..5415bf024 100644 --- a/packages/host/src/lib/socket-server.ts +++ b/packages/host/src/lib/socket-server.ts @@ -44,13 +44,13 @@ export class SocketServer extends TypedEmitter implements IComponent { let protocol: TeceMux; - this.server.on("connection", async (connection: net.Socket) => { - connection.setNoDelay(true); - connection.on("error", (err) => { + this.server.on("connection", async (runnerConnection: net.Socket) => { + runnerConnection.setNoDelay(true); + runnerConnection.on("error", (err) => { this.logger.error("Error on connection from runner", err); }); - protocol = new TeceMux(connection); + protocol = new TeceMux(runnerConnection); protocol.on("channel", async (channel: TeceMuxChannel) => { const { instanceId, channelId } = @@ -72,9 +72,9 @@ export class SocketServer extends TypedEmitter implements IComponent { .on("end", () => this.logger.debug(`Channel [${instanceId}:${channelId}] ended`)); try { - await this.handleConnection(instanceId, channelId, connection as Socket); + await this.handleConnection(instanceId, channelId, channel as unknown as Socket); } catch (err: any) { - connection.destroy(); + channel.destroy(); } }); }); diff --git a/packages/runner/src/host-client.ts b/packages/runner/src/host-client.ts index 8c938c1a1..ff434831e 100644 --- a/packages/runner/src/host-client.ts +++ b/packages/runner/src/host-client.ts @@ -41,10 +41,11 @@ class HostClient implements IHostClient { } async init(id: string): Promise { - const tunnel = net.createConnection(this.instancesServerPort, this.instancesServerHost); - const protocol = new TeceMux(tunnel); + const hostSocket = net.createConnection(this.instancesServerPort, this.instancesServerHost); + const protocol = new TeceMux(hostSocket); //protocol.logger.pipe(this.logger); + hostSocket.setNoDelay(true); const openConnections = await Promise.all( Array.from(Array(9)) From 162d4128a95f9cceffe8df81bfb01b0f70a51021 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 19 Apr 2023 21:39:58 +0000 Subject: [PATCH 103/231] Simplify TecemuxChannel --- bdd/package.json | 10 +-- .../verser/src/lib/tecemux/tecemux-channel.ts | 50 +++++------ packages/verser/src/lib/tecemux/tecemux.ts | 84 ++----------------- packages/verser/src/lib/verser-connection.ts | 5 +- 4 files changed, 41 insertions(+), 108 deletions(-) diff --git a/bdd/package.json b/bdd/package.json index 308ce85ef..cbcb40455 100644 --- a/bdd/package.json +++ b/bdd/package.json @@ -4,10 +4,10 @@ "description": "As the \"problem scope\" of the business problem that our technology solves is quite complex, we decided to use the BDD practice to support the development process. BDD is a methodology of high automation and agility. It describes a cycle of interactions with well-defined outcomes. As a result of these activities, we obtain working, tested software that has a real value.", "main": "_cucumber.js", "dependencies": { - "@scramjet/api-client": "^0.33.3", - "@scramjet/logger": "^0.33.3", - "@scramjet/obj-logger": "^0.33.3", - "@scramjet/sth-config": "^0.33.3", + "@scramjet/api-client": "^0.33.4", + "@scramjet/logger": "^0.33.4", + "@scramjet/obj-logger": "^0.33.4", + "@scramjet/sth-config": "^0.33.4", "dockerode": "^3.3.4", "find-package-json": "^1.2.0", "freeport": "^1.0.5", @@ -17,7 +17,7 @@ "devDependencies": { "@cucumber/cucumber": "^7.3.2", "@cucumber/pretty-formatter": "^1.0.0", - "@scramjet/types": "^0.33.3" + "@scramjet/types": "^0.33.4" }, "scripts": { "build:bdd": "tsc -p tsconfig.json", diff --git a/packages/verser/src/lib/tecemux/tecemux-channel.ts b/packages/verser/src/lib/tecemux/tecemux-channel.ts index 36460e5b1..83144a074 100644 --- a/packages/verser/src/lib/tecemux/tecemux-channel.ts +++ b/packages/verser/src/lib/tecemux/tecemux-channel.ts @@ -1,5 +1,5 @@ import { ObjLogger } from "@scramjet/obj-logger"; -import { Duplex } from "stream"; +import { Duplex, PassThrough } from "stream"; import { IFrameEncoder, ITeCeMux } from "./types"; import { FrameEncoder } from "./codecs"; @@ -10,30 +10,8 @@ export class TeceMuxChannel extends Duplex { encoder: IFrameEncoder; closedByFIN = false; - constructor(duplexOptions: ConstructorParameters[0], id: number, teceMux: ITeCeMux) { - super(duplexOptions); - this.logger = new ObjLogger(this); - this.allowHalfOpen = true; - this._id = id; - this.encoder = new FrameEncoder(id, teceMux, { encoding: undefined }); - } - - _write(chunk: any, encoding: BufferEncoding, next: (error?: Error | null | undefined) => void) { - this.logger.debug("WRITE channel", this._id, chunk); - - if (chunk === null) { - this.logger.info("NULL ON CHANNEL"); - - this.end(); - return false; - } - - return this.encoder.write(chunk, encoding, next); - } - - _read(_size?: number) { - this.logger.debug("READ channel", this._id); - } + private __writable = new PassThrough(); + public __readable = new PassThrough(); sendACK(sequenceNumber: number) { this.encoder.push( @@ -57,4 +35,26 @@ export class TeceMuxChannel extends Duplex { this.logger.info("Channel destroy"); } } + + constructor(duplexOptions: ConstructorParameters[0], id: number, teceMux: ITeCeMux) { + super(duplexOptions); + + this.logger = new ObjLogger(this); + this.allowHalfOpen = true; + this._id = id; + this.encoder = new FrameEncoder(id, teceMux, { encoding: undefined }); + + this.__writable.pipe(this.encoder); + + return Object.assign( + Duplex.from({ + readable: this.__readable, + writable: this.__writable + } as unknown as AsyncIterable, duplexOptions), { + ...this, + sendACK: this.sendACK, + handlerFIN: this.handlerFIN + } + ); + } } diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 9cedacf8f..25a606754 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -21,63 +21,13 @@ export class TeceMux extends TypedEmitter { logger: ObjLogger; - private async createChannel(destinationPort?: number, establish?: boolean): Promise { + private createChannel(destinationPort?: number): TeceMuxChannel { const port = destinationPort !== undefined ? destinationPort : this.channelCount; this.logger.debug("Create Channel", port); - //const encoder = new FrameEncoder(port, this, { encoding: undefined }); - - //encoder.logger.updateBaseLog({ id: this.id }); - //encoder.logger.pipe(this.logger); - const channel = new TeceMuxChannel({ allowHalfOpen: true }, port, this); - // const channel: TeceMuxChannel = Object.assign( - // new Duplex({ - // write: (chunk, encoding, next) => { - // this.logger.debug("WRITE channel", channel._id, chunk); - - // if (chunk === null) { - // this.logger.info("NULL ON CHANNEL"); - - // channel.end(); - // return false; - // } - - // return encoder.write(chunk, encoding, next); - // }, - // read: (_size) => { - // this.logger.debug("READ channel", channel._id); - // }, - // allowHalfOpen: true - // }), - // { - // _id: port, - // encoder, - // closedByFIN: false, - // sendACK: (sequenceNumber: number) => { - // channel.encoder.push( - // channel.encoder.createFrame(undefined, { - // flagsArray: ["ACK"], - // acknowledgeNumber: sequenceNumber, - // destinationPort: port - // }) - // ); - // }, - // handlerFIN: () => { - // channel.closedByFIN = true; - - // if (!channel.writableEnded) { - // channel.push(null); - // } - - // if (channel.writableEnded && channel.readableEnded) { - // channel.destroy(); - // this.logger.info("Channel destroy"); - // } - // } - // } - // ); + channel.encoder.out .pipe(this.carrierSocket, { end: false }); @@ -96,10 +46,6 @@ export class TeceMux extends TypedEmitter { this.sendFIN(channel._id); }); - if (establish) { - await channel.encoder.establishChannel(channel._id); - } - return channel; } @@ -113,21 +59,6 @@ export class TeceMux extends TypedEmitter { .on("error", (error) => { this.logger.error("Decoder error", error); }); - // .on("pause", () => { - // this.logger.warn("Decoder paused"); - // }) - // .on("close", () => { - // this.logger.warn("Decoder closed"); - // }) - // .on("end", () => { - // this.logger.warn("Decoder ended"); - // }) - // .on("abort", (error) => { - // this.logger.error("Decoder abort", error); - // }) - // .on("destroy", (error) => { - // this.logger.error("Decoder destroy", error); - // }); this.carrierDecoder.logger.updateBaseLog({ id: this.id }); this.carrierDecoder.logger.pipe(this.logger); @@ -192,7 +123,7 @@ export class TeceMux extends TypedEmitter { if (!channel) { this.logger.warn("Unknown channel"); - channel = await this.createChannel(destinationPort, false); + channel = this.createChannel(destinationPort); this.addChannel(channel, true); } @@ -201,7 +132,7 @@ export class TeceMux extends TypedEmitter { this.logger.warn("writing to channel [channel, length]", channel._id, dataLength); this.logger.warn("writing to channel [flowing, isPaused]", channel.readableFlowing, channel.isPaused()); - channel.push(new Uint8Array((frame.chunk as any).data), undefined); + channel.__readable.push(new Uint8Array((frame.chunk as any).data), undefined); } channel.sendACK(sequenceNumber); @@ -248,15 +179,14 @@ export class TeceMux extends TypedEmitter { ); } - async multiplex(opts: { channel?: number } = {}): Promise { + multiplex(opts: { channel?: number } = {}): TeceMuxChannel { const id = opts.channel !== undefined ? opts.channel : this.channelCount; - const channel = await this.createChannel(id); + const channel = this.createChannel(id); this.logger.trace("Multiplex", id); - await channel.encoder.establishChannel(id); - this.addChannel(channel, true); + channel.encoder.establishChannel(id); this.logger.trace("Multiplex ready", channel._id); return channel; diff --git a/packages/verser/src/lib/verser-connection.ts b/packages/verser/src/lib/verser-connection.ts index 79c1bd826..1518f820e 100644 --- a/packages/verser/src/lib/verser-connection.ts +++ b/packages/verser/src/lib/verser-connection.ts @@ -194,7 +194,7 @@ export class VerserConnection { async createChannel(id: number): Promise { if (!this.teceMux) throw new Error("TeCeMux not connected"); - return await this.teceMux.multiplex({ channel: id }); + return this.teceMux.multiplex({ channel: id }); } reconnect() { @@ -205,6 +205,7 @@ export class VerserConnection { }); this.agent = new Agent() as Agent & { createConnection: typeof createConnection }; // lack of types? + this.agent.createConnection = () => { try { const socket = this.teceMux!.multiplex() as unknown as Socket; @@ -219,8 +220,10 @@ export class VerserConnection { socket.setTimeout ||= (_timeout: number, _callback?: () => void) => socket; this.logger.debug("Created new muxed stream"); + return socket; } catch (e) { + this.logger.error("Create connection error", e); const ret = new Socket(); setImmediate(() => ret.emit("error", e)); From 337389b074342870bdbc6ea3798a7b4a63e209f4 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 20 Apr 2023 09:04:04 +0000 Subject: [PATCH 104/231] Fix lint --- packages/verser/src/lib/tecemux/tecemux.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 25a606754..663657d4f 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -28,7 +28,6 @@ export class TeceMux extends TypedEmitter { const channel = new TeceMuxChannel({ allowHalfOpen: true }, port, this); - channel.encoder.out .pipe(this.carrierSocket, { end: false }); @@ -186,7 +185,12 @@ export class TeceMux extends TypedEmitter { this.logger.trace("Multiplex", id); this.addChannel(channel, true); - channel.encoder.establishChannel(id); + + channel.encoder.establishChannel(id).then(() => { + this.logger.debug("Channel established", id); + }, (err) => { + this.logger.error("Channel establish error", err); + }); this.logger.trace("Multiplex ready", channel._id); return channel; From 82a770f1e10aafecc582b73e2fbe3dfd90412fdd Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 20 Apr 2023 10:36:24 +0000 Subject: [PATCH 105/231] Restore TCM channel implementation for older nodejs --- .../verser/src/lib/tecemux/tecemux-channel.ts | 50 +++++++++---------- packages/verser/src/lib/tecemux/tecemux.ts | 2 +- .../test/playgrounds/playground-tecemux.ts | 16 +----- .../verser/test/playgrounds/playground.ts | 15 +----- 4 files changed, 30 insertions(+), 53 deletions(-) diff --git a/packages/verser/src/lib/tecemux/tecemux-channel.ts b/packages/verser/src/lib/tecemux/tecemux-channel.ts index 83144a074..36460e5b1 100644 --- a/packages/verser/src/lib/tecemux/tecemux-channel.ts +++ b/packages/verser/src/lib/tecemux/tecemux-channel.ts @@ -1,5 +1,5 @@ import { ObjLogger } from "@scramjet/obj-logger"; -import { Duplex, PassThrough } from "stream"; +import { Duplex } from "stream"; import { IFrameEncoder, ITeCeMux } from "./types"; import { FrameEncoder } from "./codecs"; @@ -10,8 +10,30 @@ export class TeceMuxChannel extends Duplex { encoder: IFrameEncoder; closedByFIN = false; - private __writable = new PassThrough(); - public __readable = new PassThrough(); + constructor(duplexOptions: ConstructorParameters[0], id: number, teceMux: ITeCeMux) { + super(duplexOptions); + this.logger = new ObjLogger(this); + this.allowHalfOpen = true; + this._id = id; + this.encoder = new FrameEncoder(id, teceMux, { encoding: undefined }); + } + + _write(chunk: any, encoding: BufferEncoding, next: (error?: Error | null | undefined) => void) { + this.logger.debug("WRITE channel", this._id, chunk); + + if (chunk === null) { + this.logger.info("NULL ON CHANNEL"); + + this.end(); + return false; + } + + return this.encoder.write(chunk, encoding, next); + } + + _read(_size?: number) { + this.logger.debug("READ channel", this._id); + } sendACK(sequenceNumber: number) { this.encoder.push( @@ -35,26 +57,4 @@ export class TeceMuxChannel extends Duplex { this.logger.info("Channel destroy"); } } - - constructor(duplexOptions: ConstructorParameters[0], id: number, teceMux: ITeCeMux) { - super(duplexOptions); - - this.logger = new ObjLogger(this); - this.allowHalfOpen = true; - this._id = id; - this.encoder = new FrameEncoder(id, teceMux, { encoding: undefined }); - - this.__writable.pipe(this.encoder); - - return Object.assign( - Duplex.from({ - readable: this.__readable, - writable: this.__writable - } as unknown as AsyncIterable, duplexOptions), { - ...this, - sendACK: this.sendACK, - handlerFIN: this.handlerFIN - } - ); - } } diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 663657d4f..2f7bd0e59 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -131,7 +131,7 @@ export class TeceMux extends TypedEmitter { this.logger.warn("writing to channel [channel, length]", channel._id, dataLength); this.logger.warn("writing to channel [flowing, isPaused]", channel.readableFlowing, channel.isPaused()); - channel.__readable.push(new Uint8Array((frame.chunk as any).data), undefined); + channel.push(new Uint8Array((frame.chunk as any).data), undefined); } channel.sendACK(sequenceNumber); diff --git a/packages/verser/test/playgrounds/playground-tecemux.ts b/packages/verser/test/playgrounds/playground-tecemux.ts index fa84cf427..5c028b26a 100644 --- a/packages/verser/test/playgrounds/playground-tecemux.ts +++ b/packages/verser/test/playgrounds/playground-tecemux.ts @@ -2,9 +2,8 @@ /* eslint-disable @typescript-eslint/no-floating-promises */ /* eslint-disable no-console */ -import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; +import { ObjLogger } from "@scramjet/obj-logger"; import { IncomingMessage, createServer } from "http"; -import { DataStream } from "scramjet"; import { Socket, createConnection } from "net"; import { TeceMux } from "../../src/lib/tecemux/tecemux"; @@ -15,18 +14,7 @@ import path from "path"; (async () => { const logger = new ObjLogger("Sandbox"); - logger - .pipe( - new DataStream() - .map(prettyPrint({ colors: true })) - .map((chunk: string) => - chunk.replace( - /(:?FIN|SYN|RST|PSH|ACK|URG|ECE|CWR)|^$]/, - "\x1b[41m\$&\x1b[0m" - ) - ) - ) - .pipe(process.stdout); + logger.pipe(process.stdout); /**********************************************/ /* SERVER diff --git a/packages/verser/test/playgrounds/playground.ts b/packages/verser/test/playgrounds/playground.ts index d52e5f452..897abfa43 100644 --- a/packages/verser/test/playgrounds/playground.ts +++ b/packages/verser/test/playgrounds/playground.ts @@ -1,6 +1,5 @@ -import { DataStream } from "scramjet"; import { FrameDecoder, FrameEncoder } from "../../src/lib/tecemux/codecs"; -import { ObjLogger, prettyPrint } from "@scramjet/obj-logger"; +import { ObjLogger } from "@scramjet/obj-logger"; import { createReadStream, createWriteStream } from "fs"; import path from "path"; import { PassThrough, Transform } from "stream"; @@ -15,17 +14,7 @@ const tcm = { const logger = new ObjLogger("Sandbox"); -logger.pipe( - new DataStream() - .map(prettyPrint({ colors: true })) - .map((chunk: string) => - chunk.replace( - /(:?FIN|SYN|RST|PSH|ACK|URG|ECE|CWR)|^$]/, - "\x1b[41m\$&\x1b[0m" - ) - ) -) - .pipe(process.stdout); +logger.pipe(process.stdout); const encoder = new FrameEncoder(0, tcm); const decoder = new FrameDecoder(); From f666dfcdbf8398f4c75d8602c5034d8ee845e73c Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 26 Apr 2023 20:18:53 +0000 Subject: [PATCH 106/231] Fix M-S connection --- bdd/package.json | 10 +++++----- packages/verser/src/lib/verser-connection.ts | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bdd/package.json b/bdd/package.json index cbcb40455..44956bfda 100644 --- a/bdd/package.json +++ b/bdd/package.json @@ -4,10 +4,10 @@ "description": "As the \"problem scope\" of the business problem that our technology solves is quite complex, we decided to use the BDD practice to support the development process. BDD is a methodology of high automation and agility. It describes a cycle of interactions with well-defined outcomes. As a result of these activities, we obtain working, tested software that has a real value.", "main": "_cucumber.js", "dependencies": { - "@scramjet/api-client": "^0.33.4", - "@scramjet/logger": "^0.33.4", - "@scramjet/obj-logger": "^0.33.4", - "@scramjet/sth-config": "^0.33.4", + "@scramjet/api-client": "^0.33.5", + "@scramjet/logger": "^0.33.5", + "@scramjet/obj-logger": "^0.33.5", + "@scramjet/sth-config": "^0.33.5", "dockerode": "^3.3.4", "find-package-json": "^1.2.0", "freeport": "^1.0.5", @@ -17,7 +17,7 @@ "devDependencies": { "@cucumber/cucumber": "^7.3.2", "@cucumber/pretty-formatter": "^1.0.0", - "@scramjet/types": "^0.33.4" + "@scramjet/types": "^0.33.5" }, "scripts": { "build:bdd": "tsc -p tsconfig.json", diff --git a/packages/verser/src/lib/verser-connection.ts b/packages/verser/src/lib/verser-connection.ts index 1518f820e..f93f8c783 100644 --- a/packages/verser/src/lib/verser-connection.ts +++ b/packages/verser/src/lib/verser-connection.ts @@ -191,7 +191,7 @@ export class VerserConnection { * @param id {number} Channel id. * @returns Duplex stream. */ - async createChannel(id: number): Promise { + createChannel(id: number) { if (!this.teceMux) throw new Error("TeCeMux not connected"); return this.teceMux.multiplex({ channel: id }); From 2de58a53ff8a7c5d1b407801a19eaff23e0f09a4 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 22 May 2023 12:24:28 +0000 Subject: [PATCH 107/231] Rebase --- packages/host/src/lib/cpm-connector.ts | 4 ++-- packages/runner/src/host-client.ts | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/host/src/lib/cpm-connector.ts b/packages/host/src/lib/cpm-connector.ts index 5c0b3a889..a1ab0511a 100644 --- a/packages/host/src/lib/cpm-connector.ts +++ b/packages/host/src/lib/cpm-connector.ts @@ -343,9 +343,9 @@ export class CPMConnector extends TypedEmitter { connection.socket .once("close", async () => { - this.logger.warn("CLOSE STATUS", connection.res.statusCode); + this.logger.warn("CLOSE STATUS", connection.response.statusCode); - await this.handleConnectionClose(connection.res.statusCode || -1); + await this.handleConnectionClose(connection.response.statusCode || -1); }); } catch (error: any) { this.logger.error("Can not connect to Manager", this.cpmUrl, this.cpmId, error.message); diff --git a/packages/runner/src/host-client.ts b/packages/runner/src/host-client.ts index ff434831e..c98d26ebd 100644 --- a/packages/runner/src/host-client.ts +++ b/packages/runner/src/host-client.ts @@ -48,8 +48,7 @@ class HostClient implements IHostClient { hostSocket.setNoDelay(true); const openConnections = await Promise.all( - Array.from(Array(9)) - .map((_c, index) => protocol.multiplex({ channel: index })) + Array.from(Array(9)).map((_c, index) => protocol.multiplex({ channel: index })) ).then(async res => { return Promise.all( res.map(async (channel, index) => { From 6e06ff07e31ae948fdb404bc866dfb91058cf5e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Mon, 22 May 2023 17:23:20 +0200 Subject: [PATCH 108/231] Problem with wrong data cut resolved --- packages/python-runner/inet.py | 13 ++++++++++--- packages/python-runner/test/test_inet.py | 12 +++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/packages/python-runner/inet.py b/packages/python-runner/inet.py index 783922933..d86e5238f 100644 --- a/packages/python-runner/inet.py +++ b/packages/python-runner/inet.py @@ -127,7 +127,7 @@ def parse_flags(value): ihl: int = 5 version: int = field( default=4, converter = lambda value: value >> 4) tos: int = field(default=0) - len: int = field(default=None) + len: int = field(default=0) ids: int = field(default=0) flags_offset: int = field(default = 0) flags: int = field(default = 0, init= False, repr = lambda value: IPPacket.Flags.flags_to_str(value), \ @@ -145,7 +145,8 @@ def __attrs_post_init__(self): self.flags = self.flags_offset >> 13 #Cut data buffer to IP packet length - self.get_segment().data=self.get_segment().data[:self.len-(self.ihl*4)-20] + if self.len > 0 and self.segment: + self.get_segment().data=self.get_segment().data[:self.len-(self.ihl*4)-20] @staticmethod def calc_checksum(pkt: bytes) -> int: @@ -161,7 +162,13 @@ def calc_checksum(pkt: bytes) -> int: @classmethod def from_buffer(cls,buffer): ihl = (buffer[0] & 0xf) - return cls(ihl, *struct.unpack("!BBHHHBBH4s4s", buffer[0:ihl*4]),TCPSegment.from_buffer(buffer[ihl*4:]) if len(buffer) > ihl*4 else None) + pkt = cls(ihl, *struct.unpack("!BBHHHBBH4s4s", buffer[0:ihl*4]),TCPSegment.from_buffer(buffer[ihl*4:]) if len(buffer) > ihl*4 else None) + + #Cut data buffer to IP packet length + if pkt.segment: + pkt.get_segment().data=pkt.get_segment().data[:pkt.len-(ihl*4)-20] + + return pkt def is_flag(self,flag): return (self.flags & getattr(IPPacket.Flags, flag)) > 0 diff --git a/packages/python-runner/test/test_inet.py b/packages/python-runner/test/test_inet.py index de17ddfb8..147b67d14 100644 --- a/packages/python-runner/test/test_inet.py +++ b/packages/python-runner/test/test_inet.py @@ -1,14 +1,12 @@ import pytest from inet import TCPSegment, IPPacket, EthernetFrame - class TestIP: def test_mf_df_flags(self): data = b'E\x00\x00\x14\x00\x01`\x00@\x00\x1c\xe7\x7f\x00\x00\x01\x7f\x00\x00\x01' pkt = IPPacket.from_buffer(data) - assert pkt.flags == (IPPacket.Flags.MF | - IPPacket.Flags.DF) & ~IPPacket.Flags.RF + assert pkt.flags == (IPPacket.Flags.MF | IPPacket.Flags.DF) & ~IPPacket.Flags.RF assert pkt.is_flag('MF') == True assert pkt.is_flag('DF') == True assert pkt.is_flag('RF') == False @@ -32,8 +30,7 @@ def test_df_flags(self): pkt = EthernetFrame.from_buffer(data).get_packet() assert pkt.flags == IPPacket.Flags.DF - assert pkt.flags == (IPPacket.Flags.DF & ( - ~IPPacket.Flags.MF & ~IPPacket.Flags.RF)) + assert pkt.flags == (IPPacket.Flags.DF & (~IPPacket.Flags.MF & ~IPPacket.Flags.RF)) assert pkt.is_flag('DF') == True assert pkt.is_flag('MF') == False @@ -62,9 +59,6 @@ def test_packet_creation(self): assert pkt.src_addr == '172.25.44.3' assert pkt.dst_addr == '172.25.44.254' - # data = pkt.build().to_buffer() - - class TestTCP: def test_tcp_offset(self): data = b'\x01\xbb\xc0\xd7\xb6\x56\xa8\xb9\xd1\xac\xaa\xb1\x50\x18\x40\x00\x56\xf8\x00\x00' @@ -144,7 +138,7 @@ def test_parse_only_first_packet(self): b"\x00{'bar':'foo2'}") pkt = IPPacket.from_buffer(data) - + assert pkt.len == 53 assert pkt.get_segment().data == b"{'foo':'bar'}" From e061c0e6e06598c9985aae44dcb664b1a31bc8f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Mon, 22 May 2023 17:45:13 +0200 Subject: [PATCH 109/231] TecemuxStreamReader and TecemuxStreamWriter wrappers for asyncio.Stream --- packages/python-runner/tecemux.py | 41 +++++++++++++++++++-- packages/python-runner/test/test_tecemux.py | 6 +-- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 9478d7fdd..76acbf891 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -5,14 +5,49 @@ from inet import IPPacket, TCPSegment from hardcoded_magic_values import CommunicationChannels as CC + +class TecemuxStreamReader: + + def __init__(self, stream): + self._stream = stream + + def __getattr__(self, name): + return getattr(self._stream, name) + + async def readline(self): + return await self._stream.readline() + + async def readuntil(self, separator=b'\n'): + return await self._stream.readuntil(separator) + + async def read(self, n=-1): + return await self._stream.read(n) + + async def readexactly(self, n): + return await self._stream.readexactly(n) + +class TecemuxStreamWriter: + + def __init__(self, stream): + self._stream = stream + + def __getattr__(self, name): + return getattr(self._stream, name) + + async def wait_closed(self): + return await self._stream.wait_closed() + + async def drain(self): + return await self._stream.drain() + @define class Tecemux: @define class ChannelContext: _name: str - _reader: asyncio.StreamReader - _writer: asyncio.StreamWriter + _reader: TecemuxStreamReader + _writer: TecemuxStreamWriter _queue: asyncio.Queue _global_queue: asyncio.Queue @@ -85,7 +120,7 @@ async def prepare_socket_connection(): rsock, wsock = socket.socketpair() reader, _ = await asyncio.open_unix_connection(sock=rsock) _, writer = await asyncio.open_unix_connection(sock=wsock) - return reader,writer + return TecemuxStreamReader(reader),TecemuxStreamWriter(writer) async def prepare(self): diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index 69f376960..fe2ce819a 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -1,6 +1,6 @@ import asyncio import pytest -from tecemux import Tecemux +from tecemux import Tecemux, TecemuxStreamReader,TecemuxStreamWriter from inet import IPPacket,TCPSegment from hardcoded_magic_values import CommunicationChannels as CC from runner_tecemux import get_logger @@ -45,8 +45,8 @@ async def test_socket_connection(self): await protocol.connect(*await Tecemux.prepare_socket_connection()) - assert isinstance(protocol._reader,asyncio.StreamReader) - assert isinstance(protocol._writer,asyncio.StreamWriter) + assert isinstance(protocol._reader,TecemuxStreamReader) + assert isinstance(protocol._writer,TecemuxStreamWriter) @pytest.mark.asyncio From f1082acf7f5c669fec95c2d53079b889b1443ea7 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 25 May 2023 13:54:34 +0000 Subject: [PATCH 110/231] Pause resume tecemux channel --- .../verser/src/lib/tecemux/frames-keeper.ts | 2 +- .../verser/src/lib/tecemux/tecemux-channel.ts | 11 ++++++ packages/verser/src/lib/tecemux/tecemux.ts | 35 +++++++++++++++++-- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/packages/verser/src/lib/tecemux/frames-keeper.ts b/packages/verser/src/lib/tecemux/frames-keeper.ts index cc2fbd3b9..06d385fc1 100644 --- a/packages/verser/src/lib/tecemux/frames-keeper.ts +++ b/packages/verser/src/lib/tecemux/frames-keeper.ts @@ -3,7 +3,7 @@ import { TypedEmitter } from "@scramjet/utility"; import { FramesKeeperEvents, FramesKeeperFrame, IFramesKeeper } from "./types"; export class FramesKeeper extends TypedEmitter implements IFramesKeeper { - #MAX_FRAMES_DIFFERENCE = 5; + #MAX_FRAMES_DIFFERENCE = 3; #framesSent = new Map(); diff --git a/packages/verser/src/lib/tecemux/tecemux-channel.ts b/packages/verser/src/lib/tecemux/tecemux-channel.ts index 36460e5b1..5e75f2446 100644 --- a/packages/verser/src/lib/tecemux/tecemux-channel.ts +++ b/packages/verser/src/lib/tecemux/tecemux-channel.ts @@ -45,6 +45,17 @@ export class TeceMuxChannel extends Duplex { ); } + sendPauseACK(sequenceNumber: number) { + //console.log("Cant write more. Send pause command"); + this.encoder.push( + this.encoder.createFrame(undefined, { + flagsArray: ["ACK", "SYN"], + acknowledgeNumber: sequenceNumber, + destinationPort: this._id + }) + ); + } + handlerFIN() { this.closedByFIN = true; diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 2f7bd0e59..fa609d9d8 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -95,15 +95,27 @@ export class TeceMux extends TypedEmitter { return 2; } + let channel = this.channels.get(destinationPort); + + if (flags.ACK && flags.SYN) { + channel?.encoder.pause(); + this.framesKeeper.handleACK(acknowledgeNumber); + console.log("Pause channel command received", channel?._id); + return 0; + } + if (flags.ACK) { + if (channel?.encoder.isPaused()) { + console.log("Pause and receive ack", channel._id); + } this.logger.trace("Received ACK flag for sequenceNumber", acknowledgeNumber); this.framesKeeper.handleACK(acknowledgeNumber); + channel?.encoder.resume(); + channel?.resume(); return 0; } - let channel = this.channels.get(destinationPort); - if (flags.FIN) { this.logger.trace(`Received FIN flag [C: ${destinationPort}]`, dataLength, frame.chunk, !!channel, channel?._id); @@ -131,7 +143,24 @@ export class TeceMux extends TypedEmitter { this.logger.warn("writing to channel [channel, length]", channel._id, dataLength); this.logger.warn("writing to channel [flowing, isPaused]", channel.readableFlowing, channel.isPaused()); - channel.push(new Uint8Array((frame.chunk as any).data), undefined); + const canWrite = channel.push(new Uint8Array((frame.chunk as any).data), undefined); + + if (!canWrite) { + console.log("cant write", channel._id); + channel.sendPauseACK(sequenceNumber); + + channel.once("resume", () => { + console.log("resumed, send ack to resume"); + //channel?.encoder.resume(); + channel?.sendACK(sequenceNumber); + }); + channel.once("drain", () => { + console.log("drained, send ack to resume"); + //channel?.encoder.resume(); + channel?.sendACK(sequenceNumber); + }); + return 0; + } } channel.sendACK(sequenceNumber); From 69072bfd321bed309f35ebdf9ac73ffb8706cec8 Mon Sep 17 00:00:00 2001 From: tomekcrm Date: Mon, 24 Apr 2023 14:40:41 +0000 Subject: [PATCH 111/231] Fix: version bump of deprecated github actions --- .github/workflows/analyze-code.yml | 2 +- .github/workflows/build-docker-prerunner.yml | 4 ++-- .github/workflows/build-docker-runner-node.yml | 4 ++-- .github/workflows/build-docker-runner-python.yml | 2 +- .github/workflows/build-docker-sth.yml | 4 ++-- .github/workflows/build-refapps.yml | 4 ++-- .github/workflows/build-sth.yml | 4 ++-- .github/workflows/preinstall-deps.yml | 2 +- .github/workflows/publish-release.yml | 2 +- .github/workflows/release-test.yml | 4 ++-- .github/workflows/security-check.yml | 4 ++-- .github/workflows/test-bdd-docker.yml | 12 ++++++------ .github/workflows/test-bdd-process.yml | 6 +++--- .github/workflows/test-unit.yml | 4 ++-- 14 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.github/workflows/analyze-code.yml b/.github/workflows/analyze-code.yml index 0e7a39346..924819a72 100644 --- a/.github/workflows/analyze-code.yml +++ b/.github/workflows/analyze-code.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/checkout@v3 - name: Setup Nodejs ${{ inputs.node-version }} - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: cache: 'yarn' node-version: ${{ inputs.node-version }} diff --git a/.github/workflows/build-docker-prerunner.yml b/.github/workflows/build-docker-prerunner.yml index ce5376135..60d6fb8b9 100644 --- a/.github/workflows/build-docker-prerunner.yml +++ b/.github/workflows/build-docker-prerunner.yml @@ -25,7 +25,7 @@ jobs: ref: ${{ inputs.git-revision }} - name: Setup Nodejs ${{ inputs.node-version }} - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: ${{ inputs.node-version }} @@ -46,7 +46,7 @@ jobs: run: pigz *-docker-image.tar - name: Upload image as an artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: pre-runner-image path: '*.tar.gz' diff --git a/.github/workflows/build-docker-runner-node.yml b/.github/workflows/build-docker-runner-node.yml index 62a9b2fdb..e387d3928 100644 --- a/.github/workflows/build-docker-runner-node.yml +++ b/.github/workflows/build-docker-runner-node.yml @@ -25,7 +25,7 @@ jobs: ref: ${{ inputs.git-revision }} - name: Setup Nodejs ${{ inputs.node-version }} - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: ${{ inputs.node-version }} @@ -52,7 +52,7 @@ jobs: run: pigz *-docker-image.tar - name: Upload image as an artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: runner-node-image path: '*.tar.gz' diff --git a/.github/workflows/build-docker-runner-python.yml b/.github/workflows/build-docker-runner-python.yml index c0cfa1953..bddbf699c 100644 --- a/.github/workflows/build-docker-runner-python.yml +++ b/.github/workflows/build-docker-runner-python.yml @@ -40,7 +40,7 @@ jobs: run: pigz *-docker-image.tar - name: Upload image as an artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: runner-python-image path: '*.tar.gz' diff --git a/.github/workflows/build-docker-sth.yml b/.github/workflows/build-docker-sth.yml index b62db2976..f5451556b 100644 --- a/.github/workflows/build-docker-sth.yml +++ b/.github/workflows/build-docker-sth.yml @@ -24,7 +24,7 @@ jobs: ref: ${{ inputs.git-revision }} - name: Setup Nodejs ${{ inputs.node-version }} - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: cache: 'yarn' node-version: ${{ inputs.node-version }} @@ -51,7 +51,7 @@ jobs: run: pigz *-docker-image.tar - name: Upload image as an artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: sth-image path: '*.tar.gz' diff --git a/.github/workflows/build-refapps.yml b/.github/workflows/build-refapps.yml index 82ab59fe3..eb40a319d 100644 --- a/.github/workflows/build-refapps.yml +++ b/.github/workflows/build-refapps.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v3 - name: Setup Nodejs ${{ inputs.node-version }} - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: ${{ inputs.node-version }} @@ -32,7 +32,7 @@ jobs: run: tar --use-compress-program="pigz -0 --recursive" -cvf dist-refapps-${{ github.event.pull_request.head.sha }}-${{ inputs.node-version }}.tar.gz packages/*.tar.gz - name: Archive production artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: dist-refapps-${{ github.event.pull_request.head.sha }}-${{ inputs.node-version }}.tar.gz path: dist-refapps-${{ github.event.pull_request.head.sha }}-${{ inputs.node-version }}.tar.gz diff --git a/.github/workflows/build-sth.yml b/.github/workflows/build-sth.yml index 26bb80763..5a5cda2e5 100644 --- a/.github/workflows/build-sth.yml +++ b/.github/workflows/build-sth.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v3 - name: Setup Nodejs ${{ inputs.node-version }} - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: cache: 'yarn' node-version: ${{ inputs.node-version }} @@ -39,7 +39,7 @@ jobs: run: tar --use-compress-program="pigz --best --recursive" -cf dist-sth-${{ github.event.pull_request.head.sha }}-${{ inputs.node-version }}.tar.gz dist/. - name: Archive production artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: dist-sth-${{ github.event.pull_request.head.sha }}-${{ inputs.node-version }}.tar.gz path: dist-sth-${{ github.event.pull_request.head.sha }}-${{ inputs.node-version }}.tar.gz diff --git a/.github/workflows/preinstall-deps.yml b/.github/workflows/preinstall-deps.yml index 6124de06d..351f87989 100644 --- a/.github/workflows/preinstall-deps.yml +++ b/.github/workflows/preinstall-deps.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v3 - name: Setup Nodejs ${{ inputs.node-version }} - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: ${{ inputs.node-version }} diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index 9b7708d09..4f526418d 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -118,7 +118,7 @@ jobs: ref: v${{ inputs.version }} - name: Setup Nodejs 16 - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: cache: 'yarn' node-version: 16 diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index abe67d53e..d27ccc849 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -14,10 +14,10 @@ jobs: node-version: [ 16.x ] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Setup Nodejs ${{ inputs.node-version }} - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: ${{ inputs.node-version }} diff --git a/.github/workflows/security-check.yml b/.github/workflows/security-check.yml index 94d6d7335..3482a3f53 100644 --- a/.github/workflows/security-check.yml +++ b/.github/workflows/security-check.yml @@ -16,10 +16,10 @@ jobs: node-version: [14.x, 16.x] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Setup Nodejs ${{ inputs.node-version }} - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: ${{ inputs.node-version }} diff --git a/.github/workflows/test-bdd-docker.yml b/.github/workflows/test-bdd-docker.yml index beaaf8869..d0800fa21 100644 --- a/.github/workflows/test-bdd-docker.yml +++ b/.github/workflows/test-bdd-docker.yml @@ -28,12 +28,12 @@ jobs: - uses: actions/checkout@v3 - name: Download dist-sth-${{ github.event.pull_request.head.sha }}-${{ inputs.node-version }}.tar.gz artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: dist-sth-${{ github.event.pull_request.head.sha }}-${{ inputs.node-version }}.tar.gz - name: Download dist-refapps-${{ github.event.pull_request.head.sha }}-${{ inputs.node-version }}.tar.gz artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: dist-refapps-${{ github.event.pull_request.head.sha }}-${{ inputs.node-version }}.tar.gz @@ -41,17 +41,17 @@ jobs: run: ls dist*tar.gz | xargs -n1 tar -I pigz -xf - name: Download node runner artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: runner-node-image - name: Download python runner artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: runner-python-image - name: Download pre-runner artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: pre-runner-image @@ -62,7 +62,7 @@ jobs: run: ls -1 *-docker-image.tar | while read line; do docker load -i $line; done - name: Setup Nodejs ${{ inputs.node-version }} - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: ${{ inputs.node-version }} diff --git a/.github/workflows/test-bdd-process.yml b/.github/workflows/test-bdd-process.yml index 30dc55df4..a406d41ef 100644 --- a/.github/workflows/test-bdd-process.yml +++ b/.github/workflows/test-bdd-process.yml @@ -24,12 +24,12 @@ jobs: - uses: actions/checkout@v3 - name: Download dist-sth-${{ github.event.pull_request.head.sha }}-${{ inputs.node-version }}.tar.gz artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: dist-sth-${{ github.event.pull_request.head.sha }}-${{ inputs.node-version }}.tar.gz - name: Download dist-refapps-${{ github.event.pull_request.head.sha }}-${{ inputs.node-version }}.tar.gz artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: dist-refapps-${{ github.event.pull_request.head.sha }}-${{ inputs.node-version }}.tar.gz @@ -37,7 +37,7 @@ jobs: run: ls dist*tar.gz |xargs -n1 tar -I pigz -xf - name: Setup Nodejs ${{ inputs.node-version }} - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: ${{ inputs.node-version }} diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index d76a1d01d..8ddf21184 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -18,12 +18,12 @@ jobs: - uses: actions/checkout@v3 - name: Setup Nodejs ${{ inputs.node-version }} - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: ${{ inputs.node-version }} - name: Download dist-sth-${{ github.event.pull_request.head.sha }}-${{ inputs.node-version }}.tar.gz artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: dist-sth-${{ github.event.pull_request.head.sha }}-${{ inputs.node-version }}.tar.gz From 57b6fb1dc5bcf8d5c581c6e2f2fbf8cddb8bd28e Mon Sep 17 00:00:00 2001 From: S4adam <63969229+S4adam@users.noreply.github.com> Date: Wed, 26 Apr 2023 10:37:52 +0200 Subject: [PATCH 112/231] Fix "sequence unpack failed" error with kubernetes (#852) * change stderr redirection --------- Co-authored-by: S4Adam Co-authored-by: Budleigh Salterton --- packages/python-runner/unpack.sh | 2 +- packages/runner/unpack.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/python-runner/unpack.sh b/packages/python-runner/unpack.sh index fa88ffc4e..c9ce21220 100755 --- a/packages/python-runner/unpack.sh +++ b/packages/python-runner/unpack.sh @@ -2,4 +2,4 @@ set -e -tar zxf - -C /package 2> $PACKAGE_DIR/.fail && touch /package/.ready || touch $PACKAGE_DIR/.fail +tar zxf - -C /package && touch /package/.ready || touch $PACKAGE_DIR/.fail diff --git a/packages/runner/unpack.sh b/packages/runner/unpack.sh index 52aa1aac3..8d7d0cb7f 100755 --- a/packages/runner/unpack.sh +++ b/packages/runner/unpack.sh @@ -4,4 +4,4 @@ PACKAGE_DIR=${PACKAGE_DIR:-/package} set -e -tar zxf - -C $PACKAGE_DIR 2> $PACKAGE_DIR/.fail && touch $PACKAGE_DIR/.ready || touch $PACKAGE_DIR/.fail +tar zxf - -C $PACKAGE_DIR && touch $PACKAGE_DIR/.ready || touch $PACKAGE_DIR/.fail From d28bba50f3471fd9b3999d1c6b076e4662e3b18f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Mon, 24 Apr 2023 12:25:28 +0200 Subject: [PATCH 113/231] Fix KeyError in data --- packages/python-runner/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index 8fcbdffee..73e1d65eb 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -142,7 +142,7 @@ async def connect_control_stream(self): if code == msg_codes.STOP.value: await self.handle_stop(data) if code == msg_codes.EVENT.value: - self.emitter.emit(data['eventName'], data['message']) + self.emitter.emit(data['eventName'], data['message'] if 'message' in data else None) async def handle_stop(self, data): From a06ba5354e0e4e80e9077fcc54a4aeacececbf2c Mon Sep 17 00:00:00 2001 From: Tomasz Dzioba <63119286+tomekcrm@users.noreply.github.com> Date: Fri, 12 May 2023 14:40:38 +0200 Subject: [PATCH 114/231] Fix: readme help command --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7b5805a9e..9a21d427d 100644 --- a/README.md +++ b/README.md @@ -374,19 +374,19 @@ CLI installation was already done at the end of the [Installation](#installation ```bash npm i -g ./dist/cli # install CLI globally from the source folder -si help # show CLI commands +si --help # show CLI commands ``` - from [npm](https://www.npmjs.com/package/@scramjet/cli): ```bash npm i -g @scramjet/cli # install CLI globally from npm -si help # show CLI commands +si --help # show CLI commands ``` ![cli_npm](./images/cli_npm.png) -Running `si help` command will confirm that the installation went properly and also to see the list of available commands: +Running `si --help` command will confirm that the installation went properly and also to see the list of available commands: ```bash $ si --help From 88397ecabd031f1e31048cde3fc7361136af082f Mon Sep 17 00:00:00 2001 From: Piotr Date: Fri, 19 May 2023 10:15:36 +0000 Subject: [PATCH 115/231] added description and tags to sth --- packages/host/src/lib/cpm-connector.ts | 3 ++- packages/host/src/lib/host.ts | 3 +++ packages/sth/src/bin/hub.ts | 14 ++++++++++++++ packages/types/src/cpm-connector.ts | 3 +++ packages/types/src/rest-api-manager/common.ts | 5 ++++- packages/types/src/sth-command-options.ts | 4 ++++ packages/types/src/sth-configuration.ts | 19 +++++++++++++++++++ 7 files changed, 49 insertions(+), 2 deletions(-) diff --git a/packages/host/src/lib/cpm-connector.ts b/packages/host/src/lib/cpm-connector.ts index 08cb29041..751fc062b 100644 --- a/packages/host/src/lib/cpm-connector.ts +++ b/packages/host/src/lib/cpm-connector.ts @@ -150,10 +150,11 @@ export class CPMConnector extends TypedEmitter { this.config = config; this.logger = new ObjLogger(this); - this.verserClient = new VerserClient({ verserUrl: `${this.cpmUrl}/verser`, headers: { + "x-sth-description": typeof this.config.description !== "undefined" ? this.config.description : "", + "x-sth-tags": JSON.stringify(typeof this.config.tags !== "undefined" ? this.config.tags : []), "x-manager-id": cpmId, "x-sth-id": this.config.id || "" }, diff --git a/packages/host/src/lib/host.ts b/packages/host/src/lib/host.ts index c138bcf0c..90d16ac4a 100644 --- a/packages/host/src/lib/host.ts +++ b/packages/host/src/lib/host.ts @@ -331,6 +331,9 @@ export class Host implements IComponent { this.config.cpmUrl, this.config.cpmId, { + selfHosted: this.config.selfHosted, + description: this.config.description, + tags: this.config.tags, id: this.config.host.id, infoFilePath: this.config.host.infoFilePath, cpmSslCaPath: this.config.cpmSslCaPath, diff --git a/packages/sth/src/bin/hub.ts b/packages/sth/src/bin/hub.ts index 19099e9ad..fb1ccea5d 100755 --- a/packages/sth/src/bin/hub.ts +++ b/packages/sth/src/bin/hub.ts @@ -1,4 +1,5 @@ #!/usr/bin/env ts-node +/* eslint-disable complexity */ import { Command, Option, OptionValues } from "commander"; import { ConfigService, getRuntimeAdapterOption } from "@scramjet/sth-config"; @@ -20,6 +21,10 @@ const stringToIntSanitizer = (str : string) => { const program = new Command(); const options: OptionValues & STHCommandOptions = program + .option("-desc, --description ", "Specify sth description") + .option("--custom-name ", "Specify custom name") + .option("--tags ", "Specifies tags in the format \"tag1, tag2\"", "") + .option("-sh, --self-hosted", "Specifies if the hub is self hosted", true) .option("-c, --config ", "Specifies path to config") .option("-L, --log-level ", "Specify log level") .option("--no-colors", "Disable colors in output", false) @@ -67,6 +72,11 @@ const options: OptionValues & STHCommandOptions = program (async () => { const configService = new ConfigService(); const resolveFile = (path: string) => path && resolve(process.cwd(), path); + const tags = options.tags.length ? options.tags.split(",") : []; + + if (!tags.every((t:string) => t.length)) { + throw new Error("Tags cannot be empty"); + } if (options.config) { const configFile = FileBuilder(options.config); @@ -78,6 +88,10 @@ const options: OptionValues & STHCommandOptions = program } configService.update({ + description: options.description, + customName: options.customName, + tags: tags, + selfHosted: options.selfHosted, cpmUrl: options.cpmUrl, cpmId: options.cpmId, cpmSslCaPath: options.cpmSslCaPath, diff --git a/packages/types/src/cpm-connector.ts b/packages/types/src/cpm-connector.ts index 3baa2d0f8..2edf43609 100644 --- a/packages/types/src/cpm-connector.ts +++ b/packages/types/src/cpm-connector.ts @@ -2,6 +2,9 @@ import { STHConfiguration } from "./sth-configuration"; export type CPMConnectorOptions = { id: STHConfiguration["host"]["id"]; + description: STHConfiguration["description"]; + selfHosted: STHConfiguration["selfHosted"]; + tags: STHConfiguration["tags"]; infoFilePath: STHConfiguration["host"]["infoFilePath"]; cpmSslCaPath?: STHConfiguration["cpmSslCaPath"]; maxReconnections: STHConfiguration["cpm"]["maxReconnections"]; diff --git a/packages/types/src/rest-api-manager/common.ts b/packages/types/src/rest-api-manager/common.ts index d716ade7e..1033a1a28 100644 --- a/packages/types/src/rest-api-manager/common.ts +++ b/packages/types/src/rest-api-manager/common.ts @@ -8,7 +8,10 @@ export type ConnectedSTHInfo = { id: string, info: ConnectedSTHInfoDetails, healthy: boolean, - isConnectionActive: boolean + selfHosted: boolean, + isConnectionActive: boolean, + description?: string, + tags?: Array , }; export type HealthCheckInfo = { diff --git a/packages/types/src/sth-command-options.ts b/packages/types/src/sth-command-options.ts index 0dfa92492..018bf507b 100644 --- a/packages/types/src/sth-command-options.ts +++ b/packages/types/src/sth-command-options.ts @@ -2,6 +2,10 @@ import { LogLevel } from "./object-logger"; import { TelemetryConfig } from "./telemetry-config"; export type STHCommandOptions = { + description: string; + customName: string; + tags: string; + selfHosted: boolean; logLevel: LogLevel; colors: boolean, port: number; diff --git a/packages/types/src/sth-configuration.ts b/packages/types/src/sth-configuration.ts index 5a3e78a81..9c906c4c7 100644 --- a/packages/types/src/sth-configuration.ts +++ b/packages/types/src/sth-configuration.ts @@ -105,6 +105,25 @@ export type K8SAdapterConfiguration = { } export type STHConfiguration = { + /** + * Description set by user + */ + description?: string; + + /** + * User assigned tags + */ + tags?: Array; + + /** + * Custom name to help identify sth + */ + customName?: string; + + /** + * Is STH self hosted? + */ + selfHosted?: boolean; /** * Logging level. */ From 59ec0fd6bac5ff55ad481edf6eb70c2043f69517 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 24 Oct 2022 08:55:24 +0000 Subject: [PATCH 116/231] Example sth config --- sth-config-example.json | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 sth-config-example.json diff --git a/sth-config-example.json b/sth-config-example.json new file mode 100644 index 000000000..0dcd03d6c --- /dev/null +++ b/sth-config-example.json @@ -0,0 +1,15 @@ +{ + "host": { + "hostname": "0.0.0.0", + "port": 9000, + "instancesServerPort": 9001 + }, + "timings": { + "instanceLifetimeExtensionDelay": 10000 + }, + "cpmId": "", + "cpmUrl": "localhost:11000", + "platform": { + "token": "" + } +} From c751cc62d48566017fb0d587ddc79eb9e999efda Mon Sep 17 00:00:00 2001 From: patuwwy Date: Sun, 23 Oct 2022 00:02:21 +0000 Subject: [PATCH 117/231] Fix verser connection path --- packages/verser/src/lib/verser-client.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/verser/src/lib/verser-client.ts b/packages/verser/src/lib/verser-client.ts index cdac390a2..7727f2eec 100644 --- a/packages/verser/src/lib/verser-client.ts +++ b/packages/verser/src/lib/verser-client.ts @@ -87,15 +87,17 @@ export class VerserClient extends TypedEmitter { */ public async connect(): Promise { return new Promise((resolve, reject) => { - const { hostname, port, pathname } = - this.opts.verserUrl instanceof URL ? this.opts.verserUrl : new URL(this.opts.verserUrl); + const { hostname, port, pathname, href } = this.opts.verserUrl instanceof URL + ? this.opts.verserUrl : new URL(this.opts.verserUrl); + this.logger.info("------", { hostname, port, pathname, href }); const connectRequest = request({ agent: this.httpAgent, headers: this.opts.headers, hostname, method: "connect", path: pathname, + href, port, protocol: this.opts.https ? "https:" : "http:", ca: typeof this.opts.https === "object" ? this.opts.https.ca : undefined, From e4704962d1a20a2dcd70f28ca0e7aaba9f83074d Mon Sep 17 00:00:00 2001 From: patuwwy Date: Sat, 22 Oct 2022 22:01:19 +0000 Subject: [PATCH 118/231] Platform token --- .gitignore | 3 +++ packages/host/src/lib/cpm-connector.ts | 3 ++- packages/host/src/lib/host.ts | 1 + packages/sth/src/bin/hub.ts | 2 ++ packages/sth/sth-config-example.json | 11 +++++++++++ packages/types/src/api-expose.ts | 2 +- packages/types/src/cpm-connector.ts | 1 + packages/types/src/sth-command-options.ts | 1 + packages/types/src/sth-configuration.ts | 6 +++++- 9 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 packages/sth/sth-config-example.json diff --git a/.gitignore b/.gitignore index a38284598..3e7b29f23 100644 --- a/.gitignore +++ b/.gitignore @@ -156,3 +156,6 @@ packages/**/build.info.json bdd/timing-log.ndjson timing-log.ndjson .all-deps + +# sth config +packages/sth/sth-config.json diff --git a/packages/host/src/lib/cpm-connector.ts b/packages/host/src/lib/cpm-connector.ts index 751fc062b..29fc93a89 100644 --- a/packages/host/src/lib/cpm-connector.ts +++ b/packages/host/src/lib/cpm-connector.ts @@ -156,7 +156,8 @@ export class CPMConnector extends TypedEmitter { "x-sth-description": typeof this.config.description !== "undefined" ? this.config.description : "", "x-sth-tags": JSON.stringify(typeof this.config.tags !== "undefined" ? this.config.tags : []), "x-manager-id": cpmId, - "x-sth-id": this.config.id || "" + "x-sth-id": this.config.id || "", + ...(config.token ? { "Authorization": `Bearer ${config.token}` } : {}) }, server, https: this.isHttps diff --git a/packages/host/src/lib/host.ts b/packages/host/src/lib/host.ts index 90d16ac4a..66da293db 100644 --- a/packages/host/src/lib/host.ts +++ b/packages/host/src/lib/host.ts @@ -339,6 +339,7 @@ export class Host implements IComponent { cpmSslCaPath: this.config.cpmSslCaPath, maxReconnections: this.config.cpm.maxReconnections, reconnectionDelay: this.config.cpm.reconnectionDelay, + token: this.config.platform?.token }, this.api.server ); diff --git a/packages/sth/src/bin/hub.ts b/packages/sth/src/bin/hub.ts index fb1ccea5d..9ad1cfbbb 100755 --- a/packages/sth/src/bin/hub.ts +++ b/packages/sth/src/bin/hub.ts @@ -48,6 +48,7 @@ const options: OptionValues & STHCommandOptions = program .option("--runner-max-mem ", "Maximum mem used by runner") .option("--prerunner-image ", "Image used by prerunner") .option("--prerunner-max-mem ", "Maximum mem used by prerunner") + .option("--platform-token ", "Platform authorization token") .option("--cpm-ssl-ca-path ", "Certificate Authority for self-signed CPM SSL certificates") .option("--cpm-id ") .option("--cpm-max-reconnections ", "Maximum reconnection attempts (-1 no limit)") @@ -100,6 +101,7 @@ const options: OptionValues & STHCommandOptions = program maxReconnections: options.cpmMaxReconnections }, debug: options.debug, + platform: (options.platformToken ? { token: options.platformToken } : undefined), docker: { prerunner: { image: options.prerunnerImage, diff --git a/packages/sth/sth-config-example.json b/packages/sth/sth-config-example.json new file mode 100644 index 000000000..b6a4499eb --- /dev/null +++ b/packages/sth/sth-config-example.json @@ -0,0 +1,11 @@ +{ + + "host": { + "hostname": "0.0.0.0", + "port": 9000, + "instancesServerPort": 9001 + }, + "timings": { + "instanceLifetimeExtensionDelay": 10000 + } +} diff --git a/packages/types/src/api-expose.ts b/packages/types/src/api-expose.ts index cd9313e7a..ebe8db8ba 100644 --- a/packages/types/src/api-expose.ts +++ b/packages/types/src/api-expose.ts @@ -19,7 +19,7 @@ export type OpOptions = { rawBody?: boolean }; export type NextCallback = (err?: Error) => void; export type Middleware = (req: ParsedMessage, res: ServerResponse, next: NextCallback) => void; -export type Decorator = (req: IncomingMessage) => void; +export type Decorator = (req: IncomingMessage) => MaybePromise; /** * Configuration options for streaming endpoints diff --git a/packages/types/src/cpm-connector.ts b/packages/types/src/cpm-connector.ts index 2edf43609..501ed6dc7 100644 --- a/packages/types/src/cpm-connector.ts +++ b/packages/types/src/cpm-connector.ts @@ -9,4 +9,5 @@ export type CPMConnectorOptions = { cpmSslCaPath?: STHConfiguration["cpmSslCaPath"]; maxReconnections: STHConfiguration["cpm"]["maxReconnections"]; reconnectionDelay: STHConfiguration["cpm"]["reconnectionDelay"]; + token?: string; } diff --git a/packages/types/src/sth-command-options.ts b/packages/types/src/sth-command-options.ts index 018bf507b..fdf190077 100644 --- a/packages/types/src/sth-command-options.ts +++ b/packages/types/src/sth-command-options.ts @@ -17,6 +17,7 @@ export type STHCommandOptions = { cpmSslCaPath?: string; cpmMaxReconnections: number, cpmReconnectionDelay: number, + platformToken?: string; id?: string; runtimeAdapter: string; runnerImage: string; diff --git a/packages/types/src/sth-configuration.ts b/packages/types/src/sth-configuration.ts index 9c906c4c7..ebad764c1 100644 --- a/packages/types/src/sth-configuration.ts +++ b/packages/types/src/sth-configuration.ts @@ -151,7 +151,11 @@ export type STHConfiguration = { cpm: { maxReconnections: number, - reconnectionDelay: number, + reconnectionDelay: number + } + + platform?: { + token?: string; } /** From b99bd901e59870cbd9dc022e0d5f85f8a0b8cf45 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 15 Feb 2023 10:08:55 +0000 Subject: [PATCH 119/231] Delete duplicated config --- packages/sth/sth-config-example.json | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 packages/sth/sth-config-example.json diff --git a/packages/sth/sth-config-example.json b/packages/sth/sth-config-example.json deleted file mode 100644 index b6a4499eb..000000000 --- a/packages/sth/sth-config-example.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - - "host": { - "hostname": "0.0.0.0", - "port": 9000, - "instancesServerPort": 9001 - }, - "timings": { - "instanceLifetimeExtensionDelay": 10000 - } -} From 15492cc779b95bffe61d7b3bf6216f7c8e30dc87 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Fri, 17 Feb 2023 11:44:16 +0000 Subject: [PATCH 120/231] connect over https to MW --- packages/verser/src/lib/verser-client.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/verser/src/lib/verser-client.ts b/packages/verser/src/lib/verser-client.ts index 7727f2eec..6fded3d39 100644 --- a/packages/verser/src/lib/verser-client.ts +++ b/packages/verser/src/lib/verser-client.ts @@ -77,6 +77,9 @@ export class VerserClient extends TypedEmitter { super(); this.opts = opts; + this.opts.https ||= (this.opts.verserUrl instanceof URL + ? this.opts.verserUrl : new URL(this.opts.verserUrl)).protocol === "https:"; + this.httpAgent = this.opts.https ? new HttpsAgent() : new HttpAgent(); } From cfc34d95023abf2238189c21b9c925538ba92a01 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 20 Feb 2023 11:25:36 +0000 Subject: [PATCH 121/231] Add platform args to STH --- packages/host/src/lib/cpm-connector.ts | 7 ++++++- packages/host/src/lib/host.ts | 13 ++++++++----- packages/sth/src/bin/hub.ts | 11 ++++++++++- packages/types/src/sth-command-options.ts | 14 ++++++++------ packages/types/src/sth-configuration.ts | 9 ++++++--- packages/verser/src/lib/verser-client.ts | 1 - 6 files changed, 38 insertions(+), 17 deletions(-) diff --git a/packages/host/src/lib/cpm-connector.ts b/packages/host/src/lib/cpm-connector.ts index 29fc93a89..92a070a28 100644 --- a/packages/host/src/lib/cpm-connector.ts +++ b/packages/host/src/lib/cpm-connector.ts @@ -144,7 +144,7 @@ export class CPMConnector extends TypedEmitter { * @param {CPMConnectorOptions} config CPM connector configuration. * @param {Server} server API server to handle incoming requests. */ - constructor(private cpmHostname: string, cpmId: string, config: CPMConnectorOptions, server: http.Server) { + constructor(private cpmHostname: string, cpmId: string, org: string = "", config: CPMConnectorOptions, server: http.Server) { super(); this.cpmId = cpmId; this.config = config; @@ -153,8 +153,13 @@ export class CPMConnector extends TypedEmitter { this.verserClient = new VerserClient({ verserUrl: `${this.cpmUrl}/verser`, headers: { +<<<<<<< HEAD "x-sth-description": typeof this.config.description !== "undefined" ? this.config.description : "", "x-sth-tags": JSON.stringify(typeof this.config.tags !== "undefined" ? this.config.tags : []), +||||||| constructed merge base +======= + ...(org ? { "x-org-id": org } : {}), +>>>>>>> Add platform args to STH "x-manager-id": cpmId, "x-sth-id": this.config.id || "", ...(config.token ? { "Authorization": `Bearer ${config.token}` } : {}) diff --git a/packages/host/src/lib/host.ts b/packages/host/src/lib/host.ts index 66da293db..b780074b8 100644 --- a/packages/host/src/lib/host.ts +++ b/packages/host/src/lib/host.ts @@ -202,6 +202,8 @@ export class Host implements IComponent { prettyLog.pipe(process.stdout); + this.logger.info("config", this.config); + this.config.host.id ||= this.getId(); this.logger.updateBaseLog({ id: this.config.host.id }); this.serviceDiscovery.logger.pipe(this.logger); @@ -236,7 +238,7 @@ export class Host implements IComponent { this.api.server.timeout = 0; this.api.server.requestTimeout = 0; - if (!!this.config.cpmUrl !== !!this.config.cpmId) { + if (!!this.config.cpmId !== !!this.config.cpmUrl && !!this.config.cpmId !== !!this.config.platform?.api) { throw new HostError("CPM_CONFIGURATION_ERROR", "CPM URL and ID must be provided together"); } } @@ -326,10 +328,11 @@ export class Host implements IComponent { await this.performStartup(); await this.startListening(); - if (this.config.cpmUrl && this.config.cpmId) { + if ((this.config.cpmUrl || this.config.platform?.api) && (this.config.cpmId || this.config.platform?.space)) { this.cpmConnector = new CPMConnector( - this.config.cpmUrl, - this.config.cpmId, + this.config.platform?.api || this.config.cpmUrl, + this.config.platform?.space || this.config.cpmId, + this.config.platform?.organisation, { selfHosted: this.config.selfHosted, description: this.config.description, @@ -339,7 +342,7 @@ export class Host implements IComponent { cpmSslCaPath: this.config.cpmSslCaPath, maxReconnections: this.config.cpm.maxReconnections, reconnectionDelay: this.config.cpm.reconnectionDelay, - token: this.config.platform?.token + token: this.config.platform?.api ? this.config.platform?.apiKey : undefined }, this.api.server ); diff --git a/packages/sth/src/bin/hub.ts b/packages/sth/src/bin/hub.ts index 9ad1cfbbb..7693ab81b 100755 --- a/packages/sth/src/bin/hub.ts +++ b/packages/sth/src/bin/hub.ts @@ -32,6 +32,10 @@ const options: OptionValues & STHCommandOptions = program .option("-H, --hostname ", "API IP") .option("-E, --identify-existing", "Index existing volumes as sequences") .option("-C, --cpm-url ") + .option("--platform-api ", "Platform API url, ie. https://api.scramjet.org/api/v1") + .option("--platform-apikey ", "Platform API Key") + .option("--platform-space ", "Target Platform Space") + .option("--platform-organisation ", "Platform Organisation") .option("-I, --id ", "The id assigned to this server") .option("--runtime-adapter ", "Determines adapters used for loading and starting sequence. One of 'docker', 'process', 'kubernetes'") .option("-X, --exit-with-last-instance", "Exits host when no more instances exist.") @@ -101,7 +105,12 @@ const options: OptionValues & STHCommandOptions = program maxReconnections: options.cpmMaxReconnections }, debug: options.debug, - platform: (options.platformToken ? { token: options.platformToken } : undefined), + platform: { + apiKey: options.platformApikey, + api: options.platformApi, + space: options.platformSpace, + organisation: options.platformOrganisation + }, docker: { prerunner: { image: options.prerunnerImage, diff --git a/packages/types/src/sth-command-options.ts b/packages/types/src/sth-command-options.ts index fdf190077..42bd0ecb1 100644 --- a/packages/types/src/sth-command-options.ts +++ b/packages/types/src/sth-command-options.ts @@ -7,7 +7,7 @@ export type STHCommandOptions = { tags: string; selfHosted: boolean; logLevel: LogLevel; - colors: boolean, + colors: boolean; port: number; hostname: string; identifyExisting: boolean; @@ -15,9 +15,11 @@ export type STHCommandOptions = { cpmUrl?: string; cpmId?: string; cpmSslCaPath?: string; - cpmMaxReconnections: number, - cpmReconnectionDelay: number, - platformToken?: string; + cpmMaxReconnections: number; + cpmReconnectionDelay: number; + platformApi: string; + platformApiKey: string; + platformSpace: string; id?: string; runtimeAdapter: string; runnerImage: string; @@ -33,12 +35,12 @@ export type STHCommandOptions = { k8sQuotaName: string; k8sAuthConfigPath: string; k8sSthPodHost: string; - k8sRunnerImage: string, + k8sRunnerImage: string; k8sRunnerPyImage: string k8sSequencesRoot: string; debug: boolean; docker: boolean; - k8sRunnerCleanupTimeout: string, + k8sRunnerCleanupTimeout: string; k8sRunnerResourcesRequestsCpu: string; k8sRunnerResourcesRequestsMemory: string; k8sRunnerResourcesLimitsCpu: string; diff --git a/packages/types/src/sth-configuration.ts b/packages/types/src/sth-configuration.ts index ebad764c1..b07164ca0 100644 --- a/packages/types/src/sth-configuration.ts +++ b/packages/types/src/sth-configuration.ts @@ -152,11 +152,14 @@ export type STHConfiguration = { cpm: { maxReconnections: number, reconnectionDelay: number - } + }; platform?: { - token?: string; - } + apiKey: string; + api: string; + space: string; + organisation: string; + }; /** * Add debugging flags to runner. diff --git a/packages/verser/src/lib/verser-client.ts b/packages/verser/src/lib/verser-client.ts index 6fded3d39..1183e7367 100644 --- a/packages/verser/src/lib/verser-client.ts +++ b/packages/verser/src/lib/verser-client.ts @@ -93,7 +93,6 @@ export class VerserClient extends TypedEmitter { const { hostname, port, pathname, href } = this.opts.verserUrl instanceof URL ? this.opts.verserUrl : new URL(this.opts.verserUrl); - this.logger.info("------", { hostname, port, pathname, href }); const connectRequest = request({ agent: this.httpAgent, headers: this.opts.headers, From 4964a155aeb8c4ebd7e7587a2f157943fb52ad57 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 27 Feb 2023 13:34:11 +0000 Subject: [PATCH 122/231] Use daughter key, change addhub url --- packages/host/src/lib/cpm-connector.ts | 17 ++++++++++------- packages/host/src/lib/host.ts | 2 +- packages/types/src/cpm-connector.ts | 2 +- packages/utility/src/index.ts | 1 + 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/packages/host/src/lib/cpm-connector.ts b/packages/host/src/lib/cpm-connector.ts index 92a070a28..d35a84a48 100644 --- a/packages/host/src/lib/cpm-connector.ts +++ b/packages/host/src/lib/cpm-connector.ts @@ -19,7 +19,7 @@ import { import { StringStream } from "scramjet"; import { LoadCheck } from "@scramjet/load-check"; import { VerserClient } from "@scramjet/verser"; -import { TypedEmitter, normalizeUrl } from "@scramjet/utility"; +import { TypedEmitter, generateSTHKey, normalizeUrl } from "@scramjet/utility"; import { ObjLogger } from "@scramjet/obj-logger"; import { ReasonPhrases } from "http-status-codes"; import { DuplexStream } from "@scramjet/api-server"; @@ -150,19 +150,22 @@ export class CPMConnector extends TypedEmitter { this.config = config; this.logger = new ObjLogger(this); + + let sthKey; + + if (config.apiKey) { + sthKey = generateSTHKey(config.apiKey); + } + this.verserClient = new VerserClient({ - verserUrl: `${this.cpmUrl}/verser`, + verserUrl: `${this.cpmUrl}/addHub/${org}/${cpmId}`, headers: { -<<<<<<< HEAD "x-sth-description": typeof this.config.description !== "undefined" ? this.config.description : "", "x-sth-tags": JSON.stringify(typeof this.config.tags !== "undefined" ? this.config.tags : []), -||||||| constructed merge base -======= ...(org ? { "x-org-id": org } : {}), ->>>>>>> Add platform args to STH "x-manager-id": cpmId, "x-sth-id": this.config.id || "", - ...(config.token ? { "Authorization": `Bearer ${config.token}` } : {}) + ...(sthKey ? { "Authorization": `Digest cnonce="${sthKey}"` } : {}) }, server, https: this.isHttps diff --git a/packages/host/src/lib/host.ts b/packages/host/src/lib/host.ts index b780074b8..1f7595cd0 100644 --- a/packages/host/src/lib/host.ts +++ b/packages/host/src/lib/host.ts @@ -342,7 +342,7 @@ export class Host implements IComponent { cpmSslCaPath: this.config.cpmSslCaPath, maxReconnections: this.config.cpm.maxReconnections, reconnectionDelay: this.config.cpm.reconnectionDelay, - token: this.config.platform?.api ? this.config.platform?.apiKey : undefined + apiKey: this.config.platform?.api ? this.config.platform?.apiKey : undefined }, this.api.server ); diff --git a/packages/types/src/cpm-connector.ts b/packages/types/src/cpm-connector.ts index 501ed6dc7..92f89284c 100644 --- a/packages/types/src/cpm-connector.ts +++ b/packages/types/src/cpm-connector.ts @@ -9,5 +9,5 @@ export type CPMConnectorOptions = { cpmSslCaPath?: STHConfiguration["cpmSslCaPath"]; maxReconnections: STHConfiguration["cpm"]["maxReconnections"]; reconnectionDelay: STHConfiguration["cpm"]["reconnectionDelay"]; - token?: string; + apiKey?: string; } diff --git a/packages/utility/src/index.ts b/packages/utility/src/index.ts index c343e4e97..44d9406c6 100644 --- a/packages/utility/src/index.ts +++ b/packages/utility/src/index.ts @@ -12,3 +12,4 @@ export * from "./config"; export * from "./file"; export * from "./constants"; export * from "./validators"; +export * from "./keygen"; From 58af95be2abfb9dff4204dd1ec1ffe942196c82f Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 28 Feb 2023 11:40:35 +0000 Subject: [PATCH 123/231] Add hub type, org:space param --- packages/host/src/lib/cpm-connector.ts | 9 ++++++--- packages/host/src/lib/host.ts | 6 +++--- packages/types/src/cpm-connector.ts | 1 + packages/types/src/sth-configuration.ts | 1 + 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/host/src/lib/cpm-connector.ts b/packages/host/src/lib/cpm-connector.ts index d35a84a48..8b06258a7 100644 --- a/packages/host/src/lib/cpm-connector.ts +++ b/packages/host/src/lib/cpm-connector.ts @@ -144,8 +144,11 @@ export class CPMConnector extends TypedEmitter { * @param {CPMConnectorOptions} config CPM connector configuration. * @param {Server} server API server to handle incoming requests. */ - constructor(private cpmHostname: string, cpmId: string, org: string = "", config: CPMConnectorOptions, server: http.Server) { + constructor(private cpmHostname: string, cpm: string, config: CPMConnectorOptions, server: http.Server) { super(); + + const [orgId, cpmId] = cpm.split(":"); + this.cpmId = cpmId; this.config = config; @@ -158,13 +161,13 @@ export class CPMConnector extends TypedEmitter { } this.verserClient = new VerserClient({ - verserUrl: `${this.cpmUrl}/addHub/${org}/${cpmId}`, + verserUrl: `${this.cpmUrl}/${orgId}/${cpmId}/`, headers: { "x-sth-description": typeof this.config.description !== "undefined" ? this.config.description : "", "x-sth-tags": JSON.stringify(typeof this.config.tags !== "undefined" ? this.config.tags : []), - ...(org ? { "x-org-id": org } : {}), "x-manager-id": cpmId, "x-sth-id": this.config.id || "", + ...(orgId ? { "x-org-id": orgId } : {}), ...(sthKey ? { "Authorization": `Digest cnonce="${sthKey}"` } : {}) }, server, diff --git a/packages/host/src/lib/host.ts b/packages/host/src/lib/host.ts index 1f7595cd0..2fa0c3de5 100644 --- a/packages/host/src/lib/host.ts +++ b/packages/host/src/lib/host.ts @@ -331,8 +331,7 @@ export class Host implements IComponent { if ((this.config.cpmUrl || this.config.platform?.api) && (this.config.cpmId || this.config.platform?.space)) { this.cpmConnector = new CPMConnector( this.config.platform?.api || this.config.cpmUrl, - this.config.platform?.space || this.config.cpmId, - this.config.platform?.organisation, + this.config.platform?.space || (":" + this.config.cpmId), { selfHosted: this.config.selfHosted, description: this.config.description, @@ -342,7 +341,8 @@ export class Host implements IComponent { cpmSslCaPath: this.config.cpmSslCaPath, maxReconnections: this.config.cpm.maxReconnections, reconnectionDelay: this.config.cpm.reconnectionDelay, - apiKey: this.config.platform?.api ? this.config.platform?.apiKey : undefined + apiKey: this.config.platform?.api ? this.config.platform?.apiKey : undefined, + hostType: this.config.platform?.hostType }, this.api.server ); diff --git a/packages/types/src/cpm-connector.ts b/packages/types/src/cpm-connector.ts index 92f89284c..69456212b 100644 --- a/packages/types/src/cpm-connector.ts +++ b/packages/types/src/cpm-connector.ts @@ -10,4 +10,5 @@ export type CPMConnectorOptions = { maxReconnections: STHConfiguration["cpm"]["maxReconnections"]; reconnectionDelay: STHConfiguration["cpm"]["reconnectionDelay"]; apiKey?: string; + hostType?: NonNullable["hostType"]; } diff --git a/packages/types/src/sth-configuration.ts b/packages/types/src/sth-configuration.ts index b07164ca0..1fdcb1db8 100644 --- a/packages/types/src/sth-configuration.ts +++ b/packages/types/src/sth-configuration.ts @@ -159,6 +159,7 @@ export type STHConfiguration = { api: string; space: string; organisation: string; + hostType: "hub" | "federation" }; /** From d971abf81b3232e289d2676f6d3197d16da1b88f Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 28 Feb 2023 12:17:36 +0000 Subject: [PATCH 124/231] Update params --- packages/host/src/lib/cpm-connector.ts | 4 ++-- packages/host/src/lib/host.ts | 1 + packages/sth/src/bin/hub.ts | 10 +++++----- packages/types/src/cpm-connector.ts | 1 + packages/types/src/sth-command-options.ts | 1 + packages/types/src/sth-configuration.ts | 2 +- 6 files changed, 11 insertions(+), 8 deletions(-) diff --git a/packages/host/src/lib/cpm-connector.ts b/packages/host/src/lib/cpm-connector.ts index 8b06258a7..c291201ed 100644 --- a/packages/host/src/lib/cpm-connector.ts +++ b/packages/host/src/lib/cpm-connector.ts @@ -140,7 +140,7 @@ export class CPMConnector extends TypedEmitter { /** * @constructor * @param {string} cpmHostname CPM hostname to connect to. (e.g. "localhost:8080"). - * @param {string} cpmId CPM id to connect to. (e.g. "CPM1"). + * @param {string} cpm CPM id to connect to. (format: "org:manager"). * @param {CPMConnectorOptions} config CPM connector configuration. * @param {Server} server API server to handle incoming requests. */ @@ -161,7 +161,7 @@ export class CPMConnector extends TypedEmitter { } this.verserClient = new VerserClient({ - verserUrl: `${this.cpmUrl}/${orgId}/${cpmId}/`, + verserUrl: `${this.cpmUrl}/api/${config.apiVersion}/${orgId}/${cpmId}/`, headers: { "x-sth-description": typeof this.config.description !== "undefined" ? this.config.description : "", "x-sth-tags": JSON.stringify(typeof this.config.tags !== "undefined" ? this.config.tags : []), diff --git a/packages/host/src/lib/host.ts b/packages/host/src/lib/host.ts index 2fa0c3de5..b4fcd76ad 100644 --- a/packages/host/src/lib/host.ts +++ b/packages/host/src/lib/host.ts @@ -342,6 +342,7 @@ export class Host implements IComponent { maxReconnections: this.config.cpm.maxReconnections, reconnectionDelay: this.config.cpm.reconnectionDelay, apiKey: this.config.platform?.api ? this.config.platform?.apiKey : undefined, + apiVersion: this.config.platform?.apiVersion || "v1", hostType: this.config.platform?.hostType }, this.api.server diff --git a/packages/sth/src/bin/hub.ts b/packages/sth/src/bin/hub.ts index 7693ab81b..0e3249191 100755 --- a/packages/sth/src/bin/hub.ts +++ b/packages/sth/src/bin/hub.ts @@ -33,9 +33,9 @@ const options: OptionValues & STHCommandOptions = program .option("-E, --identify-existing", "Index existing volumes as sequences") .option("-C, --cpm-url ") .option("--platform-api ", "Platform API url, ie. https://api.scramjet.org/api/v1") - .option("--platform-apikey ", "Platform API Key") - .option("--platform-space ", "Target Platform Space") - .option("--platform-organisation ", "Platform Organisation") + .option("--platform-api-version", "Platform API version", "v1") + .option("--platform-api-key ", "Platform API Key") + .option("--platform-space ", "Target Platform Space") .option("-I, --id ", "The id assigned to this server") .option("--runtime-adapter ", "Determines adapters used for loading and starting sequence. One of 'docker', 'process', 'kubernetes'") .option("-X, --exit-with-last-instance", "Exits host when no more instances exist.") @@ -106,10 +106,10 @@ const options: OptionValues & STHCommandOptions = program }, debug: options.debug, platform: { - apiKey: options.platformApikey, + apiKey: options.platformApiKey, api: options.platformApi, space: options.platformSpace, - organisation: options.platformOrganisation + apiVersion: options.platformApiVersion }, docker: { prerunner: { diff --git a/packages/types/src/cpm-connector.ts b/packages/types/src/cpm-connector.ts index 69456212b..710439619 100644 --- a/packages/types/src/cpm-connector.ts +++ b/packages/types/src/cpm-connector.ts @@ -10,5 +10,6 @@ export type CPMConnectorOptions = { maxReconnections: STHConfiguration["cpm"]["maxReconnections"]; reconnectionDelay: STHConfiguration["cpm"]["reconnectionDelay"]; apiKey?: string; + apiVersion: string; hostType?: NonNullable["hostType"]; } diff --git a/packages/types/src/sth-command-options.ts b/packages/types/src/sth-command-options.ts index 42bd0ecb1..2ea882414 100644 --- a/packages/types/src/sth-command-options.ts +++ b/packages/types/src/sth-command-options.ts @@ -20,6 +20,7 @@ export type STHCommandOptions = { platformApi: string; platformApiKey: string; platformSpace: string; + platformApiVersion: string; id?: string; runtimeAdapter: string; runnerImage: string; diff --git a/packages/types/src/sth-configuration.ts b/packages/types/src/sth-configuration.ts index 1fdcb1db8..6044c8be5 100644 --- a/packages/types/src/sth-configuration.ts +++ b/packages/types/src/sth-configuration.ts @@ -156,9 +156,9 @@ export type STHConfiguration = { platform?: { apiKey: string; + apiVersion: string; api: string; space: string; - organisation: string; hostType: "hub" | "federation" }; From e577aa245b072301a8b64bfa8e111960320817d0 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 2 Mar 2023 23:26:59 +0000 Subject: [PATCH 125/231] si space accesskey --- packages/cli/src/lib/commands/space.ts | 12 ++++++++++++ .../middleware-api-client/src/middleware-client.ts | 8 ++++++++ 2 files changed, 20 insertions(+) diff --git a/packages/cli/src/lib/commands/space.ts b/packages/cli/src/lib/commands/space.ts index e084c808c..1a3cc04be 100644 --- a/packages/cli/src/lib/commands/space.ts +++ b/packages/cli/src/lib/commands/space.ts @@ -78,4 +78,16 @@ export const space: CommandDefinition = (program) => { await displayStream(await managerClient.getLogStream()); }); + + spaceCmd + .command("accessKey") + .description("Create access key for adding Hub to Space") + .argument("[]", "The name of the space (defaults to current space)") + .action(async (spaceName: string) => { + if (typeof spaceName === "undefined") spaceName = sessionConfig.lastSpaceId; + + const mwClient = getMiddlewareClient(); + + displayObject(await mwClient.createAccessKey(spaceName), "json"); + }); }; diff --git a/packages/middleware-api-client/src/middleware-client.ts b/packages/middleware-api-client/src/middleware-client.ts index 1ee34f0ac..f281e791a 100644 --- a/packages/middleware-api-client/src/middleware-client.ts +++ b/packages/middleware-api-client/src/middleware-client.ts @@ -54,4 +54,12 @@ export class MiddlewareClient implements ClientProvider { async getAuditStream(): Promise> { return this.client.getStream("audit"); } + + async createAccessKey(spaceId: string): Promise { + return this.client.post(`space/${spaceId}/apikey`, {}, + { + headers: { "content-type": "application/json" }, + }, + { json: true, parse: "json" }); + } } From adc45a2a1effe831fce95e4ceb31773514203ad3 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Fri, 3 Mar 2023 08:59:55 +0000 Subject: [PATCH 126/231] si space accesskey generate|revoke --- packages/cli/src/lib/commands/space.ts | 17 +++++++++++++++-- .../src/middleware-client.ts | 12 ++++++++++-- packages/sth/src/bin/hub.ts | 2 +- packages/types/src/rest-api-manager/common.ts | 8 ++++---- packages/utility/src/keygen.ts | 8 ++++++++ 5 files changed, 38 insertions(+), 9 deletions(-) create mode 100644 packages/utility/src/keygen.ts diff --git a/packages/cli/src/lib/commands/space.ts b/packages/cli/src/lib/commands/space.ts index 1a3cc04be..75f0a7094 100644 --- a/packages/cli/src/lib/commands/space.ts +++ b/packages/cli/src/lib/commands/space.ts @@ -79,8 +79,9 @@ export const space: CommandDefinition = (program) => { await displayStream(await managerClient.getLogStream()); }); - spaceCmd - .command("accessKey") + const accessKeyCmd = spaceCmd.command("accessKey"); + + accessKeyCmd.command("generate") .description("Create access key for adding Hub to Space") .argument("[]", "The name of the space (defaults to current space)") .action(async (spaceName: string) => { @@ -90,4 +91,16 @@ export const space: CommandDefinition = (program) => { displayObject(await mwClient.createAccessKey(spaceName), "json"); }); + + accessKeyCmd.command("revoke") + .description("Revokes accessKey for space") + .argument("[]", "The name of the space (defaults to current space)") + .argument("", "Access key") + .action(async (spaceName: string, accessKey: string) => { + if (typeof spaceName === "undefined") spaceName = sessionConfig.lastSpaceId; + + const mwClient = getMiddlewareClient(); + + displayObject(await mwClient.revokeAccessKey(spaceName, accessKey), "json"); + }); }; diff --git a/packages/middleware-api-client/src/middleware-client.ts b/packages/middleware-api-client/src/middleware-client.ts index f281e791a..25e6c3c68 100644 --- a/packages/middleware-api-client/src/middleware-client.ts +++ b/packages/middleware-api-client/src/middleware-client.ts @@ -56,10 +56,18 @@ export class MiddlewareClient implements ClientProvider { } async createAccessKey(spaceId: string): Promise { - return this.client.post(`space/${spaceId}/apikey`, {}, - { + return this.client.post(`space/${spaceId}/apikey`, {}, { headers: { "content-type": "application/json" }, }, { json: true, parse: "json" }); } + + async revokeAccessKey(spaceId: string, accessKey: string): Promise { + return this.client.delete(`space/${spaceId}/apikey`, { + headers: { + "content-type": "application/json", + "x-accesskey": accessKey + } + }); + } } diff --git a/packages/sth/src/bin/hub.ts b/packages/sth/src/bin/hub.ts index 0e3249191..c16765952 100755 --- a/packages/sth/src/bin/hub.ts +++ b/packages/sth/src/bin/hub.ts @@ -33,7 +33,7 @@ const options: OptionValues & STHCommandOptions = program .option("-E, --identify-existing", "Index existing volumes as sequences") .option("-C, --cpm-url ") .option("--platform-api ", "Platform API url, ie. https://api.scramjet.org/api/v1") - .option("--platform-api-version", "Platform API version", "v1") + .option("--platform-api-version ", "Platform API version", "v1") .option("--platform-api-key ", "Platform API Key") .option("--platform-space ", "Target Platform Space") .option("-I, --id ", "The id assigned to this server") diff --git a/packages/types/src/rest-api-manager/common.ts b/packages/types/src/rest-api-manager/common.ts index 1033a1a28..9fc6ea0ff 100644 --- a/packages/types/src/rest-api-manager/common.ts +++ b/packages/types/src/rest-api-manager/common.ts @@ -15,9 +15,9 @@ export type ConnectedSTHInfo = { }; export type HealthCheckInfo = { - uptime: number, - timestamp: number, + uptime: number; + timestamp: number; modules: { - [key: string]: boolean - } + [key: string]: boolean; + }; } diff --git a/packages/utility/src/keygen.ts b/packages/utility/src/keygen.ts new file mode 100644 index 000000000..c9f761210 --- /dev/null +++ b/packages/utility/src/keygen.ts @@ -0,0 +1,8 @@ +import { randomBytes, scryptSync } from "crypto"; + +export function generateSTHKey(daughterKey: string) { + const sessionId = randomBytes(12).toString("base64") + + daughterKey.substring(0, 16); + + return sessionId + scryptSync(sessionId, daughterKey, 96).toString("base64"); +} From 5f323cfd5fd1a55c5bab42d055c480a146fb56b3 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 7 Mar 2023 14:31:27 +0000 Subject: [PATCH 127/231] ManagerClient disconnect method --- packages/manager-api-client/src/manager-client.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/manager-api-client/src/manager-client.ts b/packages/manager-api-client/src/manager-client.ts index 3d3369164..4e445dc3d 100644 --- a/packages/manager-api-client/src/manager-client.ts +++ b/packages/manager-api-client/src/manager-client.ts @@ -83,4 +83,8 @@ export class ManagerClient implements ClientProvider { async deleteStoreItem(id: string) { await this.client.delete(`s3/${id}`); } + + async disconnectHubs(opts: any) { + await this.client.post(`disconnect`, opts, {}); + } } From e98bd1581e4a45c4c2a1b24c777a2aaaae98109f Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 8 Mar 2023 14:41:37 +0000 Subject: [PATCH 128/231] CLI revoke key --- packages/cli/src/lib/commands/space.ts | 46 +++++++++++++------ .../manager-api-client/src/manager-client.ts | 2 +- .../src/middleware-client.ts | 8 +++- .../src/rest-api-multi-manager/access-key.ts | 4 ++ .../types/src/rest-api-multi-manager/index.ts | 1 + 5 files changed, 46 insertions(+), 15 deletions(-) create mode 100644 packages/types/src/rest-api-multi-manager/access-key.ts diff --git a/packages/cli/src/lib/commands/space.ts b/packages/cli/src/lib/commands/space.ts index 75f0a7094..12db0831c 100644 --- a/packages/cli/src/lib/commands/space.ts +++ b/packages/cli/src/lib/commands/space.ts @@ -79,28 +79,48 @@ export const space: CommandDefinition = (program) => { await displayStream(await managerClient.getLogStream()); }); - const accessKeyCmd = spaceCmd.command("accessKey"); - - accessKeyCmd.command("generate") - .description("Create access key for adding Hub to Space") - .argument("[]", "The name of the space (defaults to current space)") - .action(async (spaceName: string) => { - if (typeof spaceName === "undefined") spaceName = sessionConfig.lastSpaceId; + const accessKeyCmd = spaceCmd + .command("access") + .description("Manages Access Keys for active Space"); + accessKeyCmd.command("create") + .description("Create Access key for adding Hub to active Space") + .action(async () => { + const spaceName = sessionConfig.lastSpaceId; const mwClient = getMiddlewareClient(); + if (!spaceName) { + throw new Error("No Space set"); + } + displayObject(await mwClient.createAccessKey(spaceName), "json"); }); - accessKeyCmd.command("revoke") - .description("Revokes accessKey for space") - .argument("[]", "The name of the space (defaults to current space)") - .argument("", "Access key") - .action(async (spaceName: string, accessKey: string) => { - if (typeof spaceName === "undefined") spaceName = sessionConfig.lastSpaceId; + accessKeyCmd.command("list") + .alias("ls") + .description("List Access Keys metadata in active Space") + .action(async () => { + const spaceName = sessionConfig.lastSpaceId; + const mwClient = getMiddlewareClient(); + + if (!spaceName) { + throw new Error("No Space set"); + } + displayObject(await mwClient.listAccessKeys(spaceName), "json"); + }); + + accessKeyCmd.command("revoke") + .description("Revokes Access Key in active Space") + .argument("", "Access Key id") + .action(async (accessKey: string) => { + const spaceName = sessionConfig.lastSpaceId; const mwClient = getMiddlewareClient(); + if (!spaceName) { + throw new Error("No Space set"); + } + displayObject(await mwClient.revokeAccessKey(spaceName, accessKey), "json"); }); }; diff --git a/packages/manager-api-client/src/manager-client.ts b/packages/manager-api-client/src/manager-client.ts index 4e445dc3d..84d61db97 100644 --- a/packages/manager-api-client/src/manager-client.ts +++ b/packages/manager-api-client/src/manager-client.ts @@ -85,6 +85,6 @@ export class ManagerClient implements ClientProvider { } async disconnectHubs(opts: any) { - await this.client.post(`disconnect`, opts, {}); + await this.client.post("disconnect", opts, {}, { json: true, parse: "json" }); } } diff --git a/packages/middleware-api-client/src/middleware-client.ts b/packages/middleware-api-client/src/middleware-client.ts index 25e6c3c68..36f89e34c 100644 --- a/packages/middleware-api-client/src/middleware-client.ts +++ b/packages/middleware-api-client/src/middleware-client.ts @@ -55,6 +55,12 @@ export class MiddlewareClient implements ClientProvider { return this.client.getStream("audit"); } + async listAccessKeys(spaceId: string): Promise { + return this.client.get(`space/${spaceId}/apikey`, { + headers: { "content-type": "application/json" }, + }); + } + async createAccessKey(spaceId: string): Promise { return this.client.post(`space/${spaceId}/apikey`, {}, { headers: { "content-type": "application/json" }, @@ -66,7 +72,7 @@ export class MiddlewareClient implements ClientProvider { return this.client.delete(`space/${spaceId}/apikey`, { headers: { "content-type": "application/json", - "x-accesskey": accessKey + "x-revoke": accessKey } }); } diff --git a/packages/types/src/rest-api-multi-manager/access-key.ts b/packages/types/src/rest-api-multi-manager/access-key.ts new file mode 100644 index 000000000..661b711d9 --- /dev/null +++ b/packages/types/src/rest-api-multi-manager/access-key.ts @@ -0,0 +1,4 @@ +export type GetAccessKeysResponse = { + created: number; + description?: string; +}[]; diff --git a/packages/types/src/rest-api-multi-manager/index.ts b/packages/types/src/rest-api-multi-manager/index.ts index c58e391bb..870cbf4a2 100644 --- a/packages/types/src/rest-api-multi-manager/index.ts +++ b/packages/types/src/rest-api-multi-manager/index.ts @@ -6,3 +6,4 @@ export * from "./get-managers"; export * from "./get-version"; export * from "./get-info"; export * from "./get-load-check"; +export * from "./access-key"; From f2b889ce6152da86b775f6d79b3a54347110223f Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 8 Mar 2023 22:45:17 +0000 Subject: [PATCH 129/231] Notify hub about disconnect --- packages/host/src/lib/cpm-connector.ts | 14 ++++++++++++-- packages/model/src/stream-handler.ts | 4 +++- packages/symbols/src/cpm-message-code.ts | 4 +++- packages/types/src/message-streams.ts | 2 +- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/packages/host/src/lib/cpm-connector.ts b/packages/host/src/lib/cpm-connector.ts index c291201ed..2edafaf62 100644 --- a/packages/host/src/lib/cpm-connector.ts +++ b/packages/host/src/lib/cpm-connector.ts @@ -105,12 +105,17 @@ export class CPMConnector extends TypedEmitter { wasConnected: boolean = false; /** - * Connection attempts counter + * Connection attempts counter. * * @type {number} */ connectionAttempts = 0; + /** + * Message of impending abandonment received. + */ + isAbandoned = false; + /** * Id of Manager (e.g. "cpm-1"). * @@ -268,6 +273,11 @@ export class CPMConnector extends TypedEmitter { this.logger.updateBaseLog({ id: this.info.id }); } + if (message[0] === CPMMessageCode.KEY_REVOKED) { + this.logger.trace("Received pre drop message"); + this.isAbandoned = true; + } + return message; }).catch((e: any) => { this.logger.error("communicationChannel error", e.message); @@ -401,7 +411,7 @@ export class CPMConnector extends TypedEmitter { * @returns {void} */ async reconnect(): Promise { - if (this.isReconnecting) { + if (this.isReconnecting || this.isAbandoned) { return; } diff --git a/packages/model/src/stream-handler.ts b/packages/model/src/stream-handler.ts index 47c5c017d..91eb252ea 100644 --- a/packages/model/src/stream-handler.ts +++ b/packages/model/src/stream-handler.ts @@ -54,6 +54,7 @@ type ControlMessageHandlerList = { [RunnerMessageCode.INPUT_CONTENT_TYPE]: ConfiguredMessageHandler[]; [RunnerMessageCode.EVENT]: ConfiguredMessageHandler[]; [CPMMessageCode.STH_ID]: ConfiguredMessageHandler[]; + [CPMMessageCode.KEY_REVOKED]: ConfiguredMessageHandler[]; }; export class CommunicationHandler implements ICommunicationHandler { @@ -87,7 +88,8 @@ export class CommunicationHandler implements ICommunicationHandler { [RunnerMessageCode.EVENT]: [], [RunnerMessageCode.PONG]: [], [RunnerMessageCode.INPUT_CONTENT_TYPE]: [], - [CPMMessageCode.STH_ID]: [] + [CPMMessageCode.STH_ID]: [], + [CPMMessageCode.KEY_REVOKED]: [], }; this.monitoringHandlerHash = { [RunnerMessageCode.ACKNOWLEDGE]: [], diff --git a/packages/symbols/src/cpm-message-code.ts b/packages/symbols/src/cpm-message-code.ts index 224d49ea4..8fcca6a63 100644 --- a/packages/symbols/src/cpm-message-code.ts +++ b/packages/symbols/src/cpm-message-code.ts @@ -11,5 +11,7 @@ export enum CPMMessageCode { TOPIC = 7007, - CONFIRM_MSG = 8000 + CONFIRM_MSG = 8000, + + KEY_REVOKED = 9001 } diff --git a/packages/types/src/message-streams.ts b/packages/types/src/message-streams.ts index d8ccbcb7b..6f09f6712 100644 --- a/packages/types/src/message-streams.ts +++ b/packages/types/src/message-streams.ts @@ -95,7 +95,7 @@ export type EncodedMessage< export type ControlMessageCode = RunnerMessageCode.KILL | RunnerMessageCode.MONITORING_RATE | RunnerMessageCode.STOP | RunnerMessageCode.EVENT | RunnerMessageCode.PONG | - CPMMessageCode.STH_ID | + CPMMessageCode.STH_ID | CPMMessageCode.KEY_REVOKED | RunnerMessageCode.INPUT_CONTENT_TYPE; export type EncodedControlMessage = EncodedMessage; From a536c48508935920b8790f6ee1766abf89c83427 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Fri, 10 Mar 2023 21:24:12 +0000 Subject: [PATCH 130/231] Add si space access create "description" --- packages/cli/src/lib/commands/space.ts | 7 ++++--- packages/middleware-api-client/src/middleware-client.ts | 6 ++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/lib/commands/space.ts b/packages/cli/src/lib/commands/space.ts index 12db0831c..72af7c7e4 100644 --- a/packages/cli/src/lib/commands/space.ts +++ b/packages/cli/src/lib/commands/space.ts @@ -84,8 +84,9 @@ export const space: CommandDefinition = (program) => { .description("Manages Access Keys for active Space"); accessKeyCmd.command("create") - .description("Create Access key for adding Hub to active Space") - .action(async () => { + .argument("", "Key description") + .description("Create Access key for adding Hubs to active Space, i.e \"Army of Darkness\"") + .action(async (description: string) => { const spaceName = sessionConfig.lastSpaceId; const mwClient = getMiddlewareClient(); @@ -93,7 +94,7 @@ export const space: CommandDefinition = (program) => { throw new Error("No Space set"); } - displayObject(await mwClient.createAccessKey(spaceName), "json"); + displayObject(await mwClient.createAccessKey(spaceName, { description }), "json"); }); accessKeyCmd.command("list") diff --git a/packages/middleware-api-client/src/middleware-client.ts b/packages/middleware-api-client/src/middleware-client.ts index 36f89e34c..08b892f7a 100644 --- a/packages/middleware-api-client/src/middleware-client.ts +++ b/packages/middleware-api-client/src/middleware-client.ts @@ -61,8 +61,10 @@ export class MiddlewareClient implements ClientProvider { }); } - async createAccessKey(spaceId: string): Promise { - return this.client.post(`space/${spaceId}/apikey`, {}, { + async createAccessKey(spaceId: string, opts: { description?: string }): Promise { + return this.client.post(`space/${spaceId}/apikey`, { + description: opts.description?.trim() + }, { headers: { "content-type": "application/json" }, }, { json: true, parse: "json" }); From ea307c59ba3334dccb0a85fbfbcc96f68e5bc4f5 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 14 Mar 2023 10:16:55 +0000 Subject: [PATCH 131/231] Add disconnect payload types --- packages/types/src/rest-api-manager/index.ts | 1 + packages/types/src/rest-api-manager/post-disconnect.ts | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 packages/types/src/rest-api-manager/post-disconnect.ts diff --git a/packages/types/src/rest-api-manager/index.ts b/packages/types/src/rest-api-manager/index.ts index bd0c477a5..b10dfb94d 100644 --- a/packages/types/src/rest-api-manager/index.ts +++ b/packages/types/src/rest-api-manager/index.ts @@ -12,3 +12,4 @@ export * from "./get-topics"; export * from "./get-store-items"; export * from "./put-store-item"; export * from "./get-entities"; +export * from "./post-disconnect"; diff --git a/packages/types/src/rest-api-manager/post-disconnect.ts b/packages/types/src/rest-api-manager/post-disconnect.ts new file mode 100644 index 000000000..ff12afd2e --- /dev/null +++ b/packages/types/src/rest-api-manager/post-disconnect.ts @@ -0,0 +1,4 @@ +export type PostDisconnectPayload = { + limit?: number; + accessKey: string; +} From 543e5616cf781ecf4e425e688e192d2d1ee5f886 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 14 Mar 2023 15:52:49 +0000 Subject: [PATCH 132/231] Disconnected due to limit --- packages/host/src/lib/cpm-connector.ts | 2 +- packages/model/src/stream-handler.ts | 2 ++ packages/symbols/src/cpm-message-code.ts | 3 ++- packages/types/src/message-streams.ts | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/host/src/lib/cpm-connector.ts b/packages/host/src/lib/cpm-connector.ts index 2edafaf62..92d7a8a41 100644 --- a/packages/host/src/lib/cpm-connector.ts +++ b/packages/host/src/lib/cpm-connector.ts @@ -273,7 +273,7 @@ export class CPMConnector extends TypedEmitter { this.logger.updateBaseLog({ id: this.info.id }); } - if (message[0] === CPMMessageCode.KEY_REVOKED) { + if (message[0] === CPMMessageCode.KEY_REVOKED || message[0] === CPMMessageCode.LIMIT_EXCEEDED) { this.logger.trace("Received pre drop message"); this.isAbandoned = true; } diff --git a/packages/model/src/stream-handler.ts b/packages/model/src/stream-handler.ts index 91eb252ea..e40d6a791 100644 --- a/packages/model/src/stream-handler.ts +++ b/packages/model/src/stream-handler.ts @@ -55,6 +55,7 @@ type ControlMessageHandlerList = { [RunnerMessageCode.EVENT]: ConfiguredMessageHandler[]; [CPMMessageCode.STH_ID]: ConfiguredMessageHandler[]; [CPMMessageCode.KEY_REVOKED]: ConfiguredMessageHandler[]; + [CPMMessageCode.LIMIT_EXCEEDED]: ConfiguredMessageHandler[]; }; export class CommunicationHandler implements ICommunicationHandler { @@ -90,6 +91,7 @@ export class CommunicationHandler implements ICommunicationHandler { [RunnerMessageCode.INPUT_CONTENT_TYPE]: [], [CPMMessageCode.STH_ID]: [], [CPMMessageCode.KEY_REVOKED]: [], + [CPMMessageCode.LIMIT_EXCEEDED]: [], }; this.monitoringHandlerHash = { [RunnerMessageCode.ACKNOWLEDGE]: [], diff --git a/packages/symbols/src/cpm-message-code.ts b/packages/symbols/src/cpm-message-code.ts index 8fcca6a63..b0acf222b 100644 --- a/packages/symbols/src/cpm-message-code.ts +++ b/packages/symbols/src/cpm-message-code.ts @@ -13,5 +13,6 @@ export enum CPMMessageCode { CONFIRM_MSG = 8000, - KEY_REVOKED = 9001 + KEY_REVOKED = 9001, + LIMIT_EXCEEDED = 9002 } diff --git a/packages/types/src/message-streams.ts b/packages/types/src/message-streams.ts index 6f09f6712..ce4b7bf34 100644 --- a/packages/types/src/message-streams.ts +++ b/packages/types/src/message-streams.ts @@ -95,7 +95,7 @@ export type EncodedMessage< export type ControlMessageCode = RunnerMessageCode.KILL | RunnerMessageCode.MONITORING_RATE | RunnerMessageCode.STOP | RunnerMessageCode.EVENT | RunnerMessageCode.PONG | - CPMMessageCode.STH_ID | CPMMessageCode.KEY_REVOKED | + CPMMessageCode.STH_ID | CPMMessageCode.KEY_REVOKED | CPMMessageCode.LIMIT_EXCEEDED | RunnerMessageCode.INPUT_CONTENT_TYPE; export type EncodedControlMessage = EncodedMessage; From 73cd7af76db1fa99c9f289378e2cbcccbc081700 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 15 Mar 2023 16:25:09 +0000 Subject: [PATCH 133/231] M Disconnect response type --- packages/manager-api-client/src/manager-client.ts | 2 +- packages/types/src/rest-api-manager/common.ts | 1 + .../types/src/rest-api-manager/post-disconnect.ts | 12 +++++++++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/manager-api-client/src/manager-client.ts b/packages/manager-api-client/src/manager-client.ts index 84d61db97..0a90c8d0f 100644 --- a/packages/manager-api-client/src/manager-client.ts +++ b/packages/manager-api-client/src/manager-client.ts @@ -85,6 +85,6 @@ export class ManagerClient implements ClientProvider { } async disconnectHubs(opts: any) { - await this.client.post("disconnect", opts, {}, { json: true, parse: "json" }); + return this.client.post("disconnect", opts, {}, { json: true, parse: "json" }); } } diff --git a/packages/types/src/rest-api-manager/common.ts b/packages/types/src/rest-api-manager/common.ts index 9fc6ea0ff..6f170174d 100644 --- a/packages/types/src/rest-api-manager/common.ts +++ b/packages/types/src/rest-api-manager/common.ts @@ -12,6 +12,7 @@ export type ConnectedSTHInfo = { isConnectionActive: boolean, description?: string, tags?: Array , + disconnectReason?: string; }; export type HealthCheckInfo = { diff --git a/packages/types/src/rest-api-manager/post-disconnect.ts b/packages/types/src/rest-api-manager/post-disconnect.ts index ff12afd2e..399e4c29a 100644 --- a/packages/types/src/rest-api-manager/post-disconnect.ts +++ b/packages/types/src/rest-api-manager/post-disconnect.ts @@ -1,4 +1,14 @@ +import { OpResponse } from "../rest-api-multi-manager"; + export type PostDisconnectPayload = { limit?: number; - accessKey: string; + accessKey?: string; } + +export type PostDisconnectResponse = OpResponse<{ + managerId: string; + disconnected: { + sthId: string; + reason: string; + }[]; +}>; From fd363a9f47285a5f503efed18cabc3f41014d618 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 20 Mar 2023 14:33:33 +0000 Subject: [PATCH 134/231] si space disconnect --- packages/cli/src/lib/commands/space.ts | 33 +++++++++++++++++-- packages/model/src/stream-handler.ts | 2 ++ packages/symbols/src/cpm-message-code.ts | 3 +- packages/types/src/message-streams.ts | 2 +- .../src/rest-api-manager/post-disconnect.ts | 1 + 5 files changed, 36 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/lib/commands/space.ts b/packages/cli/src/lib/commands/space.ts index 72af7c7e4..901765d0c 100644 --- a/packages/cli/src/lib/commands/space.ts +++ b/packages/cli/src/lib/commands/space.ts @@ -1,4 +1,5 @@ /* eslint-disable no-console */ +import { PostDisconnectPayload } from "@scramjet/types/src/rest-api-manager"; import { CommandDefinition, isProductionEnv } from "../../types"; import { profileManager, sessionConfig } from "../config"; import { displayObject, displayStream } from "../output"; @@ -79,6 +80,32 @@ export const space: CommandDefinition = (program) => { await displayStream(await managerClient.getLogStream()); }); + spaceCmd + .command("disconnect") + .description("Disconnect self hosted Hubs from space") + .argument("", "The name of the Space") + .option("--id ", "Hub Id") + .option("--all", "Disconnects all self-hosted Hubs connected to Space", false) + .action(async (spaceName: string, id: string, all: string) => { + const mwClient = getMiddlewareClient(); + const managerClient = mwClient.getManagerClient(spaceName); + let opts = { } as PostDisconnectPayload; + + if (typeof id === "string") { + opts = { id }; + } + + if (all) { + opts = { limit: 0 }; + } + + if (!Object.keys(opts).length) { + throw new Error("Missing --id or --all"); + } + + displayObject(await managerClient.disconnectHubs(opts), profileManager.getProfileConfig().format); + }); + const accessKeyCmd = spaceCmd .command("access") .description("Manages Access Keys for active Space"); @@ -94,7 +121,7 @@ export const space: CommandDefinition = (program) => { throw new Error("No Space set"); } - displayObject(await mwClient.createAccessKey(spaceName, { description }), "json"); + displayObject(await mwClient.createAccessKey(spaceName, { description }), profileManager.getProfileConfig().format); }); accessKeyCmd.command("list") @@ -108,7 +135,7 @@ export const space: CommandDefinition = (program) => { throw new Error("No Space set"); } - displayObject(await mwClient.listAccessKeys(spaceName), "json"); + displayObject(await mwClient.listAccessKeys(spaceName), profileManager.getProfileConfig().format); }); accessKeyCmd.command("revoke") @@ -122,6 +149,6 @@ export const space: CommandDefinition = (program) => { throw new Error("No Space set"); } - displayObject(await mwClient.revokeAccessKey(spaceName, accessKey), "json"); + displayObject(await mwClient.revokeAccessKey(spaceName, accessKey), profileManager.getProfileConfig().format); }); }; diff --git a/packages/model/src/stream-handler.ts b/packages/model/src/stream-handler.ts index e40d6a791..82979b732 100644 --- a/packages/model/src/stream-handler.ts +++ b/packages/model/src/stream-handler.ts @@ -56,6 +56,7 @@ type ControlMessageHandlerList = { [CPMMessageCode.STH_ID]: ConfiguredMessageHandler[]; [CPMMessageCode.KEY_REVOKED]: ConfiguredMessageHandler[]; [CPMMessageCode.LIMIT_EXCEEDED]: ConfiguredMessageHandler[]; + [CPMMessageCode.ID_DROP]: ConfiguredMessageHandler[]; }; export class CommunicationHandler implements ICommunicationHandler { @@ -92,6 +93,7 @@ export class CommunicationHandler implements ICommunicationHandler { [CPMMessageCode.STH_ID]: [], [CPMMessageCode.KEY_REVOKED]: [], [CPMMessageCode.LIMIT_EXCEEDED]: [], + [CPMMessageCode.ID_DROP]: [], }; this.monitoringHandlerHash = { [RunnerMessageCode.ACKNOWLEDGE]: [], diff --git a/packages/symbols/src/cpm-message-code.ts b/packages/symbols/src/cpm-message-code.ts index b0acf222b..83a0f1604 100644 --- a/packages/symbols/src/cpm-message-code.ts +++ b/packages/symbols/src/cpm-message-code.ts @@ -14,5 +14,6 @@ export enum CPMMessageCode { CONFIRM_MSG = 8000, KEY_REVOKED = 9001, - LIMIT_EXCEEDED = 9002 + LIMIT_EXCEEDED = 9002, + ID_DROP = 9003 } diff --git a/packages/types/src/message-streams.ts b/packages/types/src/message-streams.ts index ce4b7bf34..edc0ceef5 100644 --- a/packages/types/src/message-streams.ts +++ b/packages/types/src/message-streams.ts @@ -95,7 +95,7 @@ export type EncodedMessage< export type ControlMessageCode = RunnerMessageCode.KILL | RunnerMessageCode.MONITORING_RATE | RunnerMessageCode.STOP | RunnerMessageCode.EVENT | RunnerMessageCode.PONG | - CPMMessageCode.STH_ID | CPMMessageCode.KEY_REVOKED | CPMMessageCode.LIMIT_EXCEEDED | + CPMMessageCode.STH_ID | CPMMessageCode.KEY_REVOKED | CPMMessageCode.LIMIT_EXCEEDED | CPMMessageCode.ID_DROP | RunnerMessageCode.INPUT_CONTENT_TYPE; export type EncodedControlMessage = EncodedMessage; diff --git a/packages/types/src/rest-api-manager/post-disconnect.ts b/packages/types/src/rest-api-manager/post-disconnect.ts index 399e4c29a..15453b5e6 100644 --- a/packages/types/src/rest-api-manager/post-disconnect.ts +++ b/packages/types/src/rest-api-manager/post-disconnect.ts @@ -1,6 +1,7 @@ import { OpResponse } from "../rest-api-multi-manager"; export type PostDisconnectPayload = { + id?: string; limit?: number; accessKey?: string; } From 54b556b737f09026ff9d93e8c4e2df6ac6aa47d1 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 27 Mar 2023 13:30:56 +0000 Subject: [PATCH 135/231] Get HTTP platform connection status --- packages/host/src/lib/cpm-connector.ts | 35 +++++++++++++++--------- packages/verser/src/lib/verser-client.ts | 4 +++ 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/packages/host/src/lib/cpm-connector.ts b/packages/host/src/lib/cpm-connector.ts index 92d7a8a41..80f2f5d23 100644 --- a/packages/host/src/lib/cpm-connector.ts +++ b/packages/host/src/lib/cpm-connector.ts @@ -23,6 +23,7 @@ import { TypedEmitter, generateSTHKey, normalizeUrl } from "@scramjet/utility"; import { ObjLogger } from "@scramjet/obj-logger"; import { ReasonPhrases } from "http-status-codes"; import { DuplexStream } from "@scramjet/api-server"; +import { VerserClientConnection } from "@scramjet/verser/src/types"; type STHInformation = { id?: string; @@ -225,7 +226,7 @@ export class CPMConnector extends TypedEmitter { * @returns {string} Host id. */ getId(): string | undefined { - return this.info.id; + return this.config.id; } /** @@ -251,6 +252,8 @@ export class CPMConnector extends TypedEmitter { }; } + await this.setLoadCheckMessageSender(); + StringStream.from(duplex.input as Readable) .JSONParse() .map(async (message: EncodedControlMessage) => { @@ -294,9 +297,10 @@ export class CPMConnector extends TypedEmitter { [CPMMessageCode.NETWORK_INFO, await this.getNetworkInfo()] ); - this.emit("connect"); - await this.setLoadCheckMessageSender(); + + + this.emit("connect"); return new Promise((resolve, reject) => { duplex.on("end", () => { @@ -334,11 +338,17 @@ export class CPMConnector extends TypedEmitter { this.verserClient.updateHeaders({ "x-sth-id": this.info.id }); } - let connection; + let connection: VerserClientConnection; try { this.logger.trace("Connecting to Manager", this.cpmUrl, this.cpmId); connection = await this.verserClient.connect(); + + connection.socket + .once("close", async () => { + this.logger.warn("CLOSE STATUS", connection.res.statusCode) + await this.handleConnectionClose(connection.res.statusCode || -1); + }); } catch (error: any) { this.logger.error("Can not connect to Manager", this.cpmUrl, this.cpmId, error.message); @@ -347,12 +357,7 @@ export class CPMConnector extends TypedEmitter { return; } - this.logger.info("Connected to Manager"); - - connection.socket - .once("close", async () => { - await this.handleConnectionClose(); - }); + this.logger.info("Connected..."); /** * @TODO: Distinguish existing `connect` request and started communication (Manager handled this host @@ -388,7 +393,7 @@ export class CPMConnector extends TypedEmitter { * Handles connection close. * Tries to reconnect. */ - async handleConnectionClose() { + async handleConnectionClose(connectionStatusCode: number) { this.handleCommunicationRequestEnd(); this.connection?.removeAllListeners(); @@ -402,6 +407,10 @@ export class CPMConnector extends TypedEmitter { clearInterval(this.loadInterval); } + if (connectionStatusCode === 403) { + this.isAbandoned = true; + } + await this.reconnect(); } @@ -428,9 +437,9 @@ export class CPMConnector extends TypedEmitter { this.isReconnecting = true; await new Promise((resolve, reject) => { - setTimeout(async () => { - this.logger.info("Connection lost, retrying", this.connectionAttempts); + this.logger.info("Connection lost, retrying", this.connectionAttempts); + setTimeout(async () => { await this.connect().then(resolve, reject); }, this.config.reconnectionDelay); }); diff --git a/packages/verser/src/lib/verser-client.ts b/packages/verser/src/lib/verser-client.ts index 1183e7367..de6721fba 100644 --- a/packages/verser/src/lib/verser-client.ts +++ b/packages/verser/src/lib/verser-client.ts @@ -11,6 +11,7 @@ import { TeceMux, TeceMuxChannel } from "./tecemux/tecemux"; type Events = { error: (err: Error) => void; + close: (reason: string) => void; }; /** @@ -110,10 +111,13 @@ export class VerserClient extends TypedEmitter { reject(err); }); + connectRequest.once("connect", (res, socket, head) => { + this.logger.info("HEAD", head.toString()); connectRequest.on("connect", (response, socket) => { this.socket = socket; this.mux(); + resolve({ res, socket }); resolve({ response, socket }); }); From 4a4bd495edc0bec700a571036e1d3b981b04ed1f Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 27 Mar 2023 14:10:40 +0000 Subject: [PATCH 136/231] Fix dropping STH by id --- packages/cli/src/lib/commands/space.ts | 8 ++++---- packages/host/src/lib/cpm-connector.ts | 6 ++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/cli/src/lib/commands/space.ts b/packages/cli/src/lib/commands/space.ts index 901765d0c..578116938 100644 --- a/packages/cli/src/lib/commands/space.ts +++ b/packages/cli/src/lib/commands/space.ts @@ -86,16 +86,16 @@ export const space: CommandDefinition = (program) => { .argument("", "The name of the Space") .option("--id ", "Hub Id") .option("--all", "Disconnects all self-hosted Hubs connected to Space", false) - .action(async (spaceName: string, id: string, all: string) => { + .action(async (spaceName: string, options: { id: string, all: boolean }) => { const mwClient = getMiddlewareClient(); const managerClient = mwClient.getManagerClient(spaceName); let opts = { } as PostDisconnectPayload; - if (typeof id === "string") { - opts = { id }; + if (typeof options.id === "string") { + opts = { id: options.id }; } - if (all) { + if (options.all) { opts = { limit: 0 }; } diff --git a/packages/host/src/lib/cpm-connector.ts b/packages/host/src/lib/cpm-connector.ts index 80f2f5d23..fa586e1aa 100644 --- a/packages/host/src/lib/cpm-connector.ts +++ b/packages/host/src/lib/cpm-connector.ts @@ -297,9 +297,6 @@ export class CPMConnector extends TypedEmitter { [CPMMessageCode.NETWORK_INFO, await this.getNetworkInfo()] ); - - - this.emit("connect"); return new Promise((resolve, reject) => { @@ -346,7 +343,8 @@ export class CPMConnector extends TypedEmitter { connection.socket .once("close", async () => { - this.logger.warn("CLOSE STATUS", connection.res.statusCode) + this.logger.warn("CLOSE STATUS", connection.res.statusCode); + await this.handleConnectionClose(connection.res.statusCode || -1); }); } catch (error: any) { From bf2460dd2bbebd3b82efce2e6748bad54a18efc8 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 20 Feb 2023 11:25:36 +0000 Subject: [PATCH 137/231] Add platform args to STH --- packages/host/src/lib/cpm-connector.ts | 2 +- packages/types/src/sth-configuration.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/host/src/lib/cpm-connector.ts b/packages/host/src/lib/cpm-connector.ts index fa586e1aa..5c0b3a889 100644 --- a/packages/host/src/lib/cpm-connector.ts +++ b/packages/host/src/lib/cpm-connector.ts @@ -171,8 +171,8 @@ export class CPMConnector extends TypedEmitter { headers: { "x-sth-description": typeof this.config.description !== "undefined" ? this.config.description : "", "x-sth-tags": JSON.stringify(typeof this.config.tags !== "undefined" ? this.config.tags : []), - "x-manager-id": cpmId, "x-sth-id": this.config.id || "", + "x-manager-id": cpmId, ...(orgId ? { "x-org-id": orgId } : {}), ...(sthKey ? { "Authorization": `Digest cnonce="${sthKey}"` } : {}) }, diff --git a/packages/types/src/sth-configuration.ts b/packages/types/src/sth-configuration.ts index 6044c8be5..9de337d82 100644 --- a/packages/types/src/sth-configuration.ts +++ b/packages/types/src/sth-configuration.ts @@ -159,7 +159,7 @@ export type STHConfiguration = { apiVersion: string; api: string; space: string; - hostType: "hub" | "federation" + hostType: "hub" | "federation"; }; /** From a537a692b02b6c5e76c77384868ae0dd2cf77fba Mon Sep 17 00:00:00 2001 From: patuwwy Date: Fri, 3 Mar 2023 08:59:55 +0000 Subject: [PATCH 138/231] si space accesskey generate|revoke --- packages/cli/src/lib/commands/space.ts | 6 +++--- packages/types/src/rest-api-manager/common.ts | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/cli/src/lib/commands/space.ts b/packages/cli/src/lib/commands/space.ts index 578116938..23d278ec3 100644 --- a/packages/cli/src/lib/commands/space.ts +++ b/packages/cli/src/lib/commands/space.ts @@ -106,9 +106,9 @@ export const space: CommandDefinition = (program) => { displayObject(await managerClient.disconnectHubs(opts), profileManager.getProfileConfig().format); }); - const accessKeyCmd = spaceCmd - .command("access") - .description("Manages Access Keys for active Space"); + const accessKeyCmd = spaceCmd + .command("access") + .description("Manages Access Keys for active Space"); accessKeyCmd.command("create") .argument("", "Key description") diff --git a/packages/types/src/rest-api-manager/common.ts b/packages/types/src/rest-api-manager/common.ts index 6f170174d..bf3f4daa6 100644 --- a/packages/types/src/rest-api-manager/common.ts +++ b/packages/types/src/rest-api-manager/common.ts @@ -5,13 +5,13 @@ export type ConnectedSTHInfoDetails = { } export type ConnectedSTHInfo = { - id: string, - info: ConnectedSTHInfoDetails, - healthy: boolean, - selfHosted: boolean, - isConnectionActive: boolean, - description?: string, - tags?: Array , + id: string; + info: ConnectedSTHInfoDetails; + healthy: boolean; + selfHosted: boolean; + isConnectionActive: boolean; + description?: string; + tags?: Array; disconnectReason?: string; }; From 8f874190d4278fe37fd7e7d5ed18a9f854024469 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 28 Mar 2023 14:06:15 +0000 Subject: [PATCH 139/231] R<->STH --- packages/verser/src/lib/verser-connection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/verser/src/lib/verser-connection.ts b/packages/verser/src/lib/verser-connection.ts index f93f8c783..3d671ca72 100644 --- a/packages/verser/src/lib/verser-connection.ts +++ b/packages/verser/src/lib/verser-connection.ts @@ -194,7 +194,7 @@ export class VerserConnection { createChannel(id: number) { if (!this.teceMux) throw new Error("TeCeMux not connected"); - return this.teceMux.multiplex({ channel: id }); + return await this.teceMux.multiplex({ channel: id }); } reconnect() { From 0913bb4b4bdc8de3790e0cfbfda577079dd3abab Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 6 Apr 2023 19:25:21 +0000 Subject: [PATCH 140/231] fix channels _id --- packages/runner/src/host-client.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/runner/src/host-client.ts b/packages/runner/src/host-client.ts index c98d26ebd..95a8d5ac8 100644 --- a/packages/runner/src/host-client.ts +++ b/packages/runner/src/host-client.ts @@ -47,6 +47,8 @@ class HostClient implements IHostClient { //protocol.logger.pipe(this.logger); hostSocket.setNoDelay(true); + //protocol.logger.pipe(this.logger); + const openConnections = await Promise.all( Array.from(Array(9)).map((_c, index) => protocol.multiplex({ channel: index })) ).then(async res => { From 0cad9e4d7653bb9981689ab469bab50ee0d3c0fc Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 11 Apr 2023 13:51:40 +0000 Subject: [PATCH 141/231] Remove debug listeners --- packages/verser/src/lib/tecemux/tecemux.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 2f7bd0e59..393c69729 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -58,6 +58,21 @@ export class TeceMux extends TypedEmitter { .on("error", (error) => { this.logger.error("Decoder error", error); }); + // .on("pause", () => { + // this.logger.warn("Decoder paused"); + // }) + // .on("close", () => { + // this.logger.warn("Decoder closed"); + // }) + // .on("end", () => { + // this.logger.warn("Decoder ended"); + // }) + // .on("abort", (error) => { + // this.logger.error("Decoder abort", error); + // }) + // .on("destroy", (error) => { + // this.logger.error("Decoder destroy", error); + // }); this.carrierDecoder.logger.updateBaseLog({ id: this.id }); this.carrierDecoder.logger.pipe(this.logger); From f7813d704523a347f28cd1cdd9b9de7b2963d548 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 13 Apr 2023 20:56:19 +0000 Subject: [PATCH 142/231] Disable nagle in R-H --- packages/runner/src/host-client.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/runner/src/host-client.ts b/packages/runner/src/host-client.ts index 95a8d5ac8..aea263d6c 100644 --- a/packages/runner/src/host-client.ts +++ b/packages/runner/src/host-client.ts @@ -48,6 +48,7 @@ class HostClient implements IHostClient { hostSocket.setNoDelay(true); //protocol.logger.pipe(this.logger); + hostSocket.setNoDelay(true); const openConnections = await Promise.all( Array.from(Array(9)).map((_c, index) => protocol.multiplex({ channel: index })) From 562a68eaa5aec9f11b80f2219523b9b6ab0b3b7c Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 22 May 2023 12:24:28 +0000 Subject: [PATCH 143/231] Rebase --- packages/host/src/lib/cpm-connector.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/host/src/lib/cpm-connector.ts b/packages/host/src/lib/cpm-connector.ts index 5c0b3a889..a1ab0511a 100644 --- a/packages/host/src/lib/cpm-connector.ts +++ b/packages/host/src/lib/cpm-connector.ts @@ -343,9 +343,9 @@ export class CPMConnector extends TypedEmitter { connection.socket .once("close", async () => { - this.logger.warn("CLOSE STATUS", connection.res.statusCode); + this.logger.warn("CLOSE STATUS", connection.response.statusCode); - await this.handleConnectionClose(connection.res.statusCode || -1); + await this.handleConnectionClose(connection.response.statusCode || -1); }); } catch (error: any) { this.logger.error("Can not connect to Manager", this.cpmUrl, this.cpmId, error.message); From b00b85507cdb89d9e65a190872a87745410afbf9 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 25 May 2023 13:54:34 +0000 Subject: [PATCH 144/231] Pause resume tecemux channel --- .../verser/src/lib/tecemux/frames-keeper.ts | 2 +- .../verser/src/lib/tecemux/tecemux-channel.ts | 11 ++++++ packages/verser/src/lib/tecemux/tecemux.ts | 35 +++++++++++++++++-- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/packages/verser/src/lib/tecemux/frames-keeper.ts b/packages/verser/src/lib/tecemux/frames-keeper.ts index cc2fbd3b9..06d385fc1 100644 --- a/packages/verser/src/lib/tecemux/frames-keeper.ts +++ b/packages/verser/src/lib/tecemux/frames-keeper.ts @@ -3,7 +3,7 @@ import { TypedEmitter } from "@scramjet/utility"; import { FramesKeeperEvents, FramesKeeperFrame, IFramesKeeper } from "./types"; export class FramesKeeper extends TypedEmitter implements IFramesKeeper { - #MAX_FRAMES_DIFFERENCE = 5; + #MAX_FRAMES_DIFFERENCE = 3; #framesSent = new Map(); diff --git a/packages/verser/src/lib/tecemux/tecemux-channel.ts b/packages/verser/src/lib/tecemux/tecemux-channel.ts index 36460e5b1..5e75f2446 100644 --- a/packages/verser/src/lib/tecemux/tecemux-channel.ts +++ b/packages/verser/src/lib/tecemux/tecemux-channel.ts @@ -45,6 +45,17 @@ export class TeceMuxChannel extends Duplex { ); } + sendPauseACK(sequenceNumber: number) { + //console.log("Cant write more. Send pause command"); + this.encoder.push( + this.encoder.createFrame(undefined, { + flagsArray: ["ACK", "SYN"], + acknowledgeNumber: sequenceNumber, + destinationPort: this._id + }) + ); + } + handlerFIN() { this.closedByFIN = true; diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index 393c69729..4ef6b81ef 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -110,15 +110,27 @@ export class TeceMux extends TypedEmitter { return 2; } + let channel = this.channels.get(destinationPort); + + if (flags.ACK && flags.SYN) { + channel?.encoder.pause(); + this.framesKeeper.handleACK(acknowledgeNumber); + console.log("Pause channel command received", channel?._id); + return 0; + } + if (flags.ACK) { + if (channel?.encoder.isPaused()) { + console.log("Pause and receive ack", channel._id); + } this.logger.trace("Received ACK flag for sequenceNumber", acknowledgeNumber); this.framesKeeper.handleACK(acknowledgeNumber); + channel?.encoder.resume(); + channel?.resume(); return 0; } - let channel = this.channels.get(destinationPort); - if (flags.FIN) { this.logger.trace(`Received FIN flag [C: ${destinationPort}]`, dataLength, frame.chunk, !!channel, channel?._id); @@ -146,7 +158,24 @@ export class TeceMux extends TypedEmitter { this.logger.warn("writing to channel [channel, length]", channel._id, dataLength); this.logger.warn("writing to channel [flowing, isPaused]", channel.readableFlowing, channel.isPaused()); - channel.push(new Uint8Array((frame.chunk as any).data), undefined); + const canWrite = channel.push(new Uint8Array((frame.chunk as any).data), undefined); + + if (!canWrite) { + console.log("cant write", channel._id); + channel.sendPauseACK(sequenceNumber); + + channel.once("resume", () => { + console.log("resumed, send ack to resume"); + //channel?.encoder.resume(); + channel?.sendACK(sequenceNumber); + }); + channel.once("drain", () => { + console.log("drained, send ack to resume"); + //channel?.encoder.resume(); + channel?.sendACK(sequenceNumber); + }); + return 0; + } } channel.sendACK(sequenceNumber); From a484281aa3a0a66bf01e2c88199d2dfda09d6b0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Thu, 1 Jun 2023 10:44:48 +0200 Subject: [PATCH 145/231] Sending TCP segment during open/close channel request --- packages/python-runner/inet.py | 4 ++-- packages/python-runner/tecemux.py | 35 ++++++++++++++++++++++++------- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/packages/python-runner/inet.py b/packages/python-runner/inet.py index d86e5238f..02e89831b 100644 --- a/packages/python-runner/inet.py +++ b/packages/python-runner/inet.py @@ -136,8 +136,8 @@ def parse_flags(value): ttl: int = field(default=255) protocol: int = field(default=6) checksum: int = field(default = 0, repr = lambda value: hex(value)) - src_addr: str = field(default='', converter = lambda value: inet_ntoa(value) if isinstance(value,bytes) else value) - dst_addr: str = field(default='', converter = lambda value: inet_ntoa(value) if isinstance(value,bytes) else value) + src_addr: str = field(default='0.0.0.0', converter = lambda value: inet_ntoa(value) if isinstance(value,bytes) else value) + dst_addr: str = field(default='0.0.0.0', converter = lambda value: inet_ntoa(value) if isinstance(value,bytes) else value) segment: TCPSegment = None def __attrs_post_init__(self): diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 76acbf891..45562d3bf 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -52,6 +52,20 @@ class ChannelContext: _global_queue: asyncio.Queue _channel_paused: bool = False + _channel_opened: bool = False + + async def open(self): + channel_id = int(self._name.value) + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=channel_id,flags=['PSH'])).build().to_buffer()) + + async def close(self): + channel_id = int(self._name.value) + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=channel_id,flags=['FIN'])).build().to_buffer()) + + async def pause(self): + self._channel_paused = True + #channel_id = int(self._name.value) + #await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=channel_id,flags=['FIN'])).build().to_buffer()) async def queue_up_incoming(self,buf): @@ -60,10 +74,9 @@ async def queue_up_incoming(self,buf): await self._writer.drain() async def queue_up_outcoming(self, data): - def wrap(channel_enum, buf): channel_id = int(channel_enum.value) - return IPPacket(src_addr='0.0.0.0',dst_addr='0.0.0.0',segment=TCPSegment(dst_port=channel_id,data=buf)).build().to_buffer() + return IPPacket(segment=TCPSegment(dst_port=channel_id,data=buf)).build().to_buffer() if self._channel_paused: await self._queue.put(data) @@ -82,6 +95,10 @@ async def read(self, len): return await self._reader.read(len) async def write(self, data): + + if not self._channel_opened: + await self.open() + await self.queue_up_outcoming(data) async def drain(self): @@ -135,15 +152,18 @@ def get_channel(self, channel): return self._channels[channel] async def stop(self): + for channel in self._channels.values(): + await channel.close() + await channel._writer.drain() + channel._writer.close() + await channel._writer.wait_closed() + + await self._writer.drain() self._writer.close() await self._writer.wait_closed() self._global_stop_event.set() - for channel in self._channels.values(): - await channel._writer.drain() - channel._writer.close() - await channel._writer.wait_closed() async def wait_until_end(self): @@ -176,7 +196,8 @@ async def incoming_data_forward(self): buffer = buffer + chunk - if incoming_parser_finish_loop.is_set() and len(buffer) < MINIMAL_IP_PACKET_LENGTH: + buffer_len = len(buffer) + if incoming_parser_finish_loop.is_set() and buffer_len > 0 and buffer_len < MINIMAL_IP_PACKET_LENGTH: self._logger.warning(f'Tecemux/MAIN: [<] Too few data is waiting in global buffer but stream finished') break From 858344e0d7d59bffc1778ed407f9357a3221150b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Thu, 1 Jun 2023 13:09:28 +0200 Subject: [PATCH 146/231] Fix missing typings and minor refactor --- packages/python-runner/tecemux.py | 213 +++++++++++--------- packages/python-runner/test/test_tecemux.py | 7 +- 2 files changed, 120 insertions(+), 100 deletions(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 45562d3bf..a467ebba0 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -1,32 +1,35 @@ import asyncio +import logging import socket + from attrs import define, field from inet import IPPacket, TCPSegment from hardcoded_magic_values import CommunicationChannels as CC -class TecemuxStreamReader: +class _StreamReader: def __init__(self, stream): self._stream = stream def __getattr__(self, name): return getattr(self._stream, name) - + async def readline(self): return await self._stream.readline() - + async def readuntil(self, separator=b'\n'): return await self._stream.readuntil(separator) async def read(self, n=-1): return await self._stream.read(n) - + async def readexactly(self, n): return await self._stream.readexactly(n) - -class TecemuxStreamWriter: + + +class _StreamWriter: def __init__(self, stream): self._stream = stream @@ -39,78 +42,98 @@ async def wait_closed(self): async def drain(self): return await self._stream.drain() - + + @define -class Tecemux: - @define - class ChannelContext: - _name: str - - _reader: TecemuxStreamReader - _writer: TecemuxStreamWriter - _queue: asyncio.Queue - _global_queue: asyncio.Queue - - _channel_paused: bool = False - _channel_opened: bool = False - - async def open(self): - channel_id = int(self._name.value) - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=channel_id,flags=['PSH'])).build().to_buffer()) - - async def close(self): - channel_id = int(self._name.value) - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=channel_id,flags=['FIN'])).build().to_buffer()) - - async def pause(self): - self._channel_paused = True - #channel_id = int(self._name.value) - #await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=channel_id,flags=['FIN'])).build().to_buffer()) - - - async def queue_up_incoming(self,buf): - pkt = IPPacket().from_buffer(buf) - self._writer.write(pkt.get_segment().data) - await self._writer.drain() +class _ChannelContext: + _channel_enum: CC + _reader: _StreamReader + _writer: _StreamWriter + _global_queue: asyncio.Queue + _logger: logging.Logger = field(default=None) - async def queue_up_outcoming(self, data): - def wrap(channel_enum, buf): - channel_id = int(channel_enum.value) - return IPPacket(segment=TCPSegment(dst_port=channel_id,data=buf)).build().to_buffer() + _internal_queue: asyncio.Queue = field(init=False) + _channel_paused: bool = field(default=False, init=False) + _channel_opened: bool = field(default=False, init=False) - if self._channel_paused: - await self._queue.put(data) - else: - while not self._queue.empty(): - try: - buf = await self._queue.get() - await self._global_queue.put(wrap(self._name,data)) - except asyncio.QueueEmpty: - break - await self._global_queue.put(wrap(self._name,data)) + def __attrs_post_init__(self): + self._internal_queue = asyncio.Queue() - + def _get_channel_name(self): + return str(self._channel_enum.name) - async def read(self, len): - return await self._reader.read(len) - - async def write(self, data): + def _get_channel_id(self): + return int(self._channel_enum.value) - if not self._channel_opened: - await self.open() + async def open(self): + self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel opening request is send') + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['PSH'])).build().to_buffer()) - await self.queue_up_outcoming(data) + async def close(self): + self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel close request is send') + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['FIN'])).build().to_buffer()) - async def drain(self): - if not self._channel_paused: - await self._writer.drain() - return True + async def pause(self): + self._channel_paused = True + self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel pause confirmation is send') + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK'])).build().to_buffer()) + + async def resume(self): + self._channel_paused = False + self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel resume confirmation is send') + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK'])).build().to_buffer()) + + async def queue_up_incoming(self, buf): + pkt = IPPacket().from_buffer(buf) + + if pkt.segment.is_flag('SYN') and pkt.segment.is_flag('ACK'): + self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel pause request received') + await self.pause() + return - def set_pause(self, state): - self._channel_paused = state + self._writer.write(pkt.get_segment().data) + await self._writer.drain() + + async def queue_up_outcoming(self, data): + + def wrap(channel_enum, buf): + channel_id = int(channel_enum.value) + return IPPacket(segment=TCPSegment(dst_port=channel_id, data=buf)).build().to_buffer() + + if self._channel_paused: + self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel paused. Data queued up internally for future') + await self._internal_queue.put(data) + else: + while not self._internal_queue.empty(): + try: + buf = await self._internal_queue.get() + await self._global_queue.put(wrap(self._channel_enum, data)) + except asyncio.QueueEmpty: + self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] All data stored during pause were redirected to global queue') + break + await self._global_queue.put(wrap(self._channel_enum, data)) - def get_name(self): - return self._name.name + async def read(self, len): + return await self._reader.read(len) + + async def write(self, data): + + if not self._channel_opened: + await self.open() + + await self.queue_up_outcoming(data) + + async def drain(self): + if not self._channel_paused: + await self._writer.drain() + return True + + def set_pause(self, state): + self._channel_paused = state + + +@define +class Tecemux: _queue: asyncio.Queue = field(default=None) _reader: asyncio.StreamReader = field(default=None) @@ -118,10 +141,10 @@ def get_name(self): _incoming_data_forwarder: asyncio.coroutine = field(default=None) _outcoming_data_forwarder: asyncio.coroutine = field(default=None) - - _channels = field(default={}) - _logger = field(default=None) - _global_stop_event = asyncio.Event() + + _channels: dict = field(default={}) + _logger: logging.Logger = field(default=None) + _global_stop_event: asyncio.Event = asyncio.Event() async def connect(self, reader, writer): self._reader = reader @@ -131,26 +154,27 @@ async def connect(self, reader, writer): @staticmethod async def prepare_tcp_connection(server_host, server_port): return await asyncio.open_connection(server_host, server_port) - + @staticmethod async def prepare_socket_connection(): rsock, wsock = socket.socketpair() reader, _ = await asyncio.open_unix_connection(sock=rsock) _, writer = await asyncio.open_unix_connection(sock=wsock) - return TecemuxStreamReader(reader),TecemuxStreamWriter(writer) - + return _StreamReader(reader), _StreamWriter(writer) async def prepare(self): self._queue = asyncio.Queue() - self._channels = {channel : Tecemux.ChannelContext(channel, *await Tecemux.prepare_socket_connection(), asyncio.Queue(), self._queue) for channel in CC } + self._channels = {channel: _ChannelContext(channel, *await Tecemux.prepare_socket_connection(), self._queue, self._logger) for channel in CC} - - def set_logger(self,logger): + def set_logger(self, logger): self._logger = logger def get_channel(self, channel): return self._channels[channel] + @staticmethod + def _chunk_preview(value): + return f'{value[0:5]}... ' if len(value)>5 else f'{value}' async def stop(self): for channel in self._channels.values(): await channel.close() @@ -158,22 +182,19 @@ async def stop(self): channel._writer.close() await channel._writer.wait_closed() - await self._writer.drain() self._writer.close() await self._writer.wait_closed() - self._global_stop_event.set() - - + self._global_stop_event.set() async def wait_until_end(self): await self._global_stop_event.wait() await asyncio.gather(*[self._incoming_data_forwarder, self._outcoming_data_forwarder]) - + self._logger.debug(f'Tecemux/MAIN: [-] Finished') async def loop(self): - + loop = asyncio.get_event_loop() self._incoming_data_forwarder = loop.create_task(self.incoming_data_forward()) @@ -181,9 +202,9 @@ async def loop(self): async def incoming_data_forward(self): buffer = b'' - + incoming_parser_finish_loop = asyncio.Event() - + MINIMAL_IP_PACKET_LENGTH = 20 READ_CHUNK_SIZE = 1024 @@ -197,43 +218,43 @@ async def incoming_data_forward(self): buffer = buffer + chunk buffer_len = len(buffer) + if incoming_parser_finish_loop.is_set() and buffer_len > 0 and buffer_len < MINIMAL_IP_PACKET_LENGTH: + self._logger.warning(f'Tecemux/MAIN: [<] Too few data is waiting in global buffer but stream finished') break - + while not (len(buffer) < MINIMAL_IP_PACKET_LENGTH): - + current_packet_size = IPPacket().from_buffer(buffer).len if len(buffer) >= current_packet_size: - + self._logger.debug(f'Tecemux/MAIN: [<] Full incomming packet from Transform Hub was received') single_packet_buffer = buffer[:current_packet_size] pkt = IPPacket().from_buffer(single_packet_buffer) channel = CC(str(pkt.get_segment().dst_port)) - + await self._channels[channel].queue_up_incoming(single_packet_buffer) - - self._logger.debug(f'Tecemux/MAIN: [<] Packet forwarded to {channel.name} steam') + + self._logger.debug(f'Tecemux/MAIN: [<] Packet {Tecemux._chunk_preview(single_packet_buffer)} forwarded to {channel.name} stream') buffer = buffer[current_packet_size:] else: self._logger.warning(f'Tecemux/MAIN: [<] Not full packet received. Getting additional data chunk') break self._logger.debug(f'Tecemux/MAIN: Incomming data forwarder finished') - async def outcoming_data_forward(self): - while not self._global_stop_event.is_set(): try: chunk = self._queue.get_nowait() - self._logger.debug(f'Tecemux/MAIN: [>] Outcoming chunk "{chunk}" is waiting to send to Transform Hub') + self._logger.debug(f'Tecemux/MAIN: [>] Outcoming chunk {Tecemux._chunk_preview(chunk)} is waiting to send to Transform Hub') self._writer.write(chunk) await self._writer.drain() self._queue.task_done() - self._logger.debug(f'Tecemux/MAIN: [>] Chunk "{chunk}" was sent to Transform Hub') + self._logger.debug(f'Tecemux/MAIN: [>] Chunk {Tecemux._chunk_preview(chunk)} was sent to Transform Hub') except asyncio.QueueEmpty: await asyncio.sleep(0) diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index fe2ce819a..c0115d5e5 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -1,6 +1,5 @@ -import asyncio import pytest -from tecemux import Tecemux, TecemuxStreamReader,TecemuxStreamWriter +from tecemux import Tecemux, _StreamReader,_StreamWriter from inet import IPPacket,TCPSegment from hardcoded_magic_values import CommunicationChannels as CC from runner_tecemux import get_logger @@ -45,8 +44,8 @@ async def test_socket_connection(self): await protocol.connect(*await Tecemux.prepare_socket_connection()) - assert isinstance(protocol._reader,TecemuxStreamReader) - assert isinstance(protocol._writer,TecemuxStreamWriter) + assert isinstance(protocol._reader,_StreamReader) + assert isinstance(protocol._writer,_StreamWriter) @pytest.mark.asyncio From ef0aadbdca43d6e06193bc39b4a75523a1e2cc0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 2 Jun 2023 12:03:39 +0200 Subject: [PATCH 147/231] Add missing changes --- packages/verser/src/lib/verser-client.ts | 5 +---- packages/verser/src/lib/verser-connection.ts | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/verser/src/lib/verser-client.ts b/packages/verser/src/lib/verser-client.ts index de6721fba..3096441fa 100644 --- a/packages/verser/src/lib/verser-client.ts +++ b/packages/verser/src/lib/verser-client.ts @@ -111,13 +111,10 @@ export class VerserClient extends TypedEmitter { reject(err); }); - connectRequest.once("connect", (res, socket, head) => { - this.logger.info("HEAD", head.toString()); connectRequest.on("connect", (response, socket) => { this.socket = socket; this.mux(); - resolve({ res, socket }); resolve({ response, socket }); }); @@ -195,4 +192,4 @@ export class VerserClient extends TypedEmitter { registerChannel(channelId: number, data: RegisteredChannelCallback) { this.registeredChannels.set(channelId, data); } -} +} \ No newline at end of file diff --git a/packages/verser/src/lib/verser-connection.ts b/packages/verser/src/lib/verser-connection.ts index 3d671ca72..ff428de3e 100644 --- a/packages/verser/src/lib/verser-connection.ts +++ b/packages/verser/src/lib/verser-connection.ts @@ -194,7 +194,7 @@ export class VerserConnection { createChannel(id: number) { if (!this.teceMux) throw new Error("TeCeMux not connected"); - return await this.teceMux.multiplex({ channel: id }); + return this.teceMux.multiplex({ channel: id }); } reconnect() { @@ -257,4 +257,4 @@ export class VerserConnection { getAgent() { return this.agent as Agent; } -} +} \ No newline at end of file From 13da4b384b92c37df4aa2cb6012cd1ba2c7407bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Tue, 6 Jun 2023 16:09:40 +0200 Subject: [PATCH 148/231] Sequence number & Pause & Resume channel support --- packages/python-runner/tecemux.py | 66 ++++++++++++++------- packages/python-runner/test/test_tecemux.py | 5 +- 2 files changed, 49 insertions(+), 22 deletions(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index a467ebba0..ab0b82048 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -1,5 +1,6 @@ import asyncio import logging +import random import socket from attrs import define, field @@ -64,33 +65,38 @@ def _get_channel_name(self): def _get_channel_id(self): return int(self._channel_enum.value) + + async def send_ACK(self, sequence_number): + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK'], ack=sequence_number))) + + async def _send_pause_ACK(self, sequence_number): + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK','SYN'], ack=sequence_number))) async def open(self): self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel opening request is send') - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['PSH'])).build().to_buffer()) + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['PSH']))) async def close(self): self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel close request is send') - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['FIN'])).build().to_buffer()) - - async def pause(self): - self._channel_paused = True - self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel pause confirmation is send') - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK'])).build().to_buffer()) - - async def resume(self): - self._channel_paused = False - self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel resume confirmation is send') - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK'])).build().to_buffer()) + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['FIN']))) async def queue_up_incoming(self, buf): pkt = IPPacket().from_buffer(buf) + #if SYN & ACK flag is up, pause channel if pkt.segment.is_flag('SYN') and pkt.segment.is_flag('ACK'): self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel pause request received') - await self.pause() + self.set_pause(True) + await self.send_ACK(pkt.segment.seq) return - + + #if SYN is down & ACK flag is up, reasume channel + if not pkt.segment.is_flag('SYN') and pkt.segment.is_flag('ACK'): + self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel reasume request received') + self.set_pause(False) + await self.send_ACK(pkt.segment.seq) + return + self._writer.write(pkt.get_segment().data) await self._writer.drain() @@ -98,7 +104,7 @@ async def queue_up_outcoming(self, data): def wrap(channel_enum, buf): channel_id = int(channel_enum.value) - return IPPacket(segment=TCPSegment(dst_port=channel_id, data=buf)).build().to_buffer() + return IPPacket(segment=TCPSegment(dst_port=channel_id, data=buf)) if self._channel_paused: self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel paused. Data queued up internally for future') @@ -145,10 +151,14 @@ class Tecemux: _channels: dict = field(default={}) _logger: logging.Logger = field(default=None) _global_stop_event: asyncio.Event = asyncio.Event() + _sequence_number: int = field(default=0) + _last_sequence_received: int = field(default=0) + async def connect(self, reader, writer): self._reader = reader self._writer = writer + self._sequence_number = abs(int((random.random() * (2 ** 32)) / 2)) self._global_stop_event.clear() @staticmethod @@ -172,6 +182,12 @@ def set_logger(self, logger): def get_channel(self, channel): return self._channels[channel] + def get_channel_reader(self, channel): + return self.get_channel(channel).reader + + def get_channel_writer(self, channel): + return self.get_channel(channel).writer + @staticmethod def _chunk_preview(value): return f'{value[0:5]}... ' if len(value)>5 else f'{value}' @@ -230,12 +246,13 @@ async def incoming_data_forward(self): if len(buffer) >= current_packet_size: - self._logger.debug(f'Tecemux/MAIN: [<] Full incomming packet from Transform Hub was received') - single_packet_buffer = buffer[:current_packet_size] pkt = IPPacket().from_buffer(single_packet_buffer) - channel = CC(str(pkt.get_segment().dst_port)) + self._last_sequence_received = pkt.get_segment().seq + self._logger.debug(f'Tecemux/MAIN: [<] Full incomming packet with sequence number {self._last_sequence_received} from Transform Hub was received') + channel = CC(str(pkt.get_segment().dst_port)) + await self._channels[channel].queue_up_incoming(single_packet_buffer) self._logger.debug(f'Tecemux/MAIN: [<] Packet {Tecemux._chunk_preview(single_packet_buffer)} forwarded to {channel.name} stream') @@ -249,12 +266,21 @@ async def incoming_data_forward(self): async def outcoming_data_forward(self): while not self._global_stop_event.is_set(): try: - chunk = self._queue.get_nowait() + pkt = self._queue.get_nowait() + + #inject sequence number + if pkt.segment.seq == 0: + pkt.segment.seq = self._sequence_number + self._sequence_number += 1 + + #calc cheksum + chunk = pkt.build().to_buffer() + self._logger.debug(f'Tecemux/MAIN: [>] Outcoming chunk {Tecemux._chunk_preview(chunk)} is waiting to send to Transform Hub') self._writer.write(chunk) await self._writer.drain() self._queue.task_done() - self._logger.debug(f'Tecemux/MAIN: [>] Chunk {Tecemux._chunk_preview(chunk)} was sent to Transform Hub') + self._logger.debug(f'Tecemux/MAIN: [>] Chunk {Tecemux._chunk_preview(chunk)} with sequence number: {pkt.segment.seq} was sent to Transform Hub') except asyncio.QueueEmpty: await asyncio.sleep(0) diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index c0115d5e5..66f0c59a4 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -33,6 +33,7 @@ class TestTecemux: def test_default_init(self): protocol = Tecemux() assert isinstance(protocol, Tecemux) + assert protocol._sequence_number == 0 @pytest.mark.asyncio async def test_socket_connection(self): @@ -46,7 +47,7 @@ async def test_socket_connection(self): assert isinstance(protocol._reader,_StreamReader) assert isinstance(protocol._writer,_StreamWriter) - + assert protocol._sequence_number > 0 @pytest.mark.asyncio async def test_forward_from_main_to_channel(self, local_socket_connection): @@ -55,7 +56,7 @@ async def test_forward_from_main_to_channel(self, local_socket_connection): data_to_send ="{'foo':'bar'}" destination_channel = CC.CONTROL - pkt = IPPacket(src_addr='172.25.44.3',dst_addr='172.25.44.254',segment=TCPSegment(dst_port=int(destination_channel.value),flags=['ACK'],data=data_to_send)) + pkt = IPPacket(src_addr='172.25.44.3',dst_addr='172.25.44.254',segment=TCPSegment(dst_port=int(destination_channel.value),flags=['PSH'],data=data_to_send)) client_a._writer.write(pkt.to_buffer()) await client_a._writer.drain() From b302e1c82908bb1c40c674e450b909f5d677a8a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Wed, 7 Jun 2023 11:46:46 +0200 Subject: [PATCH 149/231] Python runner with TeceMux support --- packages/python-runner/runner.py | 33 ++++++++---- packages/python-runner/runner_tecemux.py | 65 ------------------------ 2 files changed, 24 insertions(+), 74 deletions(-) delete mode 100644 packages/python-runner/runner_tecemux.py diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index 73e1d65eb..0726e9b97 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -4,6 +4,7 @@ import codecs import json from pyee.asyncio import AsyncIOEventEmitter +from tecemux import Tecemux import importlib.util from io import DEFAULT_BUFFER_SIZE as CHUNK_SIZE import types @@ -12,6 +13,7 @@ from hardcoded_magic_values import CommunicationChannels as CC from hardcoded_magic_values import RunnerMessageCodes as msg_codes +USE_TECEMUX = os.getenv('USE_TECEMUX') or True sequence_path = os.getenv('SEQUENCE_PATH') server_port = os.getenv('INSTANCES_SERVER_PORT') @@ -34,11 +36,17 @@ def __init__(self, instance_id, sequence_path, log_setup) -> None: self.health_check = lambda: {'healthy': True} self.emitter = AsyncIOEventEmitter() self.keep_alive_requested = False - - + self.protocol = None + self.streams = {} + @staticmethod + def is_incoming(channel): + return channel in [CC.STDIN, CC.IN, CC.CONTROL] + async def main(self, server_host, server_port): - self.logger.info('Connecting to host...') - await self.init_connections(server_host, server_port) + if USE_TECEMUX: + await self.init_tecemux(server_host, server_port) + else: + await self.init_legacy_connections(server_host, server_port) # Do this early to have access to any thrown exceptions and logs. self.connect_stdio() @@ -52,8 +60,18 @@ async def main(self, server_host, server_port): self.load_sequence() await self.run_instance(config, args) + async def init_tecemux(self, server_host, server_port): + self.logger.info('Connecting to host with TeceMux...') + self.protocol = Tecemux() + await self.protocol.connect(*await Tecemux.prepare_tcp_connection(server_host, server_port)) - async def init_connections(self, host, port): + self.streams = { + channel: self.protocol.get_channel_reader(channel) if Runner.is_incoming(channel) else self.protocol.get_channel_writer(channel) + for channel in self.protocol._channels + } + + async def init_legacy_connections(self, host, port): + self.logger.info('Connecting to host with legacy method...') async def connect(channel): self.logger.debug(f'Connecting to {host}:{port}...') reader, writer = await asyncio.open_connection(host, port) @@ -69,12 +87,9 @@ async def connect(channel): conn_futures = [connect(channel) for channel in CC] connections = await asyncio.gather(*conn_futures) - def is_incoming(channel): - return channel in [CC.STDIN, CC.IN, CC.CONTROL] - # Pick read or write stream depending on channel. self.streams = { - channel: reader if is_incoming(channel) else writer + channel: reader if Runner.is_incoming(channel) else writer for channel, reader, writer in connections } diff --git a/packages/python-runner/runner_tecemux.py b/packages/python-runner/runner_tecemux.py deleted file mode 100644 index 6c6a1b18b..000000000 --- a/packages/python-runner/runner_tecemux.py +++ /dev/null @@ -1,65 +0,0 @@ -import asyncio -import sys -import codecs -from tecemux import Tecemux -from logging_setup import LoggingSetup -from hardcoded_magic_values import CommunicationChannels as CC - -def get_logger(): - if not hasattr(get_logger, "log_setup"): - get_logger.log_setup = LoggingSetup(sys.stdout) - return get_logger.log_setup.logger - -sequence_path = '/tmp/' -server_port = 8001 -server_host = 'localhost' -instance_id = '1234' - -class Runner: - def __init__(self, instance_id, sequence_path, logger) -> None: - self.instance_id = instance_id - self.seq_path = sequence_path - self.logger = logger - - self.protocol = None - - def connect_stdio(self): - - sys.stdout = codecs.getwriter('utf-8')(self.protocol.get_channel(CC.STDOUT).writer) - sys.stderr = codecs.getwriter('utf-8')(self.protocol.get_channel(CC.STDERR).writer) - sys.stdout.flush = lambda: True - sys.stderr.flush = lambda: True - - - - async def main(self, server_host, server_port): - self.logger.info('Connecting to host...') - - self.protocol = Tecemux() - self.protocol.set_logger(get_logger()) - - # self.logger.debug(f'Connecting to {server_host}:{server_port}...') - # await self.protocol.connect(*await Tecemux.prepare_tcp_connection(server_host, server_port)) - - self.logger.debug(f'Connecting locally via unix sockets') - await self.protocol.connect(*await Tecemux.prepare_socket_connection()) - - self.logger.debug('Connected.') - - await self.protocol.prepare() - - #self.connect_stdio() - - await self.protocol.loop() - - self.logger.debug("Run sequence") - - await self.protocol.stop() - - await self.protocol.wait_until_end() - - self.logger.debug("End main") - -if __name__ == '__main__': - runner = Runner(instance_id, sequence_path, get_logger()) - asyncio.run(runner.main(server_host, server_port)) From 2594af6434c0e88ca8d43746aa2493277cfbd90f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Wed, 7 Jun 2023 14:33:33 +0200 Subject: [PATCH 150/231] First try to communitate with STH --- packages/python-runner/runner.py | 17 +++++++++-- packages/python-runner/tecemux.py | 33 +++++++++++++-------- packages/python-runner/test/test_tecemux.py | 8 ++++- 3 files changed, 43 insertions(+), 15 deletions(-) diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index 0726e9b97..538478ee4 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -3,6 +3,7 @@ import os import codecs import json +import pydevd from pyee.asyncio import AsyncIOEventEmitter from tecemux import Tecemux import importlib.util @@ -13,7 +14,16 @@ from hardcoded_magic_values import CommunicationChannels as CC from hardcoded_magic_values import RunnerMessageCodes as msg_codes -USE_TECEMUX = os.getenv('USE_TECEMUX') or True + +if bool(os.environ.get('DEVELOPMENT')): + import time + while pydevd.get_global_debugger() is None or not pydevd.get_global_debugger().ready_to_run: + time.sleep(0.3) + +def debugger(): + pydevd.settrace() + +USE_TECEMUX = True sequence_path = os.getenv('SEQUENCE_PATH') server_port = os.getenv('INSTANCES_SERVER_PORT') @@ -25,6 +35,7 @@ def send_encoded_msg(stream, msg_code, data={}): message = json.dumps([msg_code.value, data]) stream.write(f'{message}\r\n'.encode()) +debugger() class Runner: def __init__(self, instance_id, sequence_path, log_setup) -> None: @@ -64,9 +75,10 @@ async def init_tecemux(self, server_host, server_port): self.logger.info('Connecting to host with TeceMux...') self.protocol = Tecemux() await self.protocol.connect(*await Tecemux.prepare_tcp_connection(server_host, server_port)) + await self.protocol.prepare() self.streams = { - channel: self.protocol.get_channel_reader(channel) if Runner.is_incoming(channel) else self.protocol.get_channel_writer(channel) + channel: self.protocol._channels[channel]._reader if Runner.is_incoming(channel) else self.protocol._channels[channel]._writer for channel in self.protocol._channels } @@ -257,6 +269,7 @@ async def connect_input_stream(self, input_stream): elif input_type == 'application/octet-stream': self.logger.debug('Opening input in binary mode...') input = Stream.read_from(self.streams[CC.IN], chunk_size=CHUNK_SIZE) + else: raise TypeError(f'Unsupported input type: {repr(input_type)}') diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index ab0b82048..76f5fde99 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -17,17 +17,26 @@ def __init__(self, stream): def __getattr__(self, name): return getattr(self._stream, name) - async def readline(self): - return await self._stream.readline() + def readline(self): + return self._stream.readline() - async def readuntil(self, separator=b'\n'): - return await self._stream.readuntil(separator) + def readuntil(self, separator=b'\n'): + return self._stream.readuntil(separator) - async def read(self, n=-1): - return await self._stream.read(n) + def read(self, n=-1): + return self._stream.read(n) - async def readexactly(self, n): - return await self._stream.readexactly(n) + def readexactly(self, n): + return self._stream.readexactly(n) + + def __aiter__(self): + return self._stream.__aiter__ + + async def __anext__(self): + val = await self._readline() + if val == b'': + raise StopAsyncIteration + return val class _StreamWriter: @@ -38,11 +47,11 @@ def __init__(self, stream): def __getattr__(self, name): return getattr(self._stream, name) - async def wait_closed(self): - return await self._stream.wait_closed() + def wait_closed(self): + return self._stream.wait_closed() - async def drain(self): - return await self._stream.drain() + def drain(self): + return self._stream.drain() @define diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index 66f0c59a4..603236d31 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -1,8 +1,14 @@ import pytest +import sys from tecemux import Tecemux, _StreamReader,_StreamWriter from inet import IPPacket,TCPSegment +from logging_setup import LoggingSetup from hardcoded_magic_values import CommunicationChannels as CC -from runner_tecemux import get_logger + +def get_logger(): + if not hasattr(get_logger, "log_setup"): + get_logger.log_setup = LoggingSetup(sys.stdout) + return get_logger.log_setup.logger @pytest.fixture() From 3c54c63e15677a93a839971d48ff3a0a771ceec4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 9 Jun 2023 14:04:35 +0200 Subject: [PATCH 151/231] TCP Options support --- packages/python-runner/inet.py | 32 +++++++++++-- packages/python-runner/runner.py | 74 +++++++++++-------------------- packages/python-runner/tecemux.py | 30 ++++++++----- 3 files changed, 71 insertions(+), 65 deletions(-) diff --git a/packages/python-runner/inet.py b/packages/python-runner/inet.py index 02e89831b..89a38de4a 100644 --- a/packages/python-runner/inet.py +++ b/packages/python-runner/inet.py @@ -6,6 +6,20 @@ @define class TCPSegment: + class Options: + EOL = 0 + NOP = 1 + MSS = 2 + WSOPT= 3 + SACKPERM = 4 + SACK = 5 + TSOPT = 8 + + @staticmethod + def parse_options(val): + return val + + class Flags: FIN = 0x01 # end of data SYN = 0x02 # synchronize sequence numbers @@ -66,10 +80,20 @@ def parse_flags(value): data: bytes = field(default=b'', repr = lambda value: f'{value[0:5]}... ' \ if len(value)>5 else f'{value}') + opt: bytes = field(default=b'', converter = lambda value: TCPSegment.Options.parse_options(value)) + @classmethod def from_buffer(cls,buffer): - return cls(*struct.unpack("!HHIIBBHHH", buffer[0:20]), buffer[20:]) - + TCP_MIN = 20 + src_port, dst_port, seq, ack, offres, flags, win, checksum, urp = struct.unpack("!HHIIBBHHH", buffer[0:TCP_MIN]) + hdr_len = (offres >> 4) * 4 + + if hdr_len <= TCP_MIN: + return cls(src_port, dst_port, seq, ack, offres, flags, win, checksum, urp, buffer[TCP_MIN:], b'') + + if hdr_len > TCP_MIN: + return cls(src_port, dst_port, seq, ack, offres, flags, win, checksum, urp, buffer[hdr_len:],buffer[TCP_MIN:hdr_len]) + def to_buffer(self): return struct.pack('!HHIIBBHHH', \ @@ -136,8 +160,8 @@ def parse_flags(value): ttl: int = field(default=255) protocol: int = field(default=6) checksum: int = field(default = 0, repr = lambda value: hex(value)) - src_addr: str = field(default='0.0.0.0', converter = lambda value: inet_ntoa(value) if isinstance(value,bytes) else value) - dst_addr: str = field(default='0.0.0.0', converter = lambda value: inet_ntoa(value) if isinstance(value,bytes) else value) + src_addr: str = field(default='10.0.0.1', converter = lambda value: inet_ntoa(value) if isinstance(value,bytes) else value) + dst_addr: str = field(default='10.0.0.2', converter = lambda value: inet_ntoa(value) if isinstance(value,bytes) else value) segment: TCPSegment = None def __attrs_post_init__(self): diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index 538478ee4..cd6b81906 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -23,17 +23,16 @@ def debugger(): pydevd.settrace() -USE_TECEMUX = True - sequence_path = os.getenv('SEQUENCE_PATH') server_port = os.getenv('INSTANCES_SERVER_PORT') server_host = os.getenv('INSTANCES_SERVER_HOST') or 'localhost' instance_id = os.getenv('INSTANCE_ID') -def send_encoded_msg(stream, msg_code, data={}): +async def send_encoded_msg(stream, msg_code, data={}): message = json.dumps([msg_code.value, data]) - stream.write(f'{message}\r\n'.encode()) + await stream.write(f'{message}\r\n'.encode()) + await stream.drain() debugger() @@ -54,10 +53,8 @@ def is_incoming(channel): return channel in [CC.STDIN, CC.IN, CC.CONTROL] async def main(self, server_host, server_port): - if USE_TECEMUX: - await self.init_tecemux(server_host, server_port) - else: - await self.init_legacy_connections(server_host, server_port) + + await self.init_tecemux(server_host, server_port) # Do this early to have access to any thrown exceptions and logs. self.connect_stdio() @@ -76,35 +73,9 @@ async def init_tecemux(self, server_host, server_port): self.protocol = Tecemux() await self.protocol.connect(*await Tecemux.prepare_tcp_connection(server_host, server_port)) await self.protocol.prepare() + await self.protocol.loop() - self.streams = { - channel: self.protocol._channels[channel]._reader if Runner.is_incoming(channel) else self.protocol._channels[channel]._writer - for channel in self.protocol._channels - } - - async def init_legacy_connections(self, host, port): - self.logger.info('Connecting to host with legacy method...') - async def connect(channel): - self.logger.debug(f'Connecting to {host}:{port}...') - reader, writer = await asyncio.open_connection(host, port) - self.logger.debug('Connected.') - - writer.write(self.instance_id.encode()) - writer.write(channel.value.encode()) - await writer.drain() - self.logger.debug(f'Sent ID {self.instance_id} on {channel.name}.') - - return (channel, reader, writer) - - conn_futures = [connect(channel) for channel in CC] - connections = await asyncio.gather(*conn_futures) - - # Pick read or write stream depending on channel. - self.streams = { - channel: reader if Runner.is_incoming(channel) else writer - for channel, reader, writer in connections - } - + self.streams = self.protocol._channels def connect_stdio(self): sys.stdout = codecs.getwriter('utf-8')(self.streams[CC.STDOUT]) @@ -121,17 +92,22 @@ def connect_log_stream(self): log_stream = codecs.getwriter('utf-8')(self.streams[CC.LOG]) self._logging_setup.switch_target(log_stream) self._logging_setup.flush_temp_handler() - self.logger.info('Log stream connected.') + self.protocol.set_logger(self.logger) + + for channel in self.protocol._channels.values(): + channel._set_logger(self.logger) + + self.logger.info('Log stream connected.') async def handshake(self): monitoring = self.streams[CC.MONITORING] control = self.streams[CC.CONTROL] self.logger.info(f'Sending PING') - send_encoded_msg(monitoring, msg_codes.PING) + await send_encoded_msg(monitoring, msg_codes.PING) - message = await control.readuntil(b'\n') + message = await control.read(8) self.logger.info(f'Got message: {message}') code, data = json.loads(message.decode()) @@ -147,7 +123,7 @@ async def handshake(self): 'requires': '', 'contentType': '' } - send_encoded_msg(monitoring, msg_codes.PANG, pang_requires_data) + await send_encoded_msg(monitoring, msg_codes.PANG, pang_requires_data) if code == msg_codes.PONG.value: self.logger.info(f'Got configuration: {data}') @@ -169,7 +145,7 @@ async def connect_control_stream(self): if code == msg_codes.STOP.value: await self.handle_stop(data) if code == msg_codes.EVENT.value: - self.emitter.emit(data['eventName'], data['message'] if 'message' in data else None) + await self.emitter.emit(data['eventName'], data['message'] if 'message' in data else None) async def handle_stop(self, data): @@ -182,10 +158,10 @@ async def handle_stop(self, data): await handler(timeout, can_keep_alive) except Exception as e: self.logger.error('Error stopping sequence', e) - send_encoded_msg(self.streams[CC.MONITORING], msg_codes.SEQUENCE_STOPPED, e) + await send_encoded_msg(self.streams[CC.MONITORING], msg_codes.SEQUENCE_STOPPED, e) if not can_keep_alive or not self.keep_alive_requested: - send_encoded_msg(self.streams[CC.MONITORING], msg_codes.SEQUENCE_STOPPED, {}) + await send_encoded_msg(self.streams[CC.MONITORING], msg_codes.SEQUENCE_STOPPED, {}) self.exit_immediately() await self.cleanup() @@ -193,7 +169,7 @@ async def handle_stop(self, data): async def setup_heartbeat(self): while True: - send_encoded_msg( + await send_encoded_msg( self.streams[CC.MONITORING], msg_codes.MONITORING, self.health_check(), @@ -228,12 +204,12 @@ async def run_instance(self, config, args): produces = getattr(result, 'provides', None) or getattr(self.sequence, 'provides', None) if produces: self.logger.info(f'Sending PANG with {produces}') - send_encoded_msg(monitoring, msg_codes.PANG, produces) + await send_encoded_msg(monitoring, msg_codes.PANG, produces) consumes = getattr(result, 'requires', None) or getattr(self.sequence, 'requires', None) if consumes: self.logger.info(f'Sending PANG with {consumes}') - send_encoded_msg(monitoring, msg_codes.PANG, consumes) + await send_encoded_msg(monitoring, msg_codes.PANG, consumes) if isinstance(result, types.AsyncGeneratorType): result = Stream.read_from(result) @@ -301,7 +277,7 @@ async def forward_output_stream(self, output): async def send_keep_alive(self, timeout: int = 0, can_keep_alive: bool = False): monitoring = self.streams[CC.MONITORING] - send_encoded_msg(monitoring, msg_codes.ALIVE) + await send_encoded_msg(monitoring, msg_codes.ALIVE) self.keep_alive_requested = True await asyncio.sleep(timeout) @@ -327,8 +303,8 @@ def set_health_check(self, health_check): def on(self, event_name, callback): self.emitter.on(event_name, callback) - def emit(self, event_name, message=''): - send_encoded_msg( + async def emit(self, event_name, message=''): + await send_encoded_msg( self.monitoring, msg_codes.EVENT, {'eventName': event_name, 'message': message} diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 76f5fde99..9f5590725 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -28,15 +28,6 @@ def read(self, n=-1): def readexactly(self, n): return self._stream.readexactly(n) - - def __aiter__(self): - return self._stream.__aiter__ - - async def __anext__(self): - val = await self._readline() - if val == b'': - raise StopAsyncIteration - return val class _StreamWriter: @@ -66,6 +57,15 @@ class _ChannelContext: _channel_paused: bool = field(default=False, init=False) _channel_opened: bool = field(default=False, init=False) + def __aiter__(self): + return self._reader._stream.__aiter__ + + async def __anext__(self): + val = await self._reader._stream.readline() + if val == b'': + raise StopAsyncIteration + return val + def __attrs_post_init__(self): self._internal_queue = asyncio.Queue() @@ -74,6 +74,9 @@ def _get_channel_name(self): def _get_channel_id(self): return int(self._channel_enum.value) + + def _set_logger(self, logger): + self._logger = logger async def send_ACK(self, sequence_number): await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK'], ack=sequence_number))) @@ -82,7 +85,7 @@ async def _send_pause_ACK(self, sequence_number): await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK','SYN'], ack=sequence_number))) async def open(self): - self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel opening request is send') + #self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel opening request is send') await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['PSH']))) async def close(self): @@ -128,8 +131,11 @@ def wrap(channel_enum, buf): break await self._global_queue.put(wrap(self._channel_enum, data)) - async def read(self, len): - return await self._reader.read(len) + def read(self, len): + return self._reader.read(len) + + def readuntil(self, separator=b'\n'): + return self._reader.readuntil(separator) async def write(self, data): From 544034584c283de4f1a4ec6bc838cd1eb73f8b42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 9 Jun 2023 16:16:40 +0200 Subject: [PATCH 152/231] Tecemux protocol minor changes --- packages/python-runner/inet.py | 6 +++--- packages/python-runner/runner.py | 2 +- packages/python-runner/tecemux.py | 13 ++++++++----- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/packages/python-runner/inet.py b/packages/python-runner/inet.py index 89a38de4a..15dd0d18a 100644 --- a/packages/python-runner/inet.py +++ b/packages/python-runner/inet.py @@ -149,15 +149,15 @@ def parse_flags(value): return res ihl: int = 5 - version: int = field( default=4, converter = lambda value: value >> 4) + version: int = field( default=69, converter = lambda value: value >> 4) tos: int = field(default=0) len: int = field(default=0) - ids: int = field(default=0) + ids: int = field(default=1) flags_offset: int = field(default = 0) flags: int = field(default = 0, init= False, repr = lambda value: IPPacket.Flags.flags_to_str(value), \ converter = lambda value: IPPacket.Flags.parse_flags(value)) offset: int = field(default = 0, init=False) - ttl: int = field(default=255) + ttl: int = field(default=64) protocol: int = field(default=6) checksum: int = field(default = 0, repr = lambda value: hex(value)) src_addr: str = field(default='10.0.0.1', converter = lambda value: inet_ntoa(value) if isinstance(value,bytes) else value) diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index cd6b81906..31de0fb0e 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -70,7 +70,7 @@ async def main(self, server_host, server_port): async def init_tecemux(self, server_host, server_port): self.logger.info('Connecting to host with TeceMux...') - self.protocol = Tecemux() + self.protocol = Tecemux(instance_id=self.instance_id) await self.protocol.connect(*await Tecemux.prepare_tcp_connection(server_host, server_port)) await self.protocol.prepare() await self.protocol.loop() diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 9f5590725..186ee66c0 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -48,11 +48,14 @@ def drain(self): @define class _ChannelContext: _channel_enum: CC + _reader: _StreamReader _writer: _StreamWriter + _global_queue: asyncio.Queue - _logger: logging.Logger = field(default=None) + _global_instance_id: str + _logger: logging.Logger = field(default=None) _internal_queue: asyncio.Queue = field(init=False) _channel_paused: bool = field(default=False, init=False) _channel_opened: bool = field(default=False, init=False) @@ -86,11 +89,11 @@ async def _send_pause_ACK(self, sequence_number): async def open(self): #self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel opening request is send') - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['PSH']))) + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'' if self._global_instance_id is None else self._global_instance_id, flags=['PSH']))) async def close(self): self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel close request is send') - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['FIN']))) + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(),data=b'' if self._global_instance_id is None else self._global_instance_id, flags=['FIN']))) async def queue_up_incoming(self, buf): pkt = IPPacket().from_buffer(buf) @@ -163,13 +166,13 @@ class Tecemux: _incoming_data_forwarder: asyncio.coroutine = field(default=None) _outcoming_data_forwarder: asyncio.coroutine = field(default=None) + _instance_id: str = field(default=None) _channels: dict = field(default={}) _logger: logging.Logger = field(default=None) _global_stop_event: asyncio.Event = asyncio.Event() _sequence_number: int = field(default=0) _last_sequence_received: int = field(default=0) - async def connect(self, reader, writer): self._reader = reader self._writer = writer @@ -189,7 +192,7 @@ async def prepare_socket_connection(): async def prepare(self): self._queue = asyncio.Queue() - self._channels = {channel: _ChannelContext(channel, *await Tecemux.prepare_socket_connection(), self._queue, self._logger) for channel in CC} + self._channels = {channel: _ChannelContext(channel, *await Tecemux.prepare_socket_connection(), self._queue, self._instance_id ,self._logger) for channel in CC} def set_logger(self, logger): self._logger = logger From 1ae7b94735ef6ba7c8b6a43137a2db3caa4aeb78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Mon, 12 Jun 2023 12:58:45 +0200 Subject: [PATCH 153/231] Fix wrong TCP header length calculation --- packages/python-runner/inet.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/python-runner/inet.py b/packages/python-runner/inet.py index 15dd0d18a..cd6553fce 100644 --- a/packages/python-runner/inet.py +++ b/packages/python-runner/inet.py @@ -74,7 +74,7 @@ def parse_flags(value): offset: int = field(default = 0, converter = lambda value: value >> 4) flags: int = field(default = 0, repr = lambda value: TCPSegment.Flags.flags_to_str(value), \ converter = lambda value: TCPSegment.Flags.parse_flags(value)) - win: int = field(default = 0) + win: int = field(default = 8192) checksum: int = field(default = 0, repr = lambda value: hex(value)) urp: int = field(default = 0) data: bytes = field(default=b'', repr = lambda value: f'{value[0:5]}... ' \ @@ -215,7 +215,9 @@ def to_buffer(self): inet_aton(self.dst_addr)) + data def _validate_tcp(self): - + + tcp_hdr_len = 5 + self.segment.offset = tcp_hdr_len << 4 self.segment.checksum = 0 tcp_segment = self.segment.to_buffer() From bfb94d9a3a95e5fa7ca33b82d00d75d1a79e9a31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Mon, 12 Jun 2023 15:28:57 +0200 Subject: [PATCH 154/231] Big/Litte-endianess switch --- packages/python-runner/inet.py | 26 ++++++++++++++++-------- packages/python-runner/test/test_inet.py | 20 +++++++++++++++++- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/packages/python-runner/inet.py b/packages/python-runner/inet.py index cd6553fce..e79ffcbcf 100644 --- a/packages/python-runner/inet.py +++ b/packages/python-runner/inet.py @@ -4,6 +4,16 @@ from socket import inet_ntoa, inet_aton from attrs import define,field +ENDIANESS = '<' + +def USE_BIGENDIAN(): + global ENDIANESS + ENDIANESS = '>' + +def USE_LITTLENDIAN(): + global ENDIANESS + ENDIANESS = '<' + @define class TCPSegment: class Options: @@ -85,7 +95,7 @@ def parse_flags(value): @classmethod def from_buffer(cls,buffer): TCP_MIN = 20 - src_port, dst_port, seq, ack, offres, flags, win, checksum, urp = struct.unpack("!HHIIBBHHH", buffer[0:TCP_MIN]) + src_port, dst_port, seq, ack, offres, flags, win, checksum, urp = struct.unpack(ENDIANESS +"HHIIBBHHH", buffer[0:TCP_MIN]) hdr_len = (offres >> 4) * 4 if hdr_len <= TCP_MIN: @@ -96,7 +106,7 @@ def from_buffer(cls,buffer): def to_buffer(self): - return struct.pack('!HHIIBBHHH', \ + return struct.pack(ENDIANESS+'HHIIBBHHH', \ self.src_port,\ self.dst_port,\ self.seq,\ @@ -186,14 +196,14 @@ def calc_checksum(pkt: bytes) -> int: @classmethod def from_buffer(cls,buffer): ihl = (buffer[0] & 0xf) - pkt = cls(ihl, *struct.unpack("!BBHHHBBH4s4s", buffer[0:ihl*4]),TCPSegment.from_buffer(buffer[ihl*4:]) if len(buffer) > ihl*4 else None) + pkt = cls(ihl, *struct.unpack(ENDIANESS+"BBHHHBBH4s4s", buffer[0:ihl*4]),TCPSegment.from_buffer(buffer[ihl*4:]) if len(buffer) > ihl*4 else None) #Cut data buffer to IP packet length if pkt.segment: pkt.get_segment().data=pkt.get_segment().data[:pkt.len-(ihl*4)-20] return pkt - + def is_flag(self,flag): return (self.flags & getattr(IPPacket.Flags, flag)) > 0 @@ -202,7 +212,7 @@ def to_buffer(self): data = self.get_segment().to_buffer() if self.segment else b'' self.len = 20 + len(data) - return struct.pack('!BBHHHBBH4s4s', \ + return struct.pack(ENDIANESS+'BBHHHBBH4s4s', \ ihl_ver, \ self.tos, \ self.len, \ @@ -222,7 +232,7 @@ def _validate_tcp(self): tcp_segment = self.segment.to_buffer() - pseudo_hdr = struct.pack("!4s4sHH", inet_aton(self.src_addr), inet_aton(self.dst_addr), self.protocol, len(tcp_segment)) + pseudo_hdr = struct.pack(ENDIANESS+"4s4sHH", inet_aton(self.src_addr), inet_aton(self.dst_addr), self.protocol, len(tcp_segment)) self.segment.checksum = IPPacket.calc_checksum(pseudo_hdr + tcp_segment) @@ -260,10 +270,10 @@ class EthernetFrame: @classmethod def from_buffer(cls,buffer): - return cls(*struct.unpack("!6s6s2s", buffer[0:14]),IPPacket.from_buffer(buffer[14:])) + return cls(*struct.unpack(ENDIANESS+"6s6s2s", buffer[0:14]),IPPacket.from_buffer(buffer[14:])) def to_buffer(self): - return struct.pack("!6s6s2s", unhexlify(self.src_mac), unhexlify(self.dst_mac),self.eth_type) + self.get_packet().to_buffer() + return struct.pack(ENDIANESS+"6s6s2s", unhexlify(self.src_mac), unhexlify(self.dst_mac),self.eth_type) + self.get_packet().to_buffer() def get_packet(self): return self.packet \ No newline at end of file diff --git a/packages/python-runner/test/test_inet.py b/packages/python-runner/test/test_inet.py index 147b67d14..868908fab 100644 --- a/packages/python-runner/test/test_inet.py +++ b/packages/python-runner/test/test_inet.py @@ -1,5 +1,8 @@ import pytest -from inet import TCPSegment, IPPacket, EthernetFrame +from inet import TCPSegment, IPPacket, EthernetFrame, USE_BIGENDIAN + + +USE_BIGENDIAN() class TestIP: def test_mf_df_flags(self): @@ -142,3 +145,18 @@ def test_parse_only_first_packet(self): assert pkt.len == 53 assert pkt.get_segment().data == b"{'foo':'bar'}" + def test_prepare_valid_tcp_with_PSH(self): + pkt = IPPacket() + pkt.segment = TCPSegment(flags=['PSH'],src_port=0, dst_port=3) + + a = pkt.build() + a_raw = a.to_buffer() + from scapy.all import TCP, IP + b = IP((IP(src='10.0.0.1', dst='10.0.0.2')/TCP(sport=0,dport=3, flags=['P'])).build()) + b_raw = b.build() + #b.build() + + c = IPPacket.from_buffer(b_raw) + + + a =5 From 64e8b2e55f1c84ef5c197a43ccfabbf13887e297 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Mon, 12 Jun 2023 19:45:45 +0200 Subject: [PATCH 155/231] To/From buffer with pseudo tcp header --- packages/python-runner/inet.py | 16 +++++ packages/python-runner/test/test_inet.py | 74 +++++++++++++++++------- 2 files changed, 70 insertions(+), 20 deletions(-) diff --git a/packages/python-runner/inet.py b/packages/python-runner/inet.py index e79ffcbcf..b46e5aff0 100644 --- a/packages/python-runner/inet.py +++ b/packages/python-runner/inet.py @@ -193,6 +193,12 @@ def calc_checksum(pkt: bytes) -> int: s = ~s return (((s >> 8) & 0xff) | s << 8) & 0xffff + @classmethod + def from_buffer_with_pseudoheader(cls,buffer): + src_addr, dst_addr, _, proto, length = struct.unpack(ENDIANESS+"4s4sBBH",bytes(buffer[0:12])) + pkt = cls(0, 0, 0, length, 0, 0, 0, proto, 0, src_addr, dst_addr, TCPSegment.from_buffer(buffer[12:]) if len(buffer) > 12 else None) + return pkt + @classmethod def from_buffer(cls,buffer): ihl = (buffer[0] & 0xf) @@ -207,6 +213,16 @@ def from_buffer(cls,buffer): def is_flag(self,flag): return (self.flags & getattr(IPPacket.Flags, flag)) > 0 + + def build_pseudoheader(self): + return struct.pack(ENDIANESS+"4s4sBBH", \ + inet_aton(self.src_addr),\ + inet_aton(self.dst_addr), \ + 0,\ + self.protocol, \ + self.len) + + def to_buffer(self): ihl_ver = (int(self.version) << 4) + int(self.ihl) diff --git a/packages/python-runner/test/test_inet.py b/packages/python-runner/test/test_inet.py index 868908fab..6d3551baf 100644 --- a/packages/python-runner/test/test_inet.py +++ b/packages/python-runner/test/test_inet.py @@ -1,12 +1,11 @@ import pytest -from inet import TCPSegment, IPPacket, EthernetFrame, USE_BIGENDIAN - - -USE_BIGENDIAN() +from inet import TCPSegment, IPPacket, EthernetFrame, USE_BIGENDIAN, USE_LITTLENDIAN class TestIP: def test_mf_df_flags(self): + USE_BIGENDIAN() + data = b'E\x00\x00\x14\x00\x01`\x00@\x00\x1c\xe7\x7f\x00\x00\x01\x7f\x00\x00\x01' pkt = IPPacket.from_buffer(data) assert pkt.flags == (IPPacket.Flags.MF | IPPacket.Flags.DF) & ~IPPacket.Flags.RF @@ -16,6 +15,8 @@ def test_mf_df_flags(self): def test_mf_flags(self): + USE_BIGENDIAN() + data = b'E\x00\x00\x14\x00\x01 \x00@\x00\\\xe7\x7f\x00\x00\x01\x7f\x00\x00\x01' pkt = IPPacket.from_buffer(data) assert pkt.flags == IPPacket.Flags.MF & ~IPPacket.Flags.RF @@ -24,6 +25,9 @@ def test_mf_flags(self): assert pkt.is_flag('RF') == False def test_df_flags(self): + + USE_BIGENDIAN() + data = (b'\x00\x23\x20\xd4\x2a\x8c\x00\x23\x20\xd4\x2a\x8c\x08\x00\x45\x00\x00\x54\x00\x00\x40\x00' b'\x40\x01\x25\x8d\x0a\x00\x00\x8f\x0a\x00\x00\x8e\x08\x00\x2e\xa0\x01\xff\x23\x73\x20\x48' b'\x4a\x4d\x00\x00\x00\x00\x78\x85\x02\x00\x00\x00\x00\x00\x10\x11\x12\x13\x14\x15\x16\x17' @@ -43,6 +47,8 @@ def test_df_flags(self): def test_checksum_calc(self): + USE_BIGENDIAN() + data = b'E\x00\x00\x14\x00\x01\x00\x00@\x00j\xd6\x01\x02\x03\x04\x05\x06\x07\x08' checksum = 27350 pkt = IPPacket.from_buffer(data) @@ -56,20 +62,48 @@ def test_checksum_calc(self): def test_packet_creation(self): + USE_BIGENDIAN() + pkt = IPPacket(src_addr='172.25.44.3', dst_addr='172.25.44.254', segment=TCPSegment(dst_port=3, flags=['ACK'])) assert pkt.src_addr == '172.25.44.3' assert pkt.dst_addr == '172.25.44.254' + + def test_pseudoheader(self): + + USE_LITTLENDIAN() + + data = bytes([10, 0, 0, 1, 10, 0, 0, 2, 0, 1, 32, 0, 0, 0, 0, 0, 249, 252, 104, 127, 0, 0, 0, 0, 0, 8, 0, 0, 149, 136, 0, 0]) + res = IPPacket.from_buffer_with_pseudoheader(data) + + assert res.src_addr == '10.0.0.1' + assert res.dst_addr == '10.0.0.2' + assert res.len == 32 + assert res.protocol == 1 + + assert res.segment.seq == 2137586937 + assert res.segment.ack == 0 + + + psh = res.build_pseudoheader() + + assert psh == data[0:12] + class TestTCP: def test_tcp_offset(self): + + USE_BIGENDIAN() + data = b'\x01\xbb\xc0\xd7\xb6\x56\xa8\xb9\xd1\xac\xaa\xb1\x50\x18\x40\x00\x56\xf8\x00\x00' segment = TCPSegment.from_buffer(data) assert segment.offset == 5 def test_prepare_segment_with_flags(self): + USE_BIGENDIAN() + segment = TCPSegment(flags=['FIN', 'SYN', 'ACK']) assert segment.is_flag('FIN') == True @@ -99,6 +133,9 @@ def test_prepare_segment_with_flags(self): assert segment.is_flag('PSH') == True def test_basic_details(self): + + USE_BIGENDIAN() + data = b'E\x00\x00(\x00\x01@\x00@\x06,\xbe\x01\x02\x03\x04\x04\x05\x06\x07\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x81m\x00\x00' pkt = IPPacket.from_buffer(data) assert pkt.segment.dst_port == 80 @@ -107,11 +144,17 @@ def test_basic_details(self): assert pkt.segment.is_flag('PSH') == False def test_checksum(self): + + USE_BIGENDIAN() + data = b'E\x00\x00(\x00\x01@\x00@\x06,\xbe\x01\x02\x03\x04\x04\x05\x06\x07\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x81m\x00\x00' pkt = IPPacket.from_buffer(data) assert pkt.segment.checksum == 33133 def test_checksum_calc(self): + + USE_BIGENDIAN() + data = b'E\x00\x00*\x00\x01\x00\x00@\x06N\x9d\n\x0b\x0c\x0e\n\x0b\x0c\r\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x00\x00\x00\x00Hi' pkt = IPPacket.from_buffer(data) @@ -122,6 +165,9 @@ def test_checksum_calc(self): assert pkt.segment.checksum == 6883 def test_unpack(self): + + USE_BIGENDIAN() + data = (b'\x00\x50\x0d\x2c\x11\x4c\x61\x8b\x38\xaf\xfe\x14\x70\x12\x16\xd0' b'\x5b\xdc\x00\x00\x02\x04\x05\x64\x01\x01\x04\x02') pkt = TCPSegment.from_buffer(data) @@ -133,6 +179,10 @@ def test_unpack(self): assert pkt.ack == 951057940 def test_parse_only_first_packet(self): + + + USE_BIGENDIAN() + data = (b"\x05\x00\x005\x00\x00\x00\x00\xff\x06\xfb\xc3\x00\x00\x00\x00\x00" b"\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00" b"\x00\x00\x00#g\x00\x00{'foo':'bar'}\x05\x00\x006\x00\x00\x00\x00" @@ -144,19 +194,3 @@ def test_parse_only_first_packet(self): assert pkt.len == 53 assert pkt.get_segment().data == b"{'foo':'bar'}" - - def test_prepare_valid_tcp_with_PSH(self): - pkt = IPPacket() - pkt.segment = TCPSegment(flags=['PSH'],src_port=0, dst_port=3) - - a = pkt.build() - a_raw = a.to_buffer() - from scapy.all import TCP, IP - b = IP((IP(src='10.0.0.1', dst='10.0.0.2')/TCP(sport=0,dport=3, flags=['P'])).build()) - b_raw = b.build() - #b.build() - - c = IPPacket.from_buffer(b_raw) - - - a =5 From 76820e3d8461783f276cf3165d8503550fc24aab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Tue, 13 Jun 2023 12:09:53 +0200 Subject: [PATCH 156/231] Tecemux with TCP pseudoheaders --- packages/python-runner/inet.py | 17 +++++++++------ packages/python-runner/tecemux.py | 24 +++++++++------------ packages/python-runner/test/test_inet.py | 2 +- packages/python-runner/test/test_tecemux.py | 2 +- 4 files changed, 23 insertions(+), 22 deletions(-) diff --git a/packages/python-runner/inet.py b/packages/python-runner/inet.py index b46e5aff0..d67266205 100644 --- a/packages/python-runner/inet.py +++ b/packages/python-runner/inet.py @@ -214,15 +214,20 @@ def is_flag(self,flag): return (self.flags & getattr(IPPacket.Flags, flag)) > 0 - def build_pseudoheader(self): + def prepare_pseudoheader(self, protocol, len): return struct.pack(ENDIANESS+"4s4sBBH", \ inet_aton(self.src_addr),\ inet_aton(self.dst_addr), \ 0,\ - self.protocol, \ - self.len) + protocol, \ + len) + def to_buffer_with_tcp_pseudoheader(self): + data = self.get_segment().to_buffer() if self.segment else b'' + + return self.prepare_pseudoheader(1, 12+len(data)) + data + def to_buffer(self): ihl_ver = (int(self.version) << 4) + int(self.ihl) @@ -246,10 +251,10 @@ def _validate_tcp(self): self.segment.offset = tcp_hdr_len << 4 self.segment.checksum = 0 - tcp_segment = self.segment.to_buffer() + tcp_segment = self.segment.to_buffer() + + pseudo_hdr = self.prepare_pseudoheader(self.protocol, len(tcp_segment)) - pseudo_hdr = struct.pack(ENDIANESS+"4s4sHH", inet_aton(self.src_addr), inet_aton(self.dst_addr), self.protocol, len(tcp_segment)) - self.segment.checksum = IPPacket.calc_checksum(pseudo_hdr + tcp_segment) return self diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 186ee66c0..80abb6b25 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -2,13 +2,14 @@ import logging import random import socket - +import struct from attrs import define, field - -from inet import IPPacket, TCPSegment +from socket import inet_ntoa, inet_aton +from inet import IPPacket, TCPSegment, USE_LITTLENDIAN from hardcoded_magic_values import CommunicationChannels as CC +USE_LITTLENDIAN() class _StreamReader: def __init__(self, stream): @@ -95,9 +96,7 @@ async def close(self): self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel close request is send') await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(),data=b'' if self._global_instance_id is None else self._global_instance_id, flags=['FIN']))) - async def queue_up_incoming(self, buf): - pkt = IPPacket().from_buffer(buf) - + async def queue_up_incoming(self, pkt): #if SYN & ACK flag is up, pause channel if pkt.segment.is_flag('SYN') and pkt.segment.is_flag('ACK'): self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel pause request received') @@ -109,8 +108,6 @@ async def queue_up_incoming(self, buf): if not pkt.segment.is_flag('SYN') and pkt.segment.is_flag('ACK'): self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel reasume request received') self.set_pause(False) - await self.send_ACK(pkt.segment.seq) - return self._writer.write(pkt.get_segment().data) await self._writer.drain() @@ -260,18 +257,18 @@ async def incoming_data_forward(self): while not (len(buffer) < MINIMAL_IP_PACKET_LENGTH): - current_packet_size = IPPacket().from_buffer(buffer).len + current_packet_size = IPPacket().from_buffer_with_pseudoheader(buffer).len if len(buffer) >= current_packet_size: single_packet_buffer = buffer[:current_packet_size] - pkt = IPPacket().from_buffer(single_packet_buffer) + pkt = IPPacket().from_buffer_with_pseudoheader(single_packet_buffer) self._last_sequence_received = pkt.get_segment().seq self._logger.debug(f'Tecemux/MAIN: [<] Full incomming packet with sequence number {self._last_sequence_received} from Transform Hub was received') channel = CC(str(pkt.get_segment().dst_port)) - await self._channels[channel].queue_up_incoming(single_packet_buffer) + await self._channels[channel].queue_up_incoming(pkt) self._logger.debug(f'Tecemux/MAIN: [<] Packet {Tecemux._chunk_preview(single_packet_buffer)} forwarded to {channel.name} stream') buffer = buffer[current_packet_size:] @@ -291,9 +288,8 @@ async def outcoming_data_forward(self): pkt.segment.seq = self._sequence_number self._sequence_number += 1 - #calc cheksum - chunk = pkt.build().to_buffer() - + chunk = pkt._validate_tcp().to_buffer_with_tcp_pseudoheader() + self._logger.debug(f'Tecemux/MAIN: [>] Outcoming chunk {Tecemux._chunk_preview(chunk)} is waiting to send to Transform Hub') self._writer.write(chunk) await self._writer.drain() diff --git a/packages/python-runner/test/test_inet.py b/packages/python-runner/test/test_inet.py index 6d3551baf..0b39d7103 100644 --- a/packages/python-runner/test/test_inet.py +++ b/packages/python-runner/test/test_inet.py @@ -87,7 +87,7 @@ def test_pseudoheader(self): assert res.segment.ack == 0 - psh = res.build_pseudoheader() + psh = res.prepare_pseudoheader(1,12 + len(data[12:])) assert psh == data[0:12] diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index 603236d31..c150eb8a6 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -64,7 +64,7 @@ async def test_forward_from_main_to_channel(self, local_socket_connection): pkt = IPPacket(src_addr='172.25.44.3',dst_addr='172.25.44.254',segment=TCPSegment(dst_port=int(destination_channel.value),flags=['PSH'],data=data_to_send)) - client_a._writer.write(pkt.to_buffer()) + client_a._writer.write(pkt.to_buffer_with_tcp_pseudoheader()) await client_a._writer.drain() assert (await client_b.get_channel(destination_channel).read(100)).decode() == data_to_send From 7ed419ef71e68b160b068c0c09bec636d016fb03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Wed, 14 Jun 2023 13:17:41 +0200 Subject: [PATCH 157/231] Runner with Tecemux --- packages/python-runner/runner.py | 47 ++++++++------- packages/python-runner/tecemux.py | 99 ++++++++++++++++--------------- 2 files changed, 74 insertions(+), 72 deletions(-) diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index 31de0fb0e..fca16b7a9 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -72,15 +72,16 @@ async def init_tecemux(self, server_host, server_port): self.logger.info('Connecting to host with TeceMux...') self.protocol = Tecemux(instance_id=self.instance_id) await self.protocol.connect(*await Tecemux.prepare_tcp_connection(server_host, server_port)) - await self.protocol.prepare() + await self.protocol.prepare(force_open=True) await self.protocol.loop() self.streams = self.protocol._channels def connect_stdio(self): - sys.stdout = codecs.getwriter('utf-8')(self.streams[CC.STDOUT]) - sys.stderr = codecs.getwriter('utf-8')(self.streams[CC.STDERR]) - sys.stdin = Stream.read_from(self.streams[CC.STDIN]).decode('utf-8') + sys.stdout = codecs.getwriter('utf-8')(self.protocol.get_channel(CC.STDOUT)) + sys.stderr = codecs.getwriter('utf-8')(self.protocol.get_channel(CC.STDERR)) + sys.stdin = Stream.read_from(self.protocol.get_channel(CC.STDIN)).decode('utf-8') + # pretend to have API compatibiliy sys.stdout.flush = lambda: True sys.stderr.flush = lambda: True @@ -89,25 +90,23 @@ def connect_stdio(self): def connect_log_stream(self): self.logger.info('Switching to main log stream...') - log_stream = codecs.getwriter('utf-8')(self.streams[CC.LOG]) + log_stream = codecs.getwriter('utf-8')(self.protocol.get_channel(CC.LOG)) self._logging_setup.switch_target(log_stream) self._logging_setup.flush_temp_handler() self.protocol.set_logger(self.logger) - - for channel in self.protocol._channels.values(): - channel._set_logger(self.logger) + [channel._set_logger(self.logger) for channel in self.protocol.get_channels()] self.logger.info('Log stream connected.') async def handshake(self): - monitoring = self.streams[CC.MONITORING] - control = self.streams[CC.CONTROL] + monitoring = self.protocol.get_channel(CC.MONITORING) + control = self.protocol.get_channel(CC.CONTROL) self.logger.info(f'Sending PING') await send_encoded_msg(monitoring, msg_codes.PING) - message = await control.read(8) + message = await control.readuntil(b'\n') self.logger.info(f'Got message: {message}') code, data = json.loads(message.decode()) @@ -135,7 +134,7 @@ async def connect_control_stream(self): control_messages = ( Stream # 128 kB is the typical size of TCP buffer. - .read_from(self.streams[CC.CONTROL], chunk_size=131072) + .read_from(self.protocol.get_channel(CC.CONTROL), chunk_size=131072) .decode('utf-8').split('\n').map(json.loads) ) async for code, data in control_messages: @@ -158,10 +157,10 @@ async def handle_stop(self, data): await handler(timeout, can_keep_alive) except Exception as e: self.logger.error('Error stopping sequence', e) - await send_encoded_msg(self.streams[CC.MONITORING], msg_codes.SEQUENCE_STOPPED, e) + await send_encoded_msg(self.protocol.get_channel(CC.MONITORING), msg_codes.SEQUENCE_STOPPED, e) if not can_keep_alive or not self.keep_alive_requested: - await send_encoded_msg(self.streams[CC.MONITORING], msg_codes.SEQUENCE_STOPPED, {}) + await send_encoded_msg(self.protocol.get_channel(CC.MONITORING), msg_codes.SEQUENCE_STOPPED, {}) self.exit_immediately() await self.cleanup() @@ -170,7 +169,7 @@ async def handle_stop(self, data): async def setup_heartbeat(self): while True: await send_encoded_msg( - self.streams[CC.MONITORING], + self.protocol.get_channel(CC.MONITORING), msg_codes.MONITORING, self.health_check(), ) @@ -199,7 +198,7 @@ async def run_instance(self, config, args): result = self.sequence.run(context, input_stream, *args) self.logger.info(f'Sending PANG') - monitoring = self.streams[CC.MONITORING] + monitoring = self.protocol.get_channel(CC.MONITORING) produces = getattr(result, 'provides', None) or getattr(self.sequence, 'provides', None) if produces: @@ -224,13 +223,15 @@ async def run_instance(self, config, args): await self.cleanup() async def cleanup(self): - self.streams[CC.LOG].write_eof() + #TODO + #self.streams[CC.LOG].write_eof() + pass async def connect_input_stream(self, input_stream): if hasattr(self.sequence, "requires"): input_type = self.sequence.requires.get('contentType') else: - raw_headers = await self.streams[CC.IN].readuntil(b'\r\n\r\n') + raw_headers = await self.protocol.get_channel(CC.IN).readuntil(b'\r\n\r\n') header_list = raw_headers.decode().rstrip().split('\r\n') headers = { key.lower(): val for key, val in [el.split(': ') for el in header_list] @@ -239,12 +240,12 @@ async def connect_input_stream(self, input_stream): input_type = headers.get('content-type') if input_type == 'text/plain': - input = Stream.read_from(self.streams[CC.IN]) + input = Stream.read_from(self.protocol.get_channel(CC.IN)) self.logger.debug('Decoding input stream...') input = input.decode('utf-8') elif input_type == 'application/octet-stream': self.logger.debug('Opening input in binary mode...') - input = Stream.read_from(self.streams[CC.IN], chunk_size=CHUNK_SIZE) + input = Stream.read_from(self.protocol.get_channel(CC.IN), chunk_size=CHUNK_SIZE) else: raise TypeError(f'Unsupported input type: {repr(input_type)}') @@ -272,11 +273,11 @@ async def forward_output_stream(self, output): self.logger.debug('Output will be converted to JSON') output = output.map(lambda chunk: (json.dumps(chunk)+'\n').encode()) - await output.write_to(self.streams[CC.OUT]) + await output.write_to(self.protocol.get_channel(CC.OUT)) async def send_keep_alive(self, timeout: int = 0, can_keep_alive: bool = False): - monitoring = self.streams[CC.MONITORING] + monitoring =self.protocol.get_channel(CC.MONITORING) await send_encoded_msg(monitoring, msg_codes.ALIVE) self.keep_alive_requested = True await asyncio.sleep(timeout) @@ -290,7 +291,7 @@ class AppContext: def __init__(self, runner, config) -> None: self.logger = runner.logger self.config = config - self.monitoring = runner.streams[CC.MONITORING] + self.monitoring = runner.protocol.get_channel(CC.MONITORING) self.runner = runner self.emitter = runner.emitter diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 80abb6b25..d80cb952d 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -2,9 +2,7 @@ import logging import random import socket -import struct from attrs import define, field -from socket import inet_ntoa, inet_aton from inet import IPPacket, TCPSegment, USE_LITTLENDIAN from hardcoded_magic_values import CommunicationChannels as CC @@ -18,19 +16,6 @@ def __init__(self, stream): def __getattr__(self, name): return getattr(self._stream, name) - def readline(self): - return self._stream.readline() - - def readuntil(self, separator=b'\n'): - return self._stream.readuntil(separator) - - def read(self, n=-1): - return self._stream.read(n) - - def readexactly(self, n): - return self._stream.readexactly(n) - - class _StreamWriter: def __init__(self, stream): @@ -38,14 +23,7 @@ def __init__(self, stream): def __getattr__(self, name): return getattr(self._stream, name) - - def wait_closed(self): - return self._stream.wait_closed() - - def drain(self): - return self._stream.drain() - - + @define class _ChannelContext: _channel_enum: CC @@ -60,15 +38,6 @@ class _ChannelContext: _internal_queue: asyncio.Queue = field(init=False) _channel_paused: bool = field(default=False, init=False) _channel_opened: bool = field(default=False, init=False) - - def __aiter__(self): - return self._reader._stream.__aiter__ - - async def __anext__(self): - val = await self._reader._stream.readline() - if val == b'': - raise StopAsyncIteration - return val def __attrs_post_init__(self): self._internal_queue = asyncio.Queue() @@ -82,6 +51,32 @@ def _get_channel_id(self): def _set_logger(self, logger): self._logger = logger + def readline(self): + return self._reader._stream.readline() + + def readuntil(self, separator=b'\n'): + return self._reader._stream.readuntil(separator) + + def read(self, n=-1): + return self._reader._stream.read(n) + + def readexactly(self, n): + return self._reader._stream.readexactly(n) + + def __aiter__(self): + return self._reader._stream.__aiter__ + + async def __anext__(self): + val = await self._reader._stream.readline() + if val == b'': + raise StopAsyncIteration + return val + + + + + + async def send_ACK(self, sequence_number): await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK'], ack=sequence_number))) @@ -90,7 +85,16 @@ async def _send_pause_ACK(self, sequence_number): async def open(self): #self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel opening request is send') - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'' if self._global_instance_id is None else self._global_instance_id, flags=['PSH']))) + + if not self._channel_opened: + + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'', flags=['PSH']))) + + buf = b'' if self._global_instance_id is None else (self._global_instance_id.encode() + str(self._get_channel_id()).encode()) + + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=buf, flags=['PSH']))) + + self._channel_opened = True async def close(self): self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel close request is send') @@ -116,7 +120,7 @@ async def queue_up_outcoming(self, data): def wrap(channel_enum, buf): channel_id = int(channel_enum.value) - return IPPacket(segment=TCPSegment(dst_port=channel_id, data=buf)) + return IPPacket(segment=TCPSegment(dst_port=channel_id, flags=['PSH'],data=buf)) if self._channel_paused: self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel paused. Data queued up internally for future') @@ -131,12 +135,6 @@ def wrap(channel_enum, buf): break await self._global_queue.put(wrap(self._channel_enum, data)) - def read(self, len): - return self._reader.read(len) - - def readuntil(self, separator=b'\n'): - return self._reader.readuntil(separator) - async def write(self, data): if not self._channel_opened: @@ -146,9 +144,12 @@ async def write(self, data): async def drain(self): if not self._channel_paused: - await self._writer.drain() + await self._writer._stream.drain() return True + def wait_closed(self): + return self._writer._stream.wait_closed() + def set_pause(self, state): self._channel_paused = state @@ -187,9 +188,12 @@ async def prepare_socket_connection(): _, writer = await asyncio.open_unix_connection(sock=wsock) return _StreamReader(reader), _StreamWriter(writer) - async def prepare(self): + async def prepare(self, force_open=False): self._queue = asyncio.Queue() - self._channels = {channel: _ChannelContext(channel, *await Tecemux.prepare_socket_connection(), self._queue, self._instance_id ,self._logger) for channel in CC} + self._channels = {channel: _ChannelContext(channel, *await Tecemux.prepare_socket_connection(), self._queue, self._instance_id ,self._logger) for channel in CC} + + if force_open: + [await channel.open() for channel in self.get_channels()] def set_logger(self, logger): self._logger = logger @@ -197,12 +201,9 @@ def set_logger(self, logger): def get_channel(self, channel): return self._channels[channel] - def get_channel_reader(self, channel): - return self.get_channel(channel).reader - - def get_channel_writer(self, channel): - return self.get_channel(channel).writer - + def get_channels(self): + return self._channels.values() + @staticmethod def _chunk_preview(value): return f'{value[0:5]}... ' if len(value)>5 else f'{value}' From 0e065601bff95dd8380224f8fcdc6c50e932dac0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Wed, 14 Jun 2023 13:27:06 +0200 Subject: [PATCH 158/231] Tecemux with only native asyncio.Stream --- packages/python-runner/tecemux.py | 48 ++++++--------------- packages/python-runner/test/test_tecemux.py | 7 +-- 2 files changed, 18 insertions(+), 37 deletions(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index d80cb952d..f1b502754 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -7,29 +7,13 @@ from hardcoded_magic_values import CommunicationChannels as CC -USE_LITTLENDIAN() -class _StreamReader: - - def __init__(self, stream): - self._stream = stream - - def __getattr__(self, name): - return getattr(self._stream, name) - -class _StreamWriter: - - def __init__(self, stream): - self._stream = stream - - def __getattr__(self, name): - return getattr(self._stream, name) - +USE_LITTLENDIAN() @define class _ChannelContext: _channel_enum: CC - _reader: _StreamReader - _writer: _StreamWriter + _reader: asyncio.StreamReader + _writer: asyncio.StreamWriter _global_queue: asyncio.Queue _global_instance_id: str @@ -52,31 +36,26 @@ def _set_logger(self, logger): self._logger = logger def readline(self): - return self._reader._stream.readline() + return self._reader.readline() def readuntil(self, separator=b'\n'): - return self._reader._stream.readuntil(separator) + return self._reader.readuntil(separator) def read(self, n=-1): - return self._reader._stream.read(n) + return self._reader.read(n) def readexactly(self, n): - return self._reader._stream.readexactly(n) + return self._reader.readexactly(n) def __aiter__(self): - return self._reader._stream.__aiter__ + return self._reader.__aiter__ async def __anext__(self): - val = await self._reader._stream.readline() + val = await self._reader.readline() if val == b'': raise StopAsyncIteration return val - - - - - async def send_ACK(self, sequence_number): await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK'], ack=sequence_number))) @@ -144,11 +123,11 @@ async def write(self, data): async def drain(self): if not self._channel_paused: - await self._writer._stream.drain() + await self._writer.drain() return True def wait_closed(self): - return self._writer._stream.wait_closed() + return self._writer.wait_closed() def set_pause(self, state): self._channel_paused = state @@ -186,7 +165,7 @@ async def prepare_socket_connection(): rsock, wsock = socket.socketpair() reader, _ = await asyncio.open_unix_connection(sock=rsock) _, writer = await asyncio.open_unix_connection(sock=wsock) - return _StreamReader(reader), _StreamWriter(writer) + return reader, writer async def prepare(self, force_open=False): self._queue = asyncio.Queue() @@ -207,8 +186,9 @@ def get_channels(self): @staticmethod def _chunk_preview(value): return f'{value[0:5]}... ' if len(value)>5 else f'{value}' + async def stop(self): - for channel in self._channels.values(): + for channel in self.get_channels(): await channel.close() await channel._writer.drain() channel._writer.close() diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index c150eb8a6..6d3382b5b 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -1,6 +1,7 @@ +import asyncio import pytest import sys -from tecemux import Tecemux, _StreamReader,_StreamWriter +from tecemux import Tecemux from inet import IPPacket,TCPSegment from logging_setup import LoggingSetup from hardcoded_magic_values import CommunicationChannels as CC @@ -51,8 +52,8 @@ async def test_socket_connection(self): await protocol.connect(*await Tecemux.prepare_socket_connection()) - assert isinstance(protocol._reader,_StreamReader) - assert isinstance(protocol._writer,_StreamWriter) + assert isinstance(protocol._reader,asyncio.StreamReader) + assert isinstance(protocol._writer,asyncio.StreamWriter) assert protocol._sequence_number > 0 @pytest.mark.asyncio From 70f2eae0f12782a9f5d9391cd8b67d267640c03a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 16 Jun 2023 11:22:14 +0200 Subject: [PATCH 159/231] calculateChecksum modulo problem fix --- packages/verser/src/lib/tecemux/codecs/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/verser/src/lib/tecemux/codecs/utils.ts b/packages/verser/src/lib/tecemux/codecs/utils.ts index 98db2e4bf..302b30dbc 100644 --- a/packages/verser/src/lib/tecemux/codecs/utils.ts +++ b/packages/verser/src/lib/tecemux/codecs/utils.ts @@ -1,7 +1,7 @@ export const calculateChecksum = (buffer: Buffer) => { let tempFrame = Buffer.concat([buffer, Buffer.alloc(0)]); - if (buffer.length % 1) { + if (buffer.length % 2) { tempFrame = Buffer.concat([buffer, Buffer.alloc(1, 0)]); } From 4d035585da73d6a27fce4d0d0c808ca9398ceebd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 16 Jun 2023 12:21:05 +0200 Subject: [PATCH 160/231] Calc checksum match with Transform Hub --- packages/python-runner/inet.py | 60 ++++++++++++++++++++---- packages/python-runner/test/test_inet.py | 17 ++++++- 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/packages/python-runner/inet.py b/packages/python-runner/inet.py index d67266205..91c613b42 100644 --- a/packages/python-runner/inet.py +++ b/packages/python-runner/inet.py @@ -184,15 +184,37 @@ def __attrs_post_init__(self): @staticmethod def calc_checksum(pkt: bytes) -> int: - #source: https://github.com/secdev/scapy if len(pkt) % 2 == 1: pkt += b"\0" - s = sum(array.array("H", pkt)) + s = sum(struct.unpack(('<' if ENDIANESS is '>' else '>')+str(len(pkt)//2)+'H',pkt)) + + #source: https://github.com/secdev/scapy s = (s >> 16) + (s & 0xffff) s += s >> 16 s = ~s return (((s >> 8) & 0xff) | s << 8) & 0xffff + def calc_checksum_for_STH(pkt): + if len(pkt) % 2 == 1: + pkt += b"\0" + elements = list(struct.unpack(ENDIANESS+str(len(pkt)//2)+'H',pkt)) + elements = elements[:14] + elements[15:] + s = sum(elements) + return s % 0x10000 + + # checksum = 0 + # i = 0 + + # if len(pkt) % 2 == 1: + # pkt += b'\0' + + # while (i <= len(pkt)-2): + # if i != 28: + # val = struct.unpack_from(' Date: Fri, 16 Jun 2023 12:22:01 +0200 Subject: [PATCH 161/231] Send ACK for PSH packet --- packages/python-runner/inet.py | 13 ------------- packages/python-runner/tecemux.py | 16 ++++++++++------ 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/packages/python-runner/inet.py b/packages/python-runner/inet.py index 91c613b42..ec7fb7918 100644 --- a/packages/python-runner/inet.py +++ b/packages/python-runner/inet.py @@ -202,19 +202,6 @@ def calc_checksum_for_STH(pkt): s = sum(elements) return s % 0x10000 - # checksum = 0 - # i = 0 - - # if len(pkt) % 2 == 1: - # pkt += b'\0' - - # while (i <= len(pkt)-2): - # if i != 28: - # val = struct.unpack_from('] Outcoming chunk {Tecemux._chunk_preview(chunk)} is waiting to send to Transform Hub') self._writer.write(chunk) From 464dfb50d34fb2f56b2e8868f7ca39ec8a4e6eb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 16 Jun 2023 12:25:58 +0200 Subject: [PATCH 162/231] Fix wrong indent --- packages/python-runner/tecemux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 37db4b107..28b486990 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -273,7 +273,7 @@ async def outcoming_data_forward(self): pkt.segment.seq = self._sequence_number self._sequence_number += 1 - chunk = pkt.build(for_STH=True).to_buffer_with_tcp_pseudoheader() + chunk = pkt.build(for_STH=True).to_buffer_with_tcp_pseudoheader() self._logger.debug(f'Tecemux/MAIN: [>] Outcoming chunk {Tecemux._chunk_preview(chunk)} is waiting to send to Transform Hub') self._writer.write(chunk) From 0a7b34935bb6206c0ff681f8e63a4093c307d752 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 16 Jun 2023 12:37:31 +0200 Subject: [PATCH 163/231] Add write_eof method --- packages/python-runner/runner.py | 4 +--- packages/python-runner/tecemux.py | 7 ++++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index fca16b7a9..e4cd367bc 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -223,9 +223,7 @@ async def run_instance(self, config, args): await self.cleanup() async def cleanup(self): - #TODO - #self.streams[CC.LOG].write_eof() - pass + self.protocol.get_channel(CC.LOG).write_eof() async def connect_input_stream(self, input_stream): if hasattr(self.sequence, "requires"): diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 28b486990..1c6237587 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -63,8 +63,6 @@ async def _send_pause_ACK(self, sequence_number): await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK','SYN'], ack=sequence_number))) async def open(self): - #self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel opening request is send') - if not self._channel_opened: await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'', flags=['PSH']))) @@ -130,6 +128,9 @@ async def drain(self): await self._writer.drain() return True + def write_eof(self): + return self._writer.write_eof() + def wait_closed(self): return self._writer.wait_closed() @@ -283,4 +284,4 @@ async def outcoming_data_forward(self): except asyncio.QueueEmpty: await asyncio.sleep(0) - self._logger.debug(f'Tecemux/MAIN: Outcoming data forwarder finished') + self._logger.debug(f'Tecemux/MAIN: Outcoming data forwarder finished') \ No newline at end of file From b88590a85b661b08e86ff405b0374a696f91ae41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 16 Jun 2023 15:33:14 +0200 Subject: [PATCH 164/231] Sequence Order as Singleton --- packages/python-runner/inet.py | 49 +++++++++++++++--------- packages/python-runner/runner.py | 2 +- packages/python-runner/tecemux.py | 6 +-- packages/python-runner/test/test_inet.py | 33 ++++++++-------- 4 files changed, 51 insertions(+), 39 deletions(-) diff --git a/packages/python-runner/inet.py b/packages/python-runner/inet.py index ec7fb7918..53e89be1c 100644 --- a/packages/python-runner/inet.py +++ b/packages/python-runner/inet.py @@ -1,18 +1,31 @@ -import array import struct from binascii import hexlify, unhexlify from socket import inet_ntoa, inet_aton from attrs import define,field -ENDIANESS = '<' +class _Singleton(type): + _instances = {} + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super(_Singleton, cls).__call__(*args, **kwargs) + return cls._instances[cls] + +class SequenceOrder(metaclass=_Singleton): + class _Order: + LITTLE_ENDIAN = '<' + BIG_ENDIAN = '>' + + endianess = _Order.LITTLE_ENDIAN -def USE_BIGENDIAN(): - global ENDIANESS - ENDIANESS = '>' + def use_big_endian(self): + self.endianess = SequenceOrder._Order.BIG_ENDIAN -def USE_LITTLENDIAN(): - global ENDIANESS - ENDIANESS = '<' + def use_little_endian(self): + self.endianess = SequenceOrder._Order.LITTLE_ENDIAN + + def get(self): + return self.endianess + @define class TCPSegment: @@ -95,7 +108,7 @@ def parse_flags(value): @classmethod def from_buffer(cls,buffer): TCP_MIN = 20 - src_port, dst_port, seq, ack, offres, flags, win, checksum, urp = struct.unpack(ENDIANESS +"HHIIBBHHH", buffer[0:TCP_MIN]) + src_port, dst_port, seq, ack, offres, flags, win, checksum, urp = struct.unpack(SequenceOrder().get() +"HHIIBBHHH", buffer[0:TCP_MIN]) hdr_len = (offres >> 4) * 4 if hdr_len <= TCP_MIN: @@ -106,7 +119,7 @@ def from_buffer(cls,buffer): def to_buffer(self): - return struct.pack(ENDIANESS+'HHIIBBHHH', \ + return struct.pack(SequenceOrder().get()+'HHIIBBHHH', \ self.src_port,\ self.dst_port,\ self.seq,\ @@ -186,7 +199,7 @@ def __attrs_post_init__(self): def calc_checksum(pkt: bytes) -> int: if len(pkt) % 2 == 1: pkt += b"\0" - s = sum(struct.unpack(('<' if ENDIANESS is '>' else '>')+str(len(pkt)//2)+'H',pkt)) + s = sum(struct.unpack(('<' if SequenceOrder().get() is '>' else '>')+str(len(pkt)//2)+'H',pkt)) #source: https://github.com/secdev/scapy s = (s >> 16) + (s & 0xffff) @@ -197,21 +210,21 @@ def calc_checksum(pkt: bytes) -> int: def calc_checksum_for_STH(pkt): if len(pkt) % 2 == 1: pkt += b"\0" - elements = list(struct.unpack(ENDIANESS+str(len(pkt)//2)+'H',pkt)) + elements = list(struct.unpack(SequenceOrder().get()+str(len(pkt)//2)+'H',pkt)) elements = elements[:14] + elements[15:] s = sum(elements) return s % 0x10000 @classmethod def from_buffer_with_pseudoheader(cls,buffer): - src_addr, dst_addr, _, proto, length = struct.unpack(ENDIANESS+"4s4sBBH",bytes(buffer[0:12])) + src_addr, dst_addr, _, proto, length = struct.unpack(SequenceOrder().get()+"4s4sBBH",bytes(buffer[0:12])) pkt = cls(0, 0, 0, length, 0, 0, 0, proto, 0, src_addr, dst_addr, TCPSegment.from_buffer(buffer[12:]) if len(buffer) > 12 else None) return pkt @classmethod def from_buffer(cls,buffer): ihl = (buffer[0] & 0xf) - pkt = cls(ihl, *struct.unpack(ENDIANESS+"BBHHHBBH4s4s", buffer[0:ihl*4]),TCPSegment.from_buffer(buffer[ihl*4:]) if len(buffer) > ihl*4 else None) + pkt = cls(ihl, *struct.unpack(SequenceOrder().get()+"BBHHHBBH4s4s", buffer[0:ihl*4]),TCPSegment.from_buffer(buffer[ihl*4:]) if len(buffer) > ihl*4 else None) #Cut data buffer to IP packet length if pkt.segment: @@ -224,7 +237,7 @@ def is_flag(self,flag): def prepare_pseudoheader(self, protocol, len): - return struct.pack(ENDIANESS+"4s4sBBH", \ + return struct.pack(SequenceOrder().get()+"4s4sBBH", \ inet_aton(self.src_addr),\ inet_aton(self.dst_addr), \ 0,\ @@ -242,7 +255,7 @@ def to_buffer(self): data = self.get_segment().to_buffer() if self.segment else b'' self.len = 20 + len(data) - return struct.pack(ENDIANESS+'BBHHHBBH4s4s', \ + return struct.pack(SequenceOrder().get()+'BBHHHBBH4s4s', \ ihl_ver, \ self.tos, \ self.len, \ @@ -318,10 +331,10 @@ class EthernetFrame: @classmethod def from_buffer(cls,buffer): - return cls(*struct.unpack(ENDIANESS+"6s6s2s", buffer[0:14]),IPPacket.from_buffer(buffer[14:])) + return cls(*struct.unpack(SequenceOrder().get()+"6s6s2s", buffer[0:14]),IPPacket.from_buffer(buffer[14:])) def to_buffer(self): - return struct.pack(ENDIANESS+"6s6s2s", unhexlify(self.src_mac), unhexlify(self.dst_mac),self.eth_type) + self.get_packet().to_buffer() + return struct.pack(SequenceOrder().get()+"6s6s2s", unhexlify(self.src_mac), unhexlify(self.dst_mac),self.eth_type) + self.get_packet().to_buffer() def get_packet(self): return self.packet \ No newline at end of file diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index e4cd367bc..6db5be21e 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -34,7 +34,7 @@ async def send_encoded_msg(stream, msg_code, data={}): await stream.write(f'{message}\r\n'.encode()) await stream.drain() -debugger() +#debugger() class Runner: def __init__(self, instance_id, sequence_path, log_setup) -> None: diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 1c6237587..a53339f00 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -3,13 +3,13 @@ import random import socket from attrs import define, field -from inet import IPPacket, TCPSegment, USE_LITTLENDIAN +from inet import IPPacket, TCPSegment, SequenceOrder from hardcoded_magic_values import CommunicationChannels as CC -USE_LITTLENDIAN() +SequenceOrder().use_little_endian() @define -class _ChannelContext: +class _ChannelContext: _channel_enum: CC _reader: asyncio.StreamReader diff --git a/packages/python-runner/test/test_inet.py b/packages/python-runner/test/test_inet.py index c55b0a627..f06c8cf73 100644 --- a/packages/python-runner/test/test_inet.py +++ b/packages/python-runner/test/test_inet.py @@ -1,10 +1,10 @@ import pytest -from inet import TCPSegment, IPPacket, EthernetFrame, USE_BIGENDIAN, USE_LITTLENDIAN +from inet import TCPSegment, IPPacket, EthernetFrame, SequenceOrder class TestIP: def test_mf_df_flags(self): - USE_BIGENDIAN() + SequenceOrder().use_big_endian() data = b'E\x00\x00\x14\x00\x01`\x00@\x00\x1c\xe7\x7f\x00\x00\x01\x7f\x00\x00\x01' pkt = IPPacket.from_buffer(data) @@ -15,7 +15,7 @@ def test_mf_df_flags(self): def test_mf_flags(self): - USE_BIGENDIAN() + SequenceOrder().use_big_endian() data = b'E\x00\x00\x14\x00\x01 \x00@\x00\\\xe7\x7f\x00\x00\x01\x7f\x00\x00\x01' pkt = IPPacket.from_buffer(data) @@ -26,7 +26,7 @@ def test_mf_flags(self): def test_df_flags(self): - USE_BIGENDIAN() + SequenceOrder().use_big_endian() data = (b'\x00\x23\x20\xd4\x2a\x8c\x00\x23\x20\xd4\x2a\x8c\x08\x00\x45\x00\x00\x54\x00\x00\x40\x00' b'\x40\x01\x25\x8d\x0a\x00\x00\x8f\x0a\x00\x00\x8e\x08\x00\x2e\xa0\x01\xff\x23\x73\x20\x48' @@ -47,7 +47,7 @@ def test_df_flags(self): def test_checksum_calc(self): - USE_BIGENDIAN() + SequenceOrder().use_big_endian() data = b'E\x00\x00\x14\x00\x01\x00\x00@\x00j\xd6\x01\x02\x03\x04\x05\x06\x07\x08' checksum = 27350 @@ -64,7 +64,7 @@ def test_checksum_calc_as_sth_expected(self): checksum_from_sth = 64434 data =bytes([10, 0, 0, 1, 10, 0, 0, 2, 0, 1, 69, 0, 0, 0, 0, 0, 32, 89, 3, 38, 0, 0, 0, 0, 0, 8, 0, 32, 178, 251, 0, 0, 97, 102, 54, 98, 50, 51, 52, 56, 45, 102, 51, 56, 56, 45, 52, 54, 56, 48, 45, 98, 54, 48, 97, 45, 97, 53, 102, 98, 50, 102, 98, 100, 100, 102, 50, 97, 48]) - USE_LITTLENDIAN() + SequenceOrder().use_little_endian() pkt = IPPacket.from_buffer_with_pseudoheader(data) assert pkt.to_buffer_with_tcp_pseudoheader() == data @@ -77,7 +77,7 @@ def test_checksum_calc_as_sth_expected(self): def test_packet_creation(self): - USE_BIGENDIAN() + SequenceOrder().use_big_endian() pkt = IPPacket(src_addr='172.25.44.3', dst_addr='172.25.44.254', segment=TCPSegment(dst_port=3, flags=['ACK'])) @@ -88,7 +88,7 @@ def test_packet_creation(self): def test_pseudoheader(self): - USE_LITTLENDIAN() + SequenceOrder().use_little_endian() data = bytes([10, 0, 0, 1, 10, 0, 0, 2, 0, 1, 32, 0, 0, 0, 0, 0, 249, 252, 104, 127, 0, 0, 0, 0, 0, 8, 0, 0, 149, 136, 0, 0]) res = IPPacket.from_buffer_with_pseudoheader(data) @@ -109,7 +109,7 @@ def test_pseudoheader(self): class TestTCP: def test_tcp_offset(self): - USE_BIGENDIAN() + SequenceOrder().use_big_endian() data = b'\x01\xbb\xc0\xd7\xb6\x56\xa8\xb9\xd1\xac\xaa\xb1\x50\x18\x40\x00\x56\xf8\x00\x00' segment = TCPSegment.from_buffer(data) @@ -117,7 +117,7 @@ def test_tcp_offset(self): def test_prepare_segment_with_flags(self): - USE_BIGENDIAN() + SequenceOrder().use_big_endian() segment = TCPSegment(flags=['FIN', 'SYN', 'ACK']) @@ -149,7 +149,7 @@ def test_prepare_segment_with_flags(self): def test_basic_details(self): - USE_BIGENDIAN() + SequenceOrder().use_big_endian() data = b'E\x00\x00(\x00\x01@\x00@\x06,\xbe\x01\x02\x03\x04\x04\x05\x06\x07\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x81m\x00\x00' pkt = IPPacket.from_buffer(data) @@ -160,7 +160,7 @@ def test_basic_details(self): def test_checksum(self): - USE_BIGENDIAN() + SequenceOrder().use_big_endian() data = b'E\x00\x00(\x00\x01@\x00@\x06,\xbe\x01\x02\x03\x04\x04\x05\x06\x07\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x81m\x00\x00' pkt = IPPacket.from_buffer(data) @@ -168,7 +168,7 @@ def test_checksum(self): def test_checksum_calc(self): - USE_BIGENDIAN() + SequenceOrder().use_big_endian() data = b'E\x00\x00*\x00\x01\x00\x00@\x06N\x9d\n\x0b\x0c\x0e\n\x0b\x0c\r\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x00\x00\x00\x00Hi' pkt = IPPacket.from_buffer(data) @@ -177,11 +177,11 @@ def test_checksum_calc(self): pkt = IPPacket.from_buffer(data).build() - assert pkt.segment.checksum == 6883 + assert pkt.segment.checksum == 6871 def test_unpack(self): - USE_BIGENDIAN() + SequenceOrder().use_big_endian() data = (b'\x00\x50\x0d\x2c\x11\x4c\x61\x8b\x38\xaf\xfe\x14\x70\x12\x16\xd0' b'\x5b\xdc\x00\x00\x02\x04\x05\x64\x01\x01\x04\x02') @@ -195,8 +195,7 @@ def test_unpack(self): def test_parse_only_first_packet(self): - - USE_BIGENDIAN() + SequenceOrder().use_big_endian() data = (b"\x05\x00\x005\x00\x00\x00\x00\xff\x06\xfb\xc3\x00\x00\x00\x00\x00" b"\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00" From 2580f1873a9252527618fede1fba5e4c2cbb5a5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 16 Jun 2023 16:57:54 +0200 Subject: [PATCH 165/231] Documentation for inet --- packages/python-runner/inet.py | 489 +++++++++++++++++++++++---------- 1 file changed, 347 insertions(+), 142 deletions(-) diff --git a/packages/python-runner/inet.py b/packages/python-runner/inet.py index 53e89be1c..481e42743 100644 --- a/packages/python-runner/inet.py +++ b/packages/python-runner/inet.py @@ -1,61 +1,109 @@ +""" inet.py - contains TCP Segment, IP Packet and Ethernet Frame support for Tecemux in Python runner +""" + import struct from binascii import hexlify, unhexlify from socket import inet_ntoa, inet_aton -from attrs import define,field +from attrs import define, field + class _Singleton(type): + """Metaclass for Singleton pattern + """ _instances = {} + def __call__(cls, *args, **kwargs): if cls not in cls._instances: - cls._instances[cls] = super(_Singleton, cls).__call__(*args, **kwargs) + cls._instances[cls] = super( + _Singleton, cls).__call__(*args, **kwargs) return cls._instances[cls] - + + class SequenceOrder(metaclass=_Singleton): + """Sets sequence order for buffer analysis + """ class _Order: + """ Contains suported orders. For internal usage only + """ LITTLE_ENDIAN = '<' BIG_ENDIAN = '>' endianess = _Order.LITTLE_ENDIAN def use_big_endian(self): + """ Sets Big-Endian order + """ self.endianess = SequenceOrder._Order.BIG_ENDIAN def use_little_endian(self): + """ Sets Little-endian order + """ self.endianess = SequenceOrder._Order.LITTLE_ENDIAN def get(self): + """ Returns order struct module format + '<' is little-endian + '>' is big-endian + + Returns: + str: return endianess for struct module + """ return self.endianess - + @define class TCPSegment: + """ Class for manage TCP segment data + """ + class Options: + """ TCP Segments options, at the end of TCP Header + """ + EOL = 0 NOP = 1 MSS = 2 - WSOPT= 3 + WSOPT = 3 SACKPERM = 4 SACK = 5 TSOPT = 8 @staticmethod def parse_options(val): + """Returns TCP option in readable format + + Args: + val (bytes): Part of TCP Header + + Returns: + str: Options in readable format + """ return val - - + class Flags: - FIN = 0x01 # end of data - SYN = 0x02 # synchronize sequence numbers - RST = 0x04 # reset connection - PSH = 0x08 # push - ACK = 0x10 # acknowledgment number set - URG = 0x20 # urgent pointer set - ECE = 0x40 # ECN echo, RFC 3168 - CWR = 0x80 # congestion window reduced - NS = 0x100 # nonce sum, RFC 3540 + """TCP Segments flags + """ + + FIN = 0x01 # end of data + SYN = 0x02 # synchronize sequence numbers + RST = 0x04 # reset connection + PSH = 0x08 # push + ACK = 0x10 # acknowledgment number set + URG = 0x20 # urgent pointer set + ECE = 0x40 # ECN echo, RFC 3168 + CWR = 0x80 # congestion window reduced + NS = 0x100 # nonce sum, RFC 3540 @staticmethod def flags_to_str(val): + """Returns TCP flags in readable format + + Args: + val (bytes): Part of TCP segment header + + Returns: + list: List of setted flags in segment + """ flags = [] if val & TCPSegment.Flags.FIN: flags.append('FIN') @@ -76,79 +124,128 @@ def flags_to_str(val): if val & TCPSegment.Flags.NS: flags.append('NS') return '+'.join(flags) - + @staticmethod def parse_flags(value): + """Checks Bit state of each flag int TCP Header + + Args: + value (None,int,bytes): TCP Header part to analyze + + Returns: + int: Integer values represents bit state of each flag + """ res = 0 if value is None: return 0 - if isinstance(value,int): + if isinstance(value, int): return value - + for flag in value: - res = res | getattr(TCPSegment.Flags, flag ) + res = res | getattr(TCPSegment.Flags, flag) return res + src_port: int = field(default=0) + dst_port: int = field(default=0) + seq: int = field(default=0) + ack: int = field(default=0) + offset: int = field(default=0, converter=lambda value: value >> 4) + flags: int = field(default=0, repr=lambda value: TCPSegment.Flags.flags_to_str(value), + converter=lambda value: TCPSegment.Flags.parse_flags(value)) + win: int = field(default=8192) + checksum: int = field(default=0, repr=lambda value: hex(value)) + urp: int = field(default=0) + data: bytes = field(default=b'', repr=lambda value: f'{value[0:5]}... ' + if len(value) > 5 else f'{value}') + + opt: bytes = field( + default=b'', converter=lambda value: TCPSegment.Options.parse_options(value)) - src_port: int = field(default = 0) - dst_port: int = field(default = 0) - seq: int = field(default = 0) - ack: int = field(default = 0) - offset: int = field(default = 0, converter = lambda value: value >> 4) - flags: int = field(default = 0, repr = lambda value: TCPSegment.Flags.flags_to_str(value), \ - converter = lambda value: TCPSegment.Flags.parse_flags(value)) - win: int = field(default = 8192) - checksum: int = field(default = 0, repr = lambda value: hex(value)) - urp: int = field(default = 0) - data: bytes = field(default=b'', repr = lambda value: f'{value[0:5]}... ' \ - if len(value)>5 else f'{value}') + @classmethod + def from_buffer(cls, buffer: bytes): + """Creates TCP Segment object from provided raw buffer - opt: bytes = field(default=b'', converter = lambda value: TCPSegment.Options.parse_options(value)) + Args: + buffer (bytes): Data from socket + + Returns: + TCPSegment: Object of TCP Segment + """ - @classmethod - def from_buffer(cls,buffer): TCP_MIN = 20 - src_port, dst_port, seq, ack, offres, flags, win, checksum, urp = struct.unpack(SequenceOrder().get() +"HHIIBBHHH", buffer[0:TCP_MIN]) + src_port, dst_port, seq, ack, offres, flags, win, checksum, urp = struct.unpack( + SequenceOrder().get() + "HHIIBBHHH", buffer[0:TCP_MIN]) hdr_len = (offres >> 4) * 4 - if hdr_len <= TCP_MIN: + if hdr_len <= TCP_MIN: return cls(src_port, dst_port, seq, ack, offres, flags, win, checksum, urp, buffer[TCP_MIN:], b'') if hdr_len > TCP_MIN: - return cls(src_port, dst_port, seq, ack, offres, flags, win, checksum, urp, buffer[hdr_len:],buffer[TCP_MIN:hdr_len]) + return cls(src_port, dst_port, seq, ack, offres, flags, win, checksum, urp, buffer[hdr_len:], buffer[TCP_MIN:hdr_len]) - def to_buffer(self): - return struct.pack(SequenceOrder().get()+'HHIIBBHHH', \ - self.src_port,\ - self.dst_port,\ - self.seq,\ - self.ack,\ - self.offset << 4,\ - self.flags,\ - self.win,\ - self.checksum,\ - self.urp) + (self.data.encode("utf-8") if isinstance(self.data,str) else self.data) - - def set_flags(self, list_of_flags): + """Build raw buffer from TCP Segment object + + Returns: + bytes: Raw buffer + """ + return struct.pack(SequenceOrder().get()+'HHIIBBHHH', + self.src_port, + self.dst_port, + self.seq, + self.ack, + self.offset << 4, + self.flags, + self.win, + self.checksum, + self.urp) + (self.data.encode("utf-8") if isinstance(self.data, str) else self.data) + + def set_flags(self, list_of_flags: int): + """Enables flags in segment + + Args: + list_of_flags (int): Single integer value represents all flag bits + + Returns: + TCPObject: Retuns self + """ self.flags = list_of_flags return self - - def is_flag(self,flag): - return (self.flags & getattr(TCPSegment.Flags, flag)) > 0 - def encapsulate(self, src_addr: str, dst_addr: str): - return IPPacket(src_addr=src_addr, dst_addr=dst_addr,segment=self) + def is_flag(self, flag: Flags) -> bool: + """Checks whether provided flag is enabled in segment + + Args: + flag (TCPSegment.Flags): Specific flag to check + + Returns: + bool: State of provided flag + """ + + return (self.flags & getattr(TCPSegment.Flags, flag)) > 0 @define class IPPacket: + """ Class for manage IP Packet data + """ + class Flags: + """TCP Segments flags + """ RF = 0x4 # reserved DF = 0x2 # don't fragment MF = 0x1 # more fragments @staticmethod def flags_to_str(val): + """Returns IP flags in readable format + + Args: + val (bytes): Part of IP Packet header + + Returns: + list: List of setted flags in packet + """ flags = [] if val & IPPacket.Flags.RF == 0: if val & IPPacket.Flags.MF != 0: @@ -158,154 +255,240 @@ def flags_to_str(val): else: flags.append('UNKNOWN') return '+'.join(flags) - + @staticmethod def parse_flags(value): + """Checks bit state of each flag in IP Header + + Args: + value (None,int,bytes): IP Header part to analyze + + Returns: + int: Integer values represents bit state of each flag + """ res = 0 if value is None: return 0 - if isinstance(value,int): + if isinstance(value, int): return value - + for flag in value: - res = res | getattr(IPPacket.Flags, flag ) + res = res | getattr(IPPacket.Flags, flag) return res - + ihl: int = 5 - version: int = field( default=69, converter = lambda value: value >> 4) + version: int = field(default=69, converter=lambda value: value >> 4) tos: int = field(default=0) len: int = field(default=0) ids: int = field(default=1) - flags_offset: int = field(default = 0) - flags: int = field(default = 0, init= False, repr = lambda value: IPPacket.Flags.flags_to_str(value), \ - converter = lambda value: IPPacket.Flags.parse_flags(value)) - offset: int = field(default = 0, init=False) + flags_offset: int = field(default=0) + flags: int = field(default=0, init=False, repr=lambda value: IPPacket.Flags.flags_to_str(value), + converter=lambda value: IPPacket.Flags.parse_flags(value)) + offset: int = field(default=0, init=False) ttl: int = field(default=64) protocol: int = field(default=6) - checksum: int = field(default = 0, repr = lambda value: hex(value)) - src_addr: str = field(default='10.0.0.1', converter = lambda value: inet_ntoa(value) if isinstance(value,bytes) else value) - dst_addr: str = field(default='10.0.0.2', converter = lambda value: inet_ntoa(value) if isinstance(value,bytes) else value) + checksum: int = field(default=0, repr=lambda value: hex(value)) + src_addr: str = field(default='10.0.0.1', converter=lambda value: inet_ntoa( + value) if isinstance(value, bytes) else value) + dst_addr: str = field(default='10.0.0.2', converter=lambda value: inet_ntoa( + value) if isinstance(value, bytes) else value) segment: TCPSegment = None def __attrs_post_init__(self): + """Internal function executes after constructor + """ + self.offset = self.flags_offset & 0x1FFF - self.flags = self.flags_offset >> 13 + self.flags = self.flags_offset >> 13 - #Cut data buffer to IP packet length + # Cut data buffer to IP packet length if self.len > 0 and self.segment: - self.get_segment().data=self.get_segment().data[:self.len-(self.ihl*4)-20] + self.get_segment().data = self.get_segment( + ).data[:self.len-(self.ihl*4)-20] @staticmethod def calc_checksum(pkt: bytes) -> int: + """Calculates checksum for provited packet + + Args: + pkt (bytes): Raw buffer + + Returns: + int: Calculated checsum + """ if len(pkt) % 2 == 1: pkt += b"\0" - s = sum(struct.unpack(('<' if SequenceOrder().get() is '>' else '>')+str(len(pkt)//2)+'H',pkt)) - - #source: https://github.com/secdev/scapy + s = sum(struct.unpack(('<' if SequenceOrder().get() + is '>' else '>')+str(len(pkt)//2)+'H', pkt)) + + # source: https://github.com/secdev/scapy s = (s >> 16) + (s & 0xffff) s += s >> 16 s = ~s return (((s >> 8) & 0xff) | s << 8) & 0xffff - - def calc_checksum_for_STH(pkt): + + @staticmethod + def calc_checksum_for_STH(pkt: bytes) -> int: + """Calculates checksum for provited packet in ormat valid for Transform Hub + + Args: + pkt (bytes): Raw buffer + + Returns: + int: Calculated checksum + """ if len(pkt) % 2 == 1: pkt += b"\0" - elements = list(struct.unpack(SequenceOrder().get()+str(len(pkt)//2)+'H',pkt)) + elements = list(struct.unpack( + SequenceOrder().get()+str(len(pkt)//2)+'H', pkt)) elements = elements[:14] + elements[15:] s = sum(elements) return s % 0x10000 - + @classmethod - def from_buffer_with_pseudoheader(cls,buffer): - src_addr, dst_addr, _, proto, length = struct.unpack(SequenceOrder().get()+"4s4sBBH",bytes(buffer[0:12])) - pkt = cls(0, 0, 0, length, 0, 0, 0, proto, 0, src_addr, dst_addr, TCPSegment.from_buffer(buffer[12:]) if len(buffer) > 12 else None) + def from_buffer_with_pseudoheader(cls, buffer:bytes): + """Creates IP Packet object and TCP Segment object + from provided raw buffer with pseudo TCP Header + + Args: + buffer (bytes): Raw buffer + + Returns: + IPPacket: IPPacket object + """ + src_addr, dst_addr, _, proto, length = struct.unpack( + SequenceOrder().get()+"4s4sBBH", bytes(buffer[0:12])) + pkt = cls(0, 0, 0, length, 0, 0, 0, proto, 0, src_addr, dst_addr, + TCPSegment.from_buffer(buffer[12:]) if len(buffer) > 12 else None) return pkt - + @classmethod - def from_buffer(cls,buffer): + def from_buffer(cls, buffer: bytes): + """Creates IP Packet object and TCP Segment object + from provided raw buffer with + + Args: + buffer (bytes): Raw buffer + + Returns: + IPPacket: IPPacket object + """ ihl = (buffer[0] & 0xf) - pkt = cls(ihl, *struct.unpack(SequenceOrder().get()+"BBHHHBBH4s4s", buffer[0:ihl*4]),TCPSegment.from_buffer(buffer[ihl*4:]) if len(buffer) > ihl*4 else None) + pkt = cls(ihl, *struct.unpack(SequenceOrder().get()+"BBHHHBBH4s4s", + buffer[0:ihl*4]), TCPSegment.from_buffer(buffer[ihl*4:]) if len(buffer) > ihl*4 else None) - #Cut data buffer to IP packet length + # Cut data buffer to IP packet length if pkt.segment: - pkt.get_segment().data=pkt.get_segment().data[:pkt.len-(ihl*4)-20] - + pkt.get_segment().data = pkt.get_segment( + ).data[:pkt.len-(ihl*4)-20] + return pkt - def is_flag(self,flag): + def is_flag(self, flag: Flags) -> bool: + """Checks whether provided flag is enabled in packet + """ return (self.flags & getattr(IPPacket.Flags, flag)) > 0 - - def prepare_pseudoheader(self, protocol, len): - return struct.pack(SequenceOrder().get()+"4s4sBBH", \ - inet_aton(self.src_addr),\ - inet_aton(self.dst_addr), \ - 0,\ - protocol, \ - len+12) - - def to_buffer_with_tcp_pseudoheader(self): - + def prepare_pseudoheader(self, protocol:int, length:int) -> bytes: + """Prepare TCP pseudoheader to calulate checksum + + Args: + protocol (int): Packet protocol value + length (int): Length of data (without header) + + Returns: + bytes: TCP Pseudoheader + """ + return struct.pack(SequenceOrder().get()+"4s4sBBH", + inet_aton(self.src_addr), + inet_aton(self.dst_addr), + 0, + protocol, + length+12) + + def to_buffer_with_tcp_pseudoheader(self) -> bytes: + """Build raw buffer from IP Packet with pseudo TCP header + + Returns: + bytes: Raw buffer + """ data = self.get_segment().to_buffer() if self.segment else b'' - + return self.prepare_pseudoheader(self.protocol, len(data)) + data - + def to_buffer(self): + """Build raw buffer from IP Packet + + Returns: + bytes: Raw buffer + """ + ihl_ver = (int(self.version) << 4) + int(self.ihl) data = self.get_segment().to_buffer() if self.segment else b'' self.len = 20 + len(data) - return struct.pack(SequenceOrder().get()+'BBHHHBBH4s4s', \ - ihl_ver, \ - self.tos, \ - self.len, \ - self.ids, - self.flags_offset, \ - self.ttl, \ - self.protocol, \ - self.checksum, \ - inet_aton(self.src_addr), \ - inet_aton(self.dst_addr)) + data - + return struct.pack(SequenceOrder().get()+'BBHHHBBH4s4s', + ihl_ver, + self.tos, + self.len, + self.ids, + self.flags_offset, + self.ttl, + self.protocol, + self.checksum, + inet_aton(self.src_addr), + inet_aton(self.dst_addr)) + data + def _validate_tcp(self): - - tcp_hdr_len = 5 + """ Calculates checksum for TCP segment part only + """ + tcp_hdr_len = 5 self.segment.offset = tcp_hdr_len << 4 self.segment.checksum = 0 - tcp_segment = self.segment.to_buffer() - + tcp_segment = self.segment.to_buffer() + pseudo_hdr = self.prepare_pseudoheader(self.protocol, len(tcp_segment)) - - self.segment.checksum = IPPacket.calc_checksum(pseudo_hdr + tcp_segment) + + self.segment.checksum = IPPacket.calc_checksum( + pseudo_hdr + tcp_segment) return self - + def _validate_tcp_for_STH(self): - - tcp_hdr_len = 5 - self.segment.offset = tcp_hdr_len << 4 + """ Calculates checksum with format valid for Transform Hub + """ + tcp_hdr_len = 5 + self.segment.offset = tcp_hdr_len << 4 self.segment.checksum = 0 self.protocol = 1 - tcp_segment = self.segment.to_buffer() - + tcp_segment = self.segment.to_buffer() + pseudo_hdr = self.prepare_pseudoheader(self.protocol, len(tcp_segment)) - - self.segment.checksum = IPPacket.calc_checksum_for_STH(pseudo_hdr + tcp_segment) + + self.segment.checksum = IPPacket.calc_checksum_for_STH( + pseudo_hdr + tcp_segment) return self - + def _validate_ip(self): + """ Calculates checksum IP Packet part only + """ self.checksum = 0 self.checksum = IPPacket.calc_checksum(self.to_buffer()[0:self.ihl*4]) return self - + def build(self, for_STH=False): + """Calculates all checksums + Args: + for_STH (bool, optional): If True, cheksums calculates with format valid for Transform Hub. + Otherise it is calcuated with RFC. Defaults to False. + """ if for_STH: if self.segment: self._validate_tcp_for_STH() @@ -313,28 +496,50 @@ def build(self, for_STH=False): if self.segment: self._validate_tcp() self._validate_ip() - - return self - - def encapsulate(self): - return EthernetFrame(b'', b'', b'', self) + return self def get_segment(self): + """Return TCP Segment + + Returns: + TCPSegment: TCP Segment object + """ return self.segment @define class EthernetFrame: - src_mac: str = field(converter = lambda value: hexlify(value)) - dst_mac: str = field(converter = lambda value: hexlify(value)) - eth_type: bytes = field(repr = lambda value: hexlify(value)) + """ Class for manage Ethernet frame data + """ + + src_mac: str = field(converter=lambda value: hexlify(value)) + dst_mac: str = field(converter=lambda value: hexlify(value)) + eth_type: bytes = field(repr=lambda value: hexlify(value)) packet: IPPacket @classmethod - def from_buffer(cls,buffer): - return cls(*struct.unpack(SequenceOrder().get()+"6s6s2s", buffer[0:14]),IPPacket.from_buffer(buffer[14:])) + def from_buffer(cls, buffer: bytes): + """Creates Ethernet frame object from provided raw buffer - def to_buffer(self): - return struct.pack(SequenceOrder().get()+"6s6s2s", unhexlify(self.src_mac), unhexlify(self.dst_mac),self.eth_type) + self.get_packet().to_buffer() + Args: + buffer (bytes): Raw buffer + + Returns: + EthernetFrame: Ethernet frame object + """ + return cls(*struct.unpack(SequenceOrder().get()+"6s6s2s", buffer[0:14]), IPPacket.from_buffer(buffer[14:])) + + def to_buffer(self) -> bytes: + """Build raw buffer from Ethernet frame object + + Returns: + _bytes: Raw buffer + """ + return struct.pack(SequenceOrder().get()+"6s6s2s", unhexlify(self.src_mac), unhexlify(self.dst_mac), self.eth_type) + self.packet.to_buffer() def get_packet(self): + """Return IP Packet object + + Returns: + IPPacket: IP Packet object + """ return self.packet \ No newline at end of file From 4555cda73e1d50d9654778d2f8f7790c34b273de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Mon, 19 Jun 2023 11:46:39 +0200 Subject: [PATCH 166/231] Tecemux docstrings --- packages/python-runner/tecemux.py | 344 +++++++++++++++----- packages/python-runner/test/test_tecemux.py | 8 +- 2 files changed, 264 insertions(+), 88 deletions(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index a53339f00..ca7ee298c 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -3,18 +3,23 @@ import random import socket from attrs import define, field +from typing import Any, Coroutine from inet import IPPacket, TCPSegment, SequenceOrder from hardcoded_magic_values import CommunicationChannels as CC +SequenceOrder().use_little_endian() + -SequenceOrder().use_little_endian() @define -class _ChannelContext: +class _ChannelContext: + """Internal class to manage single channel + """ + _channel_enum: CC - + _reader: asyncio.StreamReader _writer: asyncio.StreamWriter - + _global_queue: asyncio.Queue _global_instance_id: str @@ -22,124 +27,224 @@ class _ChannelContext: _internal_queue: asyncio.Queue = field(init=False) _channel_paused: bool = field(default=False, init=False) _channel_opened: bool = field(default=False, init=False) - - def __attrs_post_init__(self): + + def __attrs_post_init__(self) -> None: + """Internal function executes after constructor + """ + self._internal_queue = asyncio.Queue() - def _get_channel_name(self): + def _get_channel_name(self) -> str: + """Returns channel name + + Returns: + str: Channel name + """ + return str(self._channel_enum.name) - def _get_channel_id(self): + def _get_channel_id(self) -> int: + """Return channel id + + Returns: + int: Channel id + """ return int(self._channel_enum.value) - def _set_logger(self, logger): + def _set_logger(self, logger: logging.Logger) -> None: + """Sets new logger + + Args: + logger (logging.Logger): Logger object + """ self._logger = logger - - def readline(self): + + def readline(self) -> Coroutine: + """asyncio.StreamReader API. Reads chunk of data from the stream until newline (b'\n') is found. + + Returns: + Coroutine + """ + return self._reader.readline() - def readuntil(self, separator=b'\n'): + def readuntil(self, separator: bytes = b'\n') -> Coroutine: + """asyncio.StreamReader API. Reads data from the stream until ``separator`` is found. + + Args: + separator (bytes, optional): Data separator. Defaults to b'\n'. + + Returns: + Coroutine + """ + return self._reader.readuntil(separator) - def read(self, n=-1): + def read(self, n: int = -1) -> Coroutine: + """asyncio.StreamReader API. Reads up to `n` bytes from the stream. + + Args: + n (int, optional): Number of bytes to read. Defaults to -1. + + Returns: + Coroutine + """ return self._reader.read(n) - def readexactly(self, n): + def readexactly(self, n: int) -> Coroutine: + """asyncio.StreamReader API. Reads exactly `n` bytes. + + Args: + n (int): Number of bytes to read + + Returns: + Coroutine + """ return self._reader.readexactly(n) - + def __aiter__(self): + """asyncio.StreamReader API + """ return self._reader.__aiter__ async def __anext__(self): + """asyncio.StreamReader API + """ val = await self._reader.readline() if val == b'': raise StopAsyncIteration return val - async def send_ACK(self, sequence_number): + async def send_ACK(self, sequence_number: int) -> None: + """Adds to global queue an ACK packet to send. + + Args: + sequence_number (int): Value for Acknowledge field + """ await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK'], ack=sequence_number))) - - async def _send_pause_ACK(self, sequence_number): - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK','SYN'], ack=sequence_number))) - async def open(self): + async def _send_pause_ACK(self, sequence_number: int) -> None: + """"Add to global queue an pause packet to send + + Args: + sequence_number (int): Value for Acknowledge field + """ + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK', 'SYN'], ack=sequence_number))) + + async def open(self) -> None: + """Open channel. + """ + if not self._channel_opened: await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'', flags=['PSH']))) - - buf = b'' if self._global_instance_id is None else (self._global_instance_id.encode() + str(self._get_channel_id()).encode()) + + buf = b'' if self._global_instance_id is None else ( + self._global_instance_id.encode() + str(self._get_channel_id()).encode()) await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=buf, flags=['PSH']))) self._channel_opened = True - async def close(self): - self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel close request is send') - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(),data=b'' if self._global_instance_id is None else self._global_instance_id, flags=['FIN']))) + async def close(self) -> None: + """Close channel + """ + self._logger.debug( + f'Tecemux/{self._get_channel_name()}: [-] Channel close request is send') + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'' if self._global_instance_id is None else self._global_instance_id, flags=['FIN']))) + + async def queue_up_incoming(self, pkt: IPPacket) -> None: + """Redirects incomming data from provided packet to current channel - async def queue_up_incoming(self, pkt): - #if SYN & ACK flag is up, pause channel + Args: + pkt (IPPacket): Redirected packet from outside + """ + # if SYN & ACK flag is up, pause channel if pkt.get_segment().is_flag('SYN') and pkt.get_segment().is_flag('ACK'): - self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel pause request received') + self._logger.debug( + f'Tecemux/{self._get_channel_name()}: [-] Channel pause request received') self.set_pause(True) await self.send_ACK(pkt.get_segment().seq) return - - #if ACK flag is up, reasume channel + + # if ACK flag is up, reasume channel if not pkt.get_segment().is_flag('SYN') and pkt.get_segment().is_flag('ACK'): self.set_pause(False) return - #if PSH flag is up, confirm + # if PSH flag is up, confirm if pkt.segment.is_flag('PSH'): await self.send_ACK(pkt.get_segment().seq) - + self._writer.write(pkt.get_segment().data) await self._writer.drain() - async def queue_up_outcoming(self, data): + async def _queue_up_outcoming(self, data: bytes) -> None: + """Redirects raw data from currect channel to global queue. + Args: + data (bytes): Buffer to send + """ def wrap(channel_enum, buf): channel_id = int(channel_enum.value) - return IPPacket(segment=TCPSegment(dst_port=channel_id, flags=['PSH'],data=buf)) + return IPPacket(segment=TCPSegment(dst_port=channel_id, flags=['PSH'], data=buf)) if self._channel_paused: - self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] Channel paused. Data queued up internally for future') + self._logger.debug( + f'Tecemux/{self._get_channel_name()}: [-] Channel paused. Data queued up internally for future') await self._internal_queue.put(data) else: while not self._internal_queue.empty(): try: buf = await self._internal_queue.get() - await self._global_queue.put(wrap(self._channel_enum, data)) + await self._global_queue.put(wrap(self._channel_enum, buf)) except asyncio.QueueEmpty: - self._logger.debug(f'Tecemux/{self._get_channel_name()}: [-] All data stored during pause were redirected to global queue') + self._logger.debug( + f'Tecemux/{self._get_channel_name()}: [-] All data stored during pause were redirected to global queue') break await self._global_queue.put(wrap(self._channel_enum, data)) - async def write(self, data): + async def write(self, data: bytes) -> None: + """Writes data to channel + + Args: + data (bytes): Buffer to send to channel + """ if not self._channel_opened: await self.open() - await self.queue_up_outcoming(data) + await self._queue_up_outcoming(data) - async def drain(self): + async def drain(self) -> bool: + """Drain channel + """ if not self._channel_paused: await self._writer.drain() - return True - def write_eof(self): + def write_eof(self) -> None: + """asyncio.StreamWriter API + """ return self._writer.write_eof() - - def wait_closed(self): - return self._writer.wait_closed() - def set_pause(self, state): + def wait_closed(self) -> Coroutine: + """asyncio.StreamWriter API + """ + return self._writer.wait_closed() + + def set_pause(self, state: bool) -> None: + """Sets pause state for current channel + + Args: + state (bool): Pause state + """ self._channel_paused = state @define class Tecemux: + """Tecemux protocol implementation for Scramjet Transform Hub Python Runner + """ _queue: asyncio.Queue = field(default=None) _reader: asyncio.StreamReader = field(default=None) @@ -152,72 +257,133 @@ class Tecemux: _channels: dict = field(default={}) _logger: logging.Logger = field(default=None) _global_stop_event: asyncio.Event = asyncio.Event() - _sequence_number: int = field(default=0) + _sequence_number: int = field(default=0) _last_sequence_received: int = field(default=0) - - async def connect(self, reader, writer): + + async def connect(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + """Connects to Transform Hub via provided reader and writer objects + + Args: + reader (asyncio.StreamReader): Main stream reader + writer (asyncio.StreamWriter): Main stream writer + """ self._reader = reader self._writer = writer self._sequence_number = abs(int((random.random() * (2 ** 32)) / 2)) self._global_stop_event.clear() @staticmethod - async def prepare_tcp_connection(server_host, server_port): + async def prepare_tcp_connection(server_host: str, server_port: str) -> tuple: + """Prepares TCP connection to server_host:server:port + + Args: + server_host (str): Server address + server_port (str): Server port + + Returns: + tuple: Pair of StreamReader, StreamWriter + """ return await asyncio.open_connection(server_host, server_port) @staticmethod - async def prepare_socket_connection(): + async def prepare_socket_connection() -> tuple: + """Prepares Unix socket pair to create connection. + + Returns: + tuple: Pair of StreamReader, StreamWriter + """ rsock, wsock = socket.socketpair() reader, _ = await asyncio.open_unix_connection(sock=rsock) _, writer = await asyncio.open_unix_connection(sock=wsock) return reader, writer - async def prepare(self, force_open=False): + async def prepare(self, force_open: bool = False) -> None: + """Opens all channels to Transform Hub + + Args: + force_open (bool, optional): If True, all channels will be opened immediately, Otherwise, will be opened on demand. Defaults to False. + """ + self._queue = asyncio.Queue() - self._channels = {channel: _ChannelContext(channel, *await Tecemux.prepare_socket_connection(), self._queue, self._instance_id ,self._logger) for channel in CC} - + self._channels = {channel: _ChannelContext(channel, *await Tecemux.prepare_socket_connection(), self._queue, self._instance_id, self._logger) for channel in CC} + if force_open: [await channel.open() for channel in self.get_channels()] - def set_logger(self, logger): + def set_logger(self, logger: logging.Logger) -> None: + """Sets logger + + Args: + logger (logging.Logger): Logger object + """ self._logger = logger - def get_channel(self, channel): + def get_channel(self, channel: CC) -> _ChannelContext: + """Returns single channel context + + Args: + channel (CC): Channel Enum + + Returns: + _ChannelContext: Channel context + """ return self._channels[channel] - - def get_channels(self): + + def get_channels(self) -> dict: + """Returns all initiated channels + + Returns: + dict: Dict of channel's contexts + """ return self._channels.values() - + @staticmethod - def _chunk_preview(value): - return f'{value[0:5]}... ' if len(value)>5 else f'{value}' - - async def stop(self): + def _chunk_preview(value: bytes) -> str: + """Returns small string preview of byte chunk. For logs + + Args: + value (bytes): Bytes to preview + + Returns: + str: String preview + """ + return f'{value[0:5]}... ' if len(value) > 5 else f'{value}' + + async def stop(self) -> None: + """Stops protocol + """ for channel in self.get_channels(): await channel.close() - await channel._writer.drain() + await channel.drain() channel._writer.close() - await channel._writer.wait_closed() + await channel.wait_closed() await self._writer.drain() self._writer.close() await self._writer.wait_closed() self._global_stop_event.set() - async def wait_until_end(self): + async def wait_until_end(self) -> None: + """Waits until forwarders finished work. + """ await self._global_stop_event.wait() await asyncio.gather(*[self._incoming_data_forwarder, self._outcoming_data_forwarder]) self._logger.debug(f'Tecemux/MAIN: [-] Finished') - async def loop(self): - + async def loop(self) -> None: + """Main loop of Tecemux protocol. Starts forwarders tasks + """ loop = asyncio.get_event_loop() - self._incoming_data_forwarder = loop.create_task(self.incoming_data_forward()) - self._outcoming_data_forwarder = loop.create_task(self.outcoming_data_forward()) + self._incoming_data_forwarder = loop.create_task( + self.incoming_data_forward()) + self._outcoming_data_forwarder = loop.create_task( + self.outcoming_data_forward()) - async def incoming_data_forward(self): + async def incoming_data_forward(self) -> None: + """Loop for incoming data from Transform Hub + """ buffer = b'' incoming_parser_finish_loop = asyncio.Event() @@ -238,7 +404,8 @@ async def incoming_data_forward(self): if incoming_parser_finish_loop.is_set() and buffer_len > 0 and buffer_len < MINIMAL_IP_PACKET_LENGTH: - self._logger.warning(f'Tecemux/MAIN: [<] Too few data is waiting in global buffer but stream finished') + self._logger.warning( + f'Tecemux/MAIN: [<] Too few data is waiting in global buffer but stream finished') break while not (len(buffer) < MINIMAL_IP_PACKET_LENGTH): @@ -249,39 +416,48 @@ async def incoming_data_forward(self): single_packet_buffer = buffer[:current_packet_size] pkt = IPPacket().from_buffer_with_pseudoheader(single_packet_buffer) - self._last_sequence_received = pkt.get_segment().seq - self._logger.debug(f'Tecemux/MAIN: [<] Full incomming packet with sequence number {self._last_sequence_received} from Transform Hub was received') + self._last_sequence_received = pkt.get_segment().seq + self._logger.debug( + f'Tecemux/MAIN: [<] Full incomming packet with sequence number {self._last_sequence_received} from Transform Hub was received') channel = CC(str(pkt.get_segment().dst_port)) - + await self._channels[channel].queue_up_incoming(pkt) - self._logger.debug(f'Tecemux/MAIN: [<] Packet {Tecemux._chunk_preview(single_packet_buffer)} forwarded to {channel.name} stream') + self._logger.debug( + f'Tecemux/MAIN: [<] Packet {Tecemux._chunk_preview(single_packet_buffer)} forwarded to {channel.name} stream') buffer = buffer[current_packet_size:] else: - self._logger.warning(f'Tecemux/MAIN: [<] Not full packet received. Getting additional data chunk') + self._logger.warning( + f'Tecemux/MAIN: [<] Not full packet received. Getting additional data chunk') break self._logger.debug(f'Tecemux/MAIN: Incomming data forwarder finished') - async def outcoming_data_forward(self): + async def outcoming_data_forward(self) -> None: + """Loop for outcoming data to Transform Hub + """ + while not self._global_stop_event.is_set(): try: pkt = self._queue.get_nowait() - #inject sequence number + # inject sequence number if pkt.segment.seq == 0: pkt.segment.seq = self._sequence_number self._sequence_number += 1 - chunk = pkt.build(for_STH=True).to_buffer_with_tcp_pseudoheader() - - self._logger.debug(f'Tecemux/MAIN: [>] Outcoming chunk {Tecemux._chunk_preview(chunk)} is waiting to send to Transform Hub') + chunk = pkt.build( + for_STH=True).to_buffer_with_tcp_pseudoheader() + + self._logger.debug( + f'Tecemux/MAIN: [>] Outcoming chunk {Tecemux._chunk_preview(chunk)} is waiting to send to Transform Hub') self._writer.write(chunk) await self._writer.drain() self._queue.task_done() - self._logger.debug(f'Tecemux/MAIN: [>] Chunk {Tecemux._chunk_preview(chunk)} with sequence number: {pkt.segment.seq} was sent to Transform Hub') + self._logger.debug( + f'Tecemux/MAIN: [>] Chunk {Tecemux._chunk_preview(chunk)} with sequence number: {pkt.segment.seq} was sent to Transform Hub') except asyncio.QueueEmpty: await asyncio.sleep(0) - self._logger.debug(f'Tecemux/MAIN: Outcoming data forwarder finished') \ No newline at end of file + self._logger.debug(f'Tecemux/MAIN: Outcoming data forwarder finished') diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index 6d3382b5b..aad44b947 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -82,17 +82,17 @@ async def test_forward_channel_between_a_b(self, local_socket_connection): channel_alpha = CC.CONTROL - channel_beta = CC.IN - + channel_beta = CC.IN + await client_a.get_channel(channel_alpha).write("{'foo':'bar'}") await client_a._writer.drain() - + await client_a.get_channel(channel_beta).write("{'bar':'foo2'}") await client_a._writer.drain() assert (await client_b.get_channel(channel_alpha).read(100)).decode() == "{'foo':'bar'}" assert (await client_b.get_channel(channel_beta).read(100)).decode() == "{'bar':'foo2'}" - + await client_a.stop() await client_b.stop() From d24f2e714aa70069e9075d60d99f5ffa38ce7c28 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 19 Jun 2023 14:41:13 +0000 Subject: [PATCH 167/231] Fix linter issues --- packages/verser/src/lib/tecemux/codecs/utils.ts | 1 - packages/verser/src/lib/verser-client.ts | 2 +- packages/verser/src/lib/verser-connection.ts | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/verser/src/lib/tecemux/codecs/utils.ts b/packages/verser/src/lib/tecemux/codecs/utils.ts index f769d08c7..153524689 100644 --- a/packages/verser/src/lib/tecemux/codecs/utils.ts +++ b/packages/verser/src/lib/tecemux/codecs/utils.ts @@ -2,7 +2,6 @@ export const calculateChecksum = (buffer: Buffer) => { let tempFrame = Buffer.concat([buffer, Buffer.alloc(0)]); if (buffer.length % 2) { - tempFrame = Buffer.concat([buffer, Buffer.alloc(1, 0)]); } diff --git a/packages/verser/src/lib/verser-client.ts b/packages/verser/src/lib/verser-client.ts index 3096441fa..b010fe07e 100644 --- a/packages/verser/src/lib/verser-client.ts +++ b/packages/verser/src/lib/verser-client.ts @@ -192,4 +192,4 @@ export class VerserClient extends TypedEmitter { registerChannel(channelId: number, data: RegisteredChannelCallback) { this.registeredChannels.set(channelId, data); } -} \ No newline at end of file +} diff --git a/packages/verser/src/lib/verser-connection.ts b/packages/verser/src/lib/verser-connection.ts index 98735eee3..80f25fcb1 100644 --- a/packages/verser/src/lib/verser-connection.ts +++ b/packages/verser/src/lib/verser-connection.ts @@ -251,4 +251,4 @@ export class VerserConnection { getAgent() { return this.agent as Agent; } -} \ No newline at end of file +} From 668746f57ab3cfa658a51e50e14f5a1b9358e45f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Mon, 19 Jun 2023 16:41:27 +0200 Subject: [PATCH 168/231] Remove pydevd --- packages/python-runner/runner.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index 5dc87710c..95b81203e 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -3,7 +3,6 @@ import os import codecs import json -import pydevd from pyee.asyncio import AsyncIOEventEmitter from tecemux import Tecemux import importlib.util @@ -15,14 +14,6 @@ from hardcoded_magic_values import RunnerMessageCodes as msg_codes -if bool(os.environ.get('DEVELOPMENT')): - import time - while pydevd.get_global_debugger() is None or not pydevd.get_global_debugger().ready_to_run: - time.sleep(0.3) - -def debugger(): - pydevd.settrace() - sequence_path = os.getenv('SEQUENCE_PATH') server_port = os.getenv('INSTANCES_SERVER_PORT') server_host = os.getenv('INSTANCES_SERVER_HOST') or 'localhost' @@ -34,8 +25,6 @@ async def send_encoded_msg(stream, msg_code, data={}): await stream.write(f'{message}\r\n'.encode()) await stream.drain() -#debugger() - class Runner: def __init__(self, instance_id, sequence_path, log_setup) -> None: self.instance_id = instance_id From 4f16ba63eb8f5fe607478c4f836f1a87fbfff745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Mon, 19 Jun 2023 16:59:32 +0200 Subject: [PATCH 169/231] Remove init.py --- packages/python-runner/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 packages/python-runner/__init__.py diff --git a/packages/python-runner/__init__.py b/packages/python-runner/__init__.py deleted file mode 100644 index e69de29bb..000000000 From b53ae904e36befd45c6f4037db0204a8c9e89311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Mon, 19 Jun 2023 17:13:32 +0200 Subject: [PATCH 170/231] Add attrs to requirements.txt --- packages/python-runner/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/python-runner/requirements.txt b/packages/python-runner/requirements.txt index 0d96bca35..eae561f4c 100644 --- a/packages/python-runner/requirements.txt +++ b/packages/python-runner/requirements.txt @@ -1,2 +1,3 @@ +attrs==23.1.0 pyee==9.0.4 scramjet-framework-py \ No newline at end of file From ca61c780002c5e2c2bb3f55df4b0f1df950958ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Tue, 20 Jun 2023 15:48:00 +0200 Subject: [PATCH 171/231] Bump base image version for Docker container python runner --- packages/python-runner/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/python-runner/Dockerfile b/packages/python-runner/Dockerfile index 8b5f58636..8e39c8983 100644 --- a/packages/python-runner/Dockerfile +++ b/packages/python-runner/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-slim-bullseye +FROM python:3.10-slim-bullseye ENV PACKAGE_DIR=/package \ HUB_DIR=/opt/transform-hub From a63e03267e733ae1d238b5b3c89376b6796a7073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Tue, 20 Jun 2023 16:47:10 +0200 Subject: [PATCH 172/231] Additional assets for python runner --- packages/python-runner/package.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/python-runner/package.json b/packages/python-runner/package.json index 41e767f36..df5a0884c 100644 --- a/packages/python-runner/package.json +++ b/packages/python-runner/package.json @@ -13,6 +13,8 @@ "assets": [ "hardcoded_magic_values.py", "logging_setup.py", + "tecemux.py", + "inet.py", "runner.py" ], "author": "Scramjet ", From b60ebdb9c8c33f7326d0640abde3dd49e6b79fb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Wed, 21 Jun 2023 15:55:18 +0200 Subject: [PATCH 173/231] Stream write as wrapper for internal async --- packages/python-runner/tecemux.py | 34 +++++------ packages/python-runner/test/test_tecemux.py | 62 +++++++++++++++++---- 2 files changed, 68 insertions(+), 28 deletions(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index ca7ee298c..cd4670cbc 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -24,15 +24,19 @@ class _ChannelContext: _global_instance_id: str _logger: logging.Logger = field(default=None) + _event_loop:asyncio.unix_events._UnixSelectorEventLoop = field(init=False) _internal_queue: asyncio.Queue = field(init=False) _channel_paused: bool = field(default=False, init=False) _channel_opened: bool = field(default=False, init=False) + def __attrs_post_init__(self) -> None: """Internal function executes after constructor """ self._internal_queue = asyncio.Queue() + self._event_loop = asyncio.get_running_loop() + a = 5 def _get_channel_name(self) -> str: """Returns channel name @@ -80,7 +84,7 @@ def readuntil(self, separator: bytes = b'\n') -> Coroutine: return self._reader.readuntil(separator) - def read(self, n: int = -1) -> Coroutine: + def read(self, n: int = 1) -> Coroutine: """asyncio.StreamReader API. Reads up to `n` bytes from the stream. Args: @@ -105,16 +109,11 @@ def readexactly(self, n: int) -> Coroutine: def __aiter__(self): """asyncio.StreamReader API """ - return self._reader.__aiter__ - - async def __anext__(self): - """asyncio.StreamReader API - """ - val = await self._reader.readline() - if val == b'': - raise StopAsyncIteration - return val + return self._reader.__aiter__() + def __anext__(self): + return self._reader.__anext__() + async def send_ACK(self, sequence_number: int) -> None: """Adds to global queue an ACK packet to send. @@ -204,23 +203,24 @@ def wrap(channel_enum, buf): break await self._global_queue.put(wrap(self._channel_enum, data)) - async def write(self, data: bytes) -> None: + def write(self, data: bytes) -> None: """Writes data to channel Args: data (bytes): Buffer to send to channel """ + async def _async_write(data): + if not self._channel_opened: + await self.open() - if not self._channel_opened: - await self.open() + await self._queue_up_outcoming(data) - await self._queue_up_outcoming(data) + self._event_loop.create_task(_async_write(data)) - async def drain(self) -> bool: + def drain(self) -> bool: """Drain channel """ - if not self._channel_paused: - await self._writer.drain() + return self._writer.drain() def write_eof(self) -> None: """asyncio.StreamWriter API diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index aad44b947..6dc949da4 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -1,4 +1,5 @@ import asyncio +import codecs import pytest import sys from tecemux import Tecemux @@ -37,6 +38,14 @@ async def local_socket_connection(): return client_a, client_b class TestTecemux: + + async def _close_clients(self, a, b): + await a.stop() + await b.stop() + + await a.wait_until_end() + await b.wait_until_end() + def test_default_init(self): protocol = Tecemux() assert isinstance(protocol, Tecemux) @@ -70,11 +79,6 @@ async def test_forward_from_main_to_channel(self, local_socket_connection): assert (await client_b.get_channel(destination_channel).read(100)).decode() == data_to_send - await client_a.stop() - await client_b.stop() - - await client_a.wait_until_end() - await client_a.wait_until_end() @pytest.mark.asyncio async def test_forward_channel_between_a_b(self, local_socket_connection): @@ -84,17 +88,53 @@ async def test_forward_channel_between_a_b(self, local_socket_connection): channel_alpha = CC.CONTROL channel_beta = CC.IN - await client_a.get_channel(channel_alpha).write("{'foo':'bar'}") + client_a.get_channel(channel_alpha).write("{'foo':'bar'}") await client_a._writer.drain() - await client_a.get_channel(channel_beta).write("{'bar':'foo2'}") + client_a.get_channel(channel_beta).write("{'bar':'foo2'}") await client_a._writer.drain() assert (await client_b.get_channel(channel_alpha).read(100)).decode() == "{'foo':'bar'}" assert (await client_b.get_channel(channel_beta).read(100)).decode() == "{'bar':'foo2'}" - await client_a.stop() - await client_b.stop() + await self._close_clients(client_a,client_b) + + async def test_stream_write_redirection(self, local_socket_connection): + client_a, client_b = local_socket_connection + + stream = codecs.getwriter('utf-8')(client_a.get_channel(CC.HOST)) + + print("Hi!", end='', file=stream) + + assert (await client_b.get_channel(CC.HOST).read(100)).decode() == "Hi!" + + await self._close_clients(client_a,client_b) + + async def test_stream_read_redirection(self, local_socket_connection): + + from scramjet.streams import Stream + + client_a, client_b = local_socket_connection + + input_stream = codecs.getwriter('utf-8')(client_a.get_channel(CC.HOST)) + output_stream = Stream.read_from(client_b.get_channel(CC.HOST)).decode('utf-8') + + print("Test", file=input_stream) + assert await output_stream.read() == 'Test\n' + + await self._close_clients(client_a,client_b) + + async def test_stderr_write_redirection(self, local_socket_connection): + client_a, client_b = local_socket_connection + + sys.stderr = codecs.getwriter('utf-8')(client_a.get_channel(CC.STDERR)) + sys.stderr.flush = lambda: True + + print("Error", end='\n', file=sys.stderr) + + sys.stderr = sys.__stderr__ + + assert (await client_b.get_channel(CC.STDERR).read(100)).decode() == "Error\n" + + await self._close_clients(client_a,client_b) - await client_a.wait_until_end() - await client_a.wait_until_end() From a39c1e985a975b83c09e2bcb67d47df4ef95dcaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Thu, 22 Jun 2023 19:51:53 +0200 Subject: [PATCH 174/231] Excepiton details are redirected to stderr channel --- packages/python-runner/runner.py | 18 +++++++++++++++--- packages/python-runner/tecemux.py | 12 ++++++++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index 95b81203e..8bb305ff9 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -3,6 +3,8 @@ import os import codecs import json +import logging +#import debugpy from pyee.asyncio import AsyncIOEventEmitter from tecemux import Tecemux import importlib.util @@ -19,10 +21,13 @@ server_host = os.getenv('INSTANCES_SERVER_HOST') or 'localhost' instance_id = os.getenv('INSTANCE_ID') +#debugpy.listen(5678) +#debugpy.wait_for_client() +#debugpy.breakpoint() async def send_encoded_msg(stream, msg_code, data={}): message = json.dumps([msg_code.value, data]) - await stream.write(f'{message}\r\n'.encode()) + stream.write(f'{message}\r\n'.encode()) await stream.drain() class Runner: @@ -55,6 +60,7 @@ async def main(self, server_host, server_port): asyncio.create_task(self.setup_heartbeat()) self.load_sequence() + await self.protocol.sync() await self.run_instance(config, args) async def init_tecemux(self, server_host, server_port): @@ -185,7 +191,13 @@ async def run_instance(self, config, args): asyncio.create_task(self.connect_input_stream(input_stream)) self.logger.info('Running instance...') - result = self.sequence.run(context, input_stream, *args) + try: + result = self.sequence.run(context, input_stream, *args) + except Exception as e: + self.protocol.get_channel(CC.STDERR).write(str(e)+'\n') + await self.protocol.sync() + await self.protocol._writer.drain() + raise e self.logger.info(f'Sending PANG') monitoring = self.protocol.get_channel(CC.MONITORING) @@ -303,7 +315,7 @@ async def keep_alive(self, timeout: int = 0): await self.runner.send_keep_alive(timeout) log_target = open(sys.argv[1], 'a+') if len(sys.argv) > 1 else sys.stdout -log_setup = LoggingSetup(log_target) +log_setup = LoggingSetup(log_target, min_loglevel=logging.INFO) log_setup.logger.info('Starting up...') log_setup.logger.debug(f'server_host: {server_host}') diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index cd4670cbc..da811e5bb 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -28,6 +28,7 @@ class _ChannelContext: _internal_queue: asyncio.Queue = field(init=False) _channel_paused: bool = field(default=False, init=False) _channel_opened: bool = field(default=False, init=False) + _waiting_tasks: list = field(init=False) def __attrs_post_init__(self) -> None: @@ -36,7 +37,7 @@ def __attrs_post_init__(self) -> None: self._internal_queue = asyncio.Queue() self._event_loop = asyncio.get_running_loop() - a = 5 + self._waiting_tasks = [] def _get_channel_name(self) -> str: """Returns channel name @@ -215,7 +216,7 @@ async def _async_write(data): await self._queue_up_outcoming(data) - self._event_loop.create_task(_async_write(data)) + self._waiting_tasks.append(self._event_loop.create_task(_async_write(data))) def drain(self) -> bool: """Drain channel @@ -329,6 +330,13 @@ def get_channel(self, channel: CC) -> _ChannelContext: """ return self._channels[channel] + async def sync(self): + """Waits until all write tasks will be done + """ + for ch in self.get_channels(): + if len(ch._waiting_tasks) > 0: + await asyncio.wait(ch._waiting_tasks, return_when=asyncio.ALL_COMPLETED) + def get_channels(self) -> dict: """Returns all initiated channels From e6ec7d79a36f536be3f6c0ac4e8849ea47a3956d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 23 Jun 2023 12:04:35 +0200 Subject: [PATCH 175/231] Erorrs and debug messages shows correctly --- packages/python-runner/runner.py | 50 ++++++++++---------- packages/python-runner/tecemux.py | 76 +++++++++++++++++-------------- 2 files changed, 66 insertions(+), 60 deletions(-) diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index 8bb305ff9..5a15bfc91 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -16,13 +16,13 @@ from hardcoded_magic_values import RunnerMessageCodes as msg_codes -sequence_path = os.getenv('SEQUENCE_PATH') -server_port = os.getenv('INSTANCES_SERVER_PORT') -server_host = os.getenv('INSTANCES_SERVER_HOST') or 'localhost' -instance_id = os.getenv('INSTANCE_ID') +SEQUENCE_PATH = os.getenv('SEQUENCE_PATH') +SERVER_PORT = os.getenv('INSTANCES_SERVER_PORT') +SERVER_HOST = os.getenv('INSTANCES_SERVER_HOST') or 'localhost' +INSTANCE_ID = os.getenv('INSTANCE_ID') #debugpy.listen(5678) -#debugpy.wait_for_client() +#debugpy.wait_for_client() #debugpy.breakpoint() async def send_encoded_msg(stream, msg_code, data={}): @@ -41,7 +41,6 @@ def __init__(self, instance_id, sequence_path, log_setup) -> None: self.emitter = AsyncIOEventEmitter() self.keep_alive_requested = False self.protocol = None - self.streams = {} @staticmethod def is_incoming(channel): return channel in [CC.STDIN, CC.IN, CC.CONTROL] @@ -62,6 +61,7 @@ async def main(self, server_host, server_port): self.load_sequence() await self.protocol.sync() await self.run_instance(config, args) + await self.protocol.sync() async def init_tecemux(self, server_host, server_port): self.logger.info('Connecting to host with TeceMux...') @@ -70,8 +70,6 @@ async def init_tecemux(self, server_host, server_port): await self.protocol.prepare(force_open=True) await self.protocol.loop() - self.streams = self.protocol._channels - def connect_stdio(self): sys.stdout = codecs.getwriter('utf-8')(self.protocol.get_channel(CC.STDOUT)) sys.stderr = codecs.getwriter('utf-8')(self.protocol.get_channel(CC.STDERR)) @@ -98,7 +96,7 @@ async def handshake(self): monitoring = self.protocol.get_channel(CC.MONITORING) control = self.protocol.get_channel(CC.CONTROL) - self.logger.info(f'Sending PING') + self.logger.info('Sending PING') await send_encoded_msg(monitoring, msg_codes.PING) message = await control.readuntil(b'\n') @@ -112,7 +110,7 @@ async def handshake(self): if 'args' not in data: data['args'] = [] - self.logger.info(f'Sending PANG') + self.logger.info('Sending PANG') pang_requires_data = { 'requires': '', 'contentType': '' @@ -193,12 +191,12 @@ async def run_instance(self, config, args): self.logger.info('Running instance...') try: result = self.sequence.run(context, input_stream, *args) - except Exception as e: - self.protocol.get_channel(CC.STDERR).write(str(e)+'\n') + except Exception: + import traceback + self.protocol.get_channel(CC.STDERR).write(traceback.format_exc()) await self.protocol.sync() - await self.protocol._writer.drain() - raise e - + self.exit_immediately() + self.logger.info(f'Sending PANG') monitoring = self.protocol.get_channel(CC.MONITORING) @@ -314,18 +312,18 @@ async def emit(self, event_name, message=''): async def keep_alive(self, timeout: int = 0): await self.runner.send_keep_alive(timeout) -log_target = open(sys.argv[1], 'a+') if len(sys.argv) > 1 else sys.stdout -log_setup = LoggingSetup(log_target, min_loglevel=logging.INFO) +LOG_TARGET = open(sys.argv[1], 'a+',encoding='utf-8') if len(sys.argv) > 1 else sys.stdout +LOG_SETUP = LoggingSetup(LOG_TARGET, min_loglevel=logging.DEBUG) -log_setup.logger.info('Starting up...') -log_setup.logger.debug(f'server_host: {server_host}') -log_setup.logger.debug(f'server_port: {server_port}') -log_setup.logger.debug(f'instance_id: {instance_id}') -log_setup.logger.debug(f'sequence_path: {sequence_path}') +LOG_SETUP.logger.info('Starting up...') +LOG_SETUP.logger.debug(f'server_host: {SERVER_HOST}') +LOG_SETUP.logger.debug(f'server_port: {SERVER_PORT}') +LOG_SETUP.logger.debug(f'instance_id: {INSTANCE_ID}') +LOG_SETUP.logger.debug(f'sequence_path: {SEQUENCE_PATH}') -if not sequence_path or not server_port or not instance_id: - log_setup.logger.error('Undefined config variable! ') +if not SEQUENCE_PATH or not SERVER_PORT or not INSTANCE_ID: + LOG_SETUP.logger.error('Undefined config variable! ') sys.exit(2) -runner = Runner(instance_id, sequence_path, log_setup) -asyncio.run(runner.main(server_host, server_port)) +runner = Runner(INSTANCE_ID, SEQUENCE_PATH, LOG_SETUP) +asyncio.run(runner.main(SERVER_HOST, SERVER_PORT)) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index da811e5bb..d3f93267b 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -2,11 +2,14 @@ import logging import random import socket -from attrs import define, field from typing import Any, Coroutine + +from attrs import define, field from inet import IPPacket, TCPSegment, SequenceOrder from hardcoded_magic_values import CommunicationChannels as CC +TECEMUX_INTERNAL_VERBOSE_DEBUG = False + SequenceOrder().use_little_endian() @@ -24,20 +27,23 @@ class _ChannelContext: _global_instance_id: str _logger: logging.Logger = field(default=None) - _event_loop:asyncio.unix_events._UnixSelectorEventLoop = field(init=False) + _event_loop: asyncio.unix_events._UnixSelectorEventLoop = field(init=False) _internal_queue: asyncio.Queue = field(init=False) _channel_paused: bool = field(default=False, init=False) _channel_opened: bool = field(default=False, init=False) - _waiting_tasks: list = field(init=False) + _waiting_tasks: list = field(default=[], init=False) def __attrs_post_init__(self) -> None: """Internal function executes after constructor """ - self._internal_queue = asyncio.Queue() self._event_loop = asyncio.get_running_loop() - self._waiting_tasks = [] + + def _debug(self,msg): + if TECEMUX_INTERNAL_VERBOSE_DEBUG: + self._logger.debug(msg) + def _get_channel_name(self) -> str: """Returns channel name @@ -114,7 +120,7 @@ def __aiter__(self): def __anext__(self): return self._reader.__anext__() - + async def send_ACK(self, sequence_number: int) -> None: """Adds to global queue an ACK packet to send. @@ -149,8 +155,7 @@ async def open(self) -> None: async def close(self) -> None: """Close channel """ - self._logger.debug( - f'Tecemux/{self._get_channel_name()}: [-] Channel close request is send') + self._debug(f'Tecemux/{self._get_channel_name()}: [-] Channel close request is send') await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'' if self._global_instance_id is None else self._global_instance_id, flags=['FIN']))) async def queue_up_incoming(self, pkt: IPPacket) -> None: @@ -161,8 +166,7 @@ async def queue_up_incoming(self, pkt: IPPacket) -> None: """ # if SYN & ACK flag is up, pause channel if pkt.get_segment().is_flag('SYN') and pkt.get_segment().is_flag('ACK'): - self._logger.debug( - f'Tecemux/{self._get_channel_name()}: [-] Channel pause request received') + self._debug(f'Tecemux/{self._get_channel_name()}: [-] Channel pause request received') self.set_pause(True) await self.send_ACK(pkt.get_segment().seq) return @@ -190,8 +194,7 @@ def wrap(channel_enum, buf): return IPPacket(segment=TCPSegment(dst_port=channel_id, flags=['PSH'], data=buf)) if self._channel_paused: - self._logger.debug( - f'Tecemux/{self._get_channel_name()}: [-] Channel paused. Data queued up internally for future') + self._debug(f'Tecemux/{self._get_channel_name()}: [-] Channel paused. Data queued up internally for future') await self._internal_queue.put(data) else: while not self._internal_queue.empty(): @@ -199,8 +202,7 @@ def wrap(channel_enum, buf): buf = await self._internal_queue.get() await self._global_queue.put(wrap(self._channel_enum, buf)) except asyncio.QueueEmpty: - self._logger.debug( - f'Tecemux/{self._get_channel_name()}: [-] All data stored during pause were redirected to global queue') + self._debug(f'Tecemux/{self._get_channel_name()}: [-] All data stored during pause were redirected to global queue') break await self._global_queue.put(wrap(self._channel_enum, data)) @@ -216,7 +218,7 @@ async def _async_write(data): await self._queue_up_outcoming(data) - self._waiting_tasks.append(self._event_loop.create_task(_async_write(data))) + self._waiting_tasks.append(self._event_loop.create_task(_async_write(data))) #pylint:disable=E1101 def drain(self) -> bool: """Drain channel @@ -261,6 +263,15 @@ class Tecemux: _sequence_number: int = field(default=0) _last_sequence_received: int = field(default=0) + def _debug(self,msg): + if TECEMUX_INTERNAL_VERBOSE_DEBUG: + self._logger.debug(msg) + + def _warning(self,msg): + if TECEMUX_INTERNAL_VERBOSE_DEBUG: + self._logger.warning(msg) + + async def connect(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: """Connects to Transform Hub via provided reader and writer objects @@ -332,10 +343,13 @@ def get_channel(self, channel: CC) -> _ChannelContext: async def sync(self): """Waits until all write tasks will be done - """ - for ch in self.get_channels(): - if len(ch._waiting_tasks) > 0: - await asyncio.wait(ch._waiting_tasks, return_when=asyncio.ALL_COMPLETED) + """ + for channel in self.get_channels(): + if len(channel._waiting_tasks) > 0: + await asyncio.wait(channel._waiting_tasks, return_when=asyncio.ALL_COMPLETED) + + if not self._queue.empty(): + await self._queue.join() def get_channels(self) -> dict: """Returns all initiated channels @@ -377,7 +391,7 @@ async def wait_until_end(self) -> None: await self._global_stop_event.wait() await asyncio.gather(*[self._incoming_data_forwarder, self._outcoming_data_forwarder]) - self._logger.debug(f'Tecemux/MAIN: [-] Finished') + self._debug('Tecemux/MAIN: [-] Finished') async def loop(self) -> None: """Main loop of Tecemux protocol. Starts forwarders tasks @@ -412,8 +426,7 @@ async def incoming_data_forward(self) -> None: if incoming_parser_finish_loop.is_set() and buffer_len > 0 and buffer_len < MINIMAL_IP_PACKET_LENGTH: - self._logger.warning( - f'Tecemux/MAIN: [<] Too few data is waiting in global buffer but stream finished') + self._warning('Tecemux/MAIN: [<] Too few data is waiting in global buffer but stream finished') break while not (len(buffer) < MINIMAL_IP_PACKET_LENGTH): @@ -425,22 +438,19 @@ async def incoming_data_forward(self) -> None: single_packet_buffer = buffer[:current_packet_size] pkt = IPPacket().from_buffer_with_pseudoheader(single_packet_buffer) self._last_sequence_received = pkt.get_segment().seq - self._logger.debug( - f'Tecemux/MAIN: [<] Full incomming packet with sequence number {self._last_sequence_received} from Transform Hub was received') + self._debug(f'Tecemux/MAIN: [<] Full incomming packet with sequence number {self._last_sequence_received} from Transform Hub was received') channel = CC(str(pkt.get_segment().dst_port)) await self._channels[channel].queue_up_incoming(pkt) - self._logger.debug( - f'Tecemux/MAIN: [<] Packet {Tecemux._chunk_preview(single_packet_buffer)} forwarded to {channel.name} stream') + self._debug(f'Tecemux/MAIN: [<] Packet {Tecemux._chunk_preview(single_packet_buffer)} forwarded to {channel.name} stream') buffer = buffer[current_packet_size:] else: - self._logger.warning( - f'Tecemux/MAIN: [<] Not full packet received. Getting additional data chunk') + self._warning('Tecemux/MAIN: [<] Not full packet received. Getting additional data chunk') break - self._logger.debug(f'Tecemux/MAIN: Incomming data forwarder finished') + self._debug('Tecemux/MAIN: Incomming data forwarder finished') async def outcoming_data_forward(self) -> None: """Loop for outcoming data to Transform Hub @@ -458,14 +468,12 @@ async def outcoming_data_forward(self) -> None: chunk = pkt.build( for_STH=True).to_buffer_with_tcp_pseudoheader() - self._logger.debug( - f'Tecemux/MAIN: [>] Outcoming chunk {Tecemux._chunk_preview(chunk)} is waiting to send to Transform Hub') + self._debug(f'Tecemux/MAIN: [>] Outcoming chunk {Tecemux._chunk_preview(chunk)} is waiting to send to Transform Hub') self._writer.write(chunk) await self._writer.drain() self._queue.task_done() - self._logger.debug( - f'Tecemux/MAIN: [>] Chunk {Tecemux._chunk_preview(chunk)} with sequence number: {pkt.segment.seq} was sent to Transform Hub') + self._debug(f'Tecemux/MAIN: [>] Chunk {Tecemux._chunk_preview(chunk)} with sequence number: {pkt.segment.seq} was sent to Transform Hub') except asyncio.QueueEmpty: await asyncio.sleep(0) - self._logger.debug(f'Tecemux/MAIN: Outcoming data forwarder finished') + self._debug('Tecemux/MAIN: Outcoming data forwarder finished') From 276ff08f8fc2096933e85978b17c3dea2c4d4136 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 23 Jun 2023 15:11:33 +0200 Subject: [PATCH 176/231] Minor changes --- packages/python-runner/runner.py | 13 +++++++++---- packages/python-runner/tecemux.py | 4 ++-- packages/python-runner/test/test_tecemux.py | 1 + 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index 5a15bfc91..efb432ff8 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -4,7 +4,7 @@ import codecs import json import logging -#import debugpy +import debugpy from pyee.asyncio import AsyncIOEventEmitter from tecemux import Tecemux import importlib.util @@ -21,9 +21,9 @@ SERVER_HOST = os.getenv('INSTANCES_SERVER_HOST') or 'localhost' INSTANCE_ID = os.getenv('INSTANCE_ID') -#debugpy.listen(5678) -#debugpy.wait_for_client() -#debugpy.breakpoint() +debugpy.listen(5678) +debugpy.wait_for_client() +debugpy.breakpoint() async def send_encoded_msg(stream, msg_code, data={}): message = json.dumps([msg_code.value, data]) @@ -59,10 +59,15 @@ async def main(self, server_host, server_port): asyncio.create_task(self.setup_heartbeat()) self.load_sequence() + await self.protocol.sync() await self.run_instance(config, args) await self.protocol.sync() + await self.protocol.stop() + await self.protocol.wait_until_end() + + async def init_tecemux(self, server_host, server_port): self.logger.info('Connecting to host with TeceMux...') self.protocol = Tecemux(instance_id=self.instance_id) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index d3f93267b..eda699f00 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -40,7 +40,7 @@ def __attrs_post_init__(self) -> None: self._internal_queue = asyncio.Queue() self._event_loop = asyncio.get_running_loop() - def _debug(self,msg): + def _debug(self, msg): if TECEMUX_INTERNAL_VERBOSE_DEBUG: self._logger.debug(msg) @@ -91,7 +91,7 @@ def readuntil(self, separator: bytes = b'\n') -> Coroutine: return self._reader.readuntil(separator) - def read(self, n: int = 1) -> Coroutine: + def read(self, n: int = 64) -> Coroutine: """asyncio.StreamReader API. Reads up to `n` bytes from the stream. Args: diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index 6dc949da4..fbbfe5568 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -79,6 +79,7 @@ async def test_forward_from_main_to_channel(self, local_socket_connection): assert (await client_b.get_channel(destination_channel).read(100)).decode() == data_to_send + await self._close_clients(client_a, client_b) @pytest.mark.asyncio async def test_forward_channel_between_a_b(self, local_socket_connection): From 9be82a64c9aa0fcdd4b022ee20660e2536bb8260 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Mon, 26 Jun 2023 15:47:53 +0200 Subject: [PATCH 177/231] Sync barrier & simple write method --- packages/python-runner/barrier.py | 118 +++++++++++++++++++++ packages/python-runner/runner.py | 14 ++- packages/python-runner/tecemux.py | 171 +++++++++++++++++++----------- 3 files changed, 234 insertions(+), 69 deletions(-) create mode 100644 packages/python-runner/barrier.py diff --git a/packages/python-runner/barrier.py b/packages/python-runner/barrier.py new file mode 100644 index 000000000..3e0077413 --- /dev/null +++ b/packages/python-runner/barrier.py @@ -0,0 +1,118 @@ +import enum +from asyncio import Condition + + +class BrokenBarrierError(RuntimeError): + """Barrier is broken by barrier.abort() call.""" + + +class _BarrierState(enum.Enum): + FILLING = 'filling' + DRAINING = 'draining' + RESETTING = 'resetting' + BROKEN = 'broken' + + +class Barrier(): + def __init__(self, parties): + """Create a barrier, initialised to 'parties' tasks.""" + if parties < 1: + raise ValueError('parties must be > 0') + + self._cond = Condition() # notify all tasks when state changes + + self._parties = parties + self._state = _BarrierState.FILLING + self._count = 0 # count tasks in Barrier + + def __repr__(self): + res = super().__repr__() + extra = f'{self._state.value}' + if not self.broken: + extra += f', waiters:{self.n_waiting}/{self.parties}' + return f'<{res[1:-1]} [{extra}]>' + + async def __aenter__(self): + return await self.wait() + + async def __aexit__(self, *args): + pass + + async def wait(self): + async with self._cond: + await self._block() # Block while the barrier drains or resets. + try: + index = self._count + self._count += 1 + if index + 1 == self._parties: + # We release the barrier + await self._release() + else: + await self._wait() + return index + finally: + self._count -= 1 + # Wake up any tasks waiting for barrier to drain. + self._exit() + + async def _block(self): + await self._cond.wait_for( + lambda: self._state not in ( + _BarrierState.DRAINING, _BarrierState.RESETTING + ) + ) + + # see if the barrier is in a broken state + if self._state is _BarrierState.BROKEN: + raise BrokenBarrierError("Barrier aborted") + + async def _release(self): + self._state = _BarrierState.DRAINING + self._cond.notify_all() + + async def _wait(self): + await self._cond.wait_for(lambda: self._state is not _BarrierState.FILLING) + + if self._state in (_BarrierState.BROKEN, _BarrierState.RESETTING): + raise BrokenBarrierError("Abort or reset of barrier") + + def _exit(self): + # If we are the last tasks to exit the barrier, signal any tasks + # waiting for the barrier to drain. + if self._count == 0: + if self._state in (_BarrierState.RESETTING, _BarrierState.DRAINING): + self._state = _BarrierState.FILLING + self._cond.notify_all() + + async def reset(self): + async with self._cond: + if self._count > 0: + if self._state is not _BarrierState.RESETTING: + #reset the barrier, waking up tasks + self._state = _BarrierState.RESETTING + else: + self._state = _BarrierState.FILLING + self._cond.notify_all() + + async def abort(self): + async with self._cond: + self._state = _BarrierState.BROKEN + self._cond.notify_all() + + + @property + def parties(self): + """Return the number of tasks required to trip the barrier.""" + return self._parties + + @property + def n_waiting(self): + """Return the number of tasks currently waiting at the barrier.""" + if self._state is _BarrierState.FILLING: + return self._count + return 0 + + @property + def broken(self): + """Return True if the barrier is in a broken state.""" + return self._state is _BarrierState.BROKEN \ No newline at end of file diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index efb432ff8..6887e710d 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -60,9 +60,8 @@ async def main(self, server_host, server_port): self.load_sequence() - await self.protocol.sync() + await self.protocol.sync() await self.run_instance(config, args) - await self.protocol.sync() await self.protocol.stop() await self.protocol.wait_until_end() @@ -192,7 +191,7 @@ async def run_instance(self, config, args): context = AppContext(self, config) input_stream = Stream() asyncio.create_task(self.connect_input_stream(input_stream)) - + await self.protocol.sync() self.logger.info('Running instance...') try: result = self.sequence.run(context, input_stream, *args) @@ -201,7 +200,7 @@ async def run_instance(self, config, args): self.protocol.get_channel(CC.STDERR).write(traceback.format_exc()) await self.protocol.sync() self.exit_immediately() - + self.logger.info(f'Sending PANG') monitoring = self.protocol.get_channel(CC.MONITORING) @@ -220,11 +219,13 @@ async def run_instance(self, config, args): elif asyncio.iscoroutine(result): result = await result if result: + await self.protocol.sync() await self.forward_output_stream(result) else: self.logger.debug('Sequence returned no output.') self.logger.info('Finished.') + await self.protocol.sync() await self.cleanup() async def cleanup(self): @@ -243,7 +244,9 @@ async def connect_input_stream(self, input_stream): input_type = headers.get('content-type') if input_type == 'text/plain': - input = Stream.read_from(self.protocol.get_channel(CC.IN)) + await self.protocol.sync() + input = await Stream.read_from(self.protocol.get_channel(CC.IN)).to_list() + self.logger.debug('Decoding input stream...') input = input.decode('utf-8') elif input_type == 'application/octet-stream': @@ -276,6 +279,7 @@ async def forward_output_stream(self, output): self.logger.debug('Output will be converted to JSON') output = output.map(lambda chunk: (json.dumps(chunk)+'\n').encode()) + await self.protocol.sync() await output.write_to(self.protocol.get_channel(CC.OUT)) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index eda699f00..0941c16db 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -5,6 +5,7 @@ from typing import Any, Coroutine from attrs import define, field +from barrier import Barrier from inet import IPPacket, TCPSegment, SequenceOrder from hardcoded_magic_values import CommunicationChannels as CC @@ -26,19 +27,45 @@ class _ChannelContext: _global_queue: asyncio.Queue _global_instance_id: str - _logger: logging.Logger = field(default=None) - _event_loop: asyncio.unix_events._UnixSelectorEventLoop = field(init=False) - _internal_queue: asyncio.Queue = field(init=False) - _channel_paused: bool = field(default=False, init=False) - _channel_opened: bool = field(default=False, init=False) - _waiting_tasks: list = field(default=[], init=False) + _logger: logging.Logger + _event_loop: asyncio.unix_events._UnixSelectorEventLoop + _internal_queue: asyncio.Queue + _channel_paused: bool + _channel_opened: bool + _stop_event: asyncio.Event + _sync_event: asyncio.Event + _sync_barrier: Barrier + _process_task: Any + + def __init__(self, channel: CC, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter, + queue: asyncio.Queue, + instance_id: str, + logger: logging.Logger, + stop_event: asyncio.Event, + sync_event: asyncio.Event, + sync_barrier: Barrier): + + self._channel_enum = channel + self._reader = reader + self._writer = writer + self._global_queue = queue + self._global_instance_id = instance_id + self._logger = logger + self._event_loop = asyncio.get_running_loop() + self._internal_queue = asyncio.Queue() + self._channel_opened = False + self._channel_paused = False + self._stop_event = stop_event + self._sync_event = sync_event + self._sync_barrier = sync_barrier + self._process_task = asyncio.create_task(self._process_tasks()) + async def _process_tasks(self): + while not self._stop_event.is_set(): + await self._queue_up_outcoming() - def __attrs_post_init__(self) -> None: - """Internal function executes after constructor - """ - self._internal_queue = asyncio.Queue() - self._event_loop = asyncio.get_running_loop() def _debug(self, msg): if TECEMUX_INTERNAL_VERBOSE_DEBUG: @@ -183,7 +210,7 @@ async def queue_up_incoming(self, pkt: IPPacket) -> None: self._writer.write(pkt.get_segment().data) await self._writer.drain() - async def _queue_up_outcoming(self, data: bytes) -> None: + async def _queue_up_outcoming(self) -> None: """Redirects raw data from currect channel to global queue. Args: @@ -193,18 +220,22 @@ def wrap(channel_enum, buf): channel_id = int(channel_enum.value) return IPPacket(segment=TCPSegment(dst_port=channel_id, flags=['PSH'], data=buf)) - if self._channel_paused: - self._debug(f'Tecemux/{self._get_channel_name()}: [-] Channel paused. Data queued up internally for future') - await self._internal_queue.put(data) - else: + if not self._channel_paused: while not self._internal_queue.empty(): try: buf = await self._internal_queue.get() await self._global_queue.put(wrap(self._channel_enum, buf)) + self._internal_queue.task_done() except asyncio.QueueEmpty: self._debug(f'Tecemux/{self._get_channel_name()}: [-] All data stored during pause were redirected to global queue') break - await self._global_queue.put(wrap(self._channel_enum, data)) + else: + self._debug(f'Tecemux/{self._get_channel_name()}: [-] Channel paused. Data queued up internally for future') + + if self._sync_event.is_set(): + await self._sync_barrier.wait() + + await asyncio.sleep(0) def write(self, data: bytes) -> None: """Writes data to channel @@ -212,13 +243,7 @@ def write(self, data: bytes) -> None: Args: data (bytes): Buffer to send to channel """ - async def _async_write(data): - if not self._channel_opened: - await self.open() - - await self._queue_up_outcoming(data) - - self._waiting_tasks.append(self._event_loop.create_task(_async_write(data))) #pylint:disable=E1101 + self._internal_queue.put_nowait(data) def drain(self) -> bool: """Drain channel @@ -260,14 +285,16 @@ class Tecemux: _channels: dict = field(default={}) _logger: logging.Logger = field(default=None) _global_stop_event: asyncio.Event = asyncio.Event() + _global_sync_event: asyncio.Event = asyncio.Event() + _global_sync_barrier: Barrier = field(default=None, init=False) _sequence_number: int = field(default=0) _last_sequence_received: int = field(default=0) - def _debug(self,msg): + def _debug(self, msg): if TECEMUX_INTERNAL_VERBOSE_DEBUG: self._logger.debug(msg) - def _warning(self,msg): + def _warning(self, msg): if TECEMUX_INTERNAL_VERBOSE_DEBUG: self._logger.warning(msg) @@ -317,8 +344,15 @@ async def prepare(self, force_open: bool = False) -> None: """ self._queue = asyncio.Queue() - self._channels = {channel: _ChannelContext(channel, *await Tecemux.prepare_socket_connection(), self._queue, self._instance_id, self._logger) for channel in CC} - + self._global_sync_barrier = Barrier(len(CC)) + self._channels = {channel: _ChannelContext(channel, + *await Tecemux.prepare_socket_connection(), + self._queue, + self._instance_id, + self._logger, + self._global_stop_event, + self._global_sync_event, + self._global_sync_barrier) for channel in CC} if force_open: [await channel.open() for channel in self.get_channels()] @@ -344,12 +378,12 @@ def get_channel(self, channel: CC) -> _ChannelContext: async def sync(self): """Waits until all write tasks will be done """ - for channel in self.get_channels(): - if len(channel._waiting_tasks) > 0: - await asyncio.wait(channel._waiting_tasks, return_when=asyncio.ALL_COMPLETED) - + self._global_sync_event.set() + await self._global_sync_barrier.wait() + self._global_sync_event.clear() + if not self._queue.empty(): - await self._queue.join() + await self._queue.join() def get_channels(self) -> dict: """Returns all initiated channels @@ -377,20 +411,24 @@ async def stop(self) -> None: for channel in self.get_channels(): await channel.close() await channel.drain() - channel._writer.close() - await channel.wait_closed() - - await self._writer.drain() - self._writer.close() - await self._writer.wait_closed() + self.sync() self._global_stop_event.set() - async def wait_until_end(self) -> None: """Waits until forwarders finished work. """ await self._global_stop_event.wait() - await asyncio.gather(*[self._incoming_data_forwarder, self._outcoming_data_forwarder]) + for channel in self.get_channels(): + channel._process_task.cancel() + self._incoming_data_forwarder.cancel() + self._outcoming_data_forwarder.cancel() + + #await asyncio.gather(*[self._incoming_data_forwarder, self._outcoming_data_forwarder]) + await asyncio.sleep(1) + self._writer.write_eof() + await self._writer.drain() + self._writer.close() + self._debug('Tecemux/MAIN: [-] Finished') async def loop(self) -> None: @@ -414,42 +452,47 @@ async def incoming_data_forward(self) -> None: READ_CHUNK_SIZE = 1024 while not self._global_stop_event.is_set(): + try: + chunk = await asyncio.wait_for(self._reader.read(READ_CHUNK_SIZE),1) - chunk = await self._reader.read(READ_CHUNK_SIZE) - - if not chunk: - incoming_parser_finish_loop.set() + if not chunk: + incoming_parser_finish_loop.set() - buffer = buffer + chunk + buffer = buffer + chunk - buffer_len = len(buffer) + buffer_len = len(buffer) - if incoming_parser_finish_loop.is_set() and buffer_len > 0 and buffer_len < MINIMAL_IP_PACKET_LENGTH: + if incoming_parser_finish_loop.is_set() and buffer_len > 0 and buffer_len < MINIMAL_IP_PACKET_LENGTH: - self._warning('Tecemux/MAIN: [<] Too few data is waiting in global buffer but stream finished') - break + self._warning('Tecemux/MAIN: [<] Too few data is waiting in global buffer but stream finished') + break - while not (len(buffer) < MINIMAL_IP_PACKET_LENGTH): + while not (len(buffer) < MINIMAL_IP_PACKET_LENGTH): - current_packet_size = IPPacket().from_buffer_with_pseudoheader(buffer).len + current_packet_size = IPPacket().from_buffer_with_pseudoheader(buffer).len - if len(buffer) >= current_packet_size: + if len(buffer) >= current_packet_size: - single_packet_buffer = buffer[:current_packet_size] - pkt = IPPacket().from_buffer_with_pseudoheader(single_packet_buffer) - self._last_sequence_received = pkt.get_segment().seq - self._debug(f'Tecemux/MAIN: [<] Full incomming packet with sequence number {self._last_sequence_received} from Transform Hub was received') + single_packet_buffer = buffer[:current_packet_size] + pkt = IPPacket().from_buffer_with_pseudoheader(single_packet_buffer) + self._last_sequence_received = pkt.get_segment().seq + self._debug(f'Tecemux/MAIN: [<] Full incomming packet with sequence number {self._last_sequence_received} from Transform Hub was received') - channel = CC(str(pkt.get_segment().dst_port)) + channel = CC(str(pkt.get_segment().dst_port)) - await self._channels[channel].queue_up_incoming(pkt) + await self._channels[channel].queue_up_incoming(pkt) - self._debug(f'Tecemux/MAIN: [<] Packet {Tecemux._chunk_preview(single_packet_buffer)} forwarded to {channel.name} stream') - buffer = buffer[current_packet_size:] - else: - self._warning('Tecemux/MAIN: [<] Not full packet received. Getting additional data chunk') - break + self._debug(f'Tecemux/MAIN: [<] Packet {Tecemux._chunk_preview(single_packet_buffer)} forwarded to {channel.name} stream') + buffer = buffer[current_packet_size:] + else: + self._warning('Tecemux/MAIN: [<] Not full packet received. Getting additional data chunk') + break + except TimeoutError: + await asyncio.sleep(0) + except asyncio.CancelledError: + break + self._debug('Tecemux/MAIN: Incomming data forwarder finished') async def outcoming_data_forward(self) -> None: From 46fe7e40d552fbc84ecd77f82176bf5346b1db4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Tue, 27 Jun 2023 15:47:28 +0200 Subject: [PATCH 178/231] anext in __ChannelContext works with Scramjet Framework to list --- packages/python-runner/tecemux.py | 155 ++++++++++++-------- packages/python-runner/test/test_tecemux.py | 42 +++++- 2 files changed, 130 insertions(+), 67 deletions(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 0941c16db..e224328ea 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -21,25 +21,28 @@ class _ChannelContext: _channel_enum: CC - _reader: asyncio.StreamReader - _writer: asyncio.StreamWriter - _global_queue: asyncio.Queue _global_instance_id: str _logger: logging.Logger _event_loop: asyncio.unix_events._UnixSelectorEventLoop - _internal_queue: asyncio.Queue + _internal_outcoming_queue: asyncio.Queue + _internal_incoming_queue: asyncio.Queue + _channel_paused: bool _channel_opened: bool + _stop_event: asyncio.Event _sync_event: asyncio.Event _sync_barrier: Barrier - _process_task: Any + + _read_buffer: bytearray + + _eof: bool + + _outcoming_process_task: Any def __init__(self, channel: CC, - reader: asyncio.StreamReader, - writer: asyncio.StreamWriter, queue: asyncio.Queue, instance_id: str, logger: logging.Logger, @@ -48,21 +51,22 @@ def __init__(self, channel: CC, sync_barrier: Barrier): self._channel_enum = channel - self._reader = reader - self._writer = writer self._global_queue = queue self._global_instance_id = instance_id self._logger = logger self._event_loop = asyncio.get_running_loop() - self._internal_queue = asyncio.Queue() + self._internal_outcoming_queue = asyncio.Queue() + self._internal_incoming_queue = asyncio.Queue() self._channel_opened = False self._channel_paused = False self._stop_event = stop_event self._sync_event = sync_event self._sync_barrier = sync_barrier - self._process_task = asyncio.create_task(self._process_tasks()) + self._outcoming_process_task = asyncio.create_task(self._outcomming_process_tasks()) + self._eof = None + self._read_buffer = bytearray() - async def _process_tasks(self): + async def _outcomming_process_tasks(self): while not self._stop_event.is_set(): await self._queue_up_outcoming() @@ -97,56 +101,74 @@ def _set_logger(self, logger: logging.Logger) -> None: """ self._logger = logger - def readline(self) -> Coroutine: - """asyncio.StreamReader API. Reads chunk of data from the stream until newline (b'\n') is found. - - Returns: - Coroutine - """ + async def readline(self) -> str: + sep = b'\n' + line = await self.readuntil(sep) + return line + + async def readuntil(self, separator=b'\n'): + seplen = len(separator) + + offset = 0 + isep = 0 + while True: + buflen = len(self._read_buffer) + + if buflen - offset >= seplen: + isep = self._read_buffer.find(separator, offset) + + if isep != -1: + break - return self._reader.readline() + offset = buflen + 1 - seplen - def readuntil(self, separator: bytes = b'\n') -> Coroutine: - """asyncio.StreamReader API. Reads data from the stream until ``separator`` is found. + if(not await self._get_data()): + break - Args: - separator (bytes, optional): Data separator. Defaults to b'\n'. + chunk = self._read_buffer[:isep + seplen] + del self._read_buffer[:isep + seplen] + return bytes(chunk) - Returns: - Coroutine - """ - return self._reader.readuntil(separator) + async def _get_data(self,force_get=False): - def read(self, n: int = 64) -> Coroutine: - """asyncio.StreamReader API. Reads up to `n` bytes from the stream. + if self._eof and self._internal_incoming_queue.empty(): + return False - Args: - n (int, optional): Number of bytes to read. Defaults to -1. + while True: + try: + buf = self._internal_incoming_queue.get_nowait() + self._read_buffer.extend(buf) + break + except asyncio.QueueEmpty: + await asyncio.sleep(0) + await asyncio.sleep(0) + return True + - Returns: - Coroutine - """ - return self._reader.read(n) - def readexactly(self, n: int) -> Coroutine: - """asyncio.StreamReader API. Reads exactly `n` bytes. - Args: - n (int): Number of bytes to read + async def read(self, n: int = -1): - Returns: - Coroutine - """ - return self._reader.readexactly(n) + if n == 0: + return b'' + + if not self._read_buffer: + await self._get_data(force_get=True) + data = bytes(memoryview(self._read_buffer)[:n]) + del self._read_buffer[:n] + return data + def __aiter__(self): - """asyncio.StreamReader API - """ - return self._reader.__aiter__() + return self - def __anext__(self): - return self._reader.__anext__() + async def __anext__(self): + val = await self.readline() + + if val == b'': + raise StopAsyncIteration + return val async def send_ACK(self, sequence_number: int) -> None: """Adds to global queue an ACK packet to send. @@ -178,6 +200,7 @@ async def open(self) -> None: await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=buf, flags=['PSH']))) self._channel_opened = True + self._eof = False async def close(self) -> None: """Close channel @@ -207,25 +230,27 @@ async def queue_up_incoming(self, pkt: IPPacket) -> None: if pkt.segment.is_flag('PSH'): await self.send_ACK(pkt.get_segment().seq) - self._writer.write(pkt.get_segment().data) - await self._writer.drain() + if pkt.segment.is_flag('FIN'): + self._eof = True + + self._internal_incoming_queue.put_nowait(pkt.get_segment().data) async def _queue_up_outcoming(self) -> None: """Redirects raw data from currect channel to global queue. Args: - data (bytes): Buffer to send + data (bytes): Buffer to send_incoming_process_task """ def wrap(channel_enum, buf): channel_id = int(channel_enum.value) return IPPacket(segment=TCPSegment(dst_port=channel_id, flags=['PSH'], data=buf)) if not self._channel_paused: - while not self._internal_queue.empty(): + while not self._internal_outcoming_queue.empty(): try: - buf = await self._internal_queue.get() + buf = await self._internal_outcoming_queue.get() await self._global_queue.put(wrap(self._channel_enum, buf)) - self._internal_queue.task_done() + self._internal_outcoming_queue.task_done() except asyncio.QueueEmpty: self._debug(f'Tecemux/{self._get_channel_name()}: [-] All data stored during pause were redirected to global queue') break @@ -243,22 +268,22 @@ def write(self, data: bytes) -> None: Args: data (bytes): Buffer to send to channel """ - self._internal_queue.put_nowait(data) + self._internal_outcoming_queue.put_nowait(data) def drain(self) -> bool: """Drain channel """ - return self._writer.drain() + return def write_eof(self) -> None: """asyncio.StreamWriter API """ - return self._writer.write_eof() + return def wait_closed(self) -> Coroutine: """asyncio.StreamWriter API """ - return self._writer.wait_closed() + return def set_pause(self, state: bool) -> None: """Sets pause state for current channel @@ -346,7 +371,6 @@ async def prepare(self, force_open: bool = False) -> None: self._queue = asyncio.Queue() self._global_sync_barrier = Barrier(len(CC)) self._channels = {channel: _ChannelContext(channel, - *await Tecemux.prepare_socket_connection(), self._queue, self._instance_id, self._logger, @@ -410,21 +434,18 @@ async def stop(self) -> None: """ for channel in self.get_channels(): await channel.close() - await channel.drain() - self.sync() self._global_stop_event.set() async def wait_until_end(self) -> None: """Waits until forwarders finished work. """ await self._global_stop_event.wait() for channel in self.get_channels(): - channel._process_task.cancel() + channel._outcoming_process_task.cancel() self._incoming_data_forwarder.cancel() self._outcoming_data_forwarder.cancel() #await asyncio.gather(*[self._incoming_data_forwarder, self._outcoming_data_forwarder]) - await asyncio.sleep(1) self._writer.write_eof() await self._writer.drain() self._writer.close() @@ -453,7 +474,7 @@ async def incoming_data_forward(self) -> None: while not self._global_stop_event.is_set(): try: - chunk = await asyncio.wait_for(self._reader.read(READ_CHUNK_SIZE),1) + chunk = await self._reader.read(READ_CHUNK_SIZE) if not chunk: incoming_parser_finish_loop.set() @@ -488,6 +509,8 @@ async def incoming_data_forward(self) -> None: self._warning('Tecemux/MAIN: [<] Not full packet received. Getting additional data chunk') break except TimeoutError: + if self._global_stop_event.is_set(): + break await asyncio.sleep(0) except asyncio.CancelledError: @@ -501,7 +524,7 @@ async def outcoming_data_forward(self) -> None: while not self._global_stop_event.is_set(): try: - pkt = self._queue.get_nowait() + pkt = await self._queue.get() # inject sequence number if pkt.segment.seq == 0: @@ -518,5 +541,7 @@ async def outcoming_data_forward(self) -> None: self._debug(f'Tecemux/MAIN: [>] Chunk {Tecemux._chunk_preview(chunk)} with sequence number: {pkt.segment.seq} was sent to Transform Hub') except asyncio.QueueEmpty: await asyncio.sleep(0) + except asyncio.CancelledError: + pass self._debug('Tecemux/MAIN: Outcoming data forwarder finished') diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index fbbfe5568..912c0e408 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -124,7 +124,7 @@ async def test_stream_read_redirection(self, local_socket_connection): assert await output_stream.read() == 'Test\n' await self._close_clients(client_a,client_b) - + async def test_stderr_write_redirection(self, local_socket_connection): client_a, client_b = local_socket_connection @@ -135,7 +135,45 @@ async def test_stderr_write_redirection(self, local_socket_connection): sys.stderr = sys.__stderr__ - assert (await client_b.get_channel(CC.STDERR).read(100)).decode() == "Error\n" + assert (await client_b.get_channel(CC.STDERR).read(100)).decode() == "Error" await self._close_clients(client_a,client_b) + + async def test_readuntil(self, local_socket_connection): + client_a, client_b = local_socket_connection + + channel = CC.CONTROL + separator = '\r\n\r\n' + + client_a.get_channel(channel).write("{'foo':'bar'}") + await client_a._writer.drain() + + client_a.get_channel(channel).write("{'foo':'bar'}") + await client_a._writer.drain() + + client_a.get_channel(channel).write(separator + "{'should-not-be-returned'}") + await client_a._writer.drain() + + data = (await client_b.get_channel(channel).readuntil(separator.encode())).decode() + + assert data == "{'foo':'bar'}{'foo':'bar'}\r\n\r\n" + + await self._close_clients(client_a,client_b) + + async def test_wih_scramjet_framework_to_list(self, local_socket_connection): + client_a, client_b = local_socket_connection + channel = CC.CONTROL + + from scramjet.streams import Stream + + client_a.get_channel(channel).write("foo\n") + client_a.get_channel(channel).write("bar\n") + client_a.get_channel(channel).write("baz\n") + await client_a.get_channel(channel).close() + + output_list = await Stream.read_from(client_b.get_channel(channel)).to_list() + + assert output_list == [b'foo\n', b'bar\n', b'baz\n'] + + await self._close_clients(client_a,client_b) From c5727d1d10b65bcfe91c407378f34de1d144332b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Wed, 28 Jun 2023 10:56:14 +0200 Subject: [PATCH 179/231] pytest Tecemux test with pipe --- packages/python-runner/tecemux.py | 14 ++---- packages/python-runner/test/test_tecemux.py | 51 ++++++++++++++------- 2 files changed, 40 insertions(+), 25 deletions(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index e224328ea..8077496c9 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -122,39 +122,35 @@ async def readuntil(self, separator=b'\n'): offset = buflen + 1 - seplen - if(not await self._get_data()): + if (not await self._get_data()): break chunk = self._read_buffer[:isep + seplen] del self._read_buffer[:isep + seplen] return bytes(chunk) - - async def _get_data(self,force_get=False): + async def _get_data(self): if self._eof and self._internal_incoming_queue.empty(): return False while True: - try: + try: buf = self._internal_incoming_queue.get_nowait() self._read_buffer.extend(buf) break except asyncio.QueueEmpty: await asyncio.sleep(0) await asyncio.sleep(0) - return True + return True - - - async def read(self, n: int = -1): if n == 0: return b'' if not self._read_buffer: - await self._get_data(force_get=True) + await self._get_data() data = bytes(memoryview(self._read_buffer)[:n]) del self._read_buffer[:n] diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index 912c0e408..07e96a5a8 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -3,7 +3,7 @@ import pytest import sys from tecemux import Tecemux -from inet import IPPacket,TCPSegment +from inet import IPPacket, TCPSegment from logging_setup import LoggingSetup from hardcoded_magic_values import CommunicationChannels as CC @@ -21,20 +21,18 @@ async def local_socket_connection(): client_b = Tecemux() client_b.set_logger(get_logger()) - rsock_a, wsock_a = await Tecemux.prepare_socket_connection() rsock_b, wsock_b = await Tecemux.prepare_socket_connection() - await client_a.connect(rsock_a,wsock_b) - await client_b.connect(rsock_b,wsock_a) + await client_a.connect(rsock_a, wsock_b) + await client_b.connect(rsock_b, wsock_a) await client_a.prepare() await client_b.prepare() await client_a.loop() await client_b.loop() - - + return client_a, client_b class TestTecemux: @@ -61,8 +59,8 @@ async def test_socket_connection(self): await protocol.connect(*await Tecemux.prepare_socket_connection()) - assert isinstance(protocol._reader,asyncio.StreamReader) - assert isinstance(protocol._writer,asyncio.StreamWriter) + assert isinstance(protocol._reader, asyncio.StreamReader) + assert isinstance(protocol._writer, asyncio.StreamWriter) assert protocol._sequence_number > 0 @pytest.mark.asyncio @@ -72,7 +70,7 @@ async def test_forward_from_main_to_channel(self, local_socket_connection): data_to_send ="{'foo':'bar'}" destination_channel = CC.CONTROL - pkt = IPPacket(src_addr='172.25.44.3',dst_addr='172.25.44.254',segment=TCPSegment(dst_port=int(destination_channel.value),flags=['PSH'],data=data_to_send)) + pkt = IPPacket(src_addr='172.25.44.3', dst_addr='172.25.44.254', segment=TCPSegment(dst_port=int(destination_channel.value), flags=['PSH'], data=data_to_send)) client_a._writer.write(pkt.to_buffer_with_tcp_pseudoheader()) await client_a._writer.drain() @@ -85,7 +83,6 @@ async def test_forward_from_main_to_channel(self, local_socket_connection): async def test_forward_channel_between_a_b(self, local_socket_connection): client_a, client_b = local_socket_connection - channel_alpha = CC.CONTROL channel_beta = CC.IN @@ -98,7 +95,7 @@ async def test_forward_channel_between_a_b(self, local_socket_connection): assert (await client_b.get_channel(channel_alpha).read(100)).decode() == "{'foo':'bar'}" assert (await client_b.get_channel(channel_beta).read(100)).decode() == "{'bar':'foo2'}" - await self._close_clients(client_a,client_b) + await self._close_clients(client_a, client_b) async def test_stream_write_redirection(self, local_socket_connection): client_a, client_b = local_socket_connection @@ -109,7 +106,7 @@ async def test_stream_write_redirection(self, local_socket_connection): assert (await client_b.get_channel(CC.HOST).read(100)).decode() == "Hi!" - await self._close_clients(client_a,client_b) + await self._close_clients(client_a, client_b) async def test_stream_read_redirection(self, local_socket_connection): @@ -123,7 +120,7 @@ async def test_stream_read_redirection(self, local_socket_connection): print("Test", file=input_stream) assert await output_stream.read() == 'Test\n' - await self._close_clients(client_a,client_b) + await self._close_clients(client_a, client_b) async def test_stderr_write_redirection(self, local_socket_connection): client_a, client_b = local_socket_connection @@ -137,7 +134,7 @@ async def test_stderr_write_redirection(self, local_socket_connection): assert (await client_b.get_channel(CC.STDERR).read(100)).decode() == "Error" - await self._close_clients(client_a,client_b) + await self._close_clients(client_a, client_b) async def test_readuntil(self, local_socket_connection): @@ -159,7 +156,7 @@ async def test_readuntil(self, local_socket_connection): assert data == "{'foo':'bar'}{'foo':'bar'}\r\n\r\n" - await self._close_clients(client_a,client_b) + await self._close_clients(client_a, client_b) async def test_wih_scramjet_framework_to_list(self, local_socket_connection): client_a, client_b = local_socket_connection @@ -176,4 +173,26 @@ async def test_wih_scramjet_framework_to_list(self, local_socket_connection): assert output_list == [b'foo\n', b'bar\n', b'baz\n'] - await self._close_clients(client_a,client_b) + await self._close_clients(client_a, client_b) + + async def test_wih_scramjet_framework_to_pipe(self, local_socket_connection): + client_a, client_b = local_socket_connection + channel = CC.CONTROL + + from scramjet.streams import Stream + + client_a.get_channel(channel).write("foo") + client_a.get_channel(channel).write("bar\n") + client_a.get_channel(channel).write("foz") + client_a.get_channel(channel).write("baz\n") + await client_a.get_channel(channel).close() + + final_stream = Stream() + + Stream.read_from(client_b.get_channel(channel)).pipe(final_stream) + + output_list = await final_stream.to_list() + + assert output_list == [b'foobar\n', b'fozbaz\n'] + + await self._close_clients(client_a, client_b) \ No newline at end of file From ed28e2e10c63ba512bd0efb5ea3f87f1f9862489 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Wed, 28 Jun 2023 15:50:41 +0200 Subject: [PATCH 180/231] Return to asyncio.wait_for --- packages/python-runner/runner.py | 25 ++++++++++++------------- packages/python-runner/tecemux.py | 7 +++++-- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index 6887e710d..bba9511cf 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -25,10 +25,9 @@ debugpy.wait_for_client() debugpy.breakpoint() -async def send_encoded_msg(stream, msg_code, data={}): +def send_encoded_msg(stream, msg_code, data={}): message = json.dumps([msg_code.value, data]) stream.write(f'{message}\r\n'.encode()) - await stream.drain() class Runner: def __init__(self, instance_id, sequence_path, log_setup) -> None: @@ -52,7 +51,7 @@ async def main(self, server_host, server_port): # Do this early to have access to any thrown exceptions and logs. self.connect_stdio() self.connect_log_stream() - + await self.protocol.sync() config, args = await self.handshake() self.logger.info('Communication established.') asyncio.create_task(self.connect_control_stream()) @@ -101,7 +100,7 @@ async def handshake(self): control = self.protocol.get_channel(CC.CONTROL) self.logger.info('Sending PING') - await send_encoded_msg(monitoring, msg_codes.PING) + send_encoded_msg(monitoring, msg_codes.PING) message = await control.readuntil(b'\n') self.logger.info(f'Got message: {message}') @@ -119,7 +118,7 @@ async def handshake(self): 'requires': '', 'contentType': '' } - await send_encoded_msg(monitoring, msg_codes.PANG, pang_requires_data) + send_encoded_msg(monitoring, msg_codes.PANG, pang_requires_data) if code == msg_codes.PONG.value: self.logger.info(f'Got configuration: {data}') @@ -155,10 +154,10 @@ async def handle_stop(self, data): await handler(timeout, can_keep_alive) except Exception as e: self.logger.error('Error stopping sequence', e) - await send_encoded_msg(self.protocol.get_channel(CC.MONITORING), msg_codes.SEQUENCE_STOPPED, e) + send_encoded_msg(self.protocol.get_channel(CC.MONITORING), msg_codes.SEQUENCE_STOPPED, e) if not can_keep_alive or not self.keep_alive_requested: - await send_encoded_msg(self.protocol.get_channel(CC.MONITORING), msg_codes.SEQUENCE_STOPPED, {}) + send_encoded_msg(self.protocol.get_channel(CC.MONITORING), msg_codes.SEQUENCE_STOPPED, {}) self.exit_immediately() await self.cleanup() @@ -166,7 +165,7 @@ async def handle_stop(self, data): async def setup_heartbeat(self): while True: - await send_encoded_msg( + send_encoded_msg( self.protocol.get_channel(CC.MONITORING), msg_codes.MONITORING, self.health_check(), @@ -207,12 +206,12 @@ async def run_instance(self, config, args): produces = getattr(result, 'provides', None) or getattr(self.sequence, 'provides', None) if produces: self.logger.info(f'Sending PANG with {produces}') - await send_encoded_msg(monitoring, msg_codes.PANG, produces) + send_encoded_msg(monitoring, msg_codes.PANG, produces) consumes = getattr(result, 'requires', None) or getattr(self.sequence, 'requires', None) if consumes: self.logger.info(f'Sending PANG with {consumes}') - await send_encoded_msg(monitoring, msg_codes.PANG, consumes) + send_encoded_msg(monitoring, msg_codes.PANG, consumes) if isinstance(result, types.AsyncGeneratorType): result = Stream.read_from(result) @@ -285,7 +284,7 @@ async def forward_output_stream(self, output): async def send_keep_alive(self, timeout: int = 0, can_keep_alive: bool = False): monitoring =self.protocol.get_channel(CC.MONITORING) - await send_encoded_msg(monitoring, msg_codes.ALIVE) + send_encoded_msg(monitoring, msg_codes.ALIVE) self.keep_alive_requested = True await asyncio.sleep(timeout) @@ -312,7 +311,7 @@ def on(self, event_name, callback): self.emitter.on(event_name, callback) async def emit(self, event_name, message=''): - await send_encoded_msg( + send_encoded_msg( self.monitoring, msg_codes.EVENT, {'eventName': event_name, 'message': message} @@ -335,4 +334,4 @@ async def keep_alive(self, timeout: int = 0): sys.exit(2) runner = Runner(INSTANCE_ID, SEQUENCE_PATH, LOG_SETUP) -asyncio.run(runner.main(SERVER_HOST, SERVER_PORT)) +asyncio.run(runner.main(SERVER_HOST, SERVER_PORT), debug=False) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 8077496c9..e67a8bd7d 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -141,6 +141,8 @@ async def _get_data(self): break except asyncio.QueueEmpty: await asyncio.sleep(0) + except asyncio.CancelledError: + return False await asyncio.sleep(0) return True @@ -441,10 +443,11 @@ async def wait_until_end(self) -> None: self._incoming_data_forwarder.cancel() self._outcoming_data_forwarder.cancel() - #await asyncio.gather(*[self._incoming_data_forwarder, self._outcoming_data_forwarder]) + await asyncio.gather(*[self._incoming_data_forwarder, self._outcoming_data_forwarder]) self._writer.write_eof() await self._writer.drain() self._writer.close() + await self._writer.wait_closed() self._debug('Tecemux/MAIN: [-] Finished') @@ -470,7 +473,7 @@ async def incoming_data_forward(self) -> None: while not self._global_stop_event.is_set(): try: - chunk = await self._reader.read(READ_CHUNK_SIZE) + chunk = await asyncio.wait_for(self._reader.read(READ_CHUNK_SIZE),1) if not chunk: incoming_parser_finish_loop.set() From df222fe6bb821a4f3c00b92612465bfdf005b981 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Thu, 29 Jun 2023 15:50:54 +0200 Subject: [PATCH 181/231] Tecemux IN & OUT channels works --- packages/python-runner/runner.py | 9 ++++---- packages/python-runner/tecemux.py | 23 ++++++++++++++------- packages/python-runner/test/test_tecemux.py | 17 +++++++++++++++ 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index bba9511cf..652e39f38 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -21,9 +21,9 @@ SERVER_HOST = os.getenv('INSTANCES_SERVER_HOST') or 'localhost' INSTANCE_ID = os.getenv('INSTANCE_ID') -debugpy.listen(5678) -debugpy.wait_for_client() -debugpy.breakpoint() +# debugpy.listen(5678) +# debugpy.wait_for_client() +# debugpy.breakpoint() def send_encoded_msg(stream, msg_code, data={}): message = json.dumps([msg_code.value, data]) @@ -244,7 +244,7 @@ async def connect_input_stream(self, input_stream): if input_type == 'text/plain': await self.protocol.sync() - input = await Stream.read_from(self.protocol.get_channel(CC.IN)).to_list() + input = Stream.read_from(self.protocol.get_channel(CC.IN)) self.logger.debug('Decoding input stream...') input = input.decode('utf-8') @@ -278,7 +278,6 @@ async def forward_output_stream(self, output): self.logger.debug('Output will be converted to JSON') output = output.map(lambda chunk: (json.dumps(chunk)+'\n').encode()) - await self.protocol.sync() await output.write_to(self.protocol.get_channel(CC.OUT)) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index e67a8bd7d..06b287163 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -8,6 +8,7 @@ from barrier import Barrier from inet import IPPacket, TCPSegment, SequenceOrder from hardcoded_magic_values import CommunicationChannels as CC +from threading import Lock TECEMUX_INTERNAL_VERBOSE_DEBUG = False @@ -38,8 +39,8 @@ class _ChannelContext: _read_buffer: bytearray - _eof: bool - + _ended: bool + _lock: Lock _outcoming_process_task: Any def __init__(self, channel: CC, @@ -63,7 +64,8 @@ def __init__(self, channel: CC, self._sync_event = sync_event self._sync_barrier = sync_barrier self._outcoming_process_task = asyncio.create_task(self._outcomming_process_tasks()) - self._eof = None + self._ended = None + self._lock = Lock() self._read_buffer = bytearray() async def _outcomming_process_tasks(self): @@ -131,7 +133,7 @@ async def readuntil(self, separator=b'\n'): async def _get_data(self): - if self._eof and self._internal_incoming_queue.empty(): + if self._ended and self._internal_incoming_queue.empty(): return False while True: @@ -198,7 +200,7 @@ async def open(self) -> None: await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=buf, flags=['PSH']))) self._channel_opened = True - self._eof = False + self._ended = False async def close(self) -> None: """Close channel @@ -229,7 +231,8 @@ async def queue_up_incoming(self, pkt: IPPacket) -> None: await self.send_ACK(pkt.get_segment().seq) if pkt.segment.is_flag('FIN'): - self._eof = True + self._ended = True + return self._internal_incoming_queue.put_nowait(pkt.get_segment().data) @@ -266,6 +269,10 @@ def write(self, data: bytes) -> None: Args: data (bytes): Buffer to send to channel """ + if data == b'': + return + + #with self._lock: self._internal_outcoming_queue.put_nowait(data) def drain(self) -> bool: @@ -405,7 +412,7 @@ async def sync(self): self._global_sync_event.clear() if not self._queue.empty(): - await self._queue.join() + await self._queue.join() def get_channels(self) -> dict: """Returns all initiated channels @@ -473,7 +480,7 @@ async def incoming_data_forward(self) -> None: while not self._global_stop_event.is_set(): try: - chunk = await asyncio.wait_for(self._reader.read(READ_CHUNK_SIZE),1) + chunk = await self._reader.read(READ_CHUNK_SIZE) if not chunk: incoming_parser_finish_loop.set() diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index 07e96a5a8..33bdb4c98 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -195,4 +195,21 @@ async def test_wih_scramjet_framework_to_pipe(self, local_socket_connection): assert output_list == [b'foobar\n', b'fozbaz\n'] + await self._close_clients(client_a, client_b) + + + async def test_wih_scramjet_framework_write_to(self, local_socket_connection): + client_a, client_b = local_socket_connection + channel = CC.CONTROL + + data = [b'foo\n',b'bar\n',b'baz\n'] + + from scramjet.streams import Stream + + await Stream.read_from(data).write_to(client_a.get_channel(channel)) + await client_a.get_channel(channel).close() + + output = Stream.read_from(client_b.get_channel(channel)) + assert await output.to_list() == data + await self._close_clients(client_a, client_b) \ No newline at end of file From 0c652be137859125e7bda4150297679ed305935d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 30 Jun 2023 11:31:34 +0200 Subject: [PATCH 182/231] Exception handling --- packages/python-runner/runner.py | 71 ++++++++++++++++++------------- packages/python-runner/tecemux.py | 3 +- 2 files changed, 44 insertions(+), 30 deletions(-) diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index 652e39f38..795cb8752 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -21,9 +21,8 @@ SERVER_HOST = os.getenv('INSTANCES_SERVER_HOST') or 'localhost' INSTANCE_ID = os.getenv('INSTANCE_ID') -# debugpy.listen(5678) -# debugpy.wait_for_client() -# debugpy.breakpoint() +debugpy.listen(5678) +debugpy.wait_for_client() def send_encoded_msg(stream, msg_code, data={}): message = json.dumps([msg_code.value, data]) @@ -45,27 +44,37 @@ def is_incoming(channel): return channel in [CC.STDIN, CC.IN, CC.CONTROL] async def main(self, server_host, server_port): - + input_stream = Stream() await self.init_tecemux(server_host, server_port) - + debugpy.breakpoint() # Do this early to have access to any thrown exceptions and logs. self.connect_stdio() self.connect_log_stream() await self.protocol.sync() config, args = await self.handshake() self.logger.info('Communication established.') - asyncio.create_task(self.connect_control_stream()) - asyncio.create_task(self.setup_heartbeat()) + + control_stream_task = asyncio.create_task(self.connect_control_stream()) + heartbeat_task = asyncio.create_task(self.setup_heartbeat()) + connect_input_stream_task = asyncio.create_task(self.connect_input_stream(input_stream)) self.load_sequence() - + await self.protocol.sync() - await self.run_instance(config, args) + await self.run_instance(config, input_stream, args) + await self.protocol.sync() + heartbeat_task.cancel() + connect_input_stream_task.cancel() + control_stream_task.cancel() + + await asyncio.gather(*[heartbeat_task, + connect_input_stream_task, + control_stream_task]) + await self.protocol.stop() await self.protocol.wait_until_end() - async def init_tecemux(self, server_host, server_port): self.logger.info('Connecting to host with TeceMux...') self.protocol = Tecemux(instance_id=self.instance_id) @@ -133,15 +142,17 @@ async def connect_control_stream(self): .read_from(self.protocol.get_channel(CC.CONTROL), chunk_size=131072) .decode('utf-8').split('\n').map(json.loads) ) - async for code, data in control_messages: - self.logger.debug(f'Control message received: {code} {data}') - if code == msg_codes.KILL.value: - self.exit_immediately() - if code == msg_codes.STOP.value: - await self.handle_stop(data) - if code == msg_codes.EVENT.value: - await self.emitter.emit(data['eventName'], data['message'] if 'message' in data else None) - + try: + async for code, data in control_messages: + self.logger.debug(f'Control message received: {code} {data}') + if code == msg_codes.KILL.value: + self.exit_immediately() + if code == msg_codes.STOP.value: + await self.handle_stop(data) + if code == msg_codes.EVENT.value: + await self.emitter.emit(data['eventName'], data['message'] if 'message' in data else None) + except asyncio.CancelledError: + return async def handle_stop(self, data): @@ -165,12 +176,15 @@ async def handle_stop(self, data): async def setup_heartbeat(self): while True: - send_encoded_msg( - self.protocol.get_channel(CC.MONITORING), - msg_codes.MONITORING, - self.health_check(), - ) - await asyncio.sleep(1) + try: + send_encoded_msg( + self.protocol.get_channel(CC.MONITORING), + msg_codes.MONITORING, + self.health_check(), + ) + await asyncio.sleep(1) + except asyncio.CancelledError: + return def load_sequence(self): @@ -186,14 +200,12 @@ def load_sequence(self): # switch to sequence dir so that relative paths will work os.chdir(os.path.dirname(self.seq_path)) - async def run_instance(self, config, args): + async def run_instance(self, config, input, args): context = AppContext(self, config) - input_stream = Stream() - asyncio.create_task(self.connect_input_stream(input_stream)) await self.protocol.sync() self.logger.info('Running instance...') try: - result = self.sequence.run(context, input_stream, *args) + result = self.sequence.run(context, input, *args) except Exception: import traceback self.protocol.get_channel(CC.STDERR).write(traceback.format_exc()) @@ -229,6 +241,7 @@ async def run_instance(self, config, args): async def cleanup(self): self.protocol.get_channel(CC.LOG).write_eof() + async def connect_input_stream(self, input_stream): if hasattr(self.sequence, "requires"): diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 06b287163..210c58603 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -450,7 +450,8 @@ async def wait_until_end(self) -> None: self._incoming_data_forwarder.cancel() self._outcoming_data_forwarder.cancel() - await asyncio.gather(*[self._incoming_data_forwarder, self._outcoming_data_forwarder]) + await asyncio.gather(*[self._incoming_data_forwarder]) + await asyncio.gather(*[self._outcoming_data_forwarder]) self._writer.write_eof() await self._writer.drain() self._writer.close() From 8d37089d1790fb4ac1d216a20502c6e6d4ae4f3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 30 Jun 2023 13:01:42 +0200 Subject: [PATCH 183/231] Output stream ends correctly --- packages/python-runner/runner.py | 9 +++++---- packages/python-runner/tecemux.py | 18 +++++++++--------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index 795cb8752..bf402e932 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -4,7 +4,7 @@ import codecs import json import logging -import debugpy +#import debugpy from pyee.asyncio import AsyncIOEventEmitter from tecemux import Tecemux import importlib.util @@ -21,8 +21,9 @@ SERVER_HOST = os.getenv('INSTANCES_SERVER_HOST') or 'localhost' INSTANCE_ID = os.getenv('INSTANCE_ID') -debugpy.listen(5678) -debugpy.wait_for_client() +# debugpy.listen(5678) +# debugpy.wait_for_client() +# debugpy.breakpoint() def send_encoded_msg(stream, msg_code, data={}): message = json.dumps([msg_code.value, data]) @@ -46,7 +47,6 @@ def is_incoming(channel): async def main(self, server_host, server_port): input_stream = Stream() await self.init_tecemux(server_host, server_port) - debugpy.breakpoint() # Do this early to have access to any thrown exceptions and logs. self.connect_stdio() self.connect_log_stream() @@ -292,6 +292,7 @@ async def forward_output_stream(self, output): output = output.map(lambda chunk: (json.dumps(chunk)+'\n').encode()) await output.write_to(self.protocol.get_channel(CC.OUT)) + await self.protocol.get_channel(CC.OUT).end() async def send_keep_alive(self, timeout: int = 0, can_keep_alive: bool = False): diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 210c58603..afc6b6d16 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -8,7 +8,7 @@ from barrier import Barrier from inet import IPPacket, TCPSegment, SequenceOrder from hardcoded_magic_values import CommunicationChannels as CC -from threading import Lock + TECEMUX_INTERNAL_VERBOSE_DEBUG = False @@ -38,10 +38,8 @@ class _ChannelContext: _sync_barrier: Barrier _read_buffer: bytearray - _ended: bool - _lock: Lock - _outcoming_process_task: Any + _outcoming_process_task: asyncio.Task def __init__(self, channel: CC, queue: asyncio.Queue, @@ -65,7 +63,6 @@ def __init__(self, channel: CC, self._sync_barrier = sync_barrier self._outcoming_process_task = asyncio.create_task(self._outcomming_process_tasks()) self._ended = None - self._lock = Lock() self._read_buffer = bytearray() async def _outcomming_process_tasks(self): @@ -107,7 +104,7 @@ async def readline(self) -> str: sep = b'\n' line = await self.readuntil(sep) return line - + async def readuntil(self, separator=b'\n'): seplen = len(separator) @@ -202,6 +199,9 @@ async def open(self) -> None: self._channel_opened = True self._ended = False + async def end(self) -> None: + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'', flags=['PSH']))) + async def close(self) -> None: """Close channel """ @@ -271,8 +271,7 @@ def write(self, data: bytes) -> None: """ if data == b'': return - - #with self._lock: + self._internal_outcoming_queue.put_nowait(data) def drain(self) -> bool: @@ -439,6 +438,7 @@ async def stop(self) -> None: """ for channel in self.get_channels(): await channel.close() + await self.sync() self._global_stop_event.set() async def wait_until_end(self) -> None: """Waits until forwarders finished work. @@ -446,7 +446,7 @@ async def wait_until_end(self) -> None: await self._global_stop_event.wait() for channel in self.get_channels(): channel._outcoming_process_task.cancel() - + await self._queue.join() self._incoming_data_forwarder.cancel() self._outcoming_data_forwarder.cancel() From e3cb9ca1d7c805d2cb0ed79f93e028dfab73d579 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 30 Jun 2023 15:36:10 +0200 Subject: [PATCH 184/231] Runner finish improvements --- packages/python-runner/runner.py | 7 +------ packages/python-runner/tecemux.py | 29 +++++++++++++++++++++-------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index bf402e932..80a49fd11 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -4,7 +4,7 @@ import codecs import json import logging -#import debugpy +# import debugpy from pyee.asyncio import AsyncIOEventEmitter from tecemux import Tecemux import importlib.util @@ -237,10 +237,6 @@ async def run_instance(self, config, input, args): self.logger.info('Finished.') await self.protocol.sync() - await self.cleanup() - - async def cleanup(self): - self.protocol.get_channel(CC.LOG).write_eof() async def connect_input_stream(self, input_stream): @@ -292,7 +288,6 @@ async def forward_output_stream(self, output): output = output.map(lambda chunk: (json.dumps(chunk)+'\n').encode()) await output.write_to(self.protocol.get_channel(CC.OUT)) - await self.protocol.get_channel(CC.OUT).end() async def send_keep_alive(self, timeout: int = 0, can_keep_alive: bool = False): diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index afc6b6d16..d489dcaf7 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -437,26 +437,37 @@ async def stop(self) -> None: """Stops protocol """ for channel in self.get_channels(): + await channel.end() await channel.close() await self.sync() - self._global_stop_event.set() async def wait_until_end(self) -> None: """Waits until forwarders finished work. """ - await self._global_stop_event.wait() for channel in self.get_channels(): channel._outcoming_process_task.cancel() - await self._queue.join() - self._incoming_data_forwarder.cancel() - self._outcoming_data_forwarder.cancel() - await asyncio.gather(*[self._incoming_data_forwarder]) - await asyncio.gather(*[self._outcoming_data_forwarder]) + self._global_stop_event.set() + await asyncio.sleep(0.1) # to refactor + await self._global_stop_event.wait() + self._outcoming_data_forwarder.cancel() + await asyncio.gather(*[self._outcoming_data_forwarder]) self._writer.write_eof() await self._writer.drain() self._writer.close() await self._writer.wait_closed() - + self._incoming_data_forwarder.cancel() + await asyncio.gather(*[self._incoming_data_forwarder]) + + # flush commucation from STH + + while True: + try: + data = await asyncio.wait_for(self._reader.read(),1) + if not data: + break + except Exception as e: + break + self._debug('Tecemux/MAIN: [-] Finished') async def loop(self) -> None: @@ -494,6 +505,8 @@ async def incoming_data_forward(self) -> None: self._warning('Tecemux/MAIN: [<] Too few data is waiting in global buffer but stream finished') break + elif incoming_parser_finish_loop.is_set() and buffer_len == 0: + break while not (len(buffer) < MINIMAL_IP_PACKET_LENGTH): From a0b2bef049e392ba50dfc87239b525e39ade9a1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Mon, 3 Jul 2023 16:37:35 +0200 Subject: [PATCH 185/231] Better cleanup --- packages/python-runner/runner.py | 7 +- packages/python-runner/tecemux.py | 117 +++++++++++++------- packages/python-runner/test/test_inet.py | 9 ++ packages/python-runner/test/test_tecemux.py | 9 +- 4 files changed, 94 insertions(+), 48 deletions(-) diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index 80a49fd11..37745f999 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -73,7 +73,7 @@ async def main(self, server_host, server_port): control_stream_task]) await self.protocol.stop() - await self.protocol.wait_until_end() + async def init_tecemux(self, server_host, server_port): self.logger.info('Connecting to host with TeceMux...') @@ -171,8 +171,6 @@ async def handle_stop(self, data): send_encoded_msg(self.protocol.get_channel(CC.MONITORING), msg_codes.SEQUENCE_STOPPED, {}) self.exit_immediately() - await self.cleanup() - async def setup_heartbeat(self): while True: @@ -251,8 +249,9 @@ async def connect_input_stream(self, input_stream): self.logger.info(f'Input headers: {repr(headers)}') input_type = headers.get('content-type') + await self.protocol.sync() + if input_type == 'text/plain': - await self.protocol.sync() input = Stream.read_from(self.protocol.get_channel(CC.IN)) self.logger.debug('Decoding input stream...') diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index d489dcaf7..c13ab1e43 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -33,8 +33,8 @@ class _ChannelContext: _channel_paused: bool _channel_opened: bool - _stop_event: asyncio.Event - _sync_event: asyncio.Event + _stop_channel_event: asyncio.Event + _sync_channel_event: asyncio.Event _sync_barrier: Barrier _read_buffer: bytearray @@ -58,16 +58,18 @@ def __init__(self, channel: CC, self._internal_incoming_queue = asyncio.Queue() self._channel_opened = False self._channel_paused = False - self._stop_event = stop_event - self._sync_event = sync_event + self._stop_channel_event = stop_event + self._sync_channel_event = sync_event self._sync_barrier = sync_barrier self._outcoming_process_task = asyncio.create_task(self._outcomming_process_tasks()) self._ended = None self._read_buffer = bytearray() async def _outcomming_process_tasks(self): - while not self._stop_event.is_set(): + while not self._stop_channel_event.is_set(): await self._queue_up_outcoming() + await asyncio.sleep(0) + def _debug(self, msg): @@ -135,7 +137,11 @@ async def _get_data(self): while True: try: - buf = self._internal_incoming_queue.get_nowait() + buf = self._internal_incoming_queue.get_nowait() + + if buf == b'': + return False + self._read_buffer.extend(buf) break except asyncio.QueueEmpty: @@ -189,8 +195,6 @@ async def open(self) -> None: if not self._channel_opened: - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'', flags=['PSH']))) - buf = b'' if self._global_instance_id is None else ( self._global_instance_id.encode() + str(self._get_channel_id()).encode()) @@ -200,6 +204,8 @@ async def open(self) -> None: self._ended = False async def end(self) -> None: + + await self._internal_outcoming_queue.join() await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'', flags=['PSH']))) async def close(self) -> None: @@ -224,6 +230,7 @@ async def queue_up_incoming(self, pkt: IPPacket) -> None: # if ACK flag is up, reasume channel if not pkt.get_segment().is_flag('SYN') and pkt.get_segment().is_flag('ACK'): self.set_pause(False) + self._ended = False return # if PSH flag is up, confirm @@ -233,7 +240,6 @@ async def queue_up_incoming(self, pkt: IPPacket) -> None: if pkt.segment.is_flag('FIN'): self._ended = True return - self._internal_incoming_queue.put_nowait(pkt.get_segment().data) async def _queue_up_outcoming(self) -> None: @@ -258,7 +264,7 @@ def wrap(channel_enum, buf): else: self._debug(f'Tecemux/{self._get_channel_name()}: [-] Channel paused. Data queued up internally for future') - if self._sync_event.is_set(): + if self._sync_channel_event.is_set(): await self._sync_barrier.wait() await asyncio.sleep(0) @@ -313,9 +319,15 @@ class Tecemux: _instance_id: str = field(default=None) _channels: dict = field(default={}) _logger: logging.Logger = field(default=None) - _global_stop_event: asyncio.Event = asyncio.Event() - _global_sync_event: asyncio.Event = asyncio.Event() + + _global_stop_channel_event: asyncio.Event = asyncio.Event() + + _global_sync_channel_event: asyncio.Event = asyncio.Event() _global_sync_barrier: Barrier = field(default=None, init=False) + + _global_stop_outcoming_event: asyncio.Event = asyncio.Event() + _global_stop_incoming_event: asyncio.Event = asyncio.Event() + _sequence_number: int = field(default=0) _last_sequence_received: int = field(default=0) @@ -338,7 +350,10 @@ async def connect(self, reader: asyncio.StreamReader, writer: asyncio.StreamWrit self._reader = reader self._writer = writer self._sequence_number = abs(int((random.random() * (2 ** 32)) / 2)) - self._global_stop_event.clear() + self._global_stop_channel_event.clear() + self._global_stop_incoming_event.clear() + self._global_stop_outcoming_event.clear() + @staticmethod async def prepare_tcp_connection(server_host: str, server_port: str) -> tuple: @@ -378,8 +393,8 @@ async def prepare(self, force_open: bool = False) -> None: self._queue, self._instance_id, self._logger, - self._global_stop_event, - self._global_sync_event, + self._global_stop_channel_event, + self._global_sync_channel_event, self._global_sync_barrier) for channel in CC} if force_open: [await channel.open() for channel in self.get_channels()] @@ -406,9 +421,10 @@ def get_channel(self, channel: CC) -> _ChannelContext: async def sync(self): """Waits until all write tasks will be done """ - self._global_sync_event.set() + self._global_sync_channel_event.set() + await self._global_sync_channel_event.wait() await self._global_sync_barrier.wait() - self._global_sync_event.clear() + self._global_sync_channel_event.clear() if not self._queue.empty(): await self._queue.join() @@ -434,42 +450,58 @@ def _chunk_preview(value: bytes) -> str: return f'{value[0:5]}... ' if len(value) > 5 else f'{value}' async def stop(self) -> None: + + await self._finish_channels() + await self._finish_incoming() + await self._finish_outcoming() + await self._read_eof() + + + self._debug('Tecemux/MAIN: [-] Finished') + + async def _finish_channels(self) -> None: """Stops protocol """ for channel in self.get_channels(): await channel.end() await channel.close() - await self.sync() - async def wait_until_end(self) -> None: - """Waits until forwarders finished work. - """ + for channel in self.get_channels(): - channel._outcoming_process_task.cancel() + await channel._internal_outcoming_queue.join() + + self._global_stop_channel_event.set() + await self._global_stop_channel_event.wait() - self._global_stop_event.set() - await asyncio.sleep(0.1) # to refactor - await self._global_stop_event.wait() - self._outcoming_data_forwarder.cancel() + await asyncio.gather(*[channel._outcoming_process_task for channel in self.get_channels()]) + + + async def _finish_outcoming(self): + + self._global_stop_outcoming_event.set() + await self._global_stop_outcoming_event.wait() + await asyncio.gather(*[self._outcoming_data_forwarder]) self._writer.write_eof() await self._writer.drain() self._writer.close() await self._writer.wait_closed() - self._incoming_data_forwarder.cancel() + + + async def _finish_incoming(self): + self._global_stop_incoming_event.set() + await self._global_stop_incoming_event.wait() await asyncio.gather(*[self._incoming_data_forwarder]) - # flush commucation from STH - + + async def _read_eof(self): while True: try: - data = await asyncio.wait_for(self._reader.read(),1) + data = await asyncio.wait_for(self._reader.read(),0.5) if not data: break - except Exception as e: + except asyncio.TimeoutError as e: break - self._debug('Tecemux/MAIN: [-] Finished') - async def loop(self) -> None: """Main loop of Tecemux protocol. Starts forwarders tasks """ @@ -490,7 +522,7 @@ async def incoming_data_forward(self) -> None: MINIMAL_IP_PACKET_LENGTH = 20 READ_CHUNK_SIZE = 1024 - while not self._global_stop_event.is_set(): + while not self._global_stop_channel_event.is_set(): try: chunk = await self._reader.read(READ_CHUNK_SIZE) @@ -529,7 +561,7 @@ async def incoming_data_forward(self) -> None: self._warning('Tecemux/MAIN: [<] Not full packet received. Getting additional data chunk') break except TimeoutError: - if self._global_stop_event.is_set(): + if self._global_stop_channel_event.is_set(): break await asyncio.sleep(0) @@ -542,9 +574,9 @@ async def outcoming_data_forward(self) -> None: """Loop for outcoming data to Transform Hub """ - while not self._global_stop_event.is_set(): + while True: try: - pkt = await self._queue.get() + pkt = await asyncio.wait_for(self._queue.get(),1) # inject sequence number if pkt.segment.seq == 0: @@ -560,8 +592,17 @@ async def outcoming_data_forward(self) -> None: self._queue.task_done() self._debug(f'Tecemux/MAIN: [>] Chunk {Tecemux._chunk_preview(chunk)} with sequence number: {pkt.segment.seq} was sent to Transform Hub') except asyncio.QueueEmpty: + if self._global_stop_outcoming_event.is_set(): + break await asyncio.sleep(0) + + except asyncio.TimeoutError: + if self._queue.empty() and self._global_stop_outcoming_event.is_set(): + break + await asyncio.sleep(0) + except asyncio.CancelledError: - pass + break + self._debug('Tecemux/MAIN: Outcoming data forwarder finished') diff --git a/packages/python-runner/test/test_inet.py b/packages/python-runner/test/test_inet.py index f06c8cf73..42c5d9be6 100644 --- a/packages/python-runner/test/test_inet.py +++ b/packages/python-runner/test/test_inet.py @@ -85,6 +85,15 @@ def test_packet_creation(self): assert pkt.src_addr == '172.25.44.3' assert pkt.dst_addr == '172.25.44.254' + def test_pakcet_creation_with_endstring(self): + buf = IPPacket(src_addr='172.25.44.3', dst_addr='172.25.44.254', + segment=TCPSegment(dst_port=6, flags=['PSH'],data=b'\0')).build(for_STH=True).to_buffer_with_tcp_pseudoheader() + + pkt = IPPacket.from_buffer_with_pseudoheader(buf) + + assert pkt.src_addr == '172.25.44.3' + assert pkt.segment.data == b'\0' + assert pkt.dst_addr == '172.25.44.254' def test_pseudoheader(self): diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index 33bdb4c98..3e407939b 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -41,9 +41,6 @@ async def _close_clients(self, a, b): await a.stop() await b.stop() - await a.wait_until_end() - await b.wait_until_end() - def test_default_init(self): protocol = Tecemux() assert isinstance(protocol, Tecemux) @@ -167,7 +164,7 @@ async def test_wih_scramjet_framework_to_list(self, local_socket_connection): client_a.get_channel(channel).write("foo\n") client_a.get_channel(channel).write("bar\n") client_a.get_channel(channel).write("baz\n") - await client_a.get_channel(channel).close() + await client_a.get_channel(channel).end() output_list = await Stream.read_from(client_b.get_channel(channel)).to_list() @@ -185,7 +182,7 @@ async def test_wih_scramjet_framework_to_pipe(self, local_socket_connection): client_a.get_channel(channel).write("bar\n") client_a.get_channel(channel).write("foz") client_a.get_channel(channel).write("baz\n") - await client_a.get_channel(channel).close() + await client_a.get_channel(channel).end() final_stream = Stream() @@ -207,7 +204,7 @@ async def test_wih_scramjet_framework_write_to(self, local_socket_connection): from scramjet.streams import Stream await Stream.read_from(data).write_to(client_a.get_channel(channel)) - await client_a.get_channel(channel).close() + await client_a.get_channel(channel).end() output = Stream.read_from(client_b.get_channel(channel)) assert await output.to_list() == data From ecc67869bc3574011073afd7c24e6f7bfdb237cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Mon, 3 Jul 2023 16:37:59 +0200 Subject: [PATCH 186/231] Better cleanup #2 --- packages/python-runner/tecemux.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index c13ab1e43..9a7d174ae 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -230,7 +230,6 @@ async def queue_up_incoming(self, pkt: IPPacket) -> None: # if ACK flag is up, reasume channel if not pkt.get_segment().is_flag('SYN') and pkt.get_segment().is_flag('ACK'): self.set_pause(False) - self._ended = False return # if PSH flag is up, confirm @@ -319,7 +318,7 @@ class Tecemux: _instance_id: str = field(default=None) _channels: dict = field(default={}) _logger: logging.Logger = field(default=None) - + _global_stop_channel_event: asyncio.Event = asyncio.Event() _global_sync_channel_event: asyncio.Event = asyncio.Event() @@ -353,7 +352,6 @@ async def connect(self, reader: asyncio.StreamReader, writer: asyncio.StreamWrit self._global_stop_channel_event.clear() self._global_stop_incoming_event.clear() self._global_stop_outcoming_event.clear() - @staticmethod async def prepare_tcp_connection(server_host: str, server_port: str) -> tuple: @@ -450,13 +448,11 @@ def _chunk_preview(value: bytes) -> str: return f'{value[0:5]}... ' if len(value) > 5 else f'{value}' async def stop(self) -> None: - await self._finish_channels() await self._finish_incoming() await self._finish_outcoming() await self._read_eof() - - + self._debug('Tecemux/MAIN: [-] Finished') async def _finish_channels(self) -> None: @@ -465,14 +461,16 @@ async def _finish_channels(self) -> None: for channel in self.get_channels(): await channel.end() await channel.close() - + for channel in self.get_channels(): await channel._internal_outcoming_queue.join() self._global_stop_channel_event.set() await self._global_stop_channel_event.wait() - await asyncio.gather(*[channel._outcoming_process_task for channel in self.get_channels()]) + tasks = asyncio.gather(*[channel._outcoming_process_task for channel in self.get_channels()]) + tasks.cancel() + await tasks async def _finish_outcoming(self): From 4e2400088204737586e842bf12bb81c5aa45c29e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Wed, 5 Jul 2023 11:14:59 +0200 Subject: [PATCH 187/231] Close communication improvements --- packages/python-runner/runner.py | 43 ++++++++++----------- packages/python-runner/tecemux.py | 63 +++++++++++++++++++++---------- 2 files changed, 64 insertions(+), 42 deletions(-) diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index 37745f999..5bdabdc90 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -4,7 +4,6 @@ import codecs import json import logging -# import debugpy from pyee.asyncio import AsyncIOEventEmitter from tecemux import Tecemux import importlib.util @@ -21,10 +20,6 @@ SERVER_HOST = os.getenv('INSTANCES_SERVER_HOST') or 'localhost' INSTANCE_ID = os.getenv('INSTANCE_ID') -# debugpy.listen(5678) -# debugpy.wait_for_client() -# debugpy.breakpoint() - def send_encoded_msg(stream, msg_code, data={}): message = json.dumps([msg_code.value, data]) stream.write(f'{message}\r\n'.encode()) @@ -40,11 +35,9 @@ def __init__(self, instance_id, sequence_path, log_setup) -> None: self.emitter = AsyncIOEventEmitter() self.keep_alive_requested = False self.protocol = None - @staticmethod - def is_incoming(channel): - return channel in [CC.STDIN, CC.IN, CC.CONTROL] async def main(self, server_host, server_port): + asyncio.current_task().set_name('RUNNER_MAIN') input_stream = Stream() await self.init_tecemux(server_host, server_port) # Do this early to have access to any thrown exceptions and logs. @@ -59,21 +52,26 @@ async def main(self, server_host, server_port): connect_input_stream_task = asyncio.create_task(self.connect_input_stream(input_stream)) self.load_sequence() - - await self.protocol.sync() + + await self.protocol.sync() await self.run_instance(config, input_stream, args) await self.protocol.sync() heartbeat_task.cancel() - connect_input_stream_task.cancel() + await asyncio.gather(*[heartbeat_task]) + + await asyncio.gather(*[connect_input_stream_task]) + + await self.protocol.get_channel(CC.IN).sync() + await self.protocol.get_channel(CC.CONTROL).sync() + control_stream_task.cancel() + await asyncio.gather(*[control_stream_task]) - await asyncio.gather(*[heartbeat_task, - connect_input_stream_task, - control_stream_task]) - await self.protocol.stop() + [ task.cancel() if task.get_name() != 'RUNNER_MAIN' else None for task in asyncio.all_tasks()] + async def init_tecemux(self, server_host, server_port): self.logger.info('Connecting to host with TeceMux...') @@ -152,6 +150,10 @@ async def connect_control_stream(self): if code == msg_codes.EVENT.value: await self.emitter.emit(data['eventName'], data['message'] if 'message' in data else None) except asyncio.CancelledError: + task = self.protocol.get_channel(CC.CONTROL)._outcoming_process_task + task.cancel() + await asyncio.sleep(0) + await asyncio.gather(*[task]) return @@ -182,6 +184,7 @@ async def setup_heartbeat(self): ) await asyncio.sleep(1) except asyncio.CancelledError: + await self.protocol.get_channel(CC.MONITORING).sync() return @@ -199,8 +202,7 @@ def load_sequence(self): os.chdir(os.path.dirname(self.seq_path)) async def run_instance(self, config, input, args): - context = AppContext(self, config) - await self.protocol.sync() + context = AppContext(self, config) self.logger.info('Running instance...') try: result = self.sequence.run(context, input, *args) @@ -228,13 +230,11 @@ async def run_instance(self, config, input, args): elif asyncio.iscoroutine(result): result = await result if result: - await self.protocol.sync() await self.forward_output_stream(result) else: self.logger.debug('Sequence returned no output.') self.logger.info('Finished.') - await self.protocol.sync() async def connect_input_stream(self, input_stream): @@ -249,8 +249,6 @@ async def connect_input_stream(self, input_stream): self.logger.info(f'Input headers: {repr(headers)}') input_type = headers.get('content-type') - await self.protocol.sync() - if input_type == 'text/plain': input = Stream.read_from(self.protocol.get_channel(CC.IN)) @@ -266,7 +264,6 @@ async def connect_input_stream(self, input_stream): input.pipe(input_stream) self.logger.debug('Input stream forwarded to the instance.') - async def forward_output_stream(self, output): if hasattr(output, 'content_type'): content_type = output.content_type @@ -341,4 +338,4 @@ async def keep_alive(self, timeout: int = 0): sys.exit(2) runner = Runner(INSTANCE_ID, SEQUENCE_PATH, LOG_SETUP) -asyncio.run(runner.main(SERVER_HOST, SERVER_PORT), debug=False) +asyncio.run(runner.main(SERVER_HOST, SERVER_PORT)) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 9a7d174ae..90405a78d 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -61,14 +61,33 @@ def __init__(self, channel: CC, self._stop_channel_event = stop_event self._sync_channel_event = sync_event self._sync_barrier = sync_barrier - self._outcoming_process_task = asyncio.create_task(self._outcomming_process_tasks()) + self._outcoming_process_task = asyncio.create_task(self._outcomming_process_tasks(), name=f'{self._get_channel_name()}_PROCESS_TASK') self._ended = None self._read_buffer = bytearray() + async def sync(self): + + if self._is_incoming(): + await self._internal_incoming_queue.join() + + await self._internal_outcoming_queue.join() + + + + def _is_incoming(self): + return self._channel_enum in [CC.STDIN, CC.IN, CC.CONTROL] + async def _outcomming_process_tasks(self): - while not self._stop_channel_event.is_set(): - await self._queue_up_outcoming() - await asyncio.sleep(0) + while True: + try: + await self._queue_up_outcoming() + await asyncio.sleep(0) + + if self._stop_channel_event.is_set(): + return + except asyncio.CancelledError: + return + @@ -145,9 +164,15 @@ async def _get_data(self): self._read_buffer.extend(buf) break except asyncio.QueueEmpty: + await asyncio.sleep(0) + + if self._ended: + return False + except asyncio.CancelledError: return False + await asyncio.sleep(0) return True @@ -207,6 +232,7 @@ async def end(self) -> None: await self._internal_outcoming_queue.join() await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'', flags=['PSH']))) + self._ended = True async def close(self) -> None: """Close channel @@ -239,7 +265,8 @@ async def queue_up_incoming(self, pkt: IPPacket) -> None: if pkt.segment.is_flag('FIN'): self._ended = True return - self._internal_incoming_queue.put_nowait(pkt.get_segment().data) + await self._internal_incoming_queue.put(pkt.get_segment().data) + self._internal_incoming_queue.task_done() async def _queue_up_outcoming(self) -> None: """Redirects raw data from currect channel to global queue. @@ -254,12 +281,14 @@ def wrap(channel_enum, buf): if not self._channel_paused: while not self._internal_outcoming_queue.empty(): try: - buf = await self._internal_outcoming_queue.get() + buf = await asyncio.wait_for(self._internal_outcoming_queue.get(),1) await self._global_queue.put(wrap(self._channel_enum, buf)) self._internal_outcoming_queue.task_done() except asyncio.QueueEmpty: self._debug(f'Tecemux/{self._get_channel_name()}: [-] All data stored during pause were redirected to global queue') break + except asyncio.TimeoutError: + pass else: self._debug(f'Tecemux/{self._get_channel_name()}: [-] Channel paused. Data queued up internally for future') @@ -451,9 +480,6 @@ async def stop(self) -> None: await self._finish_channels() await self._finish_incoming() await self._finish_outcoming() - await self._read_eof() - - self._debug('Tecemux/MAIN: [-] Finished') async def _finish_channels(self) -> None: """Stops protocol @@ -464,22 +490,21 @@ async def _finish_channels(self) -> None: for channel in self.get_channels(): await channel._internal_outcoming_queue.join() - + self._global_stop_channel_event.set() await self._global_stop_channel_event.wait() - - tasks = asyncio.gather(*[channel._outcoming_process_task for channel in self.get_channels()]) - tasks.cancel() - await tasks - - + for channel in self.get_channels(): + try: + await asyncio.wait_for(channel._outcoming_process_task, timeout=1) + except asyncio.TimeoutError: + pass + async def _finish_outcoming(self): self._global_stop_outcoming_event.set() + await asyncio.sleep(0) await self._global_stop_outcoming_event.wait() - await asyncio.gather(*[self._outcoming_data_forwarder]) - self._writer.write_eof() await self._writer.drain() self._writer.close() await self._writer.wait_closed() @@ -575,6 +600,7 @@ async def outcoming_data_forward(self) -> None: while True: try: pkt = await asyncio.wait_for(self._queue.get(),1) + self._queue.task_done() # inject sequence number if pkt.segment.seq == 0: @@ -587,7 +613,6 @@ async def outcoming_data_forward(self) -> None: self._debug(f'Tecemux/MAIN: [>] Outcoming chunk {Tecemux._chunk_preview(chunk)} is waiting to send to Transform Hub') self._writer.write(chunk) await self._writer.drain() - self._queue.task_done() self._debug(f'Tecemux/MAIN: [>] Chunk {Tecemux._chunk_preview(chunk)} with sequence number: {pkt.segment.seq} was sent to Transform Hub') except asyncio.QueueEmpty: if self._global_stop_outcoming_event.is_set(): From b7ab9ac17095afe855a78c5e6933d0b3aca68895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Wed, 5 Jul 2023 12:53:37 +0200 Subject: [PATCH 188/231] Docs for new methods --- packages/python-runner/runner.py | 15 ++-- packages/python-runner/tecemux.py | 122 +++++++++++++++++------------- 2 files changed, 75 insertions(+), 62 deletions(-) diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index 5bdabdc90..667c05feb 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -55,19 +55,14 @@ async def main(self, server_host, server_port): await self.protocol.sync() await self.run_instance(config, input_stream, args) - await self.protocol.sync() + heartbeat_task.cancel() - await asyncio.gather(*[heartbeat_task]) + control_stream_task.cancel() + await asyncio.gather(*[connect_input_stream_task, + heartbeat_task, + control_stream_task]) - await asyncio.gather(*[connect_input_stream_task]) - - await self.protocol.get_channel(CC.IN).sync() - await self.protocol.get_channel(CC.CONTROL).sync() - - control_stream_task.cancel() - await asyncio.gather(*[control_stream_task]) - await self.protocol.stop() [ task.cancel() if task.get_name() != 'RUNNER_MAIN' else None for task in asyncio.all_tasks()] diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 90405a78d..10f0d68fe 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -2,7 +2,6 @@ import logging import random import socket -from typing import Any, Coroutine from attrs import define, field from barrier import Barrier @@ -61,23 +60,30 @@ def __init__(self, channel: CC, self._stop_channel_event = stop_event self._sync_channel_event = sync_event self._sync_barrier = sync_barrier - self._outcoming_process_task = asyncio.create_task(self._outcomming_process_tasks(), name=f'{self._get_channel_name()}_PROCESS_TASK') + self._outcoming_process_task = asyncio.create_task(self._outcoming_process_tasks(), name=f'{self._get_channel_name()}_PROCESS_TASK') self._ended = None self._read_buffer = bytearray() - async def sync(self): + async def sync(self) -> None: + """Waits until internal queues will be empty (packets will be processed) + """ if self._is_incoming(): await self._internal_incoming_queue.join() await self._internal_outcoming_queue.join() + def _is_incoming(self) -> bool: + """Returns whether channel is mostly read by Tememux - - def _is_incoming(self): + Returns: + bool: True is mostly read by Runner, False otherwise + """ return self._channel_enum in [CC.STDIN, CC.IN, CC.CONTROL] - - async def _outcomming_process_tasks(self): + + async def _outcoming_process_tasks(self) -> None: + """ Internal loop for outcoming data from channel + """ while True: try: await self._queue_up_outcoming() @@ -88,14 +94,15 @@ async def _outcomming_process_tasks(self): except asyncio.CancelledError: return - - - def _debug(self, msg): + """Wrapper for printing debug messages + + Args: + msg (str): Debug message + """ if TECEMUX_INTERNAL_VERBOSE_DEBUG: self._logger.debug(msg) - def _get_channel_name(self) -> str: """Returns channel name @@ -121,12 +128,25 @@ def _set_logger(self, logger: logging.Logger) -> None: """ self._logger = logger - async def readline(self) -> str: + async def readline(self) -> bytes: + """Reads data from current channel until '\n' appears + + Returns: + bytes: Bufer data from channel + """ sep = b'\n' line = await self.readuntil(sep) return line - async def readuntil(self, separator=b'\n'): + async def readuntil(self, separator=b'\n') -> bytes: + """Reads data from current channel until provided separator appears + + Args: + separator (bytes, optional): Defaults to b'\n'. + + Returns: + bytes: Buffer data from channel + """ seplen = len(separator) offset = 0 @@ -149,8 +169,12 @@ async def readuntil(self, separator=b'\n'): del self._read_buffer[:isep + seplen] return bytes(chunk) - async def _get_data(self): + async def _get_data(self) -> bool: + """Internal method, returns True if data are moved from queue to _read_buffer, false otherwise + Returns: + bool: True if data are moved from queue to _read_buffer, false otherwise + """ if self._ended and self._internal_incoming_queue.empty(): return False @@ -176,8 +200,15 @@ async def _get_data(self): await asyncio.sleep(0) return True - async def read(self, n: int = -1): + async def read(self, n: int = -1) -> bytes: + """Reads up to 'n' bytes of data from current channel + + Args: + n (int, optional): Number of bytes. Defaults to -1. + Returns: + bytes: Buffer data from channel + """ if n == 0: return b'' @@ -204,6 +235,7 @@ async def send_ACK(self, sequence_number: int) -> None: Args: sequence_number (int): Value for Acknowledge field """ + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK'], ack=sequence_number))) async def _send_pause_ACK(self, sequence_number: int) -> None: @@ -212,10 +244,11 @@ async def _send_pause_ACK(self, sequence_number: int) -> None: Args: sequence_number (int): Value for Acknowledge field """ + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK', 'SYN'], ack=sequence_number))) async def open(self) -> None: - """Open channel. + """Open channel """ if not self._channel_opened: @@ -229,19 +262,22 @@ async def open(self) -> None: self._ended = False async def end(self) -> None: + """Send EOF on current channel + """ await self._internal_outcoming_queue.join() await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'', flags=['PSH']))) self._ended = True async def close(self) -> None: - """Close channel + """Close current channel """ + self._debug(f'Tecemux/{self._get_channel_name()}: [-] Channel close request is send') await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'' if self._global_instance_id is None else self._global_instance_id, flags=['FIN']))) async def queue_up_incoming(self, pkt: IPPacket) -> None: - """Redirects incomming data from provided packet to current channel + """Redirects incoming data from provided packet to current channel Args: pkt (IPPacket): Redirected packet from outside @@ -308,21 +344,6 @@ def write(self, data: bytes) -> None: self._internal_outcoming_queue.put_nowait(data) - def drain(self) -> bool: - """Drain channel - """ - return - - def write_eof(self) -> None: - """asyncio.StreamWriter API - """ - return - - def wait_closed(self) -> Coroutine: - """asyncio.StreamWriter API - """ - return - def set_pause(self, state: bool) -> None: """Sets pause state for current channel @@ -445,7 +466,7 @@ def get_channel(self, channel: CC) -> _ChannelContext: """ return self._channels[channel] - async def sync(self): + async def sync(self) -> None: """Waits until all write tasks will be done """ self._global_sync_channel_event.set() @@ -477,13 +498,17 @@ def _chunk_preview(value: bytes) -> str: return f'{value[0:5]}... ' if len(value) > 5 else f'{value}' async def stop(self) -> None: + """ Stops protocol + """ + await self._finish_channels() await self._finish_incoming() await self._finish_outcoming() async def _finish_channels(self) -> None: - """Stops protocol + """ Close all channels """ + for channel in self.get_channels(): await channel.end() await channel.close() @@ -498,9 +523,11 @@ async def _finish_channels(self) -> None: await asyncio.wait_for(channel._outcoming_process_task, timeout=1) except asyncio.TimeoutError: pass - - async def _finish_outcoming(self): - + + async def _finish_outcoming(self) -> None: + """ Finish outcoming forwarder and main writer to STH + """ + self._global_stop_outcoming_event.set() await asyncio.sleep(0) await self._global_stop_outcoming_event.wait() @@ -509,22 +536,14 @@ async def _finish_outcoming(self): self._writer.close() await self._writer.wait_closed() + async def _finish_incoming(self) -> None: + """ Finish incoming forwarder + """ - async def _finish_incoming(self): self._global_stop_incoming_event.set() await self._global_stop_incoming_event.wait() await asyncio.gather(*[self._incoming_data_forwarder]) - - async def _read_eof(self): - while True: - try: - data = await asyncio.wait_for(self._reader.read(),0.5) - if not data: - break - except asyncio.TimeoutError as e: - break - async def loop(self) -> None: """Main loop of Tecemux protocol. Starts forwarders tasks """ @@ -572,7 +591,7 @@ async def incoming_data_forward(self) -> None: single_packet_buffer = buffer[:current_packet_size] pkt = IPPacket().from_buffer_with_pseudoheader(single_packet_buffer) self._last_sequence_received = pkt.get_segment().seq - self._debug(f'Tecemux/MAIN: [<] Full incomming packet with sequence number {self._last_sequence_received} from Transform Hub was received') + self._debug(f'Tecemux/MAIN: [<] Full incoming packet with sequence number {self._last_sequence_received} from Transform Hub was received') channel = CC(str(pkt.get_segment().dst_port)) @@ -591,7 +610,7 @@ async def incoming_data_forward(self) -> None: except asyncio.CancelledError: break - self._debug('Tecemux/MAIN: Incomming data forwarder finished') + self._debug('Tecemux/MAIN: Incoming data forwarder finished') async def outcoming_data_forward(self) -> None: """Loop for outcoming data to Transform Hub @@ -627,5 +646,4 @@ async def outcoming_data_forward(self) -> None: except asyncio.CancelledError: break - self._debug('Tecemux/MAIN: Outcoming data forwarder finished') From 00163a9fa17dc5377fe57c86e41c188216990af2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Wed, 5 Jul 2023 13:32:34 +0200 Subject: [PATCH 189/231] Add missing barrier.py --- packages/python-runner/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/python-runner/package.json b/packages/python-runner/package.json index 8f3d02f74..20e5c8cda 100644 --- a/packages/python-runner/package.json +++ b/packages/python-runner/package.json @@ -15,6 +15,7 @@ "logging_setup.py", "tecemux.py", "inet.py", + "barrier.py", "runner.py" ], "author": "Scramjet ", From d20ca151546c5a3a203f0d15040368ec6ee5d705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Thu, 6 Jul 2023 11:45:45 +0200 Subject: [PATCH 190/231] Fix readuntil EOF state --- packages/python-runner/tecemux.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 10f0d68fe..63a2aa252 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -164,8 +164,13 @@ async def readuntil(self, separator=b'\n') -> bytes: if (not await self._get_data()): break - - chunk = self._read_buffer[:isep + seplen] + + if self._ended: + chunk = self._read_buffer.copy() + self._read_buffer.clear() + else: + chunk = self._read_buffer[:isep + seplen] + del self._read_buffer[:isep + seplen] return bytes(chunk) @@ -191,7 +196,7 @@ async def _get_data(self) -> bool: await asyncio.sleep(0) - if self._ended: + if self._ended and self._internal_incoming_queue.empty(): return False except asyncio.CancelledError: @@ -621,6 +626,9 @@ async def outcoming_data_forward(self) -> None: pkt = await asyncio.wait_for(self._queue.get(),1) self._queue.task_done() + if pkt.get_segment().data == b'': + self._logger.debug(f'Pusty pakiet na kanale {CC(str(pkt.get_segment().dst_port))}') + # inject sequence number if pkt.segment.seq == 0: pkt.segment.seq = self._sequence_number From 2db3978549f595ef7cfd78ee49783591d7d6d0c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Thu, 6 Jul 2023 12:43:17 +0200 Subject: [PATCH 191/231] Fix readuntil EOF state #2 --- packages/python-runner/tecemux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 63a2aa252..0c091c673 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -165,7 +165,7 @@ async def readuntil(self, separator=b'\n') -> bytes: if (not await self._get_data()): break - if self._ended: + if self._ended and isep == -1: chunk = self._read_buffer.copy() self._read_buffer.clear() else: From 85552716d26facb46b66cb6ed7d81a05ef786513 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Thu, 6 Jul 2023 13:07:03 +0200 Subject: [PATCH 192/231] Fix health check in tests --- bdd/features/e2e/E2E-014-python.feature | 3 +++ packages/python-runner/tecemux.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/bdd/features/e2e/E2E-014-python.feature b/bdd/features/e2e/E2E-014-python.feature index f306c0853..a27fcdb93 100644 --- a/bdd/features/e2e/E2E-014-python.feature +++ b/bdd/features/e2e/E2E-014-python.feature @@ -33,7 +33,9 @@ Feature: Test our shiny new Python runner Given host is running When sequence "../packages/python-unhealthy-sequence.tar.gz" loaded And instance started + And wait for "1000" ms Then instance health is "false" + And host is still running @ci-instance-python @@ -64,5 +66,6 @@ Feature: Test our shiny new Python runner When sequence "../packages/python-gen-async.tar.gz" loaded And instance started And send "1" to input + And wait for "1000" ms Then "output" is "saved to db: 1" And host is still running diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 0c091c673..ae270e74d 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -626,6 +626,8 @@ async def outcoming_data_forward(self) -> None: pkt = await asyncio.wait_for(self._queue.get(),1) self._queue.task_done() + if CC(str(pkt.get_segment().dst_port)).name == 'MONITORING': + pass if pkt.get_segment().data == b'': self._logger.debug(f'Pusty pakiet na kanale {CC(str(pkt.get_segment().dst_port))}') From 9e29847ce4ca6741a4400cc7dc390bb7451090a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 7 Jul 2023 10:49:37 +0200 Subject: [PATCH 193/231] Fix problems with events and handlers --- packages/python-runner/runner.py | 81 ++++++++++++++++++------------- packages/python-runner/tecemux.py | 5 -- 2 files changed, 47 insertions(+), 39 deletions(-) diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index a481d6474..2417c516f 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -72,16 +72,21 @@ async def main(self, server_host, server_port): await self.protocol.sync() heartbeat_task.cancel() - control_stream_task.cancel() - await asyncio.gather(*[connect_input_stream_task, - heartbeat_task, - control_stream_task]) + control_stream_task.cancel() + + if not connect_input_stream_task.done(): + connect_input_stream_task.cancel() + + await asyncio.gather(*[heartbeat_task]) + await asyncio.gather(*[control_stream_task]) + await asyncio.gather(*[connect_input_stream_task]) await self.protocol.stop() + self.cancel_tasks() + def cancel_tasks(self): [ task.cancel() if task.get_name() != 'RUNNER_MAIN' else None for task in asyncio.all_tasks()] - async def init_tecemux(self, server_host, server_port): self.logger.info('Connecting to host with TeceMux...') self.protocol = Tecemux(instance_id=self.instance_id) @@ -153,11 +158,11 @@ async def connect_control_stream(self): async for code, data in control_messages: self.logger.debug(f'Control message received: {code} {data}') if code == msg_codes.KILL.value: - self.exit_immediately() + await self.exit_immediately() if code == msg_codes.STOP.value: await self.handle_stop(data) if code == msg_codes.EVENT.value: - await self.emitter.emit(data['eventName'], data['message'] if 'message' in data else None) + self.emitter.emit(data['eventName'], data['message'] if 'message' in data else None) except asyncio.CancelledError: task = self.protocol.get_channel(CC.CONTROL)._outcoming_process_task task.cancel() @@ -180,7 +185,7 @@ async def handle_stop(self, data): if not can_keep_alive or not self.keep_alive_requested: send_encoded_msg(self.protocol.get_channel(CC.MONITORING), msg_codes.SEQUENCE_STOPPED, {}) - self.exit_immediately() + await self.exit_immediately() async def setup_heartbeat(self): @@ -219,7 +224,7 @@ async def run_instance(self, config, input, args): import traceback self.protocol.get_channel(CC.STDERR).write(traceback.format_exc()) await self.protocol.sync() - self.exit_immediately() + await self.exit_immediately() self.logger.info(f'Sending PANG') @@ -248,31 +253,38 @@ async def run_instance(self, config, input, args): async def connect_input_stream(self, input_stream): - if hasattr(self.sequence, "requires"): - input_type = self.sequence.requires.get('contentType') - else: - raw_headers = await self.protocol.get_channel(CC.IN).readuntil(b'\r\n\r\n') - header_list = raw_headers.decode().rstrip().split('\r\n') - headers = { - key.lower(): val for key, val in [el.split(': ') for el in header_list] - } - self.logger.info(f'Input headers: {repr(headers)}') - input_type = headers.get('content-type') - - if input_type == 'text/plain': - input = Stream.read_from(self.protocol.get_channel(CC.IN)) - - self.logger.debug('Decoding input stream...') - input = input.decode('utf-8') - elif input_type == 'application/octet-stream': - self.logger.debug('Opening input in binary mode...') - input = Stream.read_from(self.protocol.get_channel(CC.IN), chunk_size=CHUNK_SIZE) + try: + if hasattr(self.sequence, "requires"): + input_type = self.sequence.requires.get('contentType') + else: + raw_headers = await self.protocol.get_channel(CC.IN).readuntil(b'\r\n\r\n') + header_list = raw_headers.decode().rstrip().split('\r\n') + headers = { + key.lower(): val for key, val in [el.split(': ') for el in header_list] + } + self.logger.info(f'Input headers: {repr(headers)}') + input_type = headers.get('content-type') + + if input_type == 'text/plain': + input = Stream.read_from(self.protocol.get_channel(CC.IN)) + + self.logger.debug('Decoding input stream...') + input = input.decode('utf-8') + elif input_type == 'application/octet-stream': + self.logger.debug('Opening input in binary mode...') + input = Stream.read_from(self.protocol.get_channel(CC.IN), chunk_size=CHUNK_SIZE) - else: - raise TypeError(f'Unsupported input type: {repr(input_type)}') + else: + raise TypeError(f'Unsupported input type: {repr(input_type)}') - input.pipe(input_stream) - self.logger.debug('Input stream forwarded to the instance.') + input.pipe(input_stream) + self.logger.debug('Input stream forwarded to the instance.') + except asyncio.CancelledError: + task = self.protocol.get_channel(CC.IN)._outcoming_process_task + task.cancel() + await asyncio.sleep(0) + await asyncio.gather(*[task]) + return async def forward_output_stream(self, output): @@ -305,7 +317,8 @@ async def send_keep_alive(self, timeout: int = 0, can_keep_alive: bool = False): await asyncio.sleep(timeout) - def exit_immediately(self): + async def exit_immediately(self): + await self.protocol.sync() sys.exit(1) @@ -326,7 +339,7 @@ def set_health_check(self, health_check): def on(self, event_name, callback): self.emitter.on(event_name, callback) - async def emit(self, event_name, message=''): + def emit(self, event_name, message=''): send_encoded_msg( self.monitoring, msg_codes.EVENT, diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index ae270e74d..2679b291a 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -626,11 +626,6 @@ async def outcoming_data_forward(self) -> None: pkt = await asyncio.wait_for(self._queue.get(),1) self._queue.task_done() - if CC(str(pkt.get_segment().dst_port)).name == 'MONITORING': - pass - if pkt.get_segment().data == b'': - self._logger.debug(f'Pusty pakiet na kanale {CC(str(pkt.get_segment().dst_port))}') - # inject sequence number if pkt.segment.seq == 0: pkt.segment.seq = self._sequence_number From 06dcc5d04b8d881ef628681e97a8ed21ebb6f5c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 7 Jul 2023 11:01:22 +0200 Subject: [PATCH 194/231] Longer timeout --- bdd/features/e2e/E2E-014-python.feature | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bdd/features/e2e/E2E-014-python.feature b/bdd/features/e2e/E2E-014-python.feature index a27fcdb93..2dea8d9f4 100644 --- a/bdd/features/e2e/E2E-014-python.feature +++ b/bdd/features/e2e/E2E-014-python.feature @@ -33,9 +33,8 @@ Feature: Test our shiny new Python runner Given host is running When sequence "../packages/python-unhealthy-sequence.tar.gz" loaded And instance started - And wait for "1000" ms + And wait for "3000" ms Then instance health is "false" - And host is still running @ci-instance-python @@ -66,6 +65,5 @@ Feature: Test our shiny new Python runner When sequence "../packages/python-gen-async.tar.gz" loaded And instance started And send "1" to input - And wait for "1000" ms Then "output" is "saved to db: 1" And host is still running From 9e6d00cc6417c91bf730d8f3a27c55123cfd8449 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 7 Jul 2023 11:42:46 +0200 Subject: [PATCH 195/231] Sync monitoring --- packages/python-runner/runner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index 2417c516f..d2c0591bc 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -163,6 +163,7 @@ async def connect_control_stream(self): await self.handle_stop(data) if code == msg_codes.EVENT.value: self.emitter.emit(data['eventName'], data['message'] if 'message' in data else None) + except asyncio.CancelledError: task = self.protocol.get_channel(CC.CONTROL)._outcoming_process_task task.cancel() @@ -196,6 +197,7 @@ async def setup_heartbeat(self): msg_codes.MONITORING, self.health_check(), ) + await self.protocol.get_channel(CC.MONITORING).sync() await asyncio.sleep(1) except asyncio.CancelledError: await self.protocol.get_channel(CC.MONITORING).sync() From 017017c85291da7e18056a22047e4ed1b63bb4d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 7 Jul 2023 11:57:34 +0200 Subject: [PATCH 196/231] Longer timeout --- bdd/features/e2e/E2E-015-unified.feature | 1 + 1 file changed, 1 insertion(+) diff --git a/bdd/features/e2e/E2E-015-unified.feature b/bdd/features/e2e/E2E-015-unified.feature index addee0134..45ef5c97b 100644 --- a/bdd/features/e2e/E2E-015-unified.feature +++ b/bdd/features/e2e/E2E-015-unified.feature @@ -64,6 +64,7 @@ Feature: Test our shiny new Python runner And send "topic test input" to input And find and upload sequence "topic-consumer.tar.gz" And instance started + And wait for "3000" ms Then "output" will be data named "python-topics" And host is still running From 3d19b21eb35f915d547bcfdfe8c64b408ff72fb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 7 Jul 2023 12:58:00 +0200 Subject: [PATCH 197/231] Add wait for in tests --- bdd/features/e2e/E2E-014-python.feature | 2 ++ bdd/features/e2e/E2E-015-unified.feature | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/bdd/features/e2e/E2E-014-python.feature b/bdd/features/e2e/E2E-014-python.feature index 2dea8d9f4..30f8bdf70 100644 --- a/bdd/features/e2e/E2E-014-python.feature +++ b/bdd/features/e2e/E2E-014-python.feature @@ -53,9 +53,11 @@ Feature: Test our shiny new Python runner When I execute CLI with "seq send ../packages/python-topic-producer.tar.gz" When I execute CLI with "seq start - --output-topic names3" Then I send input data "topic test input" with options "--end" + And wait for "100" ms When I execute CLI with "seq send ../packages/python-topic-consumer.tar.gz" When I execute CLI with "seq start - --input-topic names3" And I execute CLI with "inst output -" without waiting for the end + And wait for "100" ms Then I confirm data named "python-topics" will be received And host is still running diff --git a/bdd/features/e2e/E2E-015-unified.feature b/bdd/features/e2e/E2E-015-unified.feature index 45ef5c97b..a25cae67a 100644 --- a/bdd/features/e2e/E2E-015-unified.feature +++ b/bdd/features/e2e/E2E-015-unified.feature @@ -6,6 +6,7 @@ Feature: Test our shiny new Python runner When find and upload sequence "hello.tar.gz" And instance started And send "python runner" to input + And wait for "100" ms Then "output" is "Hello python runner!" And host is still running @@ -15,6 +16,7 @@ Feature: Test our shiny new Python runner When find and upload sequence "stdinout.tar.gz" And instance started And send "python runner" to stdin + And wait for "100" ms Then "stdout" is "Got on stdin: python runner" And host is still running @@ -62,9 +64,9 @@ Feature: Test our shiny new Python runner When find and upload sequence "topic-producer.tar.gz" And instance started And send "topic test input" to input + And wait for "100" ms And find and upload sequence "topic-consumer.tar.gz" And instance started - And wait for "3000" ms Then "output" will be data named "python-topics" And host is still running @@ -74,6 +76,7 @@ Feature: Test our shiny new Python runner When find and upload sequence "events.tar.gz" And instance started And send event "test-event" to instance with message "foo" + And wait for "100" ms Then instance emits event "test-response" with body """ {"eventName":"test-response","message":"reply to foo"} From c50997b768c4dd2867a40fe332755b3726db7859 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 7 Jul 2023 14:38:31 +0200 Subject: [PATCH 198/231] Fix for topics --- packages/python-runner/tecemux.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 2679b291a..2537074f8 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -525,9 +525,11 @@ async def _finish_channels(self) -> None: await self._global_stop_channel_event.wait() for channel in self.get_channels(): try: - await asyncio.wait_for(channel._outcoming_process_task, timeout=1) + await asyncio.wait_for(channel._outcoming_process_task,1) except asyncio.TimeoutError: pass + except TypeError: + pass async def _finish_outcoming(self) -> None: """ Finish outcoming forwarder and main writer to STH From ff1281c6f83440a4f00037e6d2c3f674aba2ed79 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 7 Aug 2023 12:17:56 +0000 Subject: [PATCH 199/231] Replace bpmux with tecemux in context.hub --- packages/host/src/lib/csi-controller.ts | 22 +++++++++++++--------- packages/runner/src/host-client.ts | 13 ++++++------- packages/verser/src/lib/tecemux/tecemux.ts | 4 ++-- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/packages/host/src/lib/csi-controller.ts b/packages/host/src/lib/csi-controller.ts index 0e4a5ee90..8f95b6b78 100644 --- a/packages/host/src/lib/csi-controller.ts +++ b/packages/host/src/lib/csi-controller.ts @@ -44,7 +44,9 @@ import { DuplexStream, getRouter } from "@scramjet/api-server"; import { getInstanceAdapter } from "@scramjet/adapters"; import { cancellableDefer, CancellablePromise, defer, promiseTimeout, TypedEmitter } from "@scramjet/utility"; import { ObjLogger } from "@scramjet/obj-logger"; +import { TeceMux } from "@scramjet/verser"; import { ReasonPhrases } from "http-status-codes"; +import { Socket } from "net"; /** * @TODO: Runner exits after 10secs and k8s client checks status every 500ms so we need to give it some time @@ -62,8 +64,6 @@ type Events = { terminated: (code: number) => void; }; -const BPMux = require("bpmux").BPMux; - /** * Handles all Instance lifecycle, exposes instance's HTTP API. */ @@ -74,7 +74,7 @@ export class CSIController extends TypedEmitter { private keepAliveRequested?: boolean; private _lastStats?: MonitoringMessageData; - private bpmux: any; + private hubTunnel: any; private adapter: string; get lastStats(): InstanceStats { @@ -554,13 +554,17 @@ export class CSIController extends TypedEmitter { this.hookupStreams(streams); this.createInstanceAPIRouter(); - this.bpmux = new BPMux(streams[8]); - this.bpmux.on("error", (e: any) => { - this.logger.warn("Instance client multiplex connection errored", e.message); - streams[8]?.end(); - }); - this.bpmux.on("peer_multiplex", (socket: Duplex, _data: any) => this.hostProxy.onInstanceRequest(socket)); + this.hubTunnel = new TeceMux(streams[8] as Socket); + + this.hubTunnel + .on("error", (e: any) => { + this.logger.warn("Instance client multiplex connection errored", e.message); + streams[8]?.end(); + }) + .on("channel", (socket: Duplex) => this.hostProxy.onInstanceRequest(socket)); + await once(this, "pang"); + this.initResolver?.res(); } catch (e: any) { this.initResolver?.rej(e); diff --git a/packages/runner/src/host-client.ts b/packages/runner/src/host-client.ts index c98d26ebd..04eaae106 100644 --- a/packages/runner/src/host-client.ts +++ b/packages/runner/src/host-client.ts @@ -10,8 +10,6 @@ type HostOpenConnections = [ TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel ] -const BPMux = require("bpmux").BPMux; - /** * Connects to Host and exposes streams per channel (stdin, monitor etc.) */ @@ -19,7 +17,8 @@ class HostClient implements IHostClient { private _streams?: UpstreamStreamsConfig; public agent?: Agent; logger: IObjectLogger; - bpmux: any; + hubTunnel!: TeceMux; + constructor(private instancesServerPort: number, private instancesServerHost: string) { this.logger = new ObjLogger(this); } @@ -60,10 +59,10 @@ class HostClient implements IHostClient { ); }); - this._streams = await openConnections as HostOpenConnections; + this._streams = openConnections as HostOpenConnections; try { - this.bpmux = new BPMux(this._streams[CC.PACKAGE]); + this.hubTunnel = new TeceMux(this._streams[CC.PACKAGE] as Socket); } catch (e) { // eslint-disable-next-line no-console console.error(e); @@ -75,13 +74,13 @@ class HostClient implements IHostClient { agent.createConnection = () => { try { - const socket = this.bpmux!.multiplex() as Socket; + const socket = this.hubTunnel.multiplex() as unknown as Socket; socket.on("error", () => { this.logger.error("Muxed stream error"); }); - // some libs call it but it is not here, in BPMux. + // mock Socket's setKeepAlive in TeceMuxChannel. socket.setKeepAlive ||= (_enable?: boolean, _initialDelay?: number | undefined) => socket; this.logger.info("Creating connection to verser server"); diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/verser/src/lib/tecemux/tecemux.ts index fa609d9d8..8980954fc 100644 --- a/packages/verser/src/lib/tecemux/tecemux.ts +++ b/packages/verser/src/lib/tecemux/tecemux.ts @@ -100,7 +100,7 @@ export class TeceMux extends TypedEmitter { if (flags.ACK && flags.SYN) { channel?.encoder.pause(); this.framesKeeper.handleACK(acknowledgeNumber); - console.log("Pause channel command received", channel?._id); + //console.log("Pause channel command received", channel?._id); return 0; } @@ -146,7 +146,7 @@ export class TeceMux extends TypedEmitter { const canWrite = channel.push(new Uint8Array((frame.chunk as any).data), undefined); if (!canWrite) { - console.log("cant write", channel._id); + //console.log("cant write", channel._id); channel.sendPauseACK(sequenceNumber); channel.once("resume", () => { From 8a8fbb162aa9f7c7a311d0d46b5f451a4833f1db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Thu, 10 Aug 2023 15:11:13 +0200 Subject: [PATCH 200/231] Nested tecemux in tecemux --- packages/python-runner/tecemux.py | 44 ++++++++++++++++++++- packages/python-runner/test/test_tecemux.py | 40 +++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 2537074f8..aab36503d 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -31,6 +31,8 @@ class _ChannelContext: _channel_paused: bool _channel_opened: bool + _channel_nested: bool + _channel_nested_enum: CC _stop_channel_event: asyncio.Event _sync_channel_event: asyncio.Event @@ -63,6 +65,12 @@ def __init__(self, channel: CC, self._outcoming_process_task = asyncio.create_task(self._outcoming_process_tasks(), name=f'{self._get_channel_name()}_PROCESS_TASK') self._ended = None self._read_buffer = bytearray() + self._channel_nested = False + self._channel_nested_enum = None + + def use_with_nested_channel(self, nested_channel_enum): + self._channel_nested = True + self._channel_nested_enum = nested_channel_enum async def sync(self) -> None: """Waits until internal queues will be empty (packets will be processed) @@ -263,6 +271,14 @@ async def open(self) -> None: await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=buf, flags=['PSH']))) + if self._channel_nested: + buf = b'' if self._global_instance_id is None else ( + self._global_instance_id.encode() + str(self._get_channel_id()).encode()) + + inner = IPPacket(segment=TCPSegment(dst_port=int(self._channel_nested_enum.value), data=buf, flags=['PSH'])) + inner = inner.build(for_STH=True).to_buffer_with_tcp_pseudoheader() + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=inner, flags=['PSH']))) + self._channel_opened = True self._ended = False @@ -271,7 +287,14 @@ async def end(self) -> None: """ await self._internal_outcoming_queue.join() + if self._channel_nested: + inner = IPPacket(segment=TCPSegment(dst_port=int(self._channel_nested_enum.value), data=b'', flags=['PSH'])) + inner = inner.build(for_STH=True).to_buffer_with_tcp_pseudoheader() + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=inner, flags=['PSH']))) + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'', flags=['PSH']))) + + self._ended = True async def close(self) -> None: @@ -279,6 +302,11 @@ async def close(self) -> None: """ self._debug(f'Tecemux/{self._get_channel_name()}: [-] Channel close request is send') + if self._channel_nested: + inner = IPPacket(segment=TCPSegment(dst_port=int(self._channel_nested_enum.value), data=b'', flags=['FIN'])) + inner = inner.build(for_STH=True).to_buffer_with_tcp_pseudoheader() + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=inner, flags=['PSH']))) + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'' if self._global_instance_id is None else self._global_instance_id, flags=['FIN']))) async def queue_up_incoming(self, pkt: IPPacket) -> None: @@ -287,6 +315,11 @@ async def queue_up_incoming(self, pkt: IPPacket) -> None: Args: pkt (IPPacket): Redirected packet from outside """ + if self._channel_nested: + if pkt.get_segment().data != b'': + # unpack nested packet + pkt = pkt.from_buffer_with_pseudoheader(pkt.get_segment().data) + # if SYN & ACK flag is up, pause channel if pkt.get_segment().is_flag('SYN') and pkt.get_segment().is_flag('ACK'): self._debug(f'Tecemux/{self._get_channel_name()}: [-] Channel pause request received') @@ -323,7 +356,13 @@ def wrap(channel_enum, buf): while not self._internal_outcoming_queue.empty(): try: buf = await asyncio.wait_for(self._internal_outcoming_queue.get(),1) - await self._global_queue.put(wrap(self._channel_enum, buf)) + if not self._channel_nested: + await self._global_queue.put(wrap(self._channel_enum, buf)) + else: + inner = wrap(self._channel_nested_enum, buf) + inner = inner.build(for_STH=True).to_buffer_with_tcp_pseudoheader() + + await self._global_queue.put(wrap(self._channel_enum, inner)) self._internal_outcoming_queue.task_done() except asyncio.QueueEmpty: self._debug(f'Tecemux/{self._get_channel_name()}: [-] All data stored during pause were redirected to global queue') @@ -449,6 +488,9 @@ async def prepare(self, force_open: bool = False) -> None: self._global_stop_channel_event, self._global_sync_channel_event, self._global_sync_barrier) for channel in CC} + + self.get_channel(CC.HOST).use_with_nested_channel(CC.HOST) + if force_open: [await channel.open() for channel in self.get_channels()] diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index 3e407939b..b139cdd36 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -134,6 +134,46 @@ async def test_stderr_write_redirection(self, local_socket_connection): await self._close_clients(client_a, client_b) + + async def test_nested_channel(self, local_socket_connection): + client_a, client_b = local_socket_connection + + client_a.get_channel(CC.CONTROL).use_with_nested_channel(CC.MONITORING) + client_b.get_channel(CC.CONTROL).use_with_nested_channel(CC.MONITORING) + + + await client_a.get_channel(CC.CONTROL).open() + + assert client_a._queue.qsize() == 2 + + assert client_a._queue._queue[0].get_segment().dst_port == int(CC.CONTROL.value) + assert client_a._queue._queue[0].get_segment().data == b'' + + assert client_a._queue._queue[1].get_segment().dst_port == int(CC.CONTROL.value) + assert client_a._queue._queue[1].get_segment().data != b'' + + inner_packet = client_a._queue._queue[1].get_segment().data + inner_packet = IPPacket.from_buffer_with_pseudoheader(inner_packet) + + assert inner_packet.get_segment().dst_port == int(CC.MONITORING.value) + assert inner_packet.get_segment().data == b'' + + client_a.get_channel(CC.CONTROL).write("{'foo':'bar'}\n") + await client_a.get_channel(CC.CONTROL)._internal_outcoming_queue.join() + + inner_packet = client_a._queue._queue[0].get_segment().data + inner_packet = IPPacket.from_buffer_with_pseudoheader(inner_packet) + + assert inner_packet.get_segment().data == b"{'foo':'bar'}\n" + assert inner_packet.get_segment().dst_port == int(CC.MONITORING.value) + + assert (await client_b.get_channel(CC.CONTROL).read(100)).decode() == '' + assert (await client_b.get_channel(CC.CONTROL).read(100)).decode() == '' + assert (await client_b.get_channel(CC.CONTROL).read(100)).decode() == "{'foo':'bar'}\n" + + await self._close_clients(client_a, client_b) + + async def test_readuntil(self, local_socket_connection): client_a, client_b = local_socket_connection From 4de39158d69c2bb32984506d5f6f3a252cf4fe8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Thu, 10 Aug 2023 15:36:25 +0200 Subject: [PATCH 201/231] Docstring for new function --- packages/python-runner/tecemux.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index aab36503d..f6fe55c83 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -260,6 +260,20 @@ async def _send_pause_ACK(self, sequence_number: int) -> None: await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK', 'SYN'], ack=sequence_number))) + async def _put_nested_packet(self, buf: bytes, flags: dict) -> None: + """Pack buffer to additional layer of IPPacket for nested channel and queue up to send + + Args: + buf (bytes): RAW data to send + flags (dict): TCP segment flags to set in nested packet + """ + if self._channel_nested_enum is None: + return + + inner = IPPacket(segment=TCPSegment(dst_port=int(self._channel_nested_enum.value), data=buf, flags=flags)) + inner = inner.build(for_STH=True).to_buffer_with_tcp_pseudoheader() + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=inner, flags=['PSH']))) + async def open(self) -> None: """Open channel """ @@ -273,11 +287,9 @@ async def open(self) -> None: if self._channel_nested: buf = b'' if self._global_instance_id is None else ( - self._global_instance_id.encode() + str(self._get_channel_id()).encode()) - - inner = IPPacket(segment=TCPSegment(dst_port=int(self._channel_nested_enum.value), data=buf, flags=['PSH'])) - inner = inner.build(for_STH=True).to_buffer_with_tcp_pseudoheader() - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=inner, flags=['PSH']))) + self._global_instance_id.encode() + str(self._channel_nested_enum.value).encode()) + + await self._put_nested_packet(buf, flags=['PSH']) self._channel_opened = True self._ended = False @@ -288,9 +300,7 @@ async def end(self) -> None: await self._internal_outcoming_queue.join() if self._channel_nested: - inner = IPPacket(segment=TCPSegment(dst_port=int(self._channel_nested_enum.value), data=b'', flags=['PSH'])) - inner = inner.build(for_STH=True).to_buffer_with_tcp_pseudoheader() - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=inner, flags=['PSH']))) + await self._put_nested_packet(b'', flags=['PSH']) await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'', flags=['PSH']))) @@ -303,10 +313,7 @@ async def close(self) -> None: self._debug(f'Tecemux/{self._get_channel_name()}: [-] Channel close request is send') if self._channel_nested: - inner = IPPacket(segment=TCPSegment(dst_port=int(self._channel_nested_enum.value), data=b'', flags=['FIN'])) - inner = inner.build(for_STH=True).to_buffer_with_tcp_pseudoheader() - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=inner, flags=['PSH']))) - + await self._put_nested_packet(b'', flags=['FIN']) await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'' if self._global_instance_id is None else self._global_instance_id, flags=['FIN']))) async def queue_up_incoming(self, pkt: IPPacket) -> None: From 623340ed7d7211146f3ed5dac523457e8f3b926b Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 16 Aug 2023 08:28:42 +0000 Subject: [PATCH 202/231] Removing the nested TeceMux. Requests from the sequence as channels created sequentially after handling communication channels --- packages/host/src/lib/csi-controller.ts | 12 ++---------- packages/host/src/lib/host.ts | 5 +++-- packages/host/src/lib/socket-server.ts | 11 +++++++---- packages/runner/src/host-client.ts | 9 +-------- 4 files changed, 13 insertions(+), 24 deletions(-) diff --git a/packages/host/src/lib/csi-controller.ts b/packages/host/src/lib/csi-controller.ts index 8f95b6b78..2c280172b 100644 --- a/packages/host/src/lib/csi-controller.ts +++ b/packages/host/src/lib/csi-controller.ts @@ -46,7 +46,6 @@ import { cancellableDefer, CancellablePromise, defer, promiseTimeout, TypedEmitt import { ObjLogger } from "@scramjet/obj-logger"; import { TeceMux } from "@scramjet/verser"; import { ReasonPhrases } from "http-status-codes"; -import { Socket } from "net"; /** * @TODO: Runner exits after 10secs and k8s client checks status every 500ms so we need to give it some time @@ -74,7 +73,6 @@ export class CSIController extends TypedEmitter { private keepAliveRequested?: boolean; private _lastStats?: MonitoringMessageData; - private hubTunnel: any; private adapter: string; get lastStats(): InstanceStats { @@ -549,18 +547,12 @@ export class CSIController extends TypedEmitter { this.logger.info("Instance started", this.info); } - async handleInstanceConnect(streams: DownstreamStreamsConfig) { + async handleInstanceConnect(streams: DownstreamStreamsConfig, protocol: TeceMux) { try { this.hookupStreams(streams); this.createInstanceAPIRouter(); - this.hubTunnel = new TeceMux(streams[8] as Socket); - - this.hubTunnel - .on("error", (e: any) => { - this.logger.warn("Instance client multiplex connection errored", e.message); - streams[8]?.end(); - }) + protocol .on("channel", (socket: Duplex) => this.hostProxy.onInstanceRequest(socket)); await once(this, "pang"); diff --git a/packages/host/src/lib/host.ts b/packages/host/src/lib/host.ts index 30196e313..cc3cc9067 100644 --- a/packages/host/src/lib/host.ts +++ b/packages/host/src/lib/host.ts @@ -155,11 +155,12 @@ export class Host implements IComponent { * Sets listener for connections to socket server. */ private attachListeners() { - this.socketServer.on("connect", async (id, streams) => { + this.socketServer.on("connect", async (id, streams, protocol) => { this.logger.debug("Instance connected", id); await this.instancesStore[id].handleInstanceConnect( - streams + streams, + protocol ); }); } diff --git a/packages/host/src/lib/socket-server.ts b/packages/host/src/lib/socket-server.ts index 5415bf024..99c1a2e34 100644 --- a/packages/host/src/lib/socket-server.ts +++ b/packages/host/src/lib/socket-server.ts @@ -19,7 +19,7 @@ type RunnerConnectionsInProgress = [ ]; type Events = { - connect: (id: string, streams: RunnerChannels) => void + connect: (id: string, streams: RunnerChannels, protocol: TeceMux) => void }; /** @@ -72,7 +72,7 @@ export class SocketServer extends TypedEmitter implements IComponent { .on("end", () => this.logger.debug(`Channel [${instanceId}:${channelId}] ended`)); try { - await this.handleConnection(instanceId, channelId, channel as unknown as Socket); + await this.handleCommunicationChannel(instanceId, channelId, channel as unknown as Socket, protocol); } catch (err: any) { channel.destroy(); } @@ -89,7 +89,7 @@ export class SocketServer extends TypedEmitter implements IComponent { }); } - async handleConnection(id: string, channel: number, connection: net.Socket) { + async handleCommunicationChannel(id: string, channel: number, connection: net.Socket, protocol: TeceMux) { let runner = this.runnerConnectionsInProgress.get(id); if (!runner) { @@ -102,9 +102,12 @@ export class SocketServer extends TypedEmitter implements IComponent { } else { throw new Error(`Runner(${id}) wanted to connect on already initialized channel ${channel}`); } + if (runner.every(isDefined)) { + protocol.removeAllListeners("channel"); + this.runnerConnectionsInProgress.delete(id); - this.emit("connect", id, runner as RunnerChannels); + this.emit("connect", id, runner as RunnerChannels, protocol); } } diff --git a/packages/runner/src/host-client.ts b/packages/runner/src/host-client.ts index 04eaae106..56c884403 100644 --- a/packages/runner/src/host-client.ts +++ b/packages/runner/src/host-client.ts @@ -61,20 +61,13 @@ class HostClient implements IHostClient { this._streams = openConnections as HostOpenConnections; - try { - this.hubTunnel = new TeceMux(this._streams[CC.PACKAGE] as Socket); - } catch (e) { - // eslint-disable-next-line no-console - console.error(e); - } - const agent = new Agent() as Agent & { createConnection: typeof createConnection }; // lack of types?; agent.createConnection = () => { try { - const socket = this.hubTunnel.multiplex() as unknown as Socket; + const socket = protocol.multiplex() as unknown as Socket; socket.on("error", () => { this.logger.error("Muxed stream error"); From 53d4cc3a6c57e8591b4cc6683107280c6ed7d9ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 25 Aug 2023 12:40:12 +0200 Subject: [PATCH 203/231] Remove nested channels support --- packages/python-runner/tecemux.py | 51 +------------------------------ 1 file changed, 1 insertion(+), 50 deletions(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index f6fe55c83..2537074f8 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -31,8 +31,6 @@ class _ChannelContext: _channel_paused: bool _channel_opened: bool - _channel_nested: bool - _channel_nested_enum: CC _stop_channel_event: asyncio.Event _sync_channel_event: asyncio.Event @@ -65,12 +63,6 @@ def __init__(self, channel: CC, self._outcoming_process_task = asyncio.create_task(self._outcoming_process_tasks(), name=f'{self._get_channel_name()}_PROCESS_TASK') self._ended = None self._read_buffer = bytearray() - self._channel_nested = False - self._channel_nested_enum = None - - def use_with_nested_channel(self, nested_channel_enum): - self._channel_nested = True - self._channel_nested_enum = nested_channel_enum async def sync(self) -> None: """Waits until internal queues will be empty (packets will be processed) @@ -260,20 +252,6 @@ async def _send_pause_ACK(self, sequence_number: int) -> None: await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK', 'SYN'], ack=sequence_number))) - async def _put_nested_packet(self, buf: bytes, flags: dict) -> None: - """Pack buffer to additional layer of IPPacket for nested channel and queue up to send - - Args: - buf (bytes): RAW data to send - flags (dict): TCP segment flags to set in nested packet - """ - if self._channel_nested_enum is None: - return - - inner = IPPacket(segment=TCPSegment(dst_port=int(self._channel_nested_enum.value), data=buf, flags=flags)) - inner = inner.build(for_STH=True).to_buffer_with_tcp_pseudoheader() - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=inner, flags=['PSH']))) - async def open(self) -> None: """Open channel """ @@ -285,12 +263,6 @@ async def open(self) -> None: await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=buf, flags=['PSH']))) - if self._channel_nested: - buf = b'' if self._global_instance_id is None else ( - self._global_instance_id.encode() + str(self._channel_nested_enum.value).encode()) - - await self._put_nested_packet(buf, flags=['PSH']) - self._channel_opened = True self._ended = False @@ -299,12 +271,7 @@ async def end(self) -> None: """ await self._internal_outcoming_queue.join() - if self._channel_nested: - await self._put_nested_packet(b'', flags=['PSH']) - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'', flags=['PSH']))) - - self._ended = True async def close(self) -> None: @@ -312,8 +279,6 @@ async def close(self) -> None: """ self._debug(f'Tecemux/{self._get_channel_name()}: [-] Channel close request is send') - if self._channel_nested: - await self._put_nested_packet(b'', flags=['FIN']) await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'' if self._global_instance_id is None else self._global_instance_id, flags=['FIN']))) async def queue_up_incoming(self, pkt: IPPacket) -> None: @@ -322,11 +287,6 @@ async def queue_up_incoming(self, pkt: IPPacket) -> None: Args: pkt (IPPacket): Redirected packet from outside """ - if self._channel_nested: - if pkt.get_segment().data != b'': - # unpack nested packet - pkt = pkt.from_buffer_with_pseudoheader(pkt.get_segment().data) - # if SYN & ACK flag is up, pause channel if pkt.get_segment().is_flag('SYN') and pkt.get_segment().is_flag('ACK'): self._debug(f'Tecemux/{self._get_channel_name()}: [-] Channel pause request received') @@ -363,13 +323,7 @@ def wrap(channel_enum, buf): while not self._internal_outcoming_queue.empty(): try: buf = await asyncio.wait_for(self._internal_outcoming_queue.get(),1) - if not self._channel_nested: - await self._global_queue.put(wrap(self._channel_enum, buf)) - else: - inner = wrap(self._channel_nested_enum, buf) - inner = inner.build(for_STH=True).to_buffer_with_tcp_pseudoheader() - - await self._global_queue.put(wrap(self._channel_enum, inner)) + await self._global_queue.put(wrap(self._channel_enum, buf)) self._internal_outcoming_queue.task_done() except asyncio.QueueEmpty: self._debug(f'Tecemux/{self._get_channel_name()}: [-] All data stored during pause were redirected to global queue') @@ -495,9 +449,6 @@ async def prepare(self, force_open: bool = False) -> None: self._global_stop_channel_event, self._global_sync_channel_event, self._global_sync_barrier) for channel in CC} - - self.get_channel(CC.HOST).use_with_nested_channel(CC.HOST) - if force_open: [await channel.open() for channel in self.get_channels()] From f8b160ad45a1b3703262c2332fb3c98aaa66d3fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 25 Aug 2023 16:46:38 +0200 Subject: [PATCH 204/231] Extra channels support --- packages/python-runner/tecemux.py | 53 +++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 2537074f8..92e1fc69b 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -5,6 +5,7 @@ from attrs import define, field from barrier import Barrier +from enum import Enum from inet import IPPacket, TCPSegment, SequenceOrder from hardcoded_magic_values import CommunicationChannels as CC @@ -257,9 +258,13 @@ async def open(self) -> None: """ if not self._channel_opened: + # Channels with ids 0 - 8 are have special opening procedure: send intance id with channel number - buf = b'' if self._global_instance_id is None else ( - self._global_instance_id.encode() + str(self._get_channel_id()).encode()) + if self._get_channel_id() < 9: + buf = b'' if self._global_instance_id is None else ( + self._global_instance_id.encode() + str(self._get_channel_id()).encode()) + else: + buf = b'' await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=buf, flags=['PSH']))) @@ -442,15 +447,43 @@ async def prepare(self, force_open: bool = False) -> None: self._queue = asyncio.Queue() self._global_sync_barrier = Barrier(len(CC)) - self._channels = {channel: _ChannelContext(channel, - self._queue, - self._instance_id, - self._logger, - self._global_stop_channel_event, - self._global_sync_channel_event, - self._global_sync_barrier) for channel in CC} + + [ await self.init_channel(channel, force_open) for channel in CC ] + + + async def init_channel(self, channel=None, force_open=False): + + if not isinstance(channel, CC): + + channel = self._expandCommunicationChannelEnum() + + self._channels[channel] = _ChannelContext(channel, + self._queue, + self._instance_id, + self._logger, + self._global_stop_channel_event, + self._global_sync_channel_event, + self._global_sync_barrier) + + self._global_sync_barrier._parties = len(CC) + if force_open: - [await channel.open() for channel in self.get_channels()] + await self.get_channel(channel).open() + + return channel + + + def _expandCommunicationChannelEnum(self) -> CC: + + if len(CC) >= 65535: + return None + + new_channel_name = 'HTTP_REQUEST_' + str(len(CC) - 8 + 1) + + globals()['CC'] = Enum('CC', [m.name for m in CC] + [new_channel_name], start=0) + + return getattr(CC, new_channel_name) + def set_logger(self, logger: logging.Logger) -> None: """Sets logger From 99fc58a351e60b142cb256e310788e626d3304f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Mon, 28 Aug 2023 17:52:53 +0200 Subject: [PATCH 205/231] Extra channel support --- packages/python-runner/runner.py | 2 +- packages/python-runner/tecemux.py | 186 +++++++++++--------- packages/python-runner/test/test_tecemux.py | 40 +---- 3 files changed, 114 insertions(+), 114 deletions(-) diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index d2c0591bc..6edcb3d4c 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -112,7 +112,7 @@ def connect_log_stream(self): self._logging_setup.flush_temp_handler() self.protocol.set_logger(self.logger) - [channel._set_logger(self.logger) for channel in self.protocol.get_channels()] + [channel._set_logger(self.logger) for channel in self.protocol.get_required_channels()] self.logger.info('Log stream connected.') diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 92e1fc69b..71d1b95a6 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -20,6 +20,15 @@ class _ChannelContext: """Internal class to manage single channel """ + class _ChannelState(Enum): + """Internal class describes possible channel state + """ + + PREPARED = 1 + OPENED = 2 + PAUSED = 3 + ENDED = 4 + _channel_enum: CC _global_queue: asyncio.Queue @@ -30,24 +39,23 @@ class _ChannelContext: _internal_outcoming_queue: asyncio.Queue _internal_incoming_queue: asyncio.Queue - _channel_paused: bool - _channel_opened: bool - _stop_channel_event: asyncio.Event _sync_channel_event: asyncio.Event _sync_barrier: Barrier _read_buffer: bytearray - _ended: bool _outcoming_process_task: asyncio.Task + + _state: _ChannelState - def __init__(self, channel: CC, + def __init__(self, channel, queue: asyncio.Queue, instance_id: str, logger: logging.Logger, stop_event: asyncio.Event, sync_event: asyncio.Event, - sync_barrier: Barrier): + sync_barrier: Barrier, + state: _ChannelState): self._channel_enum = channel self._global_queue = queue @@ -56,14 +64,12 @@ def __init__(self, channel: CC, self._event_loop = asyncio.get_running_loop() self._internal_outcoming_queue = asyncio.Queue() self._internal_incoming_queue = asyncio.Queue() - self._channel_opened = False - self._channel_paused = False self._stop_channel_event = stop_event self._sync_channel_event = sync_event self._sync_barrier = sync_barrier self._outcoming_process_task = asyncio.create_task(self._outcoming_process_tasks(), name=f'{self._get_channel_name()}_PROCESS_TASK') - self._ended = None self._read_buffer = bytearray() + self._state = state async def sync(self) -> None: """Waits until internal queues will be empty (packets will be processed) @@ -111,7 +117,10 @@ def _get_channel_name(self) -> str: str: Channel name """ - return str(self._channel_enum.name) + if isinstance(self._channel_enum, CC): + return str(self._channel_enum.name) + else: + return str(self._channel_enum) def _get_channel_id(self) -> int: """Return channel id @@ -119,7 +128,10 @@ def _get_channel_id(self) -> int: Returns: int: Channel id """ - return int(self._channel_enum.value) + if isinstance(self._channel_enum, CC): + return int(self._channel_enum.value) + else: + return int(self._channel_enum) def _set_logger(self, logger: logging.Logger) -> None: """Sets new logger @@ -166,7 +178,7 @@ async def readuntil(self, separator=b'\n') -> bytes: if (not await self._get_data()): break - if self._ended and isep == -1: + if self._state == _ChannelContext._ChannelState.ENDED and isep == -1: chunk = self._read_buffer.copy() self._read_buffer.clear() else: @@ -181,7 +193,7 @@ async def _get_data(self) -> bool: Returns: bool: True if data are moved from queue to _read_buffer, false otherwise """ - if self._ended and self._internal_incoming_queue.empty(): + if self._state == _ChannelContext._ChannelState.ENDED and self._internal_incoming_queue.empty(): return False while True: @@ -197,7 +209,7 @@ async def _get_data(self) -> bool: await asyncio.sleep(0) - if self._ended and self._internal_incoming_queue.empty(): + if self._state == _ChannelContext._ChannelState.ENDED and self._internal_incoming_queue.empty(): return False except asyncio.CancelledError: @@ -257,7 +269,7 @@ async def open(self) -> None: """Open channel """ - if not self._channel_opened: + if self._state == _ChannelContext._ChannelState.PREPARED: # Channels with ids 0 - 8 are have special opening procedure: send intance id with channel number if self._get_channel_id() < 9: @@ -268,8 +280,7 @@ async def open(self) -> None: await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=buf, flags=['PSH']))) - self._channel_opened = True - self._ended = False + self._state = _ChannelContext._ChannelState.OPENED async def end(self) -> None: """Send EOF on current channel @@ -277,7 +288,7 @@ async def end(self) -> None: await self._internal_outcoming_queue.join() await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'', flags=['PSH']))) - self._ended = True + self._state = _ChannelContext._ChannelState.ENDED async def close(self) -> None: """Close current channel @@ -295,13 +306,13 @@ async def queue_up_incoming(self, pkt: IPPacket) -> None: # if SYN & ACK flag is up, pause channel if pkt.get_segment().is_flag('SYN') and pkt.get_segment().is_flag('ACK'): self._debug(f'Tecemux/{self._get_channel_name()}: [-] Channel pause request received') - self.set_pause(True) + self._state = _ChannelContext._ChannelState.PAUSED await self.send_ACK(pkt.get_segment().seq) return # if ACK flag is up, reasume channel if not pkt.get_segment().is_flag('SYN') and pkt.get_segment().is_flag('ACK'): - self.set_pause(False) + self._state = _ChannelContext._ChannelState.OPENED return # if PSH flag is up, confirm @@ -309,7 +320,7 @@ async def queue_up_incoming(self, pkt: IPPacket) -> None: await self.send_ACK(pkt.get_segment().seq) if pkt.segment.is_flag('FIN'): - self._ended = True + self._state = _ChannelContext._ChannelState.ENDED return await self._internal_incoming_queue.put(pkt.get_segment().data) self._internal_incoming_queue.task_done() @@ -320,15 +331,14 @@ async def _queue_up_outcoming(self) -> None: Args: data (bytes): Buffer to send_incoming_process_task """ - def wrap(channel_enum, buf): - channel_id = int(channel_enum.value) + def wrap(channel_id, buf): return IPPacket(segment=TCPSegment(dst_port=channel_id, flags=['PSH'], data=buf)) - if not self._channel_paused: + if self._state is not _ChannelContext._ChannelState.PAUSED: while not self._internal_outcoming_queue.empty(): try: buf = await asyncio.wait_for(self._internal_outcoming_queue.get(),1) - await self._global_queue.put(wrap(self._channel_enum, buf)) + await self._global_queue.put(wrap(self._get_channel_id(), buf)) self._internal_outcoming_queue.task_done() except asyncio.QueueEmpty: self._debug(f'Tecemux/{self._get_channel_name()}: [-] All data stored during pause were redirected to global queue') @@ -354,14 +364,6 @@ def write(self, data: bytes) -> None: self._internal_outcoming_queue.put_nowait(data) - def set_pause(self, state: bool) -> None: - """Sets pause state for current channel - - Args: - state (bool): Pause state - """ - self._channel_paused = state - @define class Tecemux: @@ -376,7 +378,9 @@ class Tecemux: _outcoming_data_forwarder: asyncio.coroutine = field(default=None) _instance_id: str = field(default=None) - _channels: dict = field(default={}) + _required_channels: dict = field(default={}) + _extra_channels: dict = field(default={}) + _logger: logging.Logger = field(default=None) _global_stop_channel_event: asyncio.Event = asyncio.Event() @@ -447,43 +451,38 @@ async def prepare(self, force_open: bool = False) -> None: self._queue = asyncio.Queue() self._global_sync_barrier = Barrier(len(CC)) + self._required_channels = {channel: _ChannelContext(channel, + self._queue, + self._instance_id, + self._logger, + self._global_stop_channel_event, + self._global_sync_channel_event, + self._global_sync_barrier, + _ChannelContext._ChannelState.PREPARED) for channel in CC} + if force_open: + [await channel.open() for channel in self.get_required_channels()] - [ await self.init_channel(channel, force_open) for channel in CC ] - - - async def init_channel(self, channel=None, force_open=False): - - if not isinstance(channel, CC): - channel = self._expandCommunicationChannelEnum() - self._channels[channel] = _ChannelContext(channel, - self._queue, - self._instance_id, - self._logger, - self._global_stop_channel_event, - self._global_sync_channel_event, - self._global_sync_barrier) - - self._global_sync_barrier._parties = len(CC) - + async def open_channel(self, channel_id=None, force_open=False, initial_state=_ChannelContext._ChannelState.PREPARED): + + if channel_id is None: + channel = str(len(self._required_channels) + len(self._extra_channels)+1) + + self._extra_channels[channel] = _ChannelContext(channel, + self._queue, + self._instance_id, + self._logger, + self._global_stop_channel_event, + self._global_sync_channel_event, + self._global_sync_barrier, + initial_state) + if force_open: - await self.get_channel(channel).open() + await self._extra_channels[channel].open() + self._global_sync_barrier._parties = len(self.get_required_channels()) + len(self.get_extra_channels()) return channel - - - def _expandCommunicationChannelEnum(self) -> CC: - - if len(CC) >= 65535: - return None - - new_channel_name = 'HTTP_REQUEST_' + str(len(CC) - 8 + 1) - - globals()['CC'] = Enum('CC', [m.name for m in CC] + [new_channel_name], start=0) - - return getattr(CC, new_channel_name) - def set_logger(self, logger: logging.Logger) -> None: """Sets logger @@ -493,7 +492,7 @@ def set_logger(self, logger: logging.Logger) -> None: """ self._logger = logger - def get_channel(self, channel: CC) -> _ChannelContext: + def get_channel(self, channel) -> _ChannelContext: """Returns single channel context Args: @@ -502,7 +501,10 @@ def get_channel(self, channel: CC) -> _ChannelContext: Returns: _ChannelContext: Channel context """ - return self._channels[channel] + if isinstance(channel, CC): + return self._required_channels[channel] + else: + return self._extra_channels[channel] async def sync(self) -> None: """Waits until all write tasks will be done @@ -515,14 +517,22 @@ async def sync(self) -> None: if not self._queue.empty(): await self._queue.join() - def get_channels(self) -> dict: - """Returns all initiated channels + def get_required_channels(self) -> dict: + """Returns only required channels by STH Returns: dict: Dict of channel's contexts """ - return self._channels.values() + return self._required_channels.values() + def get_extra_channels(self) -> dict: + """Returns only extra initiated channels by runner + + Returns: + dict: Dict of channel's contexts + """ + return self._extra_channels.values() + @staticmethod def _chunk_preview(value: bytes) -> str: """Returns small string preview of byte chunk. For logs @@ -547,18 +557,25 @@ async def _finish_channels(self) -> None: """ Close all channels """ - for channel in self.get_channels(): + for channel in self.get_required_channels(): await channel.end() await channel.close() - for channel in self.get_channels(): + for channel in self.get_extra_channels(): + await channel.end() + await channel.close() + + for channel in self.get_required_channels(): await channel._internal_outcoming_queue.join() + for channel in self.get_extra_channels(): + await channel._internal_outcoming_queue.join() + self._global_stop_channel_event.set() await self._global_stop_channel_event.wait() - for channel in self.get_channels(): + for channel in self.get_required_channels(): try: - await asyncio.wait_for(channel._outcoming_process_task,1) + await asyncio.wait_for(channel._outcoming_process_task,timeout=1) except asyncio.TimeoutError: pass except TypeError: @@ -606,7 +623,7 @@ async def incoming_data_forward(self) -> None: while not self._global_stop_channel_event.is_set(): try: - chunk = await self._reader.read(READ_CHUNK_SIZE) + chunk = await asyncio.wait_for(self._reader.read(READ_CHUNK_SIZE),1) if not chunk: incoming_parser_finish_loop.set() @@ -633,16 +650,23 @@ async def incoming_data_forward(self) -> None: self._last_sequence_received = pkt.get_segment().seq self._debug(f'Tecemux/MAIN: [<] Full incoming packet with sequence number {self._last_sequence_received} from Transform Hub was received') - channel = CC(str(pkt.get_segment().dst_port)) + dst_port = pkt.get_segment().dst_port - await self._channels[channel].queue_up_incoming(pkt) + if dst_port < 9: + channel = CC(str(dst_port)) + channel_name = channel.name + await self._required_channels[channel].queue_up_incoming(pkt) + else: + channel = str(dst_port) + channel_name = f'EXTRA_CHANNEL_{channel}' + await self._extra_channels[channel].queue_up_incoming(pkt) - self._debug(f'Tecemux/MAIN: [<] Packet {Tecemux._chunk_preview(single_packet_buffer)} forwarded to {channel.name} stream') + self._debug(f'Tecemux/MAIN: [<] Packet {Tecemux._chunk_preview(single_packet_buffer)} forwarded to {channel_name} stream') buffer = buffer[current_packet_size:] else: self._warning('Tecemux/MAIN: [<] Not full packet received. Getting additional data chunk') break - except TimeoutError: + except asyncio.TimeoutError: if self._global_stop_channel_event.is_set(): break await asyncio.sleep(0) @@ -658,7 +682,7 @@ async def outcoming_data_forward(self) -> None: while True: try: - pkt = await asyncio.wait_for(self._queue.get(),1) + pkt = await asyncio.wait_for(self._queue.get(),timeout=1) self._queue.task_done() # inject sequence number @@ -680,7 +704,7 @@ async def outcoming_data_forward(self) -> None: except asyncio.TimeoutError: if self._queue.empty() and self._global_stop_outcoming_event.is_set(): - break + return await asyncio.sleep(0) except asyncio.CancelledError: diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index b139cdd36..cedbd7231 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -135,45 +135,21 @@ async def test_stderr_write_redirection(self, local_socket_connection): - async def test_nested_channel(self, local_socket_connection): + async def test_extra_channel(self, local_socket_connection): client_a, client_b = local_socket_connection - - client_a.get_channel(CC.CONTROL).use_with_nested_channel(CC.MONITORING) - client_b.get_channel(CC.CONTROL).use_with_nested_channel(CC.MONITORING) - - await client_a.get_channel(CC.CONTROL).open() - - assert client_a._queue.qsize() == 2 + test_channel = await client_a.open_channel(force_open=True) + client_a.get_channel(test_channel).write("{'foo':'bar'}\n") - assert client_a._queue._queue[0].get_segment().dst_port == int(CC.CONTROL.value) - assert client_a._queue._queue[0].get_segment().data == b'' - - assert client_a._queue._queue[1].get_segment().dst_port == int(CC.CONTROL.value) - assert client_a._queue._queue[1].get_segment().data != b'' - - inner_packet = client_a._queue._queue[1].get_segment().data - inner_packet = IPPacket.from_buffer_with_pseudoheader(inner_packet) - - assert inner_packet.get_segment().dst_port == int(CC.MONITORING.value) - assert inner_packet.get_segment().data == b'' - - client_a.get_channel(CC.CONTROL).write("{'foo':'bar'}\n") - await client_a.get_channel(CC.CONTROL)._internal_outcoming_queue.join() - - inner_packet = client_a._queue._queue[0].get_segment().data - inner_packet = IPPacket.from_buffer_with_pseudoheader(inner_packet) + await client_a._writer.drain() - assert inner_packet.get_segment().data == b"{'foo':'bar'}\n" - assert inner_packet.get_segment().dst_port == int(CC.MONITORING.value) - - assert (await client_b.get_channel(CC.CONTROL).read(100)).decode() == '' - assert (await client_b.get_channel(CC.CONTROL).read(100)).decode() == '' - assert (await client_b.get_channel(CC.CONTROL).read(100)).decode() == "{'foo':'bar'}\n" + data = await client_b.get_channel(test_channel).readuntil() + data = await client_b.get_channel(test_channel).readuntil() + + assert data.decode() == "{'foo':'bar'}\n" await self._close_clients(client_a, client_b) - async def test_readuntil(self, local_socket_connection): client_a, client_b = local_socket_connection From f023abe95433e7c2399f877670935921fb8521ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Tue, 29 Aug 2023 17:26:35 +0200 Subject: [PATCH 206/231] HTTPProxy minimal implementation --- packages/python-runner/tecemux.py | 109 ++++++++++++++++++++++++++++-- 1 file changed, 103 insertions(+), 6 deletions(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 71d1b95a6..78dcab6c5 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -24,7 +24,7 @@ class _ChannelState(Enum): """Internal class describes possible channel state """ - PREPARED = 1 + CREATED = 1 OPENED = 2 PAUSED = 3 ENDED = 4 @@ -269,7 +269,7 @@ async def open(self) -> None: """Open channel """ - if self._state == _ChannelContext._ChannelState.PREPARED: + if self._state == _ChannelContext._ChannelState.CREATED: # Channels with ids 0 - 8 are have special opening procedure: send intance id with channel number if self._get_channel_id() < 9: @@ -394,6 +394,87 @@ class Tecemux: _sequence_number: int = field(default=0) _last_sequence_received: int = field(default=0) + _proxy = field(default=None) + _proxy_task = asyncio.coroutine = field(default=None) + + class _HTTPProxy: + + def __init__(self, protocol): + self._host = '127.0.0.1' + self._proxy_socket = None + self._port = None + + + async def _from_initiator(reader, channel): + + while True: + try: + request_data = await asyncio.wait_for(reader.read(255), timeout=1) + + if request_data== b'': + continue + + channel.write(request_data) + await asyncio.sleep(0) + + except asyncio.TimeoutError: + await asyncio.sleep(0) + + except asyncio.CancelledError: + return + + + + async def _from_target(writer, channel): + + while True: + try: + response_data = await asyncio.wait_for(channel.read(), timeout=1) + + if response_data== b'': + continue + + writer.write(response_data) + await writer.drain() + + except asyncio.TimeoutError: + await asyncio.sleep(0) + + except asyncio.CancelledError: + return + + @staticmethod + async def handle_request(reader, writer, protocol): + + channel = await protocol.open_channel(force_open=True) + + from_initiator = asyncio.create_task(Tecemux._HTTPProxy._from_initiator(reader, channel)) + from_target = asyncio.create_task(Tecemux._HTTPProxy._from_target(writer, channel)) + + await asyncio.gather(from_initiator, from_target) + + async def run(self, protocol): + + self._proxy_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._proxy_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._proxy_socket.bind((self._host, 0)) + self._proxy_socket.listen() + + self._port = int(self._proxy_socket.getsockname()[1]) + + server = await asyncio.start_server(lambda r, w: Tecemux._HTTPProxy.handle_request(r, w, protocol), sock=self._proxy_socket) + + async with server: + await server.serve_forever() + + self._proxy_socket.close() + + def get_proxy_uri(self): + return f'http://{self._host}:{self._port}' + + def get_proxy_uri(self): + return self._proxy.get_proxy_uri() + def _debug(self, msg): if TECEMUX_INTERNAL_VERBOSE_DEBUG: self._logger.debug(msg) @@ -458,17 +539,23 @@ async def prepare(self, force_open: bool = False) -> None: self._global_stop_channel_event, self._global_sync_channel_event, self._global_sync_barrier, - _ChannelContext._ChannelState.PREPARED) for channel in CC} + _ChannelContext._ChannelState.CREATED) for channel in CC} if force_open: [await channel.open() for channel in self.get_required_channels()] + self._proxy = Tecemux._HTTPProxy(self) + self._proxy_task = asyncio.create_task(self._proxy.run(self)) - async def open_channel(self, channel_id=None, force_open=False, initial_state=_ChannelContext._ChannelState.PREPARED): + + async def open_channel(self, channel_id=None, force_open=False, initial_state=_ChannelContext._ChannelState.CREATED): if channel_id is None: channel = str(len(self._required_channels) + len(self._extra_channels)+1) - + + elif isinstance(channel_id, str): + channel = channel_id + self._extra_channels[channel] = _ChannelContext(channel, self._queue, self._instance_id, @@ -482,7 +569,8 @@ async def open_channel(self, channel_id=None, force_open=False, initial_state=_C await self._extra_channels[channel].open() self._global_sync_barrier._parties = len(self.get_required_channels()) + len(self.get_extra_channels()) - return channel + + return self._extra_channels[channel] def set_logger(self, logger: logging.Logger) -> None: """Sets logger @@ -549,10 +637,19 @@ async def stop(self) -> None: """ Stops protocol """ + await self._finish_proxy() await self._finish_channels() await self._finish_incoming() await self._finish_outcoming() + + async def _finish_proxy(self) -> None: + try: + self._proxy_task.cancel() + await asyncio.gather(*[self._proxy_task]) + except asyncio.CancelledError: + pass + async def _finish_channels(self) -> None: """ Close all channels """ From 85ebdffc240bbbb0a9e4bd8f63dc301ca90d5be8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Tue, 29 Aug 2023 17:27:05 +0200 Subject: [PATCH 207/231] HTTPProxy minimal tests implementation --- packages/python-runner/test/test_tecemux.py | 30 ++++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index cedbd7231..88fafc161 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -2,7 +2,7 @@ import codecs import pytest import sys -from tecemux import Tecemux +from tecemux import Tecemux, _ChannelContext from inet import IPPacket, TCPSegment from logging_setup import LoggingSetup from hardcoded_magic_values import CommunicationChannels as CC @@ -139,17 +139,39 @@ async def test_extra_channel(self, local_socket_connection): client_a, client_b = local_socket_connection test_channel = await client_a.open_channel(force_open=True) - client_a.get_channel(test_channel).write("{'foo':'bar'}\n") + test_channel.write("{'foo':'bar'}\n") await client_a._writer.drain() - data = await client_b.get_channel(test_channel).readuntil() - data = await client_b.get_channel(test_channel).readuntil() + data = await client_b.get_channel(test_channel._get_channel_name()).readuntil() + data = await client_b.get_channel(test_channel._get_channel_name()).readuntil() assert data.decode() == "{'foo':'bar'}\n" await self._close_clients(client_a, client_b) + + async def test_extra_channel_with_proxy(self, local_socket_connection): + client_a, client_b = local_socket_connection + + import aiohttp + + async with aiohttp.ClientSession() as session: + + #fake opening on client b side + side_b = await client_b.open_channel(channel_id='10',initial_state=_ChannelContext._ChannelState.OPENED) + side_b.write(b'HTTP/1.1 200 OK\r\nContent-Length: 12\r\nContent-Type: text/html\r\n\r\nHello World!\r\n') + await client_b._writer.drain() + + async with session.get("http://_/api/version", proxy=client_a.get_proxy_uri()) as resp: + + a = 5 + x = await resp.text() + x = 5 + + + + async def test_readuntil(self, local_socket_connection): client_a, client_b = local_socket_connection From 8696cde37f1f1716a140c44b09b162717c27a818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Wed, 30 Aug 2023 14:40:34 +0200 Subject: [PATCH 208/231] HTTPProxy works with aiohttp --- packages/python-runner/tecemux.py | 105 ++++++++------------ packages/python-runner/test/test_tecemux.py | 32 ++++-- 2 files changed, 65 insertions(+), 72 deletions(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 78dcab6c5..61e6e45e4 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -35,7 +35,6 @@ class _ChannelState(Enum): _global_instance_id: str _logger: logging.Logger - _event_loop: asyncio.unix_events._UnixSelectorEventLoop _internal_outcoming_queue: asyncio.Queue _internal_incoming_queue: asyncio.Queue @@ -61,7 +60,6 @@ def __init__(self, channel, self._global_queue = queue self._global_instance_id = instance_id self._logger = logger - self._event_loop = asyncio.get_running_loop() self._internal_outcoming_queue = asyncio.Queue() self._internal_incoming_queue = asyncio.Queue() self._stop_channel_event = stop_event @@ -404,55 +402,38 @@ def __init__(self, protocol): self._proxy_socket = None self._port = None - - async def _from_initiator(reader, channel): - - while True: - try: - request_data = await asyncio.wait_for(reader.read(255), timeout=1) - - if request_data== b'': - continue - - channel.write(request_data) - await asyncio.sleep(0) - - except asyncio.TimeoutError: - await asyncio.sleep(0) - - except asyncio.CancelledError: - return - + @staticmethod + async def handle_request(reader, writer, protocol): - - async def _from_target(writer, channel): - - while True: - try: - response_data = await asyncio.wait_for(channel.read(), timeout=1) + channel = await protocol.open_channel(force_open=True) - if response_data== b'': - continue + request_data = await reader.readuntil(b'\r\n\r\n') + channel.write(request_data) + + #await channel._internal_outcoming_queue.join() + #await protocol.sync() + + await asyncio.sleep(1) - writer.write(response_data) - await writer.drain() + _ = await channel.read() + + raw_response_status = await channel.readuntil(b'\r\n') + writer.write(raw_response_status) - except asyncio.TimeoutError: - await asyncio.sleep(0) - - except asyncio.CancelledError: - return + raw_response_headers = await channel.readuntil(b'\r\n\r\n') + writer.write(raw_response_headers) - @staticmethod - async def handle_request(reader, writer, protocol): - - channel = await protocol.open_channel(force_open=True) + header_list = raw_response_headers.decode().strip().split('\r\n') + + headers = { + key.lower(): val for key, val in [el.split(': ') for el in header_list] + } - from_initiator = asyncio.create_task(Tecemux._HTTPProxy._from_initiator(reader, channel)) - from_target = asyncio.create_task(Tecemux._HTTPProxy._from_target(writer, channel)) + raw_response_data = await channel.read(int(headers['content-length'])) + writer.write(raw_response_data) - await asyncio.gather(from_initiator, from_target) - + writer.write('\r\n') + await writer.drain() async def run(self, protocol): self._proxy_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -550,26 +531,26 @@ async def prepare(self, force_open: bool = False) -> None: async def open_channel(self, channel_id=None, force_open=False, initial_state=_ChannelContext._ChannelState.CREATED): - if channel_id is None: - channel = str(len(self._required_channels) + len(self._extra_channels)+1) - - elif isinstance(channel_id, str): - channel = channel_id - - self._extra_channels[channel] = _ChannelContext(channel, - self._queue, - self._instance_id, - self._logger, - self._global_stop_channel_event, - self._global_sync_channel_event, - self._global_sync_barrier, - initial_state) - - if force_open: - await self._extra_channels[channel].open() + channel = str(channel_id) if channel_id is not None else None - self._global_sync_barrier._parties = len(self.get_required_channels()) + len(self.get_extra_channels()) + if not channel in self._extra_channels.keys(): + channel = str(len(self.get_required_channels()) + len(self.get_extra_channels()) + 1 ) if channel is None else channel + + self._extra_channels[channel] = _ChannelContext(channel, + self._queue, + self._instance_id, + self._logger, + self._global_stop_channel_event, + self._global_sync_channel_event, + self._global_sync_barrier, + initial_state) + + if force_open: + await self._extra_channels[channel].open() + + self._global_sync_barrier._parties = len(self.get_required_channels()) + len(self.get_extra_channels()) + return self._extra_channels[channel] def set_logger(self, logger: logging.Logger) -> None: diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index 88fafc161..ad733c96b 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -152,22 +152,34 @@ async def test_extra_channel(self, local_socket_connection): async def test_extra_channel_with_proxy(self, local_socket_connection): - client_a, client_b = local_socket_connection + client_a, client_b = local_socket_connection + + async def _wait_for_channel(client_b): + while True: + if len(client_b._extra_channels) > 0: + opened_channel_name = list(client_b._extra_channels.keys())[0] + channel = client_b.get_channel(opened_channel_name) + _ = await channel.readuntil(b'\r\n\r\n') + await asyncio.sleep(0.5) + channel.write(b'HTTP/1.1 200 OK\r\nContent-Length: 12\r\nContent-Type: text/html\r\n\r\nHello World!\r\n') + await client_b.sync() + break + await asyncio.sleep(0) + + task = asyncio.create_task(_wait_for_channel(client_b)) import aiohttp - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession() as session: + async with session.get("http://_/api/version", proxy=client_a.get_proxy_uri()) as resp: + data = await resp.text() - #fake opening on client b side - side_b = await client_b.open_channel(channel_id='10',initial_state=_ChannelContext._ChannelState.OPENED) - side_b.write(b'HTTP/1.1 200 OK\r\nContent-Length: 12\r\nContent-Type: text/html\r\n\r\nHello World!\r\n') - await client_b._writer.drain() + await asyncio.gather(task) + + assert data == 'Hello World!' + await self._close_clients(client_a, client_b) - async with session.get("http://_/api/version", proxy=client_a.get_proxy_uri()) as resp: - a = 5 - x = await resp.text() - x = 5 From 594e464613e53f5361bde7b8448b45e28948a71a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Thu, 31 Aug 2023 12:10:56 +0200 Subject: [PATCH 209/231] Split tests into separated files --- packages/python-runner/test/conftest.py | 33 ++++++++++ packages/python-runner/test/test_proxy.py | 37 ++++++++++++ packages/python-runner/test/test_tecemux.py | 67 +-------------------- 3 files changed, 72 insertions(+), 65 deletions(-) create mode 100644 packages/python-runner/test/conftest.py create mode 100644 packages/python-runner/test/test_proxy.py diff --git a/packages/python-runner/test/conftest.py b/packages/python-runner/test/conftest.py new file mode 100644 index 000000000..eccb2af23 --- /dev/null +++ b/packages/python-runner/test/conftest.py @@ -0,0 +1,33 @@ +import pytest +import sys +from logging_setup import LoggingSetup +from tecemux import Tecemux + + +def get_logger(): + if not hasattr(get_logger, "log_setup"): + get_logger.log_setup = LoggingSetup(sys.stdout) + return get_logger.log_setup.logger + + +@pytest.fixture() +async def local_socket_connection(): + client_a = Tecemux() + client_a.set_logger(get_logger()) + + client_b = Tecemux() + client_b.set_logger(get_logger()) + + rsock_a, wsock_a = await Tecemux.prepare_socket_connection() + rsock_b, wsock_b = await Tecemux.prepare_socket_connection() + + await client_a.connect(rsock_a, wsock_b) + await client_b.connect(rsock_b, wsock_a) + + await client_a.prepare() + await client_b.prepare() + + await client_a.loop() + await client_b.loop() + + return client_a, client_b \ No newline at end of file diff --git a/packages/python-runner/test/test_proxy.py b/packages/python-runner/test/test_proxy.py new file mode 100644 index 000000000..bbd702526 --- /dev/null +++ b/packages/python-runner/test/test_proxy.py @@ -0,0 +1,37 @@ +import asyncio +import pytest + +class TestProxy: + + async def _close_clients(self, a, b): + await a.stop() + await b.stop() + + + async def test_extra_channel_with_proxy(self, local_socket_connection): + client_a, client_b = local_socket_connection + + async def _wait_for_channel(client_b): + while True: + if len(client_b._extra_channels) > 0: + opened_channel_name = list(client_b._extra_channels.keys())[0] + channel = client_b.get_channel(opened_channel_name) + _ = await channel.readuntil(b'\r\n\r\n') + await asyncio.sleep(0.5) + channel.write(b'HTTP/1.1 200 OK\r\nContent-Length: 12\r\nContent-Type: text/html\r\n\r\nHello World!\r\n') + await client_b.sync() + break + await asyncio.sleep(0) + + task = asyncio.create_task(_wait_for_channel(client_b)) + + import aiohttp + + async with aiohttp.ClientSession() as session: + async with session.get("http://_/api/version", proxy=client_a.get_proxy_uri()) as resp: + data = await resp.text() + + await asyncio.gather(task) + + assert data == 'Hello World!' + await self._close_clients(client_a, client_b) \ No newline at end of file diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index ad733c96b..42cf040ab 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -2,39 +2,10 @@ import codecs import pytest import sys -from tecemux import Tecemux, _ChannelContext +from tecemux import Tecemux from inet import IPPacket, TCPSegment -from logging_setup import LoggingSetup from hardcoded_magic_values import CommunicationChannels as CC -def get_logger(): - if not hasattr(get_logger, "log_setup"): - get_logger.log_setup = LoggingSetup(sys.stdout) - return get_logger.log_setup.logger - - -@pytest.fixture() -async def local_socket_connection(): - client_a = Tecemux() - client_a.set_logger(get_logger()) - - client_b = Tecemux() - client_b.set_logger(get_logger()) - - rsock_a, wsock_a = await Tecemux.prepare_socket_connection() - rsock_b, wsock_b = await Tecemux.prepare_socket_connection() - - await client_a.connect(rsock_a, wsock_b) - await client_b.connect(rsock_b, wsock_a) - - await client_a.prepare() - await client_b.prepare() - - await client_a.loop() - await client_b.loop() - - return client_a, client_b - class TestTecemux: async def _close_clients(self, a, b): @@ -148,41 +119,7 @@ async def test_extra_channel(self, local_socket_connection): assert data.decode() == "{'foo':'bar'}\n" - await self._close_clients(client_a, client_b) - - - async def test_extra_channel_with_proxy(self, local_socket_connection): - client_a, client_b = local_socket_connection - - async def _wait_for_channel(client_b): - while True: - if len(client_b._extra_channels) > 0: - opened_channel_name = list(client_b._extra_channels.keys())[0] - channel = client_b.get_channel(opened_channel_name) - _ = await channel.readuntil(b'\r\n\r\n') - await asyncio.sleep(0.5) - channel.write(b'HTTP/1.1 200 OK\r\nContent-Length: 12\r\nContent-Type: text/html\r\n\r\nHello World!\r\n') - await client_b.sync() - break - await asyncio.sleep(0) - - task = asyncio.create_task(_wait_for_channel(client_b)) - - import aiohttp - - async with aiohttp.ClientSession() as session: - async with session.get("http://_/api/version", proxy=client_a.get_proxy_uri()) as resp: - data = await resp.text() - - await asyncio.gather(task) - - assert data == 'Hello World!' - await self._close_clients(client_a, client_b) - - - - - + await self._close_clients(client_a, client_b) async def test_readuntil(self, local_socket_connection): client_a, client_b = local_socket_connection From a019f26ed0c9f9714007aebd997b3c814d6a2020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Thu, 31 Aug 2023 12:14:56 +0200 Subject: [PATCH 210/231] Add missing pytest.mark.asyncio --- packages/python-runner/test/test_proxy.py | 1 + packages/python-runner/test/test_tecemux.py | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/packages/python-runner/test/test_proxy.py b/packages/python-runner/test/test_proxy.py index bbd702526..87f4c26f5 100644 --- a/packages/python-runner/test/test_proxy.py +++ b/packages/python-runner/test/test_proxy.py @@ -8,6 +8,7 @@ async def _close_clients(self, a, b): await b.stop() + @pytest.mark.asyncio async def test_extra_channel_with_proxy(self, local_socket_connection): client_a, client_b = local_socket_connection diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index 42cf040ab..28fb673dd 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -65,6 +65,7 @@ async def test_forward_channel_between_a_b(self, local_socket_connection): await self._close_clients(client_a, client_b) + @pytest.mark.asyncio async def test_stream_write_redirection(self, local_socket_connection): client_a, client_b = local_socket_connection @@ -76,6 +77,7 @@ async def test_stream_write_redirection(self, local_socket_connection): await self._close_clients(client_a, client_b) + @pytest.mark.asyncio async def test_stream_read_redirection(self, local_socket_connection): from scramjet.streams import Stream @@ -89,7 +91,8 @@ async def test_stream_read_redirection(self, local_socket_connection): assert await output_stream.read() == 'Test\n' await self._close_clients(client_a, client_b) - + + @pytest.mark.asyncio async def test_stderr_write_redirection(self, local_socket_connection): client_a, client_b = local_socket_connection @@ -105,7 +108,7 @@ async def test_stderr_write_redirection(self, local_socket_connection): await self._close_clients(client_a, client_b) - + @pytest.mark.asyncio async def test_extra_channel(self, local_socket_connection): client_a, client_b = local_socket_connection @@ -119,8 +122,10 @@ async def test_extra_channel(self, local_socket_connection): assert data.decode() == "{'foo':'bar'}\n" - await self._close_clients(client_a, client_b) + await self._close_clients(client_a, client_b) + + @pytest.mark.asyncio async def test_readuntil(self, local_socket_connection): client_a, client_b = local_socket_connection @@ -142,6 +147,8 @@ async def test_readuntil(self, local_socket_connection): await self._close_clients(client_a, client_b) + + @pytest.mark.asyncio async def test_wih_scramjet_framework_to_list(self, local_socket_connection): client_a, client_b = local_socket_connection channel = CC.CONTROL @@ -159,6 +166,8 @@ async def test_wih_scramjet_framework_to_list(self, local_socket_connection): await self._close_clients(client_a, client_b) + + @pytest.mark.asyncio async def test_wih_scramjet_framework_to_pipe(self, local_socket_connection): client_a, client_b = local_socket_connection channel = CC.CONTROL @@ -182,6 +191,7 @@ async def test_wih_scramjet_framework_to_pipe(self, local_socket_connection): await self._close_clients(client_a, client_b) + @pytest.mark.asyncio async def test_wih_scramjet_framework_write_to(self, local_socket_connection): client_a, client_b = local_socket_connection channel = CC.CONTROL From af88fb6b06a608b49fad4760645981db87c92603 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Thu, 31 Aug 2023 13:22:13 +0200 Subject: [PATCH 211/231] Fix error with extra_channel creation --- packages/python-runner/tecemux.py | 20 +++++---- packages/python-runner/test/test_tecemux.py | 49 ++++++++++++++++++--- 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 61e6e45e4..5494db7c6 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -409,14 +409,7 @@ async def handle_request(reader, writer, protocol): request_data = await reader.readuntil(b'\r\n\r\n') channel.write(request_data) - - #await channel._internal_outcoming_queue.join() - #await protocol.sync() - - await asyncio.sleep(1) - _ = await channel.read() - raw_response_status = await channel.readuntil(b'\r\n') writer.write(raw_response_status) @@ -434,6 +427,8 @@ async def handle_request(reader, writer, protocol): writer.write('\r\n') await writer.drain() + + async def run(self, protocol): self._proxy_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -510,7 +505,7 @@ async def prepare(self, force_open: bool = False) -> None: Args: force_open (bool, optional): If True, all channels will be opened immediately, Otherwise, will be opened on demand. Defaults to False. """ - + self._extra_channels = {} self._queue = asyncio.Queue() self._global_sync_barrier = Barrier(len(CC)) self._required_channels = {channel: _ChannelContext(channel, @@ -737,7 +732,14 @@ async def incoming_data_forward(self) -> None: else: channel = str(dst_port) channel_name = f'EXTRA_CHANNEL_{channel}' - await self._extra_channels[channel].queue_up_incoming(pkt) + try: + await self._extra_channels[channel].queue_up_incoming(pkt) + except KeyError: + await self.open_channel(channel,initial_state=_ChannelContext._ChannelState.OPENED) + try: + await self._extra_channels[channel].queue_up_incoming(pkt) + except KeyError: + self._debug(f'Tecemux/MAIN: [<] Unknown channel') self._debug(f'Tecemux/MAIN: [<] Packet {Tecemux._chunk_preview(single_packet_buffer)} forwarded to {channel_name} stream') buffer = buffer[current_packet_size:] diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index 28fb673dd..1a58fadf8 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -2,7 +2,7 @@ import codecs import pytest import sys -from tecemux import Tecemux +from tecemux import Tecemux, _ChannelContext from inet import IPPacket, TCPSegment from hardcoded_magic_values import CommunicationChannels as CC @@ -65,6 +65,24 @@ async def test_forward_channel_between_a_b(self, local_socket_connection): await self._close_clients(client_a, client_b) + @pytest.mark.asyncio + async def test_write_and_read_from_the_same_side(self, local_socket_connection): + client_a, client_b = local_socket_connection + + channel_alpha = CC.CONTROL + + client_a.get_channel(channel_alpha).write("{'foo':'bar'}") + await client_a._writer.drain() + + data = None + + with pytest.raises(asyncio.TimeoutError): + data = await asyncio.wait_for(client_a.get_channel(channel_alpha).read(1),timeout=0.5) + + assert data is None + + await self._close_clients(client_a, client_b) + @pytest.mark.asyncio async def test_stream_write_redirection(self, local_socket_connection): client_a, client_b = local_socket_connection @@ -112,18 +130,37 @@ async def test_stderr_write_redirection(self, local_socket_connection): async def test_extra_channel(self, local_socket_connection): client_a, client_b = local_socket_connection - test_channel = await client_a.open_channel(force_open=True) + test_channel = await client_a.open_channel(initial_state=_ChannelContext._ChannelState.OPENED) test_channel.write("{'foo':'bar'}\n") - - await client_a._writer.drain() - - data = await client_b.get_channel(test_channel._get_channel_name()).readuntil() + + await asyncio.sleep(1) + data = await client_b.get_channel(test_channel._get_channel_name()).readuntil() assert data.decode() == "{'foo':'bar'}\n" await self._close_clients(client_a, client_b) + @pytest.mark.asyncio + async def test_write_and_read_from_the_same_side_on_extra_channel(self, local_socket_connection): + client_a, client_b = local_socket_connection + + + channel = await client_a.open_channel(initial_state=_ChannelContext._ChannelState.OPENED) + + channel.write("{'foo':'bar'}") + + await client_a.sync() + await client_a._writer.drain() + + data = None + + with pytest.raises(asyncio.TimeoutError): + data = await asyncio.wait_for(channel.read(1),timeout=0.5) + + assert data is None + + await self._close_clients(client_a, client_b) @pytest.mark.asyncio async def test_readuntil(self, local_socket_connection): From 57e93521f1d13cf54cf2a8ba2dfeb6a09cfdf031 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Thu, 31 Aug 2023 16:05:50 +0200 Subject: [PATCH 212/231] Add missing docstrings --- packages/python-runner/tecemux.py | 49 +++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 5494db7c6..f335c1e32 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -396,14 +396,29 @@ class Tecemux: _proxy_task = asyncio.coroutine = field(default=None) class _HTTPProxy: + """ Simple HTTP proxy implementation for internal usage by Tecemux procotol + """ def __init__(self, protocol): + """Initialize HTTP proxy + + Args: + protocol (Tecemux): Tecemux object + """ + self._host = '127.0.0.1' self._proxy_socket = None self._port = None @staticmethod - async def handle_request(reader, writer, protocol): + async def handle_request(reader, writer, protocol): + """Process single request to server + + Args: + reader (asyncio.StreamReader): Stream reader provides access to HTTP request data + writer (asyncio.StreamWriter): Stream writer give posibility to send response + protocol (Tecemux): Tecemux object + """ channel = await protocol.open_channel(force_open=True) @@ -430,6 +445,11 @@ async def handle_request(reader, writer, protocol): async def run(self, protocol): + """Starts server on random local TCP port + + Args: + protocol (Tecemux): Tecemux object + """ self._proxy_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self._proxy_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) @@ -446,9 +466,20 @@ async def run(self, protocol): self._proxy_socket.close() def get_proxy_uri(self): + """Returns proxy config URI for aiohttp + + Returns: + str: aiohttp proxy config + """ + return f'http://{self._host}:{self._port}' def get_proxy_uri(self): + """Returns proxy config URI for aiohttp + + Returns: + str: aiohttp proxy config + """ return self._proxy.get_proxy_uri() def _debug(self, msg): @@ -524,7 +555,17 @@ async def prepare(self, force_open: bool = False) -> None: - async def open_channel(self, channel_id=None, force_open=False, initial_state=_ChannelContext._ChannelState.CREATED): + async def open_channel(self, channel_id=None, force_open=False, initial_state: _ChannelContext._ChannelState=_ChannelContext._ChannelState.CREATED) -> _ChannelContext: + """Opens additional channel in Tecemux + + Args: + channel_id (CC, str, optional): Name for new channel. Defaults to None. + force_open (bool, optional): Sends empty PSH packet right now. Defaults to False. + initial_state (_ChannelState., optional): Initial channel state. Defaults to _ChannelContext._ChannelState.CREATED. + + Returns: + _ChannelContext: Openned channel + """ channel = str(channel_id) if channel_id is not None else None @@ -560,7 +601,7 @@ def get_channel(self, channel) -> _ChannelContext: """Returns single channel context Args: - channel (CC): Channel Enum + channel (str, CC): Channel name Returns: _ChannelContext: Channel context @@ -620,6 +661,8 @@ async def stop(self) -> None: async def _finish_proxy(self) -> None: + """ Stops proxy + """ try: self._proxy_task.cancel() await asyncio.gather(*[self._proxy_task]) From 4465cacf0b615d81685e6b9c8fe3fbdcbc7ba0e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 1 Sep 2023 11:24:17 +0200 Subject: [PATCH 213/231] ChannelGuard as manager for open/close channels --- packages/python-runner/tecemux.py | 102 +++++++++++++++++++--- packages/python-runner/test/test_proxy.py | 9 +- 2 files changed, 98 insertions(+), 13 deletions(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index f335c1e32..0c1ce23c4 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -1,4 +1,5 @@ import asyncio +import base64 import logging import random import socket @@ -29,6 +30,64 @@ class _ChannelState(Enum): PAUSED = 3 ENDED = 4 + class _ChannelGuard: + """Internal class to open/close channels for external usage + """ + + def __init__(self, protocol): + """Inits channel guard + + Args: + protocol (Tecemux): Tecemux object + """ + + self._protocol = protocol + self._channel_name = None + + async def __aenter__(self): + """Opens new channel and returns guard + + Returns: + _ChannelGuard: async guard object + """ + + channel = await self._protocol.open_channel(force_open=True) + self._channel_name = channel._channel_enum + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """Closes channel + + Args: + exc_type: Unused + exc_val: Unused + exc_tb: Unused + """ + await self._protocol.get_channel(self._channel_name).end() + await self._protocol.get_channel(self._channel_name).close() + del self._protocol._extra_channels[self._channel_name] + + def inject_tecemux_details_as(self, provider): + """Retuns channel details for external lib + + Args: + provider (Any): External provider + + Returns: + Any: External provider object with channel details + """ + + return provider('tecemux', self._channel_name) + + def get_proxy_uri(self): + """Returns proxy config URI for aiohttp + + Returns: + str: aiohttp config + """ + + return self._protocol._get_proxy_uri() + _channel_enum: CC _global_queue: asyncio.Queue @@ -409,6 +468,26 @@ def __init__(self, protocol): self._host = '127.0.0.1' self._proxy_socket = None self._port = None + + @staticmethod + def _get_headers_as_dict(request_headers, convert_keys_to_lowercase=False): + headers_list = request_headers.decode().strip().split('\r\n') + + return {(key.strip().lower() if convert_keys_to_lowercase + else key.strip()) : val.strip() for key,val in [el.split(': ') for el in headers_list] } + + @staticmethod + def _extract_tecemux_details(request_headers): + + headers = Tecemux._HTTPProxy._get_headers_as_dict(request_headers) + + tecemux_params = base64.b64decode(headers['Proxy-Authorization'].split(' ')[1]).decode().split(':') + + del headers['Proxy-Authorization'] + + new_headers = ('\r\n'.join(key +': '+ str(val) for key, val in headers.items())+'\r\n\r\n').encode('utf-8') + + return type('TecemuxDetails', (object,), {'user' : tecemux_params[0], 'channel_id': tecemux_params[1]})() , new_headers @staticmethod async def handle_request(reader, writer, protocol): @@ -420,10 +499,14 @@ async def handle_request(reader, writer, protocol): protocol (Tecemux): Tecemux object """ - channel = await protocol.open_channel(force_open=True) + request_status = await reader.readuntil(b'\r\n') + + tecemux_params, request_headers = Tecemux._HTTPProxy._extract_tecemux_details(await reader.readuntil(b'\r\n\r\n')) - request_data = await reader.readuntil(b'\r\n\r\n') - channel.write(request_data) + channel = protocol.get_channel(tecemux_params.channel_id) + + channel.write(request_status) + channel.write(request_headers) raw_response_status = await channel.readuntil(b'\r\n') writer.write(raw_response_status) @@ -431,16 +514,12 @@ async def handle_request(reader, writer, protocol): raw_response_headers = await channel.readuntil(b'\r\n\r\n') writer.write(raw_response_headers) - header_list = raw_response_headers.decode().strip().split('\r\n') - - headers = { - key.lower(): val for key, val in [el.split(': ') for el in header_list] - } + headers = Tecemux._HTTPProxy._get_headers_as_dict(raw_response_headers, convert_keys_to_lowercase=True) raw_response_data = await channel.read(int(headers['content-length'])) writer.write(raw_response_data) - writer.write('\r\n') + writer.write(b'\r\n') await writer.drain() @@ -474,13 +553,16 @@ def get_proxy_uri(self): return f'http://{self._host}:{self._port}' - def get_proxy_uri(self): + def _get_proxy_uri(self): """Returns proxy config URI for aiohttp Returns: str: aiohttp proxy config """ return self._proxy.get_proxy_uri() + + def get_channel_guard(self): + return _ChannelContext._ChannelGuard(self) def _debug(self, msg): if TECEMUX_INTERNAL_VERBOSE_DEBUG: diff --git a/packages/python-runner/test/test_proxy.py b/packages/python-runner/test/test_proxy.py index 87f4c26f5..408257c1d 100644 --- a/packages/python-runner/test/test_proxy.py +++ b/packages/python-runner/test/test_proxy.py @@ -28,9 +28,12 @@ async def _wait_for_channel(client_b): import aiohttp - async with aiohttp.ClientSession() as session: - async with session.get("http://_/api/version", proxy=client_a.get_proxy_uri()) as resp: - data = await resp.text() + async with client_a.get_channel_guard() as guard: + async with aiohttp.ClientSession() as session: + async with session.get("http://_/api/version", + proxy_auth=guard.inject_tecemux_details_as(aiohttp.BasicAuth), + proxy=guard.get_proxy_uri()) as resp: + data = await resp.text() await asyncio.gather(task) From 79f90260976a02662c5ad9ecb17691f357ba0216 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 1 Sep 2023 11:40:14 +0200 Subject: [PATCH 214/231] Test calculation of unused channel id --- packages/python-runner/tecemux.py | 23 +++++++++++++++++-- packages/python-runner/test/test_tecemux.py | 25 +++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py index 0c1ce23c4..a55b40287 100644 --- a/packages/python-runner/tecemux.py +++ b/packages/python-runner/tecemux.py @@ -85,7 +85,7 @@ def get_proxy_uri(self): Returns: str: aiohttp config """ - + return self._protocol._get_proxy_uri() _channel_enum: CC @@ -637,6 +637,25 @@ async def prepare(self, force_open: bool = False) -> None: + def _get_unused_extra_channel_id(self, used_channel_ids, start_from=10): + """Returns lowest, unused channel number + + Args: + used_channel_ids (list): List of *USED* channel ids + start_from (int, optional): Minimal channel id. Defaults to 10. + + Returns: + int: lowest, unused channel number + """ + used_channel_ids = sorted(set(used_channel_ids)) + if len(used_channel_ids) == 0 or used_channel_ids[0] != start_from: + return start_from + for i, v in enumerate(used_channel_ids, start_from): + if i != v: + return i + return i+1 + + async def open_channel(self, channel_id=None, force_open=False, initial_state: _ChannelContext._ChannelState=_ChannelContext._ChannelState.CREATED) -> _ChannelContext: """Opens additional channel in Tecemux @@ -653,7 +672,7 @@ async def open_channel(self, channel_id=None, force_open=False, initial_state: _ if not channel in self._extra_channels.keys(): - channel = str(len(self.get_required_channels()) + len(self.get_extra_channels()) + 1 ) if channel is None else channel + channel = str(self._get_unused_extra_channel_id([ int(id) for id in self._extra_channels.keys() ]) ) if channel is None else channel self._extra_channels[channel] = _ChannelContext(channel, self._queue, diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_tecemux.py index 1a58fadf8..2b96cb35f 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_tecemux.py @@ -141,6 +141,31 @@ async def test_extra_channel(self, local_socket_connection): await self._close_clients(client_a, client_b) + @pytest.mark.asyncio + async def test_many_extra_channels(self, local_socket_connection): + client_a, client_b = local_socket_connection + + test_func = lambda: str(client_a._get_unused_extra_channel_id([ int(id) for id in client_a._extra_channels.keys() ]) ) + + [ await client_a.open_channel(initial_state=_ChannelContext._ChannelState.OPENED) for _ in range(5)] + + assert len(client_a._extra_channels) == 5 + + assert test_func() == '15' + + del client_a._extra_channels['13'] + + assert len(client_a._extra_channels) == 4 + + assert test_func() == '13' + + await client_a.open_channel(initial_state=_ChannelContext._ChannelState.OPENED) + + assert len(client_a._extra_channels) == 5 + + await self._close_clients(client_a, client_b) + + @pytest.mark.asyncio async def test_write_and_read_from_the_same_side_on_extra_channel(self, local_socket_connection): client_a, client_b = local_socket_connection From 795cab076ddcce27c868206cb2a07c63aa6d7c30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 1 Sep 2023 16:10:38 +0200 Subject: [PATCH 215/231] Divide tecemux.py into smaller files --- packages/python-runner/package.json | 11 +- packages/python-runner/runner.py | 6 +- packages/python-runner/tecemux.py | 937 ------------------ packages/python-runner/tecemux/__init__.py | 0 .../python-runner/{ => tecemux}/barrier.py | 0 packages/python-runner/tecemux/channel.py | 425 ++++++++ .../{ => tecemux}/hardcoded_magic_values.py | 0 packages/python-runner/{ => tecemux}/inet.py | 21 +- packages/python-runner/tecemux/multiplexer.py | 426 ++++++++ packages/python-runner/tecemux/proxy.py | 102 ++ packages/python-runner/test/conftest.py | 2 +- .../{test_tecemux.py => test_channels.py} | 35 +- packages/python-runner/test/test_inet.py | 2 +- .../python-runner/test/test_multiplexer.py | 27 + 14 files changed, 1012 insertions(+), 982 deletions(-) delete mode 100644 packages/python-runner/tecemux.py create mode 100644 packages/python-runner/tecemux/__init__.py rename packages/python-runner/{ => tecemux}/barrier.py (100%) create mode 100644 packages/python-runner/tecemux/channel.py rename packages/python-runner/{ => tecemux}/hardcoded_magic_values.py (100%) rename packages/python-runner/{ => tecemux}/inet.py (98%) create mode 100644 packages/python-runner/tecemux/multiplexer.py create mode 100644 packages/python-runner/tecemux/proxy.py rename packages/python-runner/test/{test_tecemux.py => test_channels.py} (87%) create mode 100644 packages/python-runner/test/test_multiplexer.py diff --git a/packages/python-runner/package.json b/packages/python-runner/package.json index 10c55f880..a03c63bc5 100644 --- a/packages/python-runner/package.json +++ b/packages/python-runner/package.json @@ -11,11 +11,14 @@ "build:docker": "docker build -t scramjetorg/runner-py:$(git rev-parse HEAD) -f Dockerfile ../../" }, "assets": [ - "hardcoded_magic_values.py", + "tecemux/__init__.py", + "tecemux/barrier.py", + "tecemux/channel.py", + "tecemux/hardcoded_magic_values.py", + "tecemux/inet.py", + "tecemux/multiplexer.py", + "tecemux/proxy.py", "logging_setup.py", - "tecemux.py", - "inet.py", - "barrier.py", "runner.py" ], "author": "Scramjet ", diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index 6edcb3d4c..68ada9758 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -5,14 +5,14 @@ import json import logging from pyee.asyncio import AsyncIOEventEmitter -from tecemux import Tecemux import importlib.util from io import DEFAULT_BUFFER_SIZE as CHUNK_SIZE import types from scramjet.streams import Stream from logging_setup import LoggingSetup -from hardcoded_magic_values import CommunicationChannels as CC -from hardcoded_magic_values import RunnerMessageCodes as msg_codes +from tecemux.multiplexer import Tecemux +from tecemux.hardcoded_magic_values import CommunicationChannels as CC +from tecemux.hardcoded_magic_values import RunnerMessageCodes as msg_codes SEQUENCE_PATH = os.getenv('SEQUENCE_PATH') diff --git a/packages/python-runner/tecemux.py b/packages/python-runner/tecemux.py deleted file mode 100644 index a55b40287..000000000 --- a/packages/python-runner/tecemux.py +++ /dev/null @@ -1,937 +0,0 @@ -import asyncio -import base64 -import logging -import random -import socket - -from attrs import define, field -from barrier import Barrier -from enum import Enum -from inet import IPPacket, TCPSegment, SequenceOrder -from hardcoded_magic_values import CommunicationChannels as CC - - -TECEMUX_INTERNAL_VERBOSE_DEBUG = False - -SequenceOrder().use_little_endian() - - -@define -class _ChannelContext: - """Internal class to manage single channel - """ - - class _ChannelState(Enum): - """Internal class describes possible channel state - """ - - CREATED = 1 - OPENED = 2 - PAUSED = 3 - ENDED = 4 - - class _ChannelGuard: - """Internal class to open/close channels for external usage - """ - - def __init__(self, protocol): - """Inits channel guard - - Args: - protocol (Tecemux): Tecemux object - """ - - self._protocol = protocol - self._channel_name = None - - async def __aenter__(self): - """Opens new channel and returns guard - - Returns: - _ChannelGuard: async guard object - """ - - channel = await self._protocol.open_channel(force_open=True) - self._channel_name = channel._channel_enum - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - """Closes channel - - Args: - exc_type: Unused - exc_val: Unused - exc_tb: Unused - """ - await self._protocol.get_channel(self._channel_name).end() - await self._protocol.get_channel(self._channel_name).close() - del self._protocol._extra_channels[self._channel_name] - - def inject_tecemux_details_as(self, provider): - """Retuns channel details for external lib - - Args: - provider (Any): External provider - - Returns: - Any: External provider object with channel details - """ - - return provider('tecemux', self._channel_name) - - def get_proxy_uri(self): - """Returns proxy config URI for aiohttp - - Returns: - str: aiohttp config - """ - - return self._protocol._get_proxy_uri() - - _channel_enum: CC - - _global_queue: asyncio.Queue - _global_instance_id: str - - _logger: logging.Logger - _internal_outcoming_queue: asyncio.Queue - _internal_incoming_queue: asyncio.Queue - - _stop_channel_event: asyncio.Event - _sync_channel_event: asyncio.Event - _sync_barrier: Barrier - - _read_buffer: bytearray - _outcoming_process_task: asyncio.Task - - _state: _ChannelState - - def __init__(self, channel, - queue: asyncio.Queue, - instance_id: str, - logger: logging.Logger, - stop_event: asyncio.Event, - sync_event: asyncio.Event, - sync_barrier: Barrier, - state: _ChannelState): - - self._channel_enum = channel - self._global_queue = queue - self._global_instance_id = instance_id - self._logger = logger - self._internal_outcoming_queue = asyncio.Queue() - self._internal_incoming_queue = asyncio.Queue() - self._stop_channel_event = stop_event - self._sync_channel_event = sync_event - self._sync_barrier = sync_barrier - self._outcoming_process_task = asyncio.create_task(self._outcoming_process_tasks(), name=f'{self._get_channel_name()}_PROCESS_TASK') - self._read_buffer = bytearray() - self._state = state - - async def sync(self) -> None: - """Waits until internal queues will be empty (packets will be processed) - """ - - if self._is_incoming(): - await self._internal_incoming_queue.join() - - await self._internal_outcoming_queue.join() - - def _is_incoming(self) -> bool: - """Returns whether channel is mostly read by Tememux - - Returns: - bool: True is mostly read by Runner, False otherwise - """ - return self._channel_enum in [CC.STDIN, CC.IN, CC.CONTROL] - - async def _outcoming_process_tasks(self) -> None: - """ Internal loop for outcoming data from channel - """ - while True: - try: - await self._queue_up_outcoming() - await asyncio.sleep(0) - - if self._stop_channel_event.is_set(): - return - except asyncio.CancelledError: - return - - def _debug(self, msg): - """Wrapper for printing debug messages - - Args: - msg (str): Debug message - """ - if TECEMUX_INTERNAL_VERBOSE_DEBUG: - self._logger.debug(msg) - - def _get_channel_name(self) -> str: - """Returns channel name - - Returns: - str: Channel name - """ - - if isinstance(self._channel_enum, CC): - return str(self._channel_enum.name) - else: - return str(self._channel_enum) - - def _get_channel_id(self) -> int: - """Return channel id - - Returns: - int: Channel id - """ - if isinstance(self._channel_enum, CC): - return int(self._channel_enum.value) - else: - return int(self._channel_enum) - - def _set_logger(self, logger: logging.Logger) -> None: - """Sets new logger - - Args: - logger (logging.Logger): Logger object - """ - self._logger = logger - - async def readline(self) -> bytes: - """Reads data from current channel until '\n' appears - - Returns: - bytes: Bufer data from channel - """ - sep = b'\n' - line = await self.readuntil(sep) - return line - - async def readuntil(self, separator=b'\n') -> bytes: - """Reads data from current channel until provided separator appears - - Args: - separator (bytes, optional): Defaults to b'\n'. - - Returns: - bytes: Buffer data from channel - """ - seplen = len(separator) - - offset = 0 - isep = 0 - while True: - buflen = len(self._read_buffer) - - if buflen - offset >= seplen: - isep = self._read_buffer.find(separator, offset) - - if isep != -1: - break - - offset = buflen + 1 - seplen - - if (not await self._get_data()): - break - - if self._state == _ChannelContext._ChannelState.ENDED and isep == -1: - chunk = self._read_buffer.copy() - self._read_buffer.clear() - else: - chunk = self._read_buffer[:isep + seplen] - - del self._read_buffer[:isep + seplen] - return bytes(chunk) - - async def _get_data(self) -> bool: - """Internal method, returns True if data are moved from queue to _read_buffer, false otherwise - - Returns: - bool: True if data are moved from queue to _read_buffer, false otherwise - """ - if self._state == _ChannelContext._ChannelState.ENDED and self._internal_incoming_queue.empty(): - return False - - while True: - try: - buf = self._internal_incoming_queue.get_nowait() - - if buf == b'': - return False - - self._read_buffer.extend(buf) - break - except asyncio.QueueEmpty: - - await asyncio.sleep(0) - - if self._state == _ChannelContext._ChannelState.ENDED and self._internal_incoming_queue.empty(): - return False - - except asyncio.CancelledError: - return False - - await asyncio.sleep(0) - return True - - async def read(self, n: int = -1) -> bytes: - """Reads up to 'n' bytes of data from current channel - - Args: - n (int, optional): Number of bytes. Defaults to -1. - - Returns: - bytes: Buffer data from channel - """ - if n == 0: - return b'' - - if not self._read_buffer: - await self._get_data() - - data = bytes(memoryview(self._read_buffer)[:n]) - del self._read_buffer[:n] - return data - - def __aiter__(self): - return self - - async def __anext__(self): - val = await self.readline() - - if val == b'': - raise StopAsyncIteration - return val - - async def send_ACK(self, sequence_number: int) -> None: - """Adds to global queue an ACK packet to send. - - Args: - sequence_number (int): Value for Acknowledge field - """ - - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK'], ack=sequence_number))) - - async def _send_pause_ACK(self, sequence_number: int) -> None: - """"Add to global queue an pause packet to send - - Args: - sequence_number (int): Value for Acknowledge field - """ - - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK', 'SYN'], ack=sequence_number))) - - async def open(self) -> None: - """Open channel - """ - - if self._state == _ChannelContext._ChannelState.CREATED: - # Channels with ids 0 - 8 are have special opening procedure: send intance id with channel number - - if self._get_channel_id() < 9: - buf = b'' if self._global_instance_id is None else ( - self._global_instance_id.encode() + str(self._get_channel_id()).encode()) - else: - buf = b'' - - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=buf, flags=['PSH']))) - - self._state = _ChannelContext._ChannelState.OPENED - - async def end(self) -> None: - """Send EOF on current channel - """ - - await self._internal_outcoming_queue.join() - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'', flags=['PSH']))) - self._state = _ChannelContext._ChannelState.ENDED - - async def close(self) -> None: - """Close current channel - """ - - self._debug(f'Tecemux/{self._get_channel_name()}: [-] Channel close request is send') - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'' if self._global_instance_id is None else self._global_instance_id, flags=['FIN']))) - - async def queue_up_incoming(self, pkt: IPPacket) -> None: - """Redirects incoming data from provided packet to current channel - - Args: - pkt (IPPacket): Redirected packet from outside - """ - # if SYN & ACK flag is up, pause channel - if pkt.get_segment().is_flag('SYN') and pkt.get_segment().is_flag('ACK'): - self._debug(f'Tecemux/{self._get_channel_name()}: [-] Channel pause request received') - self._state = _ChannelContext._ChannelState.PAUSED - await self.send_ACK(pkt.get_segment().seq) - return - - # if ACK flag is up, reasume channel - if not pkt.get_segment().is_flag('SYN') and pkt.get_segment().is_flag('ACK'): - self._state = _ChannelContext._ChannelState.OPENED - return - - # if PSH flag is up, confirm - if pkt.segment.is_flag('PSH'): - await self.send_ACK(pkt.get_segment().seq) - - if pkt.segment.is_flag('FIN'): - self._state = _ChannelContext._ChannelState.ENDED - return - await self._internal_incoming_queue.put(pkt.get_segment().data) - self._internal_incoming_queue.task_done() - - async def _queue_up_outcoming(self) -> None: - """Redirects raw data from currect channel to global queue. - - Args: - data (bytes): Buffer to send_incoming_process_task - """ - def wrap(channel_id, buf): - return IPPacket(segment=TCPSegment(dst_port=channel_id, flags=['PSH'], data=buf)) - - if self._state is not _ChannelContext._ChannelState.PAUSED: - while not self._internal_outcoming_queue.empty(): - try: - buf = await asyncio.wait_for(self._internal_outcoming_queue.get(),1) - await self._global_queue.put(wrap(self._get_channel_id(), buf)) - self._internal_outcoming_queue.task_done() - except asyncio.QueueEmpty: - self._debug(f'Tecemux/{self._get_channel_name()}: [-] All data stored during pause were redirected to global queue') - break - except asyncio.TimeoutError: - pass - else: - self._debug(f'Tecemux/{self._get_channel_name()}: [-] Channel paused. Data queued up internally for future') - - if self._sync_channel_event.is_set(): - await self._sync_barrier.wait() - - await asyncio.sleep(0) - - def write(self, data: bytes) -> None: - """Writes data to channel - - Args: - data (bytes): Buffer to send to channel - """ - if data == b'': - return - - self._internal_outcoming_queue.put_nowait(data) - - -@define -class Tecemux: - """Tecemux protocol implementation for Scramjet Transform Hub Python Runner - """ - - _queue: asyncio.Queue = field(default=None) - _reader: asyncio.StreamReader = field(default=None) - _writer: asyncio.StreamWriter = field(default=None) - - _incoming_data_forwarder: asyncio.coroutine = field(default=None) - _outcoming_data_forwarder: asyncio.coroutine = field(default=None) - - _instance_id: str = field(default=None) - _required_channels: dict = field(default={}) - _extra_channels: dict = field(default={}) - - _logger: logging.Logger = field(default=None) - - _global_stop_channel_event: asyncio.Event = asyncio.Event() - - _global_sync_channel_event: asyncio.Event = asyncio.Event() - _global_sync_barrier: Barrier = field(default=None, init=False) - - _global_stop_outcoming_event: asyncio.Event = asyncio.Event() - _global_stop_incoming_event: asyncio.Event = asyncio.Event() - - _sequence_number: int = field(default=0) - _last_sequence_received: int = field(default=0) - - _proxy = field(default=None) - _proxy_task = asyncio.coroutine = field(default=None) - - class _HTTPProxy: - """ Simple HTTP proxy implementation for internal usage by Tecemux procotol - """ - - def __init__(self, protocol): - """Initialize HTTP proxy - - Args: - protocol (Tecemux): Tecemux object - """ - - self._host = '127.0.0.1' - self._proxy_socket = None - self._port = None - - @staticmethod - def _get_headers_as_dict(request_headers, convert_keys_to_lowercase=False): - headers_list = request_headers.decode().strip().split('\r\n') - - return {(key.strip().lower() if convert_keys_to_lowercase - else key.strip()) : val.strip() for key,val in [el.split(': ') for el in headers_list] } - - @staticmethod - def _extract_tecemux_details(request_headers): - - headers = Tecemux._HTTPProxy._get_headers_as_dict(request_headers) - - tecemux_params = base64.b64decode(headers['Proxy-Authorization'].split(' ')[1]).decode().split(':') - - del headers['Proxy-Authorization'] - - new_headers = ('\r\n'.join(key +': '+ str(val) for key, val in headers.items())+'\r\n\r\n').encode('utf-8') - - return type('TecemuxDetails', (object,), {'user' : tecemux_params[0], 'channel_id': tecemux_params[1]})() , new_headers - - @staticmethod - async def handle_request(reader, writer, protocol): - """Process single request to server - - Args: - reader (asyncio.StreamReader): Stream reader provides access to HTTP request data - writer (asyncio.StreamWriter): Stream writer give posibility to send response - protocol (Tecemux): Tecemux object - """ - - request_status = await reader.readuntil(b'\r\n') - - tecemux_params, request_headers = Tecemux._HTTPProxy._extract_tecemux_details(await reader.readuntil(b'\r\n\r\n')) - - channel = protocol.get_channel(tecemux_params.channel_id) - - channel.write(request_status) - channel.write(request_headers) - - raw_response_status = await channel.readuntil(b'\r\n') - writer.write(raw_response_status) - - raw_response_headers = await channel.readuntil(b'\r\n\r\n') - writer.write(raw_response_headers) - - headers = Tecemux._HTTPProxy._get_headers_as_dict(raw_response_headers, convert_keys_to_lowercase=True) - - raw_response_data = await channel.read(int(headers['content-length'])) - writer.write(raw_response_data) - - writer.write(b'\r\n') - await writer.drain() - - - async def run(self, protocol): - """Starts server on random local TCP port - - Args: - protocol (Tecemux): Tecemux object - """ - - self._proxy_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self._proxy_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - self._proxy_socket.bind((self._host, 0)) - self._proxy_socket.listen() - - self._port = int(self._proxy_socket.getsockname()[1]) - - server = await asyncio.start_server(lambda r, w: Tecemux._HTTPProxy.handle_request(r, w, protocol), sock=self._proxy_socket) - - async with server: - await server.serve_forever() - - self._proxy_socket.close() - - def get_proxy_uri(self): - """Returns proxy config URI for aiohttp - - Returns: - str: aiohttp proxy config - """ - - return f'http://{self._host}:{self._port}' - - def _get_proxy_uri(self): - """Returns proxy config URI for aiohttp - - Returns: - str: aiohttp proxy config - """ - return self._proxy.get_proxy_uri() - - def get_channel_guard(self): - return _ChannelContext._ChannelGuard(self) - - def _debug(self, msg): - if TECEMUX_INTERNAL_VERBOSE_DEBUG: - self._logger.debug(msg) - - def _warning(self, msg): - if TECEMUX_INTERNAL_VERBOSE_DEBUG: - self._logger.warning(msg) - - - async def connect(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - """Connects to Transform Hub via provided reader and writer objects - - Args: - reader (asyncio.StreamReader): Main stream reader - writer (asyncio.StreamWriter): Main stream writer - """ - self._reader = reader - self._writer = writer - self._sequence_number = abs(int((random.random() * (2 ** 32)) / 2)) - self._global_stop_channel_event.clear() - self._global_stop_incoming_event.clear() - self._global_stop_outcoming_event.clear() - - @staticmethod - async def prepare_tcp_connection(server_host: str, server_port: str) -> tuple: - """Prepares TCP connection to server_host:server:port - - Args: - server_host (str): Server address - server_port (str): Server port - - Returns: - tuple: Pair of StreamReader, StreamWriter - """ - return await asyncio.open_connection(server_host, server_port) - - @staticmethod - async def prepare_socket_connection() -> tuple: - """Prepares Unix socket pair to create connection. - - Returns: - tuple: Pair of StreamReader, StreamWriter - """ - rsock, wsock = socket.socketpair() - reader, _ = await asyncio.open_unix_connection(sock=rsock) - _, writer = await asyncio.open_unix_connection(sock=wsock) - return reader, writer - - async def prepare(self, force_open: bool = False) -> None: - """Opens all channels to Transform Hub - - Args: - force_open (bool, optional): If True, all channels will be opened immediately, Otherwise, will be opened on demand. Defaults to False. - """ - self._extra_channels = {} - self._queue = asyncio.Queue() - self._global_sync_barrier = Barrier(len(CC)) - self._required_channels = {channel: _ChannelContext(channel, - self._queue, - self._instance_id, - self._logger, - self._global_stop_channel_event, - self._global_sync_channel_event, - self._global_sync_barrier, - _ChannelContext._ChannelState.CREATED) for channel in CC} - if force_open: - [await channel.open() for channel in self.get_required_channels()] - - self._proxy = Tecemux._HTTPProxy(self) - self._proxy_task = asyncio.create_task(self._proxy.run(self)) - - - - def _get_unused_extra_channel_id(self, used_channel_ids, start_from=10): - """Returns lowest, unused channel number - - Args: - used_channel_ids (list): List of *USED* channel ids - start_from (int, optional): Minimal channel id. Defaults to 10. - - Returns: - int: lowest, unused channel number - """ - used_channel_ids = sorted(set(used_channel_ids)) - if len(used_channel_ids) == 0 or used_channel_ids[0] != start_from: - return start_from - for i, v in enumerate(used_channel_ids, start_from): - if i != v: - return i - return i+1 - - - async def open_channel(self, channel_id=None, force_open=False, initial_state: _ChannelContext._ChannelState=_ChannelContext._ChannelState.CREATED) -> _ChannelContext: - """Opens additional channel in Tecemux - - Args: - channel_id (CC, str, optional): Name for new channel. Defaults to None. - force_open (bool, optional): Sends empty PSH packet right now. Defaults to False. - initial_state (_ChannelState., optional): Initial channel state. Defaults to _ChannelContext._ChannelState.CREATED. - - Returns: - _ChannelContext: Openned channel - """ - - channel = str(channel_id) if channel_id is not None else None - - if not channel in self._extra_channels.keys(): - - channel = str(self._get_unused_extra_channel_id([ int(id) for id in self._extra_channels.keys() ]) ) if channel is None else channel - - self._extra_channels[channel] = _ChannelContext(channel, - self._queue, - self._instance_id, - self._logger, - self._global_stop_channel_event, - self._global_sync_channel_event, - self._global_sync_barrier, - initial_state) - - if force_open: - await self._extra_channels[channel].open() - - self._global_sync_barrier._parties = len(self.get_required_channels()) + len(self.get_extra_channels()) - - return self._extra_channels[channel] - - def set_logger(self, logger: logging.Logger) -> None: - """Sets logger - - Args: - logger (logging.Logger): Logger object - """ - self._logger = logger - - def get_channel(self, channel) -> _ChannelContext: - """Returns single channel context - - Args: - channel (str, CC): Channel name - - Returns: - _ChannelContext: Channel context - """ - if isinstance(channel, CC): - return self._required_channels[channel] - else: - return self._extra_channels[channel] - - async def sync(self) -> None: - """Waits until all write tasks will be done - """ - self._global_sync_channel_event.set() - await self._global_sync_channel_event.wait() - await self._global_sync_barrier.wait() - self._global_sync_channel_event.clear() - - if not self._queue.empty(): - await self._queue.join() - - def get_required_channels(self) -> dict: - """Returns only required channels by STH - - Returns: - dict: Dict of channel's contexts - """ - return self._required_channels.values() - - def get_extra_channels(self) -> dict: - """Returns only extra initiated channels by runner - - Returns: - dict: Dict of channel's contexts - """ - return self._extra_channels.values() - - @staticmethod - def _chunk_preview(value: bytes) -> str: - """Returns small string preview of byte chunk. For logs - - Args: - value (bytes): Bytes to preview - - Returns: - str: String preview - """ - return f'{value[0:5]}... ' if len(value) > 5 else f'{value}' - - async def stop(self) -> None: - """ Stops protocol - """ - - await self._finish_proxy() - await self._finish_channels() - await self._finish_incoming() - await self._finish_outcoming() - - - async def _finish_proxy(self) -> None: - """ Stops proxy - """ - try: - self._proxy_task.cancel() - await asyncio.gather(*[self._proxy_task]) - except asyncio.CancelledError: - pass - - async def _finish_channels(self) -> None: - """ Close all channels - """ - - for channel in self.get_required_channels(): - await channel.end() - await channel.close() - - for channel in self.get_extra_channels(): - await channel.end() - await channel.close() - - for channel in self.get_required_channels(): - await channel._internal_outcoming_queue.join() - - for channel in self.get_extra_channels(): - await channel._internal_outcoming_queue.join() - - self._global_stop_channel_event.set() - await self._global_stop_channel_event.wait() - for channel in self.get_required_channels(): - try: - await asyncio.wait_for(channel._outcoming_process_task,timeout=1) - except asyncio.TimeoutError: - pass - except TypeError: - pass - - async def _finish_outcoming(self) -> None: - """ Finish outcoming forwarder and main writer to STH - """ - - self._global_stop_outcoming_event.set() - await asyncio.sleep(0) - await self._global_stop_outcoming_event.wait() - await asyncio.gather(*[self._outcoming_data_forwarder]) - await self._writer.drain() - self._writer.close() - await self._writer.wait_closed() - - async def _finish_incoming(self) -> None: - """ Finish incoming forwarder - """ - - self._global_stop_incoming_event.set() - await self._global_stop_incoming_event.wait() - await asyncio.gather(*[self._incoming_data_forwarder]) - - async def loop(self) -> None: - """Main loop of Tecemux protocol. Starts forwarders tasks - """ - loop = asyncio.get_event_loop() - - self._incoming_data_forwarder = loop.create_task( - self.incoming_data_forward()) - self._outcoming_data_forwarder = loop.create_task( - self.outcoming_data_forward()) - - async def incoming_data_forward(self) -> None: - """Loop for incoming data from Transform Hub - """ - buffer = b'' - - incoming_parser_finish_loop = asyncio.Event() - - MINIMAL_IP_PACKET_LENGTH = 20 - READ_CHUNK_SIZE = 1024 - - while not self._global_stop_channel_event.is_set(): - try: - chunk = await asyncio.wait_for(self._reader.read(READ_CHUNK_SIZE),1) - - if not chunk: - incoming_parser_finish_loop.set() - - buffer = buffer + chunk - - buffer_len = len(buffer) - - if incoming_parser_finish_loop.is_set() and buffer_len > 0 and buffer_len < MINIMAL_IP_PACKET_LENGTH: - - self._warning('Tecemux/MAIN: [<] Too few data is waiting in global buffer but stream finished') - break - elif incoming_parser_finish_loop.is_set() and buffer_len == 0: - break - - while not (len(buffer) < MINIMAL_IP_PACKET_LENGTH): - - current_packet_size = IPPacket().from_buffer_with_pseudoheader(buffer).len - - if len(buffer) >= current_packet_size: - - single_packet_buffer = buffer[:current_packet_size] - pkt = IPPacket().from_buffer_with_pseudoheader(single_packet_buffer) - self._last_sequence_received = pkt.get_segment().seq - self._debug(f'Tecemux/MAIN: [<] Full incoming packet with sequence number {self._last_sequence_received} from Transform Hub was received') - - dst_port = pkt.get_segment().dst_port - - if dst_port < 9: - channel = CC(str(dst_port)) - channel_name = channel.name - await self._required_channels[channel].queue_up_incoming(pkt) - else: - channel = str(dst_port) - channel_name = f'EXTRA_CHANNEL_{channel}' - try: - await self._extra_channels[channel].queue_up_incoming(pkt) - except KeyError: - await self.open_channel(channel,initial_state=_ChannelContext._ChannelState.OPENED) - try: - await self._extra_channels[channel].queue_up_incoming(pkt) - except KeyError: - self._debug(f'Tecemux/MAIN: [<] Unknown channel') - - self._debug(f'Tecemux/MAIN: [<] Packet {Tecemux._chunk_preview(single_packet_buffer)} forwarded to {channel_name} stream') - buffer = buffer[current_packet_size:] - else: - self._warning('Tecemux/MAIN: [<] Not full packet received. Getting additional data chunk') - break - except asyncio.TimeoutError: - if self._global_stop_channel_event.is_set(): - break - await asyncio.sleep(0) - - except asyncio.CancelledError: - break - - self._debug('Tecemux/MAIN: Incoming data forwarder finished') - - async def outcoming_data_forward(self) -> None: - """Loop for outcoming data to Transform Hub - """ - - while True: - try: - pkt = await asyncio.wait_for(self._queue.get(),timeout=1) - self._queue.task_done() - - # inject sequence number - if pkt.segment.seq == 0: - pkt.segment.seq = self._sequence_number - self._sequence_number += 1 - - chunk = pkt.build( - for_STH=True).to_buffer_with_tcp_pseudoheader() - - self._debug(f'Tecemux/MAIN: [>] Outcoming chunk {Tecemux._chunk_preview(chunk)} is waiting to send to Transform Hub') - self._writer.write(chunk) - await self._writer.drain() - self._debug(f'Tecemux/MAIN: [>] Chunk {Tecemux._chunk_preview(chunk)} with sequence number: {pkt.segment.seq} was sent to Transform Hub') - except asyncio.QueueEmpty: - if self._global_stop_outcoming_event.is_set(): - break - await asyncio.sleep(0) - - except asyncio.TimeoutError: - if self._queue.empty() and self._global_stop_outcoming_event.is_set(): - return - await asyncio.sleep(0) - - except asyncio.CancelledError: - break - - self._debug('Tecemux/MAIN: Outcoming data forwarder finished') diff --git a/packages/python-runner/tecemux/__init__.py b/packages/python-runner/tecemux/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/python-runner/barrier.py b/packages/python-runner/tecemux/barrier.py similarity index 100% rename from packages/python-runner/barrier.py rename to packages/python-runner/tecemux/barrier.py diff --git a/packages/python-runner/tecemux/channel.py b/packages/python-runner/tecemux/channel.py new file mode 100644 index 000000000..0aefa58cb --- /dev/null +++ b/packages/python-runner/tecemux/channel.py @@ -0,0 +1,425 @@ +""" channel.py - contains Tecemux channel management in Python +""" + +import asyncio +import logging + + +from attrs import define +from enum import Enum + +from .barrier import Barrier +from .inet import IPPacket, TCPSegment, SequenceOrder +from .hardcoded_magic_values import CommunicationChannels as CC + +SequenceOrder().use_little_endian() + +TECEMUX_INTERNAL_VERBOSE_DEBUG = False + + +class ChannelState(Enum): + """Internal class describes possible channel state + """ + + CREATED = 1 + OPENED = 2 + PAUSED = 3 + ENDED = 4 + + +class ChannelGuard: + """Internal class to open/close channels for external usage + """ + + def __init__(self, protocol): + """Inits channel guard + + Args: + protocol (Tecemux): Tecemux object + """ + + self._protocol = protocol + self._channel_name = None + + async def __aenter__(self): + """Opens new channel and returns guard + + Returns: + _ChannelGuard: async guard object + """ + + channel = await self._protocol.open_channel(force_open=True) + self._channel_name = channel._channel_enum + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """Closes channel + + Args: + exc_type: Unused + exc_val: Unused + exc_tb: Unused + """ + await self._protocol.get_channel(self._channel_name).end() + await self._protocol.get_channel(self._channel_name).close() + del self._protocol._extra_channels[self._channel_name] + + def inject_tecemux_details_as(self, provider): + """Retuns channel details for external lib + + Args: + provider (Any): External provider + + Returns: + Any: External provider object with channel details + """ + + return provider('tecemux', self._channel_name) + + def get_proxy_uri(self): + """Returns proxy config URI for aiohttp + + Returns: + str: aiohttp config + """ + + return self._protocol._get_proxy_uri() + + +@define +class ChannelContext: + """Internal class to manage single channel + """ + + _channel_enum: CC + + _global_queue: asyncio.Queue + _global_instance_id: str + + _logger: logging.Logger + _internal_outcoming_queue: asyncio.Queue + _internal_incoming_queue: asyncio.Queue + + _stop_channel_event: asyncio.Event + _sync_channel_event: asyncio.Event + _sync_barrier: Barrier + + _read_buffer: bytearray + _outcoming_process_task: asyncio.Task + + _state: ChannelState + + def __init__(self, channel, + queue: asyncio.Queue, + instance_id: str, + logger: logging.Logger, + stop_event: asyncio.Event, + sync_event: asyncio.Event, + sync_barrier: Barrier, + state: ChannelState): + + self._channel_enum = channel + self._global_queue = queue + self._global_instance_id = instance_id + self._logger = logger + self._internal_outcoming_queue = asyncio.Queue() + self._internal_incoming_queue = asyncio.Queue() + self._stop_channel_event = stop_event + self._sync_channel_event = sync_event + self._sync_barrier = sync_barrier + self._outcoming_process_task = asyncio.create_task(self._outcoming_process_tasks(), name=f'{self._get_channel_name()}_PROCESS_TASK') + self._read_buffer = bytearray() + self._state = state + + async def sync(self) -> None: + """Waits until internal queues will be empty (packets will be processed) + """ + + if self._is_incoming(): + await self._internal_incoming_queue.join() + + await self._internal_outcoming_queue.join() + + def _is_incoming(self) -> bool: + """Returns whether channel is mostly read by Tememux + + Returns: + bool: True is mostly read by Runner, False otherwise + """ + return self._channel_enum in [CC.STDIN, CC.IN, CC.CONTROL] + + async def _outcoming_process_tasks(self) -> None: + """ Internal loop for outcoming data from channel + """ + while True: + try: + await self._queue_up_outcoming() + await asyncio.sleep(0) + + if self._stop_channel_event.is_set(): + return + except asyncio.CancelledError: + return + + def _debug(self, msg): + """Wrapper for printing debug messages + + Args: + msg (str): Debug message + """ + if TECEMUX_INTERNAL_VERBOSE_DEBUG: + self._logger.debug(msg) + + def _get_channel_name(self) -> str: + """Returns channel name + + Returns: + str: Channel name + """ + + if isinstance(self._channel_enum, CC): + return str(self._channel_enum.name) + else: + return str(self._channel_enum) + + def _get_channel_id(self) -> int: + """Return channel id + + Returns: + int: Channel id + """ + if isinstance(self._channel_enum, CC): + return int(self._channel_enum.value) + else: + return int(self._channel_enum) + + def _set_logger(self, logger: logging.Logger) -> None: + """Sets new logger + + Args: + logger (logging.Logger): Logger object + """ + self._logger = logger + + async def readline(self) -> bytes: + """Reads data from current channel until '\n' appears + + Returns: + bytes: Bufer data from channel + """ + sep = b'\n' + line = await self.readuntil(sep) + return line + + async def readuntil(self, separator=b'\n') -> bytes: + """Reads data from current channel until provided separator appears + + Args: + separator (bytes, optional): Defaults to b'\n'. + + Returns: + bytes: Buffer data from channel + """ + seplen = len(separator) + + offset = 0 + isep = 0 + while True: + buflen = len(self._read_buffer) + + if buflen - offset >= seplen: + isep = self._read_buffer.find(separator, offset) + + if isep != -1: + break + + offset = buflen + 1 - seplen + + if (not await self._get_data()): + break + + if self._state == ChannelState.ENDED and isep == -1: + chunk = self._read_buffer.copy() + self._read_buffer.clear() + else: + chunk = self._read_buffer[:isep + seplen] + + del self._read_buffer[:isep + seplen] + return bytes(chunk) + + async def _get_data(self) -> bool: + """Internal method, returns True if data are moved from queue to _read_buffer, false otherwise + + Returns: + bool: True if data are moved from queue to _read_buffer, false otherwise + """ + if self._state == ChannelState.ENDED and self._internal_incoming_queue.empty(): + return False + + while True: + try: + buf = self._internal_incoming_queue.get_nowait() + + if buf == b'': + return False + + self._read_buffer.extend(buf) + break + except asyncio.QueueEmpty: + + await asyncio.sleep(0) + + if self._state == ChannelState.ENDED and self._internal_incoming_queue.empty(): + return False + + except asyncio.CancelledError: + return False + + await asyncio.sleep(0) + return True + + async def read(self, n: int = -1) -> bytes: + """Reads up to 'n' bytes of data from current channel + + Args: + n (int, optional): Number of bytes. Defaults to -1. + + Returns: + bytes: Buffer data from channel + """ + if n == 0: + return b'' + + if not self._read_buffer: + await self._get_data() + + data = bytes(memoryview(self._read_buffer)[:n]) + del self._read_buffer[:n] + return data + + def __aiter__(self): + return self + + async def __anext__(self): + val = await self.readline() + + if val == b'': + raise StopAsyncIteration + return val + + async def send_ACK(self, sequence_number: int) -> None: + """Adds to global queue an ACK packet to send. + + Args: + sequence_number (int): Value for Acknowledge field + """ + + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK'], ack=sequence_number))) + + async def _send_pause_ACK(self, sequence_number: int) -> None: + """"Add to global queue an pause packet to send + + Args: + sequence_number (int): Value for Acknowledge field + """ + + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK', 'SYN'], ack=sequence_number))) + + async def open(self) -> None: + """Open channel + """ + + if self._state == ChannelState.CREATED: + # Channels with ids 0 - 8 are have special opening procedure: send intance id with channel number + + if self._get_channel_id() < 9: + buf = b'' if self._global_instance_id is None else ( + self._global_instance_id.encode() + str(self._get_channel_id()).encode()) + else: + buf = b'' + + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=buf, flags=['PSH']))) + + self._state = ChannelState.OPENED + + async def end(self) -> None: + """Send EOF on current channel + """ + + await self._internal_outcoming_queue.join() + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'', flags=['PSH']))) + self._state = ChannelState.ENDED + + async def close(self) -> None: + """Close current channel + """ + + self._debug(f'Tecemux/{self._get_channel_name()}: [-] Channel close request is send') + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'' if self._global_instance_id is None else self._global_instance_id, flags=['FIN']))) + + async def queue_up_incoming(self, pkt: IPPacket) -> None: + """Redirects incoming data from provided packet to current channel + + Args: + pkt (IPPacket): Redirected packet from outside + """ + # if SYN & ACK flag is up, pause channel + if pkt.get_segment().is_flag('SYN') and pkt.get_segment().is_flag('ACK'): + self._debug(f'Tecemux/{self._get_channel_name()}: [-] Channel pause request received') + self._state = ChannelState.PAUSED + await self.send_ACK(pkt.get_segment().seq) + return + + # if ACK flag is up, reasume channel + if not pkt.get_segment().is_flag('SYN') and pkt.get_segment().is_flag('ACK'): + self._state = ChannelState.OPENED + return + + # if PSH flag is up, confirm + if pkt.segment.is_flag('PSH'): + await self.send_ACK(pkt.get_segment().seq) + + if pkt.segment.is_flag('FIN'): + self._state = ChannelState.ENDED + return + await self._internal_incoming_queue.put(pkt.get_segment().data) + self._internal_incoming_queue.task_done() + + async def _queue_up_outcoming(self) -> None: + """Redirects raw data from currect channel to global queue. + + Args: + data (bytes): Buffer to send_incoming_process_task + """ + def wrap(channel_id, buf): + return IPPacket(segment=TCPSegment(dst_port=channel_id, flags=['PSH'], data=buf)) + + if self._state is not ChannelState.PAUSED: + while not self._internal_outcoming_queue.empty(): + try: + buf = await asyncio.wait_for(self._internal_outcoming_queue.get(),1) + await self._global_queue.put(wrap(self._get_channel_id(), buf)) + self._internal_outcoming_queue.task_done() + except asyncio.QueueEmpty: + self._debug(f'Tecemux/{self._get_channel_name()}: [-] All data stored during pause were redirected to global queue') + break + except asyncio.TimeoutError: + pass + else: + self._debug(f'Tecemux/{self._get_channel_name()}: [-] Channel paused. Data queued up internally for future') + + if self._sync_channel_event.is_set(): + await self._sync_barrier.wait() + + await asyncio.sleep(0) + + def write(self, data: bytes) -> None: + """Writes data to channel + + Args: + data (bytes): Buffer to send to channel + """ + if data == b'': + return + + self._internal_outcoming_queue.put_nowait(data) diff --git a/packages/python-runner/hardcoded_magic_values.py b/packages/python-runner/tecemux/hardcoded_magic_values.py similarity index 100% rename from packages/python-runner/hardcoded_magic_values.py rename to packages/python-runner/tecemux/hardcoded_magic_values.py diff --git a/packages/python-runner/inet.py b/packages/python-runner/tecemux/inet.py similarity index 98% rename from packages/python-runner/inet.py rename to packages/python-runner/tecemux/inet.py index 481e42743..d492bb866 100644 --- a/packages/python-runner/inet.py +++ b/packages/python-runner/tecemux/inet.py @@ -47,7 +47,7 @@ def get(self): Returns: str: return endianess for struct module - """ + """ return self.endianess @@ -77,7 +77,7 @@ def parse_options(val): Returns: str: Options in readable format - """ + """ return val class Flags: @@ -224,6 +224,7 @@ def is_flag(self, flag: Flags) -> bool: return (self.flags & getattr(TCPSegment.Flags, flag)) > 0 + @define class IPPacket: """ Class for manage IP Packet data @@ -265,7 +266,7 @@ def parse_flags(value): Returns: int: Integer values represents bit state of each flag - """ + """ res = 0 if value is None: return 0 @@ -372,7 +373,7 @@ def from_buffer(cls, buffer: bytes): Returns: IPPacket: IPPacket object - """ + """ ihl = (buffer[0] & 0xf) pkt = cls(ihl, *struct.unpack(SequenceOrder().get()+"BBHHHBBH4s4s", buffer[0:ihl*4]), TCPSegment.from_buffer(buffer[ihl*4:]) if len(buffer) > ihl*4 else None) @@ -498,6 +499,7 @@ def build(self, for_STH=False): self._validate_ip() return self + def get_segment(self): """Return TCP Segment @@ -506,6 +508,7 @@ def get_segment(self): """ return self.segment + @define class EthernetFrame: """ Class for manage Ethernet frame data @@ -532,14 +535,14 @@ def to_buffer(self) -> bytes: """Build raw buffer from Ethernet frame object Returns: - _bytes: Raw buffer - """ + bytes: Raw buffer + """ return struct.pack(SequenceOrder().get()+"6s6s2s", unhexlify(self.src_mac), unhexlify(self.dst_mac), self.eth_type) + self.packet.to_buffer() - + def get_packet(self): """Return IP Packet object Returns: IPPacket: IP Packet object - """ - return self.packet \ No newline at end of file + """ + return self.packet diff --git a/packages/python-runner/tecemux/multiplexer.py b/packages/python-runner/tecemux/multiplexer.py new file mode 100644 index 000000000..66e399f1f --- /dev/null +++ b/packages/python-runner/tecemux/multiplexer.py @@ -0,0 +1,426 @@ +""" multiplexer.py - entry point for tecemux implementation for Python +""" + +import asyncio +import logging +import random +import socket + +from attrs import define, field +from .barrier import Barrier +from .inet import IPPacket +from .hardcoded_magic_values import CommunicationChannels as CC +from .channel import ChannelContext, ChannelGuard, ChannelState +from .proxy import HTTPProxy +TECEMUX_INTERNAL_VERBOSE_DEBUG = False + + +@define +class Tecemux: + """Tecemux protocol implementation for Scramjet Transform Hub Python Runner + """ + + _queue: asyncio.Queue = field(default=None) + _reader: asyncio.StreamReader = field(default=None) + _writer: asyncio.StreamWriter = field(default=None) + + _incoming_data_forwarder: asyncio.coroutine = field(default=None) + _outcoming_data_forwarder: asyncio.coroutine = field(default=None) + + _instance_id: str = field(default=None) + _required_channels: dict = field(default={}) + _extra_channels: dict = field(default={}) + + _logger: logging.Logger = field(default=None) + + _global_stop_channel_event: asyncio.Event = asyncio.Event() + + _global_sync_channel_event: asyncio.Event = asyncio.Event() + _global_sync_barrier: Barrier = field(default=None, init=False) + + _global_stop_outcoming_event: asyncio.Event = asyncio.Event() + _global_stop_incoming_event: asyncio.Event = asyncio.Event() + + _sequence_number: int = field(default=0) + _last_sequence_received: int = field(default=0) + + _proxy = field(default=None) + _proxy_task = asyncio.coroutine = field(default=None) + + def _get_proxy_uri(self): + """Returns proxy config URI for aiohttp + + Returns: + str: aiohttp proxy config + """ + return self._proxy.get_proxy_uri() + + def get_channel_guard(self): + return ChannelGuard(self) + + def _debug(self, msg): + if TECEMUX_INTERNAL_VERBOSE_DEBUG: + self._logger.debug(msg) + + def _warning(self, msg): + if TECEMUX_INTERNAL_VERBOSE_DEBUG: + self._logger.warning(msg) + + async def connect(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + """Connects to Transform Hub via provided reader and writer objects + + Args: + reader (asyncio.StreamReader): Main stream reader + writer (asyncio.StreamWriter): Main stream writer + """ + self._reader = reader + self._writer = writer + self._sequence_number = abs(int((random.random() * (2 ** 32)) / 2)) + self._global_stop_channel_event.clear() + self._global_stop_incoming_event.clear() + self._global_stop_outcoming_event.clear() + + @staticmethod + async def prepare_tcp_connection(server_host: str, server_port: str) -> tuple: + """Prepares TCP connection to server_host:server:port + + Args: + server_host (str): Server address + server_port (str): Server port + + Returns: + tuple: Pair of StreamReader, StreamWriter + """ + return await asyncio.open_connection(server_host, server_port) + + @staticmethod + async def prepare_socket_connection() -> tuple: + """Prepares Unix socket pair to create connection. + + Returns: + tuple: Pair of StreamReader, StreamWriter + """ + rsock, wsock = socket.socketpair() + reader, _ = await asyncio.open_unix_connection(sock=rsock) + _, writer = await asyncio.open_unix_connection(sock=wsock) + return reader, writer + + async def prepare(self, force_open: bool = False) -> None: + """Opens all channels to Transform Hub + + Args: + force_open (bool, optional): If True, all channels will be opened immediately, Otherwise, will be opened on demand. Defaults to False. + """ + self._extra_channels = {} + self._queue = asyncio.Queue() + self._global_sync_barrier = Barrier(len(CC)) + self._required_channels = {channel: ChannelContext(channel, + self._queue, + self._instance_id, + self._logger, + self._global_stop_channel_event, + self._global_sync_channel_event, + self._global_sync_barrier, + ChannelState.CREATED) for channel in CC} + if force_open: + [await channel.open() for channel in self.get_required_channels()] + + self._proxy = HTTPProxy() + self._proxy_task = asyncio.create_task(self._proxy.run(self)) + + def _get_unused_extra_channel_id(self, used_channel_ids, start_from=10): + """Returns lowest, unused channel number + + Args: + used_channel_ids (list): List of *USED* channel ids + start_from (int, optional): Minimal channel id. Defaults to 10. + + Returns: + int: lowest, unused channel number + """ + used_channel_ids = sorted(set(used_channel_ids)) + if len(used_channel_ids) == 0 or used_channel_ids[0] != start_from: + return start_from + for i, v in enumerate(used_channel_ids, start_from): + if i != v: + return i + return i+1 + + async def open_channel(self, channel_id=None, force_open=False, initial_state:ChannelState = ChannelState.CREATED) -> ChannelContext: + """Opens additional channel in Tecemux + + Args: + channel_id (CC, str, optional): Name for new channel. Defaults to None. + force_open (bool, optional): Sends empty PSH packet right now. Defaults to False. + initial_state (_ChannelState., optional): Initial channel state. Defaults to ChannelState.CREATED. + + Returns: + ChannelContext: Openned channel + """ + + channel = str(channel_id) if channel_id is not None else None + + if not channel in self._extra_channels.keys(): + + channel = str(self._get_unused_extra_channel_id([ int(id) for id in self._extra_channels.keys() ]) ) if channel is None else channel + + self._extra_channels[channel] = ChannelContext(channel, + self._queue, + self._instance_id, + self._logger, + self._global_stop_channel_event, + self._global_sync_channel_event, + self._global_sync_barrier, + initial_state) + + if force_open: + await self._extra_channels[channel].open() + + self._global_sync_barrier._parties = len(self.get_required_channels()) + len(self.get_extra_channels()) + + return self._extra_channels[channel] + + def set_logger(self, logger: logging.Logger) -> None: + """Sets logger + + Args: + logger (logging.Logger): Logger object + """ + self._logger = logger + + def get_channel(self, channel) -> ChannelContext: + """Returns single channel context + + Args: + channel (str, CC): Channel name + + Returns: + ChannelContext: Channel context + """ + if isinstance(channel, CC): + return self._required_channels[channel] + else: + return self._extra_channels[channel] + + async def sync(self) -> None: + """Waits until all write tasks will be done + """ + self._global_sync_channel_event.set() + await self._global_sync_channel_event.wait() + await self._global_sync_barrier.wait() + self._global_sync_channel_event.clear() + + if not self._queue.empty(): + await self._queue.join() + + def get_required_channels(self) -> dict: + """Returns only required channels by STH + + Returns: + dict: Dict of channel's contexts + """ + return self._required_channels.values() + + def get_extra_channels(self) -> dict: + """Returns only extra initiated channels by runner + + Returns: + dict: Dict of channel's contexts + """ + return self._extra_channels.values() + + @staticmethod + def _chunk_preview(value: bytes) -> str: + """Returns small string preview of byte chunk. For logs + + Args: + value (bytes): Bytes to preview + + Returns: + str: String preview + """ + return f'{value[0:5]}... ' if len(value) > 5 else f'{value}' + + async def stop(self) -> None: + """ Stops protocol + """ + + await self._finish_proxy() + await self._finish_channels() + await self._finish_incoming() + await self._finish_outcoming() + + async def _finish_proxy(self) -> None: + """ Stops proxy + """ + try: + self._proxy_task.cancel() + await asyncio.gather(*[self._proxy_task]) + except asyncio.CancelledError: + pass + + async def _finish_channels(self) -> None: + """ Close all channels + """ + + for channel in self.get_required_channels(): + await channel.end() + await channel.close() + + for channel in self.get_extra_channels(): + await channel.end() + await channel.close() + + for channel in self.get_required_channels(): + await channel._internal_outcoming_queue.join() + + for channel in self.get_extra_channels(): + await channel._internal_outcoming_queue.join() + + self._global_stop_channel_event.set() + await self._global_stop_channel_event.wait() + for channel in self.get_required_channels(): + try: + await asyncio.wait_for(channel._outcoming_process_task,timeout=1) + except asyncio.TimeoutError: + pass + except TypeError: + pass + + async def _finish_outcoming(self) -> None: + """ Finish outcoming forwarder and main writer to STH + """ + + self._global_stop_outcoming_event.set() + await asyncio.sleep(0) + await self._global_stop_outcoming_event.wait() + await asyncio.gather(*[self._outcoming_data_forwarder]) + await self._writer.drain() + self._writer.close() + await self._writer.wait_closed() + + async def _finish_incoming(self) -> None: + """ Finish incoming forwarder + """ + + self._global_stop_incoming_event.set() + await self._global_stop_incoming_event.wait() + await asyncio.gather(*[self._incoming_data_forwarder]) + + async def loop(self) -> None: + """Main loop of Tecemux protocol. Starts forwarders tasks + """ + loop = asyncio.get_event_loop() + + self._incoming_data_forwarder = loop.create_task( + self.incoming_data_forward()) + self._outcoming_data_forwarder = loop.create_task( + self.outcoming_data_forward()) + + async def incoming_data_forward(self) -> None: + """Loop for incoming data from Transform Hub + """ + buffer = b'' + + incoming_parser_finish_loop = asyncio.Event() + + MINIMAL_IP_PACKET_LENGTH = 20 + READ_CHUNK_SIZE = 1024 + + while not self._global_stop_channel_event.is_set(): + try: + chunk = await asyncio.wait_for(self._reader.read(READ_CHUNK_SIZE),1) + + if not chunk: + incoming_parser_finish_loop.set() + + buffer = buffer + chunk + + buffer_len = len(buffer) + + if incoming_parser_finish_loop.is_set() and buffer_len > 0 and buffer_len < MINIMAL_IP_PACKET_LENGTH: + + self._warning('Tecemux/MAIN: [<] Too few data is waiting in global buffer but stream finished') + break + elif incoming_parser_finish_loop.is_set() and buffer_len == 0: + break + + while not (len(buffer) < MINIMAL_IP_PACKET_LENGTH): + + current_packet_size = IPPacket().from_buffer_with_pseudoheader(buffer).len + + if len(buffer) >= current_packet_size: + + single_packet_buffer = buffer[:current_packet_size] + pkt = IPPacket().from_buffer_with_pseudoheader(single_packet_buffer) + self._last_sequence_received = pkt.get_segment().seq + self._debug(f'Tecemux/MAIN: [<] Full incoming packet with sequence number {self._last_sequence_received} from Transform Hub was received') + + dst_port = pkt.get_segment().dst_port + + if dst_port < 9: + channel = CC(str(dst_port)) + channel_name = channel.name + await self._required_channels[channel].queue_up_incoming(pkt) + else: + channel = str(dst_port) + channel_name = f'EXTRA_CHANNEL_{channel}' + try: + await self._extra_channels[channel].queue_up_incoming(pkt) + except KeyError: + await self.open_channel(channel, initial_state=ChannelState.OPENED) + try: + await self._extra_channels[channel].queue_up_incoming(pkt) + except KeyError: + self._debug(f'Tecemux/MAIN: [<] Unknown channel') + + self._debug(f'Tecemux/MAIN: [<] Packet {Tecemux._chunk_preview(single_packet_buffer)} forwarded to {channel_name} stream') + buffer = buffer[current_packet_size:] + else: + self._warning('Tecemux/MAIN: [<] Not full packet received. Getting additional data chunk') + break + except asyncio.TimeoutError: + if self._global_stop_channel_event.is_set(): + break + await asyncio.sleep(0) + + except asyncio.CancelledError: + break + + self._debug('Tecemux/MAIN: Incoming data forwarder finished') + + async def outcoming_data_forward(self) -> None: + """Loop for outcoming data to Transform Hub + """ + + while True: + try: + pkt = await asyncio.wait_for(self._queue.get(),timeout=1) + self._queue.task_done() + + # inject sequence number + if pkt.segment.seq == 0: + pkt.segment.seq = self._sequence_number + self._sequence_number += 1 + + chunk = pkt.build( + for_STH=True).to_buffer_with_tcp_pseudoheader() + + self._debug(f'Tecemux/MAIN: [>] Outcoming chunk {Tecemux._chunk_preview(chunk)} is waiting to send to Transform Hub') + self._writer.write(chunk) + await self._writer.drain() + self._debug(f'Tecemux/MAIN: [>] Chunk {Tecemux._chunk_preview(chunk)} with sequence number: {pkt.segment.seq} was sent to Transform Hub') + except asyncio.QueueEmpty: + if self._global_stop_outcoming_event.is_set(): + break + await asyncio.sleep(0) + + except asyncio.TimeoutError: + if self._queue.empty() and self._global_stop_outcoming_event.is_set(): + return + await asyncio.sleep(0) + + except asyncio.CancelledError: + break + + self._debug('Tecemux/MAIN: Outcoming data forwarder finished') diff --git a/packages/python-runner/tecemux/proxy.py b/packages/python-runner/tecemux/proxy.py new file mode 100644 index 000000000..3ed47895c --- /dev/null +++ b/packages/python-runner/tecemux/proxy.py @@ -0,0 +1,102 @@ +""" proxy.py - contains Tecemux HTTP proxy to connect with 3rd party Python packages +""" + +import asyncio +import base64 +import socket + + +class HTTPProxy: + """ Simple HTTP proxy implementation for internal usage by Tecemux procotol + """ + + def __init__(self): + """Initialize HTTP proxy + """ + + self._host = '127.0.0.1' + self._proxy_socket = None + self._port = None + + @staticmethod + def _get_headers_as_dict(request_headers, convert_keys_to_lowercase=False): + headers_list = request_headers.decode().strip().split('\r\n') + + return {(key.strip().lower() if convert_keys_to_lowercase + else key.strip()) : val.strip() for key,val in [el.split(': ') for el in headers_list] } + + @staticmethod + def _extract_tecemux_details(request_headers): + + headers = HTTPProxy._get_headers_as_dict(request_headers) + + tecemux_params = base64.b64decode(headers['Proxy-Authorization'].split(' ')[1]).decode().split(':') + + del headers['Proxy-Authorization'] + + new_headers = ('\r\n'.join(key +': '+ str(val) for key, val in headers.items())+'\r\n\r\n').encode('utf-8') + + return type('TecemuxDetails', (object,), {'user' : tecemux_params[0], 'channel_id': tecemux_params[1]})() , new_headers + + @staticmethod + async def handle_request(reader, writer, protocol): + """Process single request to server + + Args: + reader (asyncio.StreamReader): Stream reader provides access to HTTP request data + writer (asyncio.StreamWriter): Stream writer give posibility to send response + protocol (Tecemux): Tecemux object + """ + + request_status = await reader.readuntil(b'\r\n') + + tecemux_params, request_headers = HTTPProxy._extract_tecemux_details(await reader.readuntil(b'\r\n\r\n')) + + channel = protocol.get_channel(tecemux_params.channel_id) + + channel.write(request_status) + channel.write(request_headers) + + raw_response_status = await channel.readuntil(b'\r\n') + writer.write(raw_response_status) + + raw_response_headers = await channel.readuntil(b'\r\n\r\n') + writer.write(raw_response_headers) + + headers = HTTPProxy._get_headers_as_dict(raw_response_headers, convert_keys_to_lowercase=True) + + raw_response_data = await channel.read(int(headers['content-length'])) + writer.write(raw_response_data) + + writer.write(b'\r\n') + await writer.drain() + + async def run(self, protocol): + """Starts server on random local TCP port + + Args: + protocol (Tecemux): Tecemux object + """ + + self._proxy_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._proxy_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._proxy_socket.bind((self._host, 0)) + self._proxy_socket.listen() + + self._port = int(self._proxy_socket.getsockname()[1]) + + server = await asyncio.start_server(lambda r, w: HTTPProxy.handle_request(r, w, protocol), sock=self._proxy_socket) + + async with server: + await server.serve_forever() + + self._proxy_socket.close() + + def get_proxy_uri(self): + """Returns proxy config URI for aiohttp + + Returns: + str: aiohttp proxy config + """ + + return f'http://{self._host}:{self._port}' diff --git a/packages/python-runner/test/conftest.py b/packages/python-runner/test/conftest.py index eccb2af23..0ebdaa9c4 100644 --- a/packages/python-runner/test/conftest.py +++ b/packages/python-runner/test/conftest.py @@ -1,7 +1,7 @@ import pytest import sys from logging_setup import LoggingSetup -from tecemux import Tecemux +from tecemux.multiplexer import Tecemux def get_logger(): diff --git a/packages/python-runner/test/test_tecemux.py b/packages/python-runner/test/test_channels.py similarity index 87% rename from packages/python-runner/test/test_tecemux.py rename to packages/python-runner/test/test_channels.py index 2b96cb35f..1b13c9937 100644 --- a/packages/python-runner/test/test_tecemux.py +++ b/packages/python-runner/test/test_channels.py @@ -2,35 +2,16 @@ import codecs import pytest import sys -from tecemux import Tecemux, _ChannelContext -from inet import IPPacket, TCPSegment -from hardcoded_magic_values import CommunicationChannels as CC +from tecemux.channel import ChannelState +from tecemux.inet import IPPacket, TCPSegment +from tecemux.hardcoded_magic_values import CommunicationChannels as CC -class TestTecemux: +class TestChannels: async def _close_clients(self, a, b): await a.stop() await b.stop() - - def test_default_init(self): - protocol = Tecemux() - assert isinstance(protocol, Tecemux) - assert protocol._sequence_number == 0 - - @pytest.mark.asyncio - async def test_socket_connection(self): - protocol = Tecemux() - - assert protocol._reader == None - assert protocol._writer == None - - await protocol.connect(*await Tecemux.prepare_socket_connection()) - - assert isinstance(protocol._reader, asyncio.StreamReader) - assert isinstance(protocol._writer, asyncio.StreamWriter) - assert protocol._sequence_number > 0 - @pytest.mark.asyncio async def test_forward_from_main_to_channel(self, local_socket_connection): client_a, client_b = local_socket_connection @@ -130,7 +111,7 @@ async def test_stderr_write_redirection(self, local_socket_connection): async def test_extra_channel(self, local_socket_connection): client_a, client_b = local_socket_connection - test_channel = await client_a.open_channel(initial_state=_ChannelContext._ChannelState.OPENED) + test_channel = await client_a.open_channel(initial_state=ChannelState.OPENED) test_channel.write("{'foo':'bar'}\n") await asyncio.sleep(1) @@ -147,7 +128,7 @@ async def test_many_extra_channels(self, local_socket_connection): test_func = lambda: str(client_a._get_unused_extra_channel_id([ int(id) for id in client_a._extra_channels.keys() ]) ) - [ await client_a.open_channel(initial_state=_ChannelContext._ChannelState.OPENED) for _ in range(5)] + [ await client_a.open_channel(initial_state=ChannelState.OPENED) for _ in range(5)] assert len(client_a._extra_channels) == 5 @@ -159,7 +140,7 @@ async def test_many_extra_channels(self, local_socket_connection): assert test_func() == '13' - await client_a.open_channel(initial_state=_ChannelContext._ChannelState.OPENED) + await client_a.open_channel(initial_state=ChannelState.OPENED) assert len(client_a._extra_channels) == 5 @@ -171,7 +152,7 @@ async def test_write_and_read_from_the_same_side_on_extra_channel(self, local_so client_a, client_b = local_socket_connection - channel = await client_a.open_channel(initial_state=_ChannelContext._ChannelState.OPENED) + channel = await client_a.open_channel(initial_state=ChannelState.OPENED) channel.write("{'foo':'bar'}") diff --git a/packages/python-runner/test/test_inet.py b/packages/python-runner/test/test_inet.py index 42c5d9be6..f5d543dbb 100644 --- a/packages/python-runner/test/test_inet.py +++ b/packages/python-runner/test/test_inet.py @@ -1,5 +1,5 @@ import pytest -from inet import TCPSegment, IPPacket, EthernetFrame, SequenceOrder +from tecemux.inet import TCPSegment, IPPacket, EthernetFrame, SequenceOrder class TestIP: def test_mf_df_flags(self): diff --git a/packages/python-runner/test/test_multiplexer.py b/packages/python-runner/test/test_multiplexer.py new file mode 100644 index 000000000..e5b21628a --- /dev/null +++ b/packages/python-runner/test/test_multiplexer.py @@ -0,0 +1,27 @@ +import asyncio +import pytest +from tecemux.multiplexer import Tecemux + +class TestMultiplexer: + async def _close_clients(self, a, b): + await a.stop() + await b.stop() + + def test_default_init(self): + protocol = Tecemux() + assert isinstance(protocol, Tecemux) + assert protocol._sequence_number == 0 + + @pytest.mark.asyncio + async def test_socket_connection(self): + + protocol = Tecemux() + + assert protocol._reader == None + assert protocol._writer == None + + await protocol.connect(*await Tecemux.prepare_socket_connection()) + + assert isinstance(protocol._reader, asyncio.StreamReader) + assert isinstance(protocol._writer, asyncio.StreamWriter) + assert protocol._sequence_number > 0 \ No newline at end of file From 9962668a09cac2fc9fb069cf36c2ff02f3be07d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Fri, 1 Sep 2023 16:16:20 +0200 Subject: [PATCH 216/231] Move Python tecemux tests --- packages/python-runner/{ => tecemux}/test/__init__.py | 0 packages/python-runner/{ => tecemux}/test/conftest.py | 0 packages/python-runner/{ => tecemux}/test/test_channels.py | 0 packages/python-runner/{ => tecemux}/test/test_inet.py | 0 packages/python-runner/{ => tecemux}/test/test_multiplexer.py | 0 packages/python-runner/{ => tecemux}/test/test_proxy.py | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename packages/python-runner/{ => tecemux}/test/__init__.py (100%) rename packages/python-runner/{ => tecemux}/test/conftest.py (100%) rename packages/python-runner/{ => tecemux}/test/test_channels.py (100%) rename packages/python-runner/{ => tecemux}/test/test_inet.py (100%) rename packages/python-runner/{ => tecemux}/test/test_multiplexer.py (100%) rename packages/python-runner/{ => tecemux}/test/test_proxy.py (100%) diff --git a/packages/python-runner/test/__init__.py b/packages/python-runner/tecemux/test/__init__.py similarity index 100% rename from packages/python-runner/test/__init__.py rename to packages/python-runner/tecemux/test/__init__.py diff --git a/packages/python-runner/test/conftest.py b/packages/python-runner/tecemux/test/conftest.py similarity index 100% rename from packages/python-runner/test/conftest.py rename to packages/python-runner/tecemux/test/conftest.py diff --git a/packages/python-runner/test/test_channels.py b/packages/python-runner/tecemux/test/test_channels.py similarity index 100% rename from packages/python-runner/test/test_channels.py rename to packages/python-runner/tecemux/test/test_channels.py diff --git a/packages/python-runner/test/test_inet.py b/packages/python-runner/tecemux/test/test_inet.py similarity index 100% rename from packages/python-runner/test/test_inet.py rename to packages/python-runner/tecemux/test/test_inet.py diff --git a/packages/python-runner/test/test_multiplexer.py b/packages/python-runner/tecemux/test/test_multiplexer.py similarity index 100% rename from packages/python-runner/test/test_multiplexer.py rename to packages/python-runner/tecemux/test/test_multiplexer.py diff --git a/packages/python-runner/test/test_proxy.py b/packages/python-runner/tecemux/test/test_proxy.py similarity index 100% rename from packages/python-runner/test/test_proxy.py rename to packages/python-runner/tecemux/test/test_proxy.py From e8ee060ff5ca2e101783673e7f459e316f29316d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Mon, 4 Sep 2023 11:21:55 +0200 Subject: [PATCH 217/231] Format code --- .vscode/settings.json | 14 ++- packages/python-runner/tecemux/barrier.py | 11 +- packages/python-runner/tecemux/channel.py | 28 +++-- .../tecemux/hardcoded_magic_values.py | 2 + packages/python-runner/tecemux/inet.py | 32 +++--- packages/python-runner/tecemux/multiplexer.py | 70 ++++++------ packages/python-runner/tecemux/proxy.py | 18 +-- .../python-runner/tecemux/test/conftest.py | 2 +- .../tecemux/test/test_channels.py | 92 ++++++++-------- .../python-runner/tecemux/test/test_inet.py | 104 ++++++++++-------- .../tecemux/test/test_multiplexer.py | 11 +- .../python-runner/tecemux/test/test_proxy.py | 19 ++-- 12 files changed, 223 insertions(+), 180 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index c2e6effdb..fbc0848fd 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -13,5 +13,17 @@ "markdown", "latex", "plaintext" - ] + ], + "python.testing.pytestArgs": [ + "packages" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "python.linting.flake8Enabled": true, + "python.linting.enabled": true, + "flake8.args": [ + "--max-line-length=120", + "--ignore=F401", + ], + } diff --git a/packages/python-runner/tecemux/barrier.py b/packages/python-runner/tecemux/barrier.py index 3e0077413..42a3fe1ac 100644 --- a/packages/python-runner/tecemux/barrier.py +++ b/packages/python-runner/tecemux/barrier.py @@ -1,6 +1,8 @@ import enum from asyncio import Condition +# asyncio.barrier - only for compatibility with Python older than 3.11 + class BrokenBarrierError(RuntimeError): """Barrier is broken by barrier.abort() call.""" @@ -19,7 +21,7 @@ def __init__(self, parties): if parties < 1: raise ValueError('parties must be > 0') - self._cond = Condition() # notify all tasks when state changes + self._cond = Condition() # notify all tasks when state changes self._parties = parties self._state = _BarrierState.FILLING @@ -40,7 +42,7 @@ async def __aexit__(self, *args): async def wait(self): async with self._cond: - await self._block() # Block while the barrier drains or resets. + await self._block() # Block while the barrier drains or resets. try: index = self._count self._count += 1 @@ -88,7 +90,7 @@ async def reset(self): async with self._cond: if self._count > 0: if self._state is not _BarrierState.RESETTING: - #reset the barrier, waking up tasks + # reset the barrier, waking up tasks self._state = _BarrierState.RESETTING else: self._state = _BarrierState.FILLING @@ -99,7 +101,6 @@ async def abort(self): self._state = _BarrierState.BROKEN self._cond.notify_all() - @property def parties(self): """Return the number of tasks required to trip the barrier.""" @@ -115,4 +116,4 @@ def n_waiting(self): @property def broken(self): """Return True if the barrier is in a broken state.""" - return self._state is _BarrierState.BROKEN \ No newline at end of file + return self._state is _BarrierState.BROKEN diff --git a/packages/python-runner/tecemux/channel.py b/packages/python-runner/tecemux/channel.py index 0aefa58cb..f53763ff2 100644 --- a/packages/python-runner/tecemux/channel.py +++ b/packages/python-runner/tecemux/channel.py @@ -111,7 +111,7 @@ class ChannelContext: def __init__(self, channel, queue: asyncio.Queue, - instance_id: str, + instance_id: str, logger: logging.Logger, stop_event: asyncio.Event, sync_event: asyncio.Event, @@ -127,7 +127,8 @@ def __init__(self, channel, self._stop_channel_event = stop_event self._sync_channel_event = sync_event self._sync_barrier = sync_barrier - self._outcoming_process_task = asyncio.create_task(self._outcoming_process_tasks(), name=f'{self._get_channel_name()}_PROCESS_TASK') + self._outcoming_process_task = asyncio.create_task(self._outcoming_process_tasks(), + name=f'{self._get_channel_name()}_PROCESS_TASK') self._read_buffer = bytearray() self._state = state @@ -258,7 +259,7 @@ async def _get_data(self) -> bool: while True: try: - buf = self._internal_incoming_queue.get_nowait() + buf = self._internal_incoming_queue.get_nowait() if buf == b'': return False @@ -314,7 +315,8 @@ async def send_ACK(self, sequence_number: int) -> None: sequence_number (int): Value for Acknowledge field """ - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK'], ack=sequence_number))) + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), + flags=['ACK'], ack=sequence_number))) async def _send_pause_ACK(self, sequence_number: int) -> None: """"Add to global queue an pause packet to send @@ -323,7 +325,8 @@ async def _send_pause_ACK(self, sequence_number: int) -> None: sequence_number (int): Value for Acknowledge field """ - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), flags=['ACK', 'SYN'], ack=sequence_number))) + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), + flags=['ACK', 'SYN'], ack=sequence_number))) async def open(self) -> None: """Open channel @@ -338,7 +341,8 @@ async def open(self) -> None: else: buf = b'' - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=buf, flags=['PSH']))) + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), + data=buf, flags=['PSH']))) self._state = ChannelState.OPENED @@ -347,7 +351,8 @@ async def end(self) -> None: """ await self._internal_outcoming_queue.join() - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'', flags=['PSH']))) + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), + data=b'', flags=['PSH']))) self._state = ChannelState.ENDED async def close(self) -> None: @@ -355,7 +360,9 @@ async def close(self) -> None: """ self._debug(f'Tecemux/{self._get_channel_name()}: [-] Channel close request is send') - await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), data=b'' if self._global_instance_id is None else self._global_instance_id, flags=['FIN']))) + await self._global_queue.put(IPPacket(segment=TCPSegment(dst_port=self._get_channel_id(), + data=b'' if self._global_instance_id is None + else self._global_instance_id, flags=['FIN']))) async def queue_up_incoming(self, pkt: IPPacket) -> None: """Redirects incoming data from provided packet to current channel @@ -397,11 +404,12 @@ def wrap(channel_id, buf): if self._state is not ChannelState.PAUSED: while not self._internal_outcoming_queue.empty(): try: - buf = await asyncio.wait_for(self._internal_outcoming_queue.get(),1) + buf = await asyncio.wait_for(self._internal_outcoming_queue.get(), 1) await self._global_queue.put(wrap(self._get_channel_id(), buf)) self._internal_outcoming_queue.task_done() except asyncio.QueueEmpty: - self._debug(f'Tecemux/{self._get_channel_name()}: [-] All data stored during pause were redirected to global queue') + self._debug(f'Tecemux/{self._get_channel_name()}:' + '[-] All data stored during pause were redirected to global queue') break except asyncio.TimeoutError: pass diff --git a/packages/python-runner/tecemux/hardcoded_magic_values.py b/packages/python-runner/tecemux/hardcoded_magic_values.py index 12625a6f8..639c69932 100644 --- a/packages/python-runner/tecemux/hardcoded_magic_values.py +++ b/packages/python-runner/tecemux/hardcoded_magic_values.py @@ -1,5 +1,6 @@ from enum import Enum + # See packages/symbols/src/communication-channel.ts class CommunicationChannels(Enum): STDIN = "0" @@ -12,6 +13,7 @@ class CommunicationChannels(Enum): LOG = "7" HOST = "8" + # See packages/symbols/src/runner-message-code.ts class RunnerMessageCodes(Enum): PING = 3000 diff --git a/packages/python-runner/tecemux/inet.py b/packages/python-runner/tecemux/inet.py index d492bb866..fac8cc0cb 100644 --- a/packages/python-runner/tecemux/inet.py +++ b/packages/python-runner/tecemux/inet.py @@ -24,7 +24,7 @@ class SequenceOrder(metaclass=_Singleton): """ class _Order: """ Contains suported orders. For internal usage only - """ + """ LITTLE_ENDIAN = '<' BIG_ENDIAN = '>' @@ -58,7 +58,7 @@ class TCPSegment: class Options: """ TCP Segments options, at the end of TCP Header - """ + """ EOL = 0 NOP = 1 @@ -181,10 +181,11 @@ def from_buffer(cls, buffer: bytes): return cls(src_port, dst_port, seq, ack, offres, flags, win, checksum, urp, buffer[TCP_MIN:], b'') if hdr_len > TCP_MIN: - return cls(src_port, dst_port, seq, ack, offres, flags, win, checksum, urp, buffer[hdr_len:], buffer[TCP_MIN:hdr_len]) + return cls(src_port, dst_port, seq, ack, offres, flags, win, + checksum, urp, buffer[hdr_len:], buffer[TCP_MIN:hdr_len]) def to_buffer(self): - """Build raw buffer from TCP Segment object + """Build raw buffer from TCP Segment object Returns: bytes: Raw buffer @@ -208,7 +209,7 @@ def set_flags(self, list_of_flags: int): Returns: TCPObject: Retuns self - """ + """ self.flags = list_of_flags return self @@ -312,15 +313,15 @@ def calc_checksum(pkt: bytes) -> int: """Calculates checksum for provited packet Args: - pkt (bytes): Raw buffer + pkt (bytes): Raw buffer Returns: int: Calculated checsum - """ + """ if len(pkt) % 2 == 1: pkt += b"\0" s = sum(struct.unpack(('<' if SequenceOrder().get() - is '>' else '>')+str(len(pkt)//2)+'H', pkt)) + == '>' else '>')+str(len(pkt)//2)+'H', pkt)) # source: https://github.com/secdev/scapy s = (s >> 16) + (s & 0xffff) @@ -337,7 +338,7 @@ def calc_checksum_for_STH(pkt: bytes) -> int: Returns: int: Calculated checksum - """ + """ if len(pkt) % 2 == 1: pkt += b"\0" elements = list(struct.unpack( @@ -347,8 +348,8 @@ def calc_checksum_for_STH(pkt: bytes) -> int: return s % 0x10000 @classmethod - def from_buffer_with_pseudoheader(cls, buffer:bytes): - """Creates IP Packet object and TCP Segment object + def from_buffer_with_pseudoheader(cls, buffer: bytes): + """Creates IP Packet object and TCP Segment object from provided raw buffer with pseudo TCP Header Args: @@ -365,7 +366,7 @@ def from_buffer_with_pseudoheader(cls, buffer:bytes): @classmethod def from_buffer(cls, buffer: bytes): - """Creates IP Packet object and TCP Segment object + """Creates IP Packet object and TCP Segment object from provided raw buffer with Args: @@ -390,7 +391,7 @@ def is_flag(self, flag: Flags) -> bool: """ return (self.flags & getattr(IPPacket.Flags, flag)) > 0 - def prepare_pseudoheader(self, protocol:int, length:int) -> bytes: + def prepare_pseudoheader(self, protocol: int, length: int) -> bytes: """Prepare TCP pseudoheader to calulate checksum Args: @@ -487,7 +488,7 @@ def build(self, for_STH=False): """Calculates all checksums Args: - for_STH (bool, optional): If True, cheksums calculates with format valid for Transform Hub. + for_STH (bool, optional): If True, cheksums calculates with format valid for Transform Hub. Otherise it is calcuated with RFC. Defaults to False. """ if for_STH: @@ -537,7 +538,8 @@ def to_buffer(self) -> bytes: Returns: bytes: Raw buffer """ - return struct.pack(SequenceOrder().get()+"6s6s2s", unhexlify(self.src_mac), unhexlify(self.dst_mac), self.eth_type) + self.packet.to_buffer() + return struct.pack(SequenceOrder().get()+"6s6s2s", unhexlify(self.src_mac), + unhexlify(self.dst_mac), self.eth_type) + self.packet.to_buffer() def get_packet(self): """Return IP Packet object diff --git a/packages/python-runner/tecemux/multiplexer.py b/packages/python-runner/tecemux/multiplexer.py index 66e399f1f..9b57c6434 100644 --- a/packages/python-runner/tecemux/multiplexer.py +++ b/packages/python-runner/tecemux/multiplexer.py @@ -109,19 +109,20 @@ async def prepare(self, force_open: bool = False) -> None: """Opens all channels to Transform Hub Args: - force_open (bool, optional): If True, all channels will be opened immediately, Otherwise, will be opened on demand. Defaults to False. + force_open (bool, optional): If True, all channels will be opened immediately, + Otherwise, will be opened on demand. Defaults to False. """ self._extra_channels = {} self._queue = asyncio.Queue() self._global_sync_barrier = Barrier(len(CC)) self._required_channels = {channel: ChannelContext(channel, - self._queue, - self._instance_id, - self._logger, - self._global_stop_channel_event, - self._global_sync_channel_event, - self._global_sync_barrier, - ChannelState.CREATED) for channel in CC} + self._queue, + self._instance_id, + self._logger, + self._global_stop_channel_event, + self._global_sync_channel_event, + self._global_sync_barrier, + ChannelState.CREATED) for channel in CC} if force_open: [await channel.open() for channel in self.get_required_channels()] @@ -137,7 +138,7 @@ def _get_unused_extra_channel_id(self, used_channel_ids, start_from=10): Returns: int: lowest, unused channel number - """ + """ used_channel_ids = sorted(set(used_channel_ids)) if len(used_channel_ids) == 0 or used_channel_ids[0] != start_from: return start_from @@ -146,7 +147,8 @@ def _get_unused_extra_channel_id(self, used_channel_ids, start_from=10): return i return i+1 - async def open_channel(self, channel_id=None, force_open=False, initial_state:ChannelState = ChannelState.CREATED) -> ChannelContext: + async def open_channel(self, channel_id=None, force_open=False, + initial_state: ChannelState = ChannelState.CREATED) -> ChannelContext: """Opens additional channel in Tecemux Args: @@ -160,18 +162,20 @@ async def open_channel(self, channel_id=None, force_open=False, initial_state:Ch channel = str(channel_id) if channel_id is not None else None - if not channel in self._extra_channels.keys(): + if channel not in self._extra_channels.keys(): - channel = str(self._get_unused_extra_channel_id([ int(id) for id in self._extra_channels.keys() ]) ) if channel is None else channel + channel = str(self._get_unused_extra_channel_id([int(id) + for id in self._extra_channels.keys()])) \ + if channel is None else channel self._extra_channels[channel] = ChannelContext(channel, - self._queue, - self._instance_id, - self._logger, - self._global_stop_channel_event, - self._global_sync_channel_event, - self._global_sync_barrier, - initial_state) + self._queue, + self._instance_id, + self._logger, + self._global_stop_channel_event, + self._global_sync_channel_event, + self._global_sync_barrier, + initial_state) if force_open: await self._extra_channels[channel].open() @@ -222,7 +226,7 @@ def get_required_channels(self) -> dict: return self._required_channels.values() def get_extra_channels(self) -> dict: - """Returns only extra initiated channels by runner + """Returns only extra initiated channels by runner Returns: dict: Dict of channel's contexts @@ -245,7 +249,7 @@ async def stop(self) -> None: """ Stops protocol """ - await self._finish_proxy() + await self._finish_proxy() await self._finish_channels() await self._finish_incoming() await self._finish_outcoming() @@ -269,7 +273,7 @@ async def _finish_channels(self) -> None: for channel in self.get_extra_channels(): await channel.end() - await channel.close() + await channel.close() for channel in self.get_required_channels(): await channel._internal_outcoming_queue.join() @@ -281,7 +285,7 @@ async def _finish_channels(self) -> None: await self._global_stop_channel_event.wait() for channel in self.get_required_channels(): try: - await asyncio.wait_for(channel._outcoming_process_task,timeout=1) + await asyncio.wait_for(channel._outcoming_process_task, timeout=1) except asyncio.TimeoutError: pass except TypeError: @@ -294,7 +298,7 @@ async def _finish_outcoming(self) -> None: self._global_stop_outcoming_event.set() await asyncio.sleep(0) await self._global_stop_outcoming_event.wait() - await asyncio.gather(*[self._outcoming_data_forwarder]) + await asyncio.gather(*[self._outcoming_data_forwarder]) await self._writer.drain() self._writer.close() await self._writer.wait_closed() @@ -329,7 +333,7 @@ async def incoming_data_forward(self) -> None: while not self._global_stop_channel_event.is_set(): try: - chunk = await asyncio.wait_for(self._reader.read(READ_CHUNK_SIZE),1) + chunk = await asyncio.wait_for(self._reader.read(READ_CHUNK_SIZE), 1) if not chunk: incoming_parser_finish_loop.set() @@ -354,7 +358,8 @@ async def incoming_data_forward(self) -> None: single_packet_buffer = buffer[:current_packet_size] pkt = IPPacket().from_buffer_with_pseudoheader(single_packet_buffer) self._last_sequence_received = pkt.get_segment().seq - self._debug(f'Tecemux/MAIN: [<] Full incoming packet with sequence number {self._last_sequence_received} from Transform Hub was received') + self._debug(f'Tecemux/MAIN: [<] Full incoming packet with sequence number' + f'{self._last_sequence_received} from Transform Hub was received') dst_port = pkt.get_segment().dst_port @@ -372,9 +377,10 @@ async def incoming_data_forward(self) -> None: try: await self._extra_channels[channel].queue_up_incoming(pkt) except KeyError: - self._debug(f'Tecemux/MAIN: [<] Unknown channel') + self._debug('Tecemux/MAIN: [<] Unknown channel') - self._debug(f'Tecemux/MAIN: [<] Packet {Tecemux._chunk_preview(single_packet_buffer)} forwarded to {channel_name} stream') + self._debug(f'Tecemux/MAIN: [<] Packet {Tecemux._chunk_preview(single_packet_buffer)}' + f' forwarded to {channel_name} stream') buffer = buffer[current_packet_size:] else: self._warning('Tecemux/MAIN: [<] Not full packet received. Getting additional data chunk') @@ -395,7 +401,7 @@ async def outcoming_data_forward(self) -> None: while True: try: - pkt = await asyncio.wait_for(self._queue.get(),timeout=1) + pkt = await asyncio.wait_for(self._queue.get(), timeout=1) self._queue.task_done() # inject sequence number @@ -406,10 +412,12 @@ async def outcoming_data_forward(self) -> None: chunk = pkt.build( for_STH=True).to_buffer_with_tcp_pseudoheader() - self._debug(f'Tecemux/MAIN: [>] Outcoming chunk {Tecemux._chunk_preview(chunk)} is waiting to send to Transform Hub') + self._debug(f'Tecemux/MAIN: [>] Outcoming chunk {Tecemux._chunk_preview(chunk)}' + ' is waiting to send to Transform Hub') self._writer.write(chunk) await self._writer.drain() - self._debug(f'Tecemux/MAIN: [>] Chunk {Tecemux._chunk_preview(chunk)} with sequence number: {pkt.segment.seq} was sent to Transform Hub') + self._debug(f'Tecemux/MAIN: [>] Chunk {Tecemux._chunk_preview(chunk)} with sequence number:' + f' {pkt.segment.seq} was sent to Transform Hub') except asyncio.QueueEmpty: if self._global_stop_outcoming_event.is_set(): break diff --git a/packages/python-runner/tecemux/proxy.py b/packages/python-runner/tecemux/proxy.py index 3ed47895c..fa602519c 100644 --- a/packages/python-runner/tecemux/proxy.py +++ b/packages/python-runner/tecemux/proxy.py @@ -22,28 +22,29 @@ def __init__(self): def _get_headers_as_dict(request_headers, convert_keys_to_lowercase=False): headers_list = request_headers.decode().strip().split('\r\n') - return {(key.strip().lower() if convert_keys_to_lowercase - else key.strip()) : val.strip() for key,val in [el.split(': ') for el in headers_list] } + return {(key.strip().lower() if convert_keys_to_lowercase + else key.strip()): val.strip() for key, val in [el.split(': ') for el in headers_list]} @staticmethod def _extract_tecemux_details(request_headers): headers = HTTPProxy._get_headers_as_dict(request_headers) - tecemux_params = base64.b64decode(headers['Proxy-Authorization'].split(' ')[1]).decode().split(':') + tecemux_params = base64.b64decode(headers['Proxy-Authorization'].split(' ')[1]).decode().split(':') del headers['Proxy-Authorization'] - new_headers = ('\r\n'.join(key +': '+ str(val) for key, val in headers.items())+'\r\n\r\n').encode('utf-8') + new_headers = ('\r\n'.join(key + ': ' + str(val) for key, val in headers.items())+'\r\n\r\n').encode('utf-8') - return type('TecemuxDetails', (object,), {'user' : tecemux_params[0], 'channel_id': tecemux_params[1]})() , new_headers + return type('TecemuxDetails', (object,), {'user': tecemux_params[0], + 'channel_id': tecemux_params[1]})(), new_headers @staticmethod async def handle_request(reader, writer, protocol): """Process single request to server Args: - reader (asyncio.StreamReader): Stream reader provides access to HTTP request data + reader (asyncio.StreamReader): Stream reader provides access to HTTP request data writer (asyncio.StreamWriter): Stream writer give posibility to send response protocol (Tecemux): Tecemux object """ @@ -76,7 +77,7 @@ async def run(self, protocol): Args: protocol (Tecemux): Tecemux object - """ + """ self._proxy_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self._proxy_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) @@ -85,7 +86,8 @@ async def run(self, protocol): self._port = int(self._proxy_socket.getsockname()[1]) - server = await asyncio.start_server(lambda r, w: HTTPProxy.handle_request(r, w, protocol), sock=self._proxy_socket) + server = await asyncio.start_server(lambda r, w: HTTPProxy.handle_request(r, w, protocol), + sock=self._proxy_socket) async with server: await server.serve_forever() diff --git a/packages/python-runner/tecemux/test/conftest.py b/packages/python-runner/tecemux/test/conftest.py index 0ebdaa9c4..8b797ea6f 100644 --- a/packages/python-runner/tecemux/test/conftest.py +++ b/packages/python-runner/tecemux/test/conftest.py @@ -30,4 +30,4 @@ async def local_socket_connection(): await client_a.loop() await client_b.loop() - return client_a, client_b \ No newline at end of file + return client_a, client_b diff --git a/packages/python-runner/tecemux/test/test_channels.py b/packages/python-runner/tecemux/test/test_channels.py index 1b13c9937..df619d105 100644 --- a/packages/python-runner/tecemux/test/test_channels.py +++ b/packages/python-runner/tecemux/test/test_channels.py @@ -6,21 +6,23 @@ from tecemux.inet import IPPacket, TCPSegment from tecemux.hardcoded_magic_values import CommunicationChannels as CC + class TestChannels: async def _close_clients(self, a, b): await a.stop() await b.stop() - + @pytest.mark.asyncio async def test_forward_from_main_to_channel(self, local_socket_connection): client_a, client_b = local_socket_connection - data_to_send ="{'foo':'bar'}" + data_to_send = "{'foo':'bar'}" destination_channel = CC.CONTROL - - pkt = IPPacket(src_addr='172.25.44.3', dst_addr='172.25.44.254', segment=TCPSegment(dst_port=int(destination_channel.value), flags=['PSH'], data=data_to_send)) - + + pkt = IPPacket(src_addr='172.25.44.3', dst_addr='172.25.44.254', + segment=TCPSegment(dst_port=int(destination_channel.value), flags=['PSH'], data=data_to_send)) + client_a._writer.write(pkt.to_buffer_with_tcp_pseudoheader()) await client_a._writer.drain() @@ -58,87 +60,88 @@ async def test_write_and_read_from_the_same_side(self, local_socket_connection): data = None with pytest.raises(asyncio.TimeoutError): - data = await asyncio.wait_for(client_a.get_channel(channel_alpha).read(1),timeout=0.5) + data = await asyncio.wait_for(client_a.get_channel(channel_alpha).read(1), + timeout=0.5) assert data is None - + await self._close_clients(client_a, client_b) @pytest.mark.asyncio async def test_stream_write_redirection(self, local_socket_connection): client_a, client_b = local_socket_connection - + stream = codecs.getwriter('utf-8')(client_a.get_channel(CC.HOST)) print("Hi!", end='', file=stream) - + assert (await client_b.get_channel(CC.HOST).read(100)).decode() == "Hi!" await self._close_clients(client_a, client_b) @pytest.mark.asyncio async def test_stream_read_redirection(self, local_socket_connection): - + from scramjet.streams import Stream - + client_a, client_b = local_socket_connection - + input_stream = codecs.getwriter('utf-8')(client_a.get_channel(CC.HOST)) output_stream = Stream.read_from(client_b.get_channel(CC.HOST)).decode('utf-8') print("Test", file=input_stream) assert await output_stream.read() == 'Test\n' - + await self._close_clients(client_a, client_b) - + @pytest.mark.asyncio async def test_stderr_write_redirection(self, local_socket_connection): client_a, client_b = local_socket_connection - + sys.stderr = codecs.getwriter('utf-8')(client_a.get_channel(CC.STDERR)) sys.stderr.flush = lambda: True print("Error", end='\n', file=sys.stderr) sys.stderr = sys.__stderr__ - + assert (await client_b.get_channel(CC.STDERR).read(100)).decode() == "Error" await self._close_clients(client_a, client_b) - @pytest.mark.asyncio async def test_extra_channel(self, local_socket_connection): client_a, client_b = local_socket_connection - + test_channel = await client_a.open_channel(initial_state=ChannelState.OPENED) test_channel.write("{'foo':'bar'}\n") await asyncio.sleep(1) data = await client_b.get_channel(test_channel._get_channel_name()).readuntil() - + assert data.decode() == "{'foo':'bar'}\n" await self._close_clients(client_a, client_b) - + @pytest.mark.asyncio async def test_many_extra_channels(self, local_socket_connection): client_a, client_b = local_socket_connection - - test_func = lambda: str(client_a._get_unused_extra_channel_id([ int(id) for id in client_a._extra_channels.keys() ]) ) - [ await client_a.open_channel(initial_state=ChannelState.OPENED) for _ in range(5)] + def test_func(c): + return str(c._get_unused_extra_channel_id([int(id) for id in client_a._extra_channels.keys()])) + + [await client_a.open_channel(initial_state=ChannelState.OPENED) for _ in range(5)] assert len(client_a._extra_channels) == 5 - assert test_func() == '15' + assert test_func(client_a) == '15' del client_a._extra_channels['13'] assert len(client_a._extra_channels) == 4 - assert test_func() == '13' + assert test_func(client_a) == '13' await client_a.open_channel(initial_state=ChannelState.OPENED) @@ -146,26 +149,24 @@ async def test_many_extra_channels(self, local_socket_connection): await self._close_clients(client_a, client_b) - @pytest.mark.asyncio async def test_write_and_read_from_the_same_side_on_extra_channel(self, local_socket_connection): client_a, client_b = local_socket_connection - channel = await client_a.open_channel(initial_state=ChannelState.OPENED) - + channel.write("{'foo':'bar'}") - + await client_a.sync() await client_a._writer.drain() data = None - + with pytest.raises(asyncio.TimeoutError): - data = await asyncio.wait_for(channel.read(1),timeout=0.5) + data = await asyncio.wait_for(channel.read(1), timeout=0.5) assert data is None - + await self._close_clients(client_a, client_b) @pytest.mark.asyncio @@ -187,21 +188,20 @@ async def test_readuntil(self, local_socket_connection): data = (await client_b.get_channel(channel).readuntil(separator.encode())).decode() assert data == "{'foo':'bar'}{'foo':'bar'}\r\n\r\n" - + await self._close_clients(client_a, client_b) - @pytest.mark.asyncio async def test_wih_scramjet_framework_to_list(self, local_socket_connection): client_a, client_b = local_socket_connection channel = CC.CONTROL - + from scramjet.streams import Stream - + client_a.get_channel(channel).write("foo\n") client_a.get_channel(channel).write("bar\n") client_a.get_channel(channel).write("baz\n") - await client_a.get_channel(channel).end() + await client_a.get_channel(channel).end() output_list = await Stream.read_from(client_b.get_channel(channel)).to_list() @@ -209,19 +209,18 @@ async def test_wih_scramjet_framework_to_list(self, local_socket_connection): await self._close_clients(client_a, client_b) - @pytest.mark.asyncio async def test_wih_scramjet_framework_to_pipe(self, local_socket_connection): client_a, client_b = local_socket_connection channel = CC.CONTROL - + from scramjet.streams import Stream - + client_a.get_channel(channel).write("foo") client_a.get_channel(channel).write("bar\n") client_a.get_channel(channel).write("foz") client_a.get_channel(channel).write("baz\n") - await client_a.get_channel(channel).end() + await client_a.get_channel(channel).end() final_stream = Stream() @@ -233,20 +232,19 @@ async def test_wih_scramjet_framework_to_pipe(self, local_socket_connection): await self._close_clients(client_a, client_b) - @pytest.mark.asyncio async def test_wih_scramjet_framework_write_to(self, local_socket_connection): client_a, client_b = local_socket_connection channel = CC.CONTROL - - data = [b'foo\n',b'bar\n',b'baz\n'] + + data = [b'foo\n', b'bar\n', b'baz\n'] from scramjet.streams import Stream await Stream.read_from(data).write_to(client_a.get_channel(channel)) - await client_a.get_channel(channel).end() + await client_a.get_channel(channel).end() output = Stream.read_from(client_b.get_channel(channel)) assert await output.to_list() == data - - await self._close_clients(client_a, client_b) \ No newline at end of file + + await self._close_clients(client_a, client_b) diff --git a/packages/python-runner/tecemux/test/test_inet.py b/packages/python-runner/tecemux/test/test_inet.py index f5d543dbb..69d85c728 100644 --- a/packages/python-runner/tecemux/test/test_inet.py +++ b/packages/python-runner/tecemux/test/test_inet.py @@ -1,6 +1,7 @@ import pytest from tecemux.inet import TCPSegment, IPPacket, EthernetFrame, SequenceOrder + class TestIP: def test_mf_df_flags(self): @@ -9,9 +10,9 @@ def test_mf_df_flags(self): data = b'E\x00\x00\x14\x00\x01`\x00@\x00\x1c\xe7\x7f\x00\x00\x01\x7f\x00\x00\x01' pkt = IPPacket.from_buffer(data) assert pkt.flags == (IPPacket.Flags.MF | IPPacket.Flags.DF) & ~IPPacket.Flags.RF - assert pkt.is_flag('MF') == True - assert pkt.is_flag('DF') == True - assert pkt.is_flag('RF') == False + assert pkt.is_flag('MF') is True + assert pkt.is_flag('DF') is True + assert pkt.is_flag('RF') is False def test_mf_flags(self): @@ -20,28 +21,28 @@ def test_mf_flags(self): data = b'E\x00\x00\x14\x00\x01 \x00@\x00\\\xe7\x7f\x00\x00\x01\x7f\x00\x00\x01' pkt = IPPacket.from_buffer(data) assert pkt.flags == IPPacket.Flags.MF & ~IPPacket.Flags.RF - assert pkt.is_flag('MF') == True - assert pkt.is_flag('DF') == False - assert pkt.is_flag('RF') == False + assert pkt.is_flag('MF') is True + assert pkt.is_flag('DF') is False + assert pkt.is_flag('RF') is False def test_df_flags(self): SequenceOrder().use_big_endian() data = (b'\x00\x23\x20\xd4\x2a\x8c\x00\x23\x20\xd4\x2a\x8c\x08\x00\x45\x00\x00\x54\x00\x00\x40\x00' - b'\x40\x01\x25\x8d\x0a\x00\x00\x8f\x0a\x00\x00\x8e\x08\x00\x2e\xa0\x01\xff\x23\x73\x20\x48' - b'\x4a\x4d\x00\x00\x00\x00\x78\x85\x02\x00\x00\x00\x00\x00\x10\x11\x12\x13\x14\x15\x16\x17' - b'\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d' - b'\x2e\x2f\x30\x31\x32\x33\x34\x35\x36\x37') + b'\x40\x01\x25\x8d\x0a\x00\x00\x8f\x0a\x00\x00\x8e\x08\x00\x2e\xa0\x01\xff\x23\x73\x20\x48' + b'\x4a\x4d\x00\x00\x00\x00\x78\x85\x02\x00\x00\x00\x00\x00\x10\x11\x12\x13\x14\x15\x16\x17' + b'\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d' + b'\x2e\x2f\x30\x31\x32\x33\x34\x35\x36\x37') pkt = EthernetFrame.from_buffer(data).get_packet() assert pkt.flags == IPPacket.Flags.DF assert pkt.flags == (IPPacket.Flags.DF & (~IPPacket.Flags.MF & ~IPPacket.Flags.RF)) - assert pkt.is_flag('DF') == True - assert pkt.is_flag('MF') == False - assert pkt.is_flag('RF') == False + assert pkt.is_flag('DF') is True + assert pkt.is_flag('MF') is False + assert pkt.is_flag('RF') is False assert pkt.offset == 0 @@ -62,7 +63,9 @@ def test_checksum_calc(self): def test_checksum_calc_as_sth_expected(self): checksum_from_sth = 64434 - data =bytes([10, 0, 0, 1, 10, 0, 0, 2, 0, 1, 69, 0, 0, 0, 0, 0, 32, 89, 3, 38, 0, 0, 0, 0, 0, 8, 0, 32, 178, 251, 0, 0, 97, 102, 54, 98, 50, 51, 52, 56, 45, 102, 51, 56, 56, 45, 52, 54, 56, 48, 45, 98, 54, 48, 97, 45, 97, 53, 102, 98, 50, 102, 98, 100, 100, 102, 50, 97, 48]) + data = bytes([10, 0, 0, 1, 10, 0, 0, 2, 0, 1, 69, 0, 0, 0, 0, 0, 32, 89, 3, 38, 0, 0, 0, 0, 0, 8, 0, 32, + 178, 251, 0, 0, 97, 102, 54, 98, 50, 51, 52, 56, 45, 102, 51, 56, 56, 45, 52, 54, 56, 48, + 45, 98, 54, 48, 97, 45, 97, 53, 102, 98, 50, 102, 98, 100, 100, 102, 50, 97, 48]) SequenceOrder().use_little_endian() @@ -71,7 +74,7 @@ def test_checksum_calc_as_sth_expected(self): pkt.segment.checksum = 0 assert pkt.segment.checksum == 0 - + new_pkt = pkt.build(for_STH=True) assert new_pkt.segment.checksum == checksum_from_sth @@ -87,8 +90,9 @@ def test_packet_creation(self): def test_pakcet_creation_with_endstring(self): buf = IPPacket(src_addr='172.25.44.3', dst_addr='172.25.44.254', - segment=TCPSegment(dst_port=6, flags=['PSH'],data=b'\0')).build(for_STH=True).to_buffer_with_tcp_pseudoheader() - + segment=TCPSegment(dst_port=6, flags=['PSH'], + data=b'\0')).build(for_STH=True).to_buffer_with_tcp_pseudoheader() + pkt = IPPacket.from_buffer_with_pseudoheader(buf) assert pkt.src_addr == '172.25.44.3' @@ -96,10 +100,11 @@ def test_pakcet_creation_with_endstring(self): assert pkt.dst_addr == '172.25.44.254' def test_pseudoheader(self): - + SequenceOrder().use_little_endian() - data = bytes([10, 0, 0, 1, 10, 0, 0, 2, 0, 1, 32, 0, 0, 0, 0, 0, 249, 252, 104, 127, 0, 0, 0, 0, 0, 8, 0, 0, 149, 136, 0, 0]) + data = bytes([10, 0, 0, 1, 10, 0, 0, 2, 0, 1, 32, 0, 0, 0, 0, 0, 249, 252, 104, 127, + 0, 0, 0, 0, 0, 8, 0, 0, 149, 136, 0, 0]) res = IPPacket.from_buffer_with_pseudoheader(data) assert res.src_addr == '10.0.0.1' @@ -108,13 +113,13 @@ def test_pseudoheader(self): assert res.protocol == 1 assert res.segment.seq == 2137586937 - assert res.segment.ack == 0 - + assert res.segment.ack == 0 - psh = res.prepare_pseudoheader(1,len(data[12:])) + psh = res.prepare_pseudoheader(1, len(data[12:])) assert psh == data[0:12] + class TestTCP: def test_tcp_offset(self): @@ -130,56 +135,59 @@ def test_prepare_segment_with_flags(self): segment = TCPSegment(flags=['FIN', 'SYN', 'ACK']) - assert segment.is_flag('FIN') == True - assert segment.is_flag('SYN') == True - assert segment.is_flag('ACK') == True - assert segment.is_flag('PSH') == False + assert segment.is_flag('FIN') is True + assert segment.is_flag('SYN') is True + assert segment.is_flag('ACK') is True + assert segment.is_flag('PSH') is False segment = TCPSegment(flags=['SYN', 'PSH']) - assert segment.is_flag('FIN') == False - assert segment.is_flag('SYN') == True - assert segment.is_flag('ACK') == False - assert segment.is_flag('PSH') == True + assert segment.is_flag('FIN') is False + assert segment.is_flag('SYN') is True + assert segment.is_flag('ACK') is False + assert segment.is_flag('PSH') is True segment = TCPSegment().set_flags(['FIN', 'SYN', 'ACK']) - assert segment.is_flag('FIN') == True - assert segment.is_flag('SYN') == True - assert segment.is_flag('ACK') == True - assert segment.is_flag('PSH') == False + assert segment.is_flag('FIN') is True + assert segment.is_flag('SYN') is True + assert segment.is_flag('ACK') is True + assert segment.is_flag('PSH') is False segment = TCPSegment().set_flags(['SYN', 'PSH']) - assert segment.is_flag('FIN') == False - assert segment.is_flag('SYN') == True - assert segment.is_flag('ACK') == False - assert segment.is_flag('PSH') == True + assert segment.is_flag('FIN') is False + assert segment.is_flag('SYN') is True + assert segment.is_flag('ACK') is False + assert segment.is_flag('PSH') is True def test_basic_details(self): - + SequenceOrder().use_big_endian() - data = b'E\x00\x00(\x00\x01@\x00@\x06,\xbe\x01\x02\x03\x04\x04\x05\x06\x07\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x81m\x00\x00' + data = (b'E\x00\x00(\x00\x01@\x00@\x06,\xbe\x01\x02\x03\x04\x04\x05\x06\x07' + b'\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x81m\x00\x00') pkt = IPPacket.from_buffer(data) assert pkt.segment.dst_port == 80 assert pkt.segment.flags & TCPSegment.Flags.SYN - assert pkt.segment.is_flag('SYN') == True - assert pkt.segment.is_flag('PSH') == False + assert pkt.segment.is_flag('SYN') is True + assert pkt.segment.is_flag('PSH') is False def test_checksum(self): - + SequenceOrder().use_big_endian() - data = b'E\x00\x00(\x00\x01@\x00@\x06,\xbe\x01\x02\x03\x04\x04\x05\x06\x07\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x81m\x00\x00' + data = (b'E\x00\x00(\x00\x01@\x00@\x06,\xbe\x01\x02\x03\x04\x04\x05\x06\x07\x00' + b'\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x81m\x00\x00') pkt = IPPacket.from_buffer(data) assert pkt.segment.checksum == 33133 def test_checksum_calc(self): - + SequenceOrder().use_big_endian() - data = b'E\x00\x00*\x00\x01\x00\x00@\x06N\x9d\n\x0b\x0c\x0e\n\x0b\x0c\r\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x00\x00\x00\x00Hi' + data = (b'E\x00\x00*\x00\x01\x00\x00@\x06N\x9d\n\x0b\x0c\x0e\n\x0b\x0c\r\x00\x14' + b'\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x00\x00\x00\x00Hi') pkt = IPPacket.from_buffer(data) assert pkt.segment.checksum == 0 @@ -189,7 +197,7 @@ def test_checksum_calc(self): assert pkt.segment.checksum == 6871 def test_unpack(self): - + SequenceOrder().use_big_endian() data = (b'\x00\x50\x0d\x2c\x11\x4c\x61\x8b\x38\xaf\xfe\x14\x70\x12\x16\xd0' @@ -203,7 +211,7 @@ def test_unpack(self): assert pkt.ack == 951057940 def test_parse_only_first_packet(self): - + SequenceOrder().use_big_endian() data = (b"\x05\x00\x005\x00\x00\x00\x00\xff\x06\xfb\xc3\x00\x00\x00\x00\x00" diff --git a/packages/python-runner/tecemux/test/test_multiplexer.py b/packages/python-runner/tecemux/test/test_multiplexer.py index e5b21628a..e7c7b6007 100644 --- a/packages/python-runner/tecemux/test/test_multiplexer.py +++ b/packages/python-runner/tecemux/test/test_multiplexer.py @@ -2,6 +2,7 @@ import pytest from tecemux.multiplexer import Tecemux + class TestMultiplexer: async def _close_clients(self, a, b): await a.stop() @@ -14,14 +15,14 @@ def test_default_init(self): @pytest.mark.asyncio async def test_socket_connection(self): - + protocol = Tecemux() - assert protocol._reader == None - assert protocol._writer == None - + assert protocol._reader is None + assert protocol._writer is None + await protocol.connect(*await Tecemux.prepare_socket_connection()) assert isinstance(protocol._reader, asyncio.StreamReader) assert isinstance(protocol._writer, asyncio.StreamWriter) - assert protocol._sequence_number > 0 \ No newline at end of file + assert protocol._sequence_number > 0 diff --git a/packages/python-runner/tecemux/test/test_proxy.py b/packages/python-runner/tecemux/test/test_proxy.py index 408257c1d..2fc39a427 100644 --- a/packages/python-runner/tecemux/test/test_proxy.py +++ b/packages/python-runner/tecemux/test/test_proxy.py @@ -1,41 +1,42 @@ import asyncio import pytest + class TestProxy: async def _close_clients(self, a, b): await a.stop() await b.stop() - @pytest.mark.asyncio async def test_extra_channel_with_proxy(self, local_socket_connection): - client_a, client_b = local_socket_connection + client_a, client_b = local_socket_connection async def _wait_for_channel(client_b): - while True: + while True: if len(client_b._extra_channels) > 0: opened_channel_name = list(client_b._extra_channels.keys())[0] channel = client_b.get_channel(opened_channel_name) _ = await channel.readuntil(b'\r\n\r\n') await asyncio.sleep(0.5) - channel.write(b'HTTP/1.1 200 OK\r\nContent-Length: 12\r\nContent-Type: text/html\r\n\r\nHello World!\r\n') + channel.write(b'HTTP/1.1 200 OK\r\nContent-Length: 12\r\n' + b'Content-Type: text/html\r\n\r\nHello World!\r\n') await client_b.sync() break await asyncio.sleep(0) - + task = asyncio.create_task(_wait_for_channel(client_b)) import aiohttp async with client_a.get_channel_guard() as guard: - async with aiohttp.ClientSession() as session: - async with session.get("http://_/api/version", + async with aiohttp.ClientSession() as session: + async with session.get("http://_/api/version", proxy_auth=guard.inject_tecemux_details_as(aiohttp.BasicAuth), proxy=guard.get_proxy_uri()) as resp: data = await resp.text() - + await asyncio.gather(task) assert data == 'Hello World!' - await self._close_clients(client_a, client_b) \ No newline at end of file + await self._close_clients(client_a, client_b) From 0028afe8626550ed6e0ca0e85c98914958e27094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Mon, 4 Sep 2023 11:27:09 +0200 Subject: [PATCH 218/231] Format code #2 --- packages/python-runner/logging_setup.py | 6 +- packages/python-runner/runner.py | 112 +++++++++++------------- 2 files changed, 54 insertions(+), 64 deletions(-) diff --git a/packages/python-runner/logging_setup.py b/packages/python-runner/logging_setup.py index 39a265e02..3a7ba7a2a 100644 --- a/packages/python-runner/logging_setup.py +++ b/packages/python-runner/logging_setup.py @@ -9,7 +9,7 @@ def formatMessage(self, record): # record.created is a float in seconds, we want ms. # Note that using timestamp gives us UTC, as desired. return json.dumps({ - "ts": round(record.created*1000), + "ts": round(record.created * 1000), "level": record.levelname, "from": record.name, "msg": record.message, @@ -29,7 +29,6 @@ def __init__(self, target, min_loglevel=logging.DEBUG) -> None: self.create_handlers(min_loglevel) self.adjust_levels() - def create_handlers(self, min_loglevel): formatter = JsonFormatter() @@ -43,7 +42,6 @@ def create_handlers(self, min_loglevel): self._temp_handler.setFormatter(formatter) self.logger.addHandler(self._temp_handler) - def adjust_levels(self): # Adjust level names according to transform hub convention. logging.addLevelName(logging.WARNING, "WARN") @@ -52,14 +50,12 @@ def adjust_levels(self): # Un-deprecate "warn", as transform-hub uses "warn" rather than "warning". self.logger.warn = self.logger.warning - def switch_target(self, new_target): old_target = self._target self._main_handler.setStream(new_target) self._target = new_target old_target.close() - def flush_temp_handler(self): self._temp_handler.setTarget(self._main_handler) self._temp_handler.close() diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index 68ada9758..e88d16a7c 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -20,10 +20,12 @@ SERVER_HOST = os.getenv('INSTANCES_SERVER_HOST') or 'localhost' INSTANCE_ID = os.getenv('INSTANCE_ID') + def send_encoded_msg(stream, msg_code, data={}): message = json.dumps([msg_code.value, data]) stream.write(f'{message}\r\n'.encode()) + class StderrRedirector: """A workaround class to write to both sys.stderr and the Instance stderr endpoint.""" def __init__(self, stream): @@ -48,8 +50,8 @@ def __init__(self, instance_id, sequence_path, log_setup) -> None: self.health_check = lambda: {'healthy': True} self.emitter = AsyncIOEventEmitter() self.keep_alive_requested = False - self.protocol = None - + self.multiplexer = None + async def main(self, server_host, server_port): asyncio.current_task().set_name('RUNNER_MAIN') input_stream = Stream() @@ -57,68 +59,67 @@ async def main(self, server_host, server_port): # Do this early to have access to any thrown exceptions and logs. self.connect_stdio() self.connect_log_stream() - await self.protocol.sync() + await self.multiplexer.sync() config, args = await self.handshake() self.logger.info('Communication established.') - + control_stream_task = asyncio.create_task(self.connect_control_stream()) heartbeat_task = asyncio.create_task(self.setup_heartbeat()) connect_input_stream_task = asyncio.create_task(self.connect_input_stream(input_stream)) self.load_sequence() - await self.protocol.sync() + await self.multiplexer.sync() await self.run_instance(config, input_stream, args) - await self.protocol.sync() + await self.multiplexer.sync() heartbeat_task.cancel() control_stream_task.cancel() - + if not connect_input_stream_task.done(): connect_input_stream_task.cancel() await asyncio.gather(*[heartbeat_task]) await asyncio.gather(*[control_stream_task]) await asyncio.gather(*[connect_input_stream_task]) - - await self.protocol.stop() + + await self.multiplexer.stop() self.cancel_tasks() def cancel_tasks(self): - [ task.cancel() if task.get_name() != 'RUNNER_MAIN' else None for task in asyncio.all_tasks()] + [task.cancel() if task.get_name() != 'RUNNER_MAIN' else None for task in asyncio.all_tasks()] async def init_tecemux(self, server_host, server_port): self.logger.info('Connecting to host with TeceMux...') - self.protocol = Tecemux(instance_id=self.instance_id) - await self.protocol.connect(*await Tecemux.prepare_tcp_connection(server_host, server_port)) - await self.protocol.prepare(force_open=True) - await self.protocol.loop() + self.multiplexer = Tecemux(instance_id=self.instance_id) + await self.multiplexer.connect(*await Tecemux.prepare_tcp_connection(server_host, server_port)) + await self.multiplexer.prepare(force_open=True) + await self.multiplexer.loop() def connect_stdio(self): - sys.stdout = codecs.getwriter('utf-8')(self.protocol.get_channel(CC.STDOUT)) - sys.stderr = StderrRedirector(codecs.getwriter('utf-8')(self.protocol.get_channel(CC.STDERR))) - sys.stdin = Stream.read_from(self.protocol.get_channel(CC.STDIN)).decode('utf-8') - + sys.stdout = codecs.getwriter('utf-8')(self.multiplexer.get_channel(CC.STDOUT)) + sys.stderr = StderrRedirector(codecs.getwriter('utf-8')(self.multiplexer.get_channel(CC.STDERR))) + sys.stdin = Stream.read_from(self.multiplexer.get_channel(CC.STDIN)).decode('utf-8') + # pretend to have API compatibiliy sys.stdout.flush = lambda: True sys.stderr.flush = lambda: True self.logger.info('Stdio connected.') - def connect_log_stream(self): self.logger.info('Switching to main log stream...') - log_stream = codecs.getwriter('utf-8')(self.protocol.get_channel(CC.LOG)) + log_stream = codecs.getwriter('utf-8')(self.multiplexer.get_channel(CC.LOG)) self._logging_setup.switch_target(log_stream) self._logging_setup.flush_temp_handler() - self.protocol.set_logger(self.logger) - [channel._set_logger(self.logger) for channel in self.protocol.get_required_channels()] + self.multiplexer.set_logger(self.logger) + [channel._set_logger(self.logger) for channel in self.multiplexer.get_required_channels()] self.logger.info('Log stream connected.') async def handshake(self): - monitoring = self.protocol.get_channel(CC.MONITORING) - control = self.protocol.get_channel(CC.CONTROL) + monitoring = self.multiplexer.get_channel(CC.MONITORING) + control = self.multiplexer.get_channel(CC.CONTROL) self.logger.info('Sending PING') send_encoded_msg(monitoring, msg_codes.PING) @@ -128,7 +129,7 @@ async def handshake(self): code, data = json.loads(message.decode()) if not data: - data = {"appConfig":{},"args":[]} + data = {"appConfig": {}, "args": []} if 'appConfig' not in data: data['appConfig'] = {} if 'args' not in data: @@ -145,14 +146,12 @@ async def handshake(self): self.logger.info(f'Got configuration: {data}') return data['appConfig'], data['args'] - async def connect_control_stream(self): # Control stream carries ndjson, so it's enough to split into lines. control_messages = ( - Stream - # 128 kB is the typical size of TCP buffer. - .read_from(self.protocol.get_channel(CC.CONTROL), chunk_size=131072) - .decode('utf-8').split('\n').map(json.loads) + # 128 kB is the typical size of TCP buffer. + Stream.read_from(self.multiplexer.get_channel(CC.CONTROL), chunk_size=131072) + .decode('utf-8').split('\n').map(json.loads) ) try: async for code, data in control_messages: @@ -165,45 +164,42 @@ async def connect_control_stream(self): self.emitter.emit(data['eventName'], data['message'] if 'message' in data else None) except asyncio.CancelledError: - task = self.protocol.get_channel(CC.CONTROL)._outcoming_process_task + task = self.multiplexer.get_channel(CC.CONTROL)._outcoming_process_task task.cancel() await asyncio.sleep(0) await asyncio.gather(*[task]) return - async def handle_stop(self, data): self.logger.info(f'Gracefully shutting down...{data}') self.keep_alive_requested = False timeout = data.get('timeout') / 1000 can_keep_alive = data.get('canCallKeepalive') - try: + try: for handler in self.stop_handlers: await handler(timeout, can_keep_alive) except Exception as e: self.logger.error('Error stopping sequence', e) - send_encoded_msg(self.protocol.get_channel(CC.MONITORING), msg_codes.SEQUENCE_STOPPED, e) + send_encoded_msg(self.multiplexer.get_channel(CC.MONITORING), msg_codes.SEQUENCE_STOPPED, e) if not can_keep_alive or not self.keep_alive_requested: - send_encoded_msg(self.protocol.get_channel(CC.MONITORING), msg_codes.SEQUENCE_STOPPED, {}) + send_encoded_msg(self.multiplexer.get_channel(CC.MONITORING), msg_codes.SEQUENCE_STOPPED, {}) await self.exit_immediately() - async def setup_heartbeat(self): while True: try: send_encoded_msg( - self.protocol.get_channel(CC.MONITORING), + self.multiplexer.get_channel(CC.MONITORING), msg_codes.MONITORING, self.health_check(), ) - await self.protocol.get_channel(CC.MONITORING).sync() + await self.multiplexer.get_channel(CC.MONITORING).sync() await asyncio.sleep(1) except asyncio.CancelledError: - await self.protocol.get_channel(CC.MONITORING).sync() + await self.multiplexer.get_channel(CC.MONITORING).sync() return - def load_sequence(self): # Add sequence directory to sys.path module_dir = os.path.dirname(self.seq_path) @@ -218,19 +214,19 @@ def load_sequence(self): os.chdir(os.path.dirname(self.seq_path)) async def run_instance(self, config, input, args): - context = AppContext(self, config) + context = AppContext(self, config) self.logger.info('Running instance...') try: result = self.sequence.run(context, input, *args) except Exception: import traceback - self.protocol.get_channel(CC.STDERR).write(traceback.format_exc()) - await self.protocol.sync() + self.multiplexer.get_channel(CC.STDERR).write(traceback.format_exc()) + await self.multiplexer.sync() await self.exit_immediately() - - self.logger.info(f'Sending PANG') - monitoring = self.protocol.get_channel(CC.MONITORING) + self.logger.info('Sending PANG') + + monitoring = self.multiplexer.get_channel(CC.MONITORING) produces = getattr(result, 'provides', None) or getattr(self.sequence, 'provides', None) if produces: @@ -252,14 +248,13 @@ async def run_instance(self, config, input, args): self.logger.debug('Sequence returned no output.') self.logger.info('Finished.') - async def connect_input_stream(self, input_stream): try: if hasattr(self.sequence, "requires"): input_type = self.sequence.requires.get('contentType') else: - raw_headers = await self.protocol.get_channel(CC.IN).readuntil(b'\r\n\r\n') + raw_headers = await self.multiplexer.get_channel(CC.IN).readuntil(b'\r\n\r\n') header_list = raw_headers.decode().rstrip().split('\r\n') headers = { key.lower(): val for key, val in [el.split(': ') for el in header_list] @@ -268,13 +263,13 @@ async def connect_input_stream(self, input_stream): input_type = headers.get('content-type') if input_type == 'text/plain': - input = Stream.read_from(self.protocol.get_channel(CC.IN)) + input = Stream.read_from(self.multiplexer.get_channel(CC.IN)) self.logger.debug('Decoding input stream...') input = input.decode('utf-8') elif input_type == 'application/octet-stream': self.logger.debug('Opening input in binary mode...') - input = Stream.read_from(self.protocol.get_channel(CC.IN), chunk_size=CHUNK_SIZE) + input = Stream.read_from(self.multiplexer.get_channel(CC.IN), chunk_size=CHUNK_SIZE) else: raise TypeError(f'Unsupported input type: {repr(input_type)}') @@ -282,7 +277,7 @@ async def connect_input_stream(self, input_stream): input.pipe(input_stream) self.logger.debug('Input stream forwarded to the instance.') except asyncio.CancelledError: - task = self.protocol.get_channel(CC.IN)._outcoming_process_task + task = self.multiplexer.get_channel(CC.IN)._outcoming_process_task task.cancel() await asyncio.sleep(0) await asyncio.gather(*[task]) @@ -307,20 +302,18 @@ async def forward_output_stream(self, output): output = output.map(lambda s: s.encode()) if content_type == 'application/x-ndjson': self.logger.debug('Output will be converted to JSON') - output = output.map(lambda chunk: (json.dumps(chunk)+'\n').encode()) + output = output.map(lambda chunk: (json.dumps(chunk) + '\n').encode()) - await output.write_to(self.protocol.get_channel(CC.OUT)) + await output.write_to(self.multiplexer.get_channel(CC.OUT)) - async def send_keep_alive(self, timeout: int = 0, can_keep_alive: bool = False): - monitoring =self.protocol.get_channel(CC.MONITORING) + monitoring = self.multiplexer.get_channel(CC.MONITORING) send_encoded_msg(monitoring, msg_codes.ALIVE) self.keep_alive_requested = True await asyncio.sleep(timeout) - async def exit_immediately(self): - await self.protocol.sync() + await self.multiplexer.sync() sys.exit(1) @@ -347,11 +340,12 @@ def emit(self, event_name, message=''): msg_codes.EVENT, {'eventName': event_name, 'message': message} ) - + async def keep_alive(self, timeout: int = 0): await self.runner.send_keep_alive(timeout) -LOG_TARGET = open(sys.argv[1], 'a+',encoding='utf-8') if len(sys.argv) > 1 else sys.stdout + +LOG_TARGET = open(sys.argv[1], 'a+', encoding='utf-8') if len(sys.argv) > 1 else sys.stdout LOG_SETUP = LoggingSetup(LOG_TARGET, min_loglevel=logging.DEBUG) LOG_SETUP.logger.info('Starting up...') From 7270e666c13c453d228b144e8c411700f44a23f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Mon, 4 Sep 2023 11:34:11 +0200 Subject: [PATCH 219/231] Protocol to Multiplexer name change --- packages/python-runner/tecemux/channel.py | 16 ++++++++-------- packages/python-runner/tecemux/multiplexer.py | 2 +- packages/python-runner/tecemux/proxy.py | 14 +++++++------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/python-runner/tecemux/channel.py b/packages/python-runner/tecemux/channel.py index f53763ff2..02ef824bd 100644 --- a/packages/python-runner/tecemux/channel.py +++ b/packages/python-runner/tecemux/channel.py @@ -31,14 +31,14 @@ class ChannelGuard: """Internal class to open/close channels for external usage """ - def __init__(self, protocol): + def __init__(self, multiplexer): """Inits channel guard Args: - protocol (Tecemux): Tecemux object + multiplexer (Tecemux): Tecemux object """ - self._protocol = protocol + self._multiplexer = multiplexer self._channel_name = None async def __aenter__(self): @@ -48,7 +48,7 @@ async def __aenter__(self): _ChannelGuard: async guard object """ - channel = await self._protocol.open_channel(force_open=True) + channel = await self._multiplexer.open_channel(force_open=True) self._channel_name = channel._channel_enum return self @@ -60,9 +60,9 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): exc_val: Unused exc_tb: Unused """ - await self._protocol.get_channel(self._channel_name).end() - await self._protocol.get_channel(self._channel_name).close() - del self._protocol._extra_channels[self._channel_name] + await self._multiplexer.get_channel(self._channel_name).end() + await self._multiplexer.get_channel(self._channel_name).close() + del self._multiplexer._extra_channels[self._channel_name] def inject_tecemux_details_as(self, provider): """Retuns channel details for external lib @@ -83,7 +83,7 @@ def get_proxy_uri(self): str: aiohttp config """ - return self._protocol._get_proxy_uri() + return self._multiplexer._get_proxy_uri() @define diff --git a/packages/python-runner/tecemux/multiplexer.py b/packages/python-runner/tecemux/multiplexer.py index 9b57c6434..91a43bd47 100644 --- a/packages/python-runner/tecemux/multiplexer.py +++ b/packages/python-runner/tecemux/multiplexer.py @@ -145,7 +145,7 @@ def _get_unused_extra_channel_id(self, used_channel_ids, start_from=10): for i, v in enumerate(used_channel_ids, start_from): if i != v: return i - return i+1 + return i + 1 async def open_channel(self, channel_id=None, force_open=False, initial_state: ChannelState = ChannelState.CREATED) -> ChannelContext: diff --git a/packages/python-runner/tecemux/proxy.py b/packages/python-runner/tecemux/proxy.py index fa602519c..04e9c151c 100644 --- a/packages/python-runner/tecemux/proxy.py +++ b/packages/python-runner/tecemux/proxy.py @@ -34,26 +34,26 @@ def _extract_tecemux_details(request_headers): del headers['Proxy-Authorization'] - new_headers = ('\r\n'.join(key + ': ' + str(val) for key, val in headers.items())+'\r\n\r\n').encode('utf-8') + new_headers = ('\r\n'.join(key + ': ' + str(val) for key, val in headers.items()) + '\r\n\r\n').encode('utf-8') return type('TecemuxDetails', (object,), {'user': tecemux_params[0], 'channel_id': tecemux_params[1]})(), new_headers @staticmethod - async def handle_request(reader, writer, protocol): + async def handle_request(reader, writer, multiplexer): """Process single request to server Args: reader (asyncio.StreamReader): Stream reader provides access to HTTP request data writer (asyncio.StreamWriter): Stream writer give posibility to send response - protocol (Tecemux): Tecemux object + multiplexer (Tecemux): Tecemux object """ request_status = await reader.readuntil(b'\r\n') tecemux_params, request_headers = HTTPProxy._extract_tecemux_details(await reader.readuntil(b'\r\n\r\n')) - channel = protocol.get_channel(tecemux_params.channel_id) + channel = multiplexer.get_channel(tecemux_params.channel_id) channel.write(request_status) channel.write(request_headers) @@ -72,11 +72,11 @@ async def handle_request(reader, writer, protocol): writer.write(b'\r\n') await writer.drain() - async def run(self, protocol): + async def run(self, multiplexer): """Starts server on random local TCP port Args: - protocol (Tecemux): Tecemux object + multiplexer (Tecemux): Tecemux object """ self._proxy_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -86,7 +86,7 @@ async def run(self, protocol): self._port = int(self._proxy_socket.getsockname()[1]) - server = await asyncio.start_server(lambda r, w: HTTPProxy.handle_request(r, w, protocol), + server = await asyncio.start_server(lambda r, w: HTTPProxy.handle_request(r, w, multiplexer), sock=self._proxy_socket) async with server: From 6d39e174f1130635f8fd4f0a8e1caa0b86b5ca35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Mon, 4 Sep 2023 11:51:32 +0200 Subject: [PATCH 220/231] Format code #3 --- packages/python-runner/tecemux/inet.py | 30 +++++++++---------- packages/python-runner/tecemux/multiplexer.py | 5 ++-- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/packages/python-runner/tecemux/inet.py b/packages/python-runner/tecemux/inet.py index fac8cc0cb..ae6d9649d 100644 --- a/packages/python-runner/tecemux/inet.py +++ b/packages/python-runner/tecemux/inet.py @@ -190,7 +190,7 @@ def to_buffer(self): Returns: bytes: Raw buffer """ - return struct.pack(SequenceOrder().get()+'HHIIBBHHH', + return struct.pack(SequenceOrder().get() + 'HHIIBBHHH', self.src_port, self.dst_port, self.seq, @@ -306,7 +306,7 @@ def __attrs_post_init__(self): # Cut data buffer to IP packet length if self.len > 0 and self.segment: self.get_segment().data = self.get_segment( - ).data[:self.len-(self.ihl*4)-20] + ).data[:self.len - (self.ihl * 4) - 20] @staticmethod def calc_checksum(pkt: bytes) -> int: @@ -320,8 +320,8 @@ def calc_checksum(pkt: bytes) -> int: """ if len(pkt) % 2 == 1: pkt += b"\0" - s = sum(struct.unpack(('<' if SequenceOrder().get() - == '>' else '>')+str(len(pkt)//2)+'H', pkt)) + s = sum(struct.unpack(('<' if SequenceOrder().get() == '>' + else '>') + str(len(pkt) // 2) + 'H', pkt)) # source: https://github.com/secdev/scapy s = (s >> 16) + (s & 0xffff) @@ -342,7 +342,7 @@ def calc_checksum_for_STH(pkt: bytes) -> int: if len(pkt) % 2 == 1: pkt += b"\0" elements = list(struct.unpack( - SequenceOrder().get()+str(len(pkt)//2)+'H', pkt)) + SequenceOrder().get() + str(len(pkt) // 2) + 'H', pkt)) elements = elements[:14] + elements[15:] s = sum(elements) return s % 0x10000 @@ -359,7 +359,7 @@ def from_buffer_with_pseudoheader(cls, buffer: bytes): IPPacket: IPPacket object """ src_addr, dst_addr, _, proto, length = struct.unpack( - SequenceOrder().get()+"4s4sBBH", bytes(buffer[0:12])) + SequenceOrder().get() + "4s4sBBH", bytes(buffer[0:12])) pkt = cls(0, 0, 0, length, 0, 0, 0, proto, 0, src_addr, dst_addr, TCPSegment.from_buffer(buffer[12:]) if len(buffer) > 12 else None) return pkt @@ -376,13 +376,13 @@ def from_buffer(cls, buffer: bytes): IPPacket: IPPacket object """ ihl = (buffer[0] & 0xf) - pkt = cls(ihl, *struct.unpack(SequenceOrder().get()+"BBHHHBBH4s4s", - buffer[0:ihl*4]), TCPSegment.from_buffer(buffer[ihl*4:]) if len(buffer) > ihl*4 else None) + pkt = cls(ihl, *struct.unpack(SequenceOrder().get() + "BBHHHBBH4s4s", + buffer[0:ihl * 4]), TCPSegment.from_buffer(buffer[ihl * 4:]) if len(buffer) > ihl * 4 else None) # Cut data buffer to IP packet length if pkt.segment: pkt.get_segment().data = pkt.get_segment( - ).data[:pkt.len-(ihl*4)-20] + ).data[:pkt.len - (ihl * 4) - 20] return pkt @@ -401,12 +401,12 @@ def prepare_pseudoheader(self, protocol: int, length: int) -> bytes: Returns: bytes: TCP Pseudoheader """ - return struct.pack(SequenceOrder().get()+"4s4sBBH", + return struct.pack(SequenceOrder().get() + "4s4sBBH", inet_aton(self.src_addr), inet_aton(self.dst_addr), 0, protocol, - length+12) + length + 12) def to_buffer_with_tcp_pseudoheader(self) -> bytes: """Build raw buffer from IP Packet with pseudo TCP header @@ -429,7 +429,7 @@ def to_buffer(self): data = self.get_segment().to_buffer() if self.segment else b'' self.len = 20 + len(data) - return struct.pack(SequenceOrder().get()+'BBHHHBBH4s4s', + return struct.pack(SequenceOrder().get() + 'BBHHHBBH4s4s', ihl_ver, self.tos, self.len, @@ -480,7 +480,7 @@ def _validate_ip(self): self.checksum = 0 - self.checksum = IPPacket.calc_checksum(self.to_buffer()[0:self.ihl*4]) + self.checksum = IPPacket.calc_checksum(self.to_buffer()[0:self.ihl * 4]) return self @@ -530,7 +530,7 @@ def from_buffer(cls, buffer: bytes): Returns: EthernetFrame: Ethernet frame object """ - return cls(*struct.unpack(SequenceOrder().get()+"6s6s2s", buffer[0:14]), IPPacket.from_buffer(buffer[14:])) + return cls(*struct.unpack(SequenceOrder().get() + "6s6s2s", buffer[0:14]), IPPacket.from_buffer(buffer[14:])) def to_buffer(self) -> bytes: """Build raw buffer from Ethernet frame object @@ -538,7 +538,7 @@ def to_buffer(self) -> bytes: Returns: bytes: Raw buffer """ - return struct.pack(SequenceOrder().get()+"6s6s2s", unhexlify(self.src_mac), + return struct.pack(SequenceOrder().get() + "6s6s2s", unhexlify(self.src_mac), unhexlify(self.dst_mac), self.eth_type) + self.packet.to_buffer() def get_packet(self): diff --git a/packages/python-runner/tecemux/multiplexer.py b/packages/python-runner/tecemux/multiplexer.py index 91a43bd47..e775835cf 100644 --- a/packages/python-runner/tecemux/multiplexer.py +++ b/packages/python-runner/tecemux/multiplexer.py @@ -164,9 +164,8 @@ async def open_channel(self, channel_id=None, force_open=False, if channel not in self._extra_channels.keys(): - channel = str(self._get_unused_extra_channel_id([int(id) - for id in self._extra_channels.keys()])) \ - if channel is None else channel + channel = str(self._get_unused_extra_channel_id( + [int(id) for id in self._extra_channels.keys()])) if channel is None else channel self._extra_channels[channel] = ChannelContext(channel, self._queue, From c611a11c550ca58a334a58ed77a307b84e4d0b63 Mon Sep 17 00:00:00 2001 From: S4Adam Date: Tue, 5 Sep 2023 09:21:23 +0200 Subject: [PATCH 221/231] streams sync() fix --- packages/python-runner/runner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index e88d16a7c..b37e3fa85 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -65,6 +65,7 @@ async def main(self, server_host, server_port): control_stream_task = asyncio.create_task(self.connect_control_stream()) heartbeat_task = asyncio.create_task(self.setup_heartbeat()) + await self.multiplexer.sync() connect_input_stream_task = asyncio.create_task(self.connect_input_stream(input_stream)) self.load_sequence() From 33b700bd10c740fa6c4ba828496dcd6bda0a6a88 Mon Sep 17 00:00:00 2001 From: S4Adam Date: Tue, 5 Sep 2023 10:56:20 +0200 Subject: [PATCH 222/231] leftover - rename protocol to multiplexer --- packages/python-runner/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index b37e3fa85..9870375f9 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -322,7 +322,7 @@ class AppContext: def __init__(self, runner, config) -> None: self.logger = runner.logger self.config = config - self.monitoring = runner.protocol.get_channel(CC.MONITORING) + self.monitoring = runner.multiplexer.get_channel(CC.MONITORING) self.runner = runner self.emitter = runner.emitter From 5a03ca4028d635bf0c8b63dd89f583e716b19051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Gierwia=C5=82o?= Date: Tue, 5 Sep 2023 13:10:54 +0200 Subject: [PATCH 223/231] Setup.py for tecemux --- .gitignore | 3 ++- packages/python-runner/Dockerfile | 1 + packages/python-runner/package.json | 7 ------- packages/python-runner/requirements.txt | 3 ++- packages/python-runner/tecemux/setup.py | 11 +++++++++++ 5 files changed, 16 insertions(+), 9 deletions(-) create mode 100644 packages/python-runner/tecemux/setup.py diff --git a/.gitignore b/.gitignore index 6a7ff80a5..9e464f9d3 100644 --- a/.gitignore +++ b/.gitignore @@ -127,7 +127,8 @@ bdd/*-test-result.txt # pre-runner sample-package packages/pre-runner/sample-package/ - +packages/python-runner/tecemux/build/ +*.egg-info # sample packages *.tar.gz diff --git a/packages/python-runner/Dockerfile b/packages/python-runner/Dockerfile index 8e39c8983..7e9908ac2 100644 --- a/packages/python-runner/Dockerfile +++ b/packages/python-runner/Dockerfile @@ -15,6 +15,7 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* COPY packages/python-runner/*.py ./ +COPY packages/python-runner/tecemux ./tecemux COPY packages/python-runner/requirements.txt ./requirements.txt COPY packages/python-runner/docker-entrypoint.sh /usr/local/bin/ COPY packages/runner/unpack.sh /usr/local/bin/ diff --git a/packages/python-runner/package.json b/packages/python-runner/package.json index b6c9b247f..be5d94059 100644 --- a/packages/python-runner/package.json +++ b/packages/python-runner/package.json @@ -11,13 +11,6 @@ "build:docker": "docker build -t scramjetorg/runner-py:$(git rev-parse HEAD) -f Dockerfile ../../" }, "assets": [ - "tecemux/__init__.py", - "tecemux/barrier.py", - "tecemux/channel.py", - "tecemux/hardcoded_magic_values.py", - "tecemux/inet.py", - "tecemux/multiplexer.py", - "tecemux/proxy.py", "logging_setup.py", "runner.py" ], diff --git a/packages/python-runner/requirements.txt b/packages/python-runner/requirements.txt index eae561f4c..60d3e7921 100644 --- a/packages/python-runner/requirements.txt +++ b/packages/python-runner/requirements.txt @@ -1,3 +1,4 @@ attrs==23.1.0 pyee==9.0.4 -scramjet-framework-py \ No newline at end of file +scramjet-framework-py +tecemux/ \ No newline at end of file diff --git a/packages/python-runner/tecemux/setup.py b/packages/python-runner/tecemux/setup.py new file mode 100644 index 000000000..b8efb2267 --- /dev/null +++ b/packages/python-runner/tecemux/setup.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python + +from setuptools import setup, find_packages + +setup(name='tecemux', + version='0.8', + description='Scramjet Tecemux for Python', + author='Scramjet', + author_email='opensource@scramjet.org', + packages=["tecemux"], + package_dir={"tecemux": "."}) From 116e0b942a8ed79bafc2ead6a62cdfd39ffb2ec7 Mon Sep 17 00:00:00 2001 From: S4Adam Date: Tue, 19 Sep 2023 14:50:41 +0200 Subject: [PATCH 224/231] [py] Proxy url handling --- packages/python-runner/tecemux/proxy.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/python-runner/tecemux/proxy.py b/packages/python-runner/tecemux/proxy.py index 04e9c151c..1fb18f2b3 100644 --- a/packages/python-runner/tecemux/proxy.py +++ b/packages/python-runner/tecemux/proxy.py @@ -50,7 +50,8 @@ async def handle_request(reader, writer, multiplexer): """ request_status = await reader.readuntil(b'\r\n') - + # Workaround... + request_status = bytes('GET /', encoding='utf-8' )+'/'.join(request_status.decode().split('/')[3:]).encode('utf-8') tecemux_params, request_headers = HTTPProxy._extract_tecemux_details(await reader.readuntil(b'\r\n\r\n')) channel = multiplexer.get_channel(tecemux_params.channel_id) @@ -65,9 +66,17 @@ async def handle_request(reader, writer, multiplexer): writer.write(raw_response_headers) headers = HTTPProxy._get_headers_as_dict(raw_response_headers, convert_keys_to_lowercase=True) - - raw_response_data = await channel.read(int(headers['content-length'])) - writer.write(raw_response_data) + if 'content-length' in headers: + writer.write(await channel.read(int(headers['content-length']))) + elif 'transfer-encoding' in headers and headers['transfer-encoding'] == 'chunked': + while True: + try: + raw_response_data = await asyncio.wait_for(channel.read(8), timeout=3) + if not raw_response_data: + break + writer.write(raw_response_data) + except asyncio.TimeoutError: + break writer.write(b'\r\n') await writer.drain() From 1c83e75c1eacf1169eefbfde3dce8ceac9fe604e Mon Sep 17 00:00:00 2001 From: patuwwy Date: Fri, 3 Nov 2023 14:14:50 +0000 Subject: [PATCH 225/231] Move Tecemux from Verser --- packages/host/package.json | 1 + packages/host/src/lib/csi-controller.ts | 2 +- packages/host/src/lib/socket-server.ts | 2 +- packages/runner/Dockerfile | 1 + packages/runner/package.json | 2 +- packages/runner/src/host-client.ts | 5 +- packages/sth/Dockerfile | 1 + packages/tecemux/.eslintrc.js | 7 + packages/tecemux/README.md | 62 ++++++++ packages/tecemux/package.json | 46 ++++++ .../src}/codecs/frame-decoder.ts | 0 .../src}/codecs/frame-encoder.ts | 0 .../tecemux => tecemux/src}/codecs/index.ts | 0 .../tecemux => tecemux/src}/codecs/utils.ts | 0 .../lib/tecemux => tecemux/src}/constants.ts | 0 .../tecemux => tecemux/src}/frames-keeper.ts | 0 .../src/lib/tecemux => tecemux/src}/index.ts | 0 packages/tecemux/src/readme.mtpl | 19 +++ .../src}/tecemux-channel.ts | 0 .../lib/tecemux => tecemux/src}/tecemux.ts | 0 .../src/lib/tecemux => tecemux/src}/types.ts | 0 .../tecemux => tecemux/src}/utils/index.ts | 0 packages/tecemux/test/.keep | 0 .../test/cert/cleanup-localhost-cert.sh | 2 + .../tecemux/test/cert/gen-localhost-cert.sh | 28 ++++ packages/tecemux/test/http-connection.spec.ts | 135 ++++++++++++++++ .../test/playgrounds/playground-tecemux.ts | 144 ++++++++++++++++++ .../tecemux/test/playgrounds/playground.ts | 69 +++++++++ .../tecemux/test/tecemux-transfer.spec.ts | 86 +++++++++++ packages/tecemux/tsconfig.build.json | 9 ++ packages/tecemux/tsconfig.json | 22 +++ packages/verser/package.json | 2 +- packages/verser/src/index.ts | 1 - packages/verser/src/lib/verser-client.ts | 2 +- packages/verser/src/lib/verser-connection.ts | 2 +- 35 files changed, 641 insertions(+), 9 deletions(-) create mode 100644 packages/tecemux/.eslintrc.js create mode 100644 packages/tecemux/README.md create mode 100644 packages/tecemux/package.json rename packages/{verser/src/lib/tecemux => tecemux/src}/codecs/frame-decoder.ts (100%) rename packages/{verser/src/lib/tecemux => tecemux/src}/codecs/frame-encoder.ts (100%) rename packages/{verser/src/lib/tecemux => tecemux/src}/codecs/index.ts (100%) rename packages/{verser/src/lib/tecemux => tecemux/src}/codecs/utils.ts (100%) rename packages/{verser/src/lib/tecemux => tecemux/src}/constants.ts (100%) rename packages/{verser/src/lib/tecemux => tecemux/src}/frames-keeper.ts (100%) rename packages/{verser/src/lib/tecemux => tecemux/src}/index.ts (100%) create mode 100644 packages/tecemux/src/readme.mtpl rename packages/{verser/src/lib/tecemux => tecemux/src}/tecemux-channel.ts (100%) rename packages/{verser/src/lib/tecemux => tecemux/src}/tecemux.ts (100%) rename packages/{verser/src/lib/tecemux => tecemux/src}/types.ts (100%) rename packages/{verser/src/lib/tecemux => tecemux/src}/utils/index.ts (100%) create mode 100644 packages/tecemux/test/.keep create mode 100755 packages/tecemux/test/cert/cleanup-localhost-cert.sh create mode 100755 packages/tecemux/test/cert/gen-localhost-cert.sh create mode 100644 packages/tecemux/test/http-connection.spec.ts create mode 100644 packages/tecemux/test/playgrounds/playground-tecemux.ts create mode 100644 packages/tecemux/test/playgrounds/playground.ts create mode 100644 packages/tecemux/test/tecemux-transfer.spec.ts create mode 100644 packages/tecemux/tsconfig.build.json create mode 100644 packages/tecemux/tsconfig.json diff --git a/packages/host/package.json b/packages/host/package.json index 65a3fbf21..aee6df2e7 100644 --- a/packages/host/package.json +++ b/packages/host/package.json @@ -28,6 +28,7 @@ "@scramjet/telemetry": "^0.35.3", "@scramjet/utility": "^0.35.3", "@scramjet/verser": "^0.35.3", + "@scramjet/tecemux": "^0.35.3", "bpmux": "^8.2.1", "ext-ip": "^0.3.9", "find-package-json": "^1.2.0", diff --git a/packages/host/src/lib/csi-controller.ts b/packages/host/src/lib/csi-controller.ts index 3ecaa3799..a404746e2 100644 --- a/packages/host/src/lib/csi-controller.ts +++ b/packages/host/src/lib/csi-controller.ts @@ -44,7 +44,7 @@ import { DuplexStream, getRouter } from "@scramjet/api-server"; import { getInstanceAdapter } from "@scramjet/adapters"; import { cancellableDefer, CancellablePromise, defer, promiseTimeout, TypedEmitter } from "@scramjet/utility"; import { ObjLogger } from "@scramjet/obj-logger"; -import { TeceMux } from "@scramjet/verser"; +import { TeceMux } from "@scramjet/tecemux"; import { ReasonPhrases } from "http-status-codes"; /** diff --git a/packages/host/src/lib/socket-server.ts b/packages/host/src/lib/socket-server.ts index 99c1a2e34..c9e38b11b 100644 --- a/packages/host/src/lib/socket-server.ts +++ b/packages/host/src/lib/socket-server.ts @@ -4,7 +4,7 @@ import net, { Socket } from "net"; import { isDefined, TypedEmitter } from "@scramjet/utility"; import { ObjLogger } from "@scramjet/obj-logger"; -import { TeceMux, TeceMuxChannel } from "@scramjet/verser"; +import { TeceMux, TeceMuxChannel } from "@scramjet/tecemux"; type MaybeChannel = TeceMuxChannel | Socket | null; diff --git a/packages/runner/Dockerfile b/packages/runner/Dockerfile index 8a5deb669..478b8f316 100644 --- a/packages/runner/Dockerfile +++ b/packages/runner/Dockerfile @@ -28,6 +28,7 @@ COPY ./dist/obj-logger ./dist/obj-logger COPY ./dist/model ./dist/model COPY ./dist/runner ./dist/runner COPY ./dist/verser ./dist/verser +COPY ./dist/tecemux ./dist/tecemux COPY ./dist/package.json ./dist/package.json FROM target diff --git a/packages/runner/package.json b/packages/runner/package.json index 5f2362497..7e466c917 100644 --- a/packages/runner/package.json +++ b/packages/runner/package.json @@ -23,7 +23,7 @@ "@scramjet/obj-logger": "^0.35.2", "@scramjet/symbols": "^0.35.2", "@scramjet/utility": "^0.35.2", - "@scramjet/verser": "^0.35.2", + "@scramjet/tecemux": "^0.35.2", "scramjet": "^4.36.9" }, "devDependencies": { diff --git a/packages/runner/src/host-client.ts b/packages/runner/src/host-client.ts index 56c884403..a66472589 100644 --- a/packages/runner/src/host-client.ts +++ b/packages/runner/src/host-client.ts @@ -4,10 +4,11 @@ import { CommunicationChannel as CC } from "@scramjet/symbols"; import net, { createConnection, Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; import { Agent } from "http"; -import { TeceMux, TeceMuxChannel } from "@scramjet/verser"; +import { TeceMux, TeceMuxChannel } from "@scramjet/tecemux"; type HostOpenConnections = [ - TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel + TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, + TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel, TeceMuxChannel ] /** diff --git a/packages/sth/Dockerfile b/packages/sth/Dockerfile index ad24eb014..1fb8f39be 100644 --- a/packages/sth/Dockerfile +++ b/packages/sth/Dockerfile @@ -17,6 +17,7 @@ COPY ./dist/types ./dist/types COPY ./dist/load-check ./dist/load-check COPY ./dist/utility ./dist/utility COPY ./dist/verser ./dist/verser +COPY ./dist/tecemux ./dist/tecemux COPY ./dist/telemetry ./dist/telemetry COPY ./dist/package.json ./dist/package.json COPY LICENSE ./ diff --git a/packages/tecemux/.eslintrc.js b/packages/tecemux/.eslintrc.js new file mode 100644 index 000000000..fc7819a8a --- /dev/null +++ b/packages/tecemux/.eslintrc.js @@ -0,0 +1,7 @@ +module.exports = { + ignorePatterns: [".eslintrc.js", "types.ts"], + parserOptions:{ + project: "./tsconfig.json", + tsconfigRootDir: __dirname + } +}; diff --git a/packages/tecemux/README.md b/packages/tecemux/README.md new file mode 100644 index 000000000..f5f2ec3d0 --- /dev/null +++ b/packages/tecemux/README.md @@ -0,0 +1,62 @@ +# Scramjet Verser + +![Scramjet Transform Hub](https://assets.scramjet.org/sth-logo.svg "Scramjet Transform Hub Logo") + +This package provides a reverse server functionality. + +```bash +npm install -g @scramjet/verser +``` + +## Docs + +See the code documentation here: [scramjetorg/transform-hub/docs/verser/modules.md](https://github.com/scramjetorg/transform-hub/tree/HEAD/docs/verser/modules.md) + +## Scramjet Transform Hub + +This package is part of [Scramjet Transform Hub](https://www.npmjs.org/package/@scramjet/sth). + +Scramjet Transform Hub is a deployment and execution platform. Once installed on a server, it will allow you to start your programs and keep them running on a remote machine. You will be able to start programs in the background or connect to them and see their output directly on your terminal. You will be able to pipe your local data to the program, as if it was running from your terminal. You can start your server in AWS, Google Cloud or Azure, start it on your local machine, install it on a Raspberry Pi or wherever else you'd like. + +## Use cases + +There's no limit what you can use it for. You want a stock checker? A chat bot? Maybe you'd like to automate your home? Retrieve sensor data? Maybe you have a lot of data and want to transfer and wrangle it? You have a database of cities and you'd like to enrich your data? You do machine learning and you want to train your set while the data is fetched in real time? Hey, you want to use it for something else and ask us if that's a good use? Ask us [via email](mailto:get@scramjet.org) or hop on our [Scramjet Discord](https://scr.je/join-community-mg1)! + +## Some important links + +* Scramjet, the company behind [Transform Hub](https://scramjet.org) +* The [Scramjet Framework - functional reactive stream processing framework](https://framework.scramjet.org) +* The [Transform Hub repo on github](https://github.com/scramjetorg/transform-hub) +* You can see the [Scramjet Transform Hub API docs here](https://github.com/scramjetorg/transform-hub/tree/HEAD/docs/api-client/README.md) +* You can see the [CLI documentation here](https://github.com/scramjetorg/transform-hub/tree/HEAD/packages/cli/README.md), but `si help` should also be quite effective. +* Don't forget to ⭐ this repo if you like it, `subscribe` to releases and keep visiting us for new versions and updates. +* You can [open an issue - file a bug report or a feature request here](https://github.com/scramjetorg/transform-hub/issues/new/choose) + +## License and contributions + +This module is licensed under AGPL-3.0 license. + +The Scramjet Transform Hub project is dual-licensed under the AGPL-3.0 and MIT licenses. Parts of the project that are linked with your programs are MIT licensed, the rest is AGPL. + +## Contributions + +We accept valid contributions and we will be publishing a more specific project roadmap so contributors can propose features and also help us implement them. We kindly ask you that contributed commits are Signed-Off `git commit --sign-off`. + +We provide support for contributors via test cases. If you expect a certain type of workflow to be officially supported, please specify and implement a test case in `Gherkin` format in `bdd` directory and include it in your pull request. More info about our BDD test you will find [here](https://github.com/scramjetorg/transform-hub/tree/HEAD/bdd/README.md). + +### Help wanted 👩‍🎓🧑👱‍♀️ + +The project need's your help! There's lots of work to do and we have a lot of plans. If you want to help and be part of the Scramjet team, please reach out to us, [on discord](https://scr.je/join-community-mg1) or email us: [opensource@scramjet.org](mailto:opensource@scramjet.org). + +### Donation 💸 + +Do you like this project? It helped you to reduce time spent on delivering your solution? You are welcome to buy us a coffee ☕ Thanks a lot! 😉 + +[You can sponsor us on github](https://github.com/sponsors/scramjetorg) + +* There's also a Paypal donation link if you prefer that: + +[![paypal](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=7F7V65C43EBMW) + + + diff --git a/packages/tecemux/package.json b/packages/tecemux/package.json new file mode 100644 index 000000000..eacfbb3e4 --- /dev/null +++ b/packages/tecemux/package.json @@ -0,0 +1,46 @@ +{ + "name": "@scramjet/tecemux", + "version": "0.35.3", + "description": "This package is part of Scramjet Transform Hub. The package provides a communication protocol used among Scramjet modules.", + "main": "./src/index.ts", + "scripts": { + "start": "ts-node ./src/bin/index", + "build": "../../scripts/build-all.js --config-name=tsconfig.build.json --copy-dist", + "build:docs": "typedoc", + "clean": "rm -rf ./dist .bic_cache", + "test": "npm run test:ava", + "test:ava": "ava" + }, + "author": "Scramjet ", + "license": "AGPL-3.0", + "dependencies": { + "@scramjet/obj-logger": "^0.35.3", + "@scramjet/utility": "^0.35.3", + "bpmux": "^8.2.1" + }, + "devDependencies": { + "@scramjet/api-server": "^0.35.3", + "@scramjet/types": "^0.35.3", + "@types/node": "15.12.5", + "ava": "^3.15.0", + "ts-node": "^10.9.1", + "typedoc": "0.23.17", + "typedoc-plugin-markdown": "3.13.6", + "typescript": "~4.7.4" + }, + "ava": { + "extensions": [ + "ts" + ], + "files": [ + "**/*.spec.ts" + ], + "require": [ + "ts-node/register" + ] + }, + "repository": { + "type": "git", + "url": "https://github.com/scramjetorg/transform-hub.git" + } +} diff --git a/packages/verser/src/lib/tecemux/codecs/frame-decoder.ts b/packages/tecemux/src/codecs/frame-decoder.ts similarity index 100% rename from packages/verser/src/lib/tecemux/codecs/frame-decoder.ts rename to packages/tecemux/src/codecs/frame-decoder.ts diff --git a/packages/verser/src/lib/tecemux/codecs/frame-encoder.ts b/packages/tecemux/src/codecs/frame-encoder.ts similarity index 100% rename from packages/verser/src/lib/tecemux/codecs/frame-encoder.ts rename to packages/tecemux/src/codecs/frame-encoder.ts diff --git a/packages/verser/src/lib/tecemux/codecs/index.ts b/packages/tecemux/src/codecs/index.ts similarity index 100% rename from packages/verser/src/lib/tecemux/codecs/index.ts rename to packages/tecemux/src/codecs/index.ts diff --git a/packages/verser/src/lib/tecemux/codecs/utils.ts b/packages/tecemux/src/codecs/utils.ts similarity index 100% rename from packages/verser/src/lib/tecemux/codecs/utils.ts rename to packages/tecemux/src/codecs/utils.ts diff --git a/packages/verser/src/lib/tecemux/constants.ts b/packages/tecemux/src/constants.ts similarity index 100% rename from packages/verser/src/lib/tecemux/constants.ts rename to packages/tecemux/src/constants.ts diff --git a/packages/verser/src/lib/tecemux/frames-keeper.ts b/packages/tecemux/src/frames-keeper.ts similarity index 100% rename from packages/verser/src/lib/tecemux/frames-keeper.ts rename to packages/tecemux/src/frames-keeper.ts diff --git a/packages/verser/src/lib/tecemux/index.ts b/packages/tecemux/src/index.ts similarity index 100% rename from packages/verser/src/lib/tecemux/index.ts rename to packages/tecemux/src/index.ts diff --git a/packages/tecemux/src/readme.mtpl b/packages/tecemux/src/readme.mtpl new file mode 100644 index 000000000..781404c6d --- /dev/null +++ b/packages/tecemux/src/readme.mtpl @@ -0,0 +1,19 @@ +# Scramjet Verser + +![Scramjet Transform Hub](https://assets.scramjet.org/sth-logo.svg "Scramjet Transform Hub Logo") + +This package provides a reverse server functionality. + +```bash +npm install -g @scramjet/verser +``` + +## Docs + +>!docs verser/modules.md & + +>@sth +>@use-cases +>@links +>@license-agpl +>@contrib diff --git a/packages/verser/src/lib/tecemux/tecemux-channel.ts b/packages/tecemux/src/tecemux-channel.ts similarity index 100% rename from packages/verser/src/lib/tecemux/tecemux-channel.ts rename to packages/tecemux/src/tecemux-channel.ts diff --git a/packages/verser/src/lib/tecemux/tecemux.ts b/packages/tecemux/src/tecemux.ts similarity index 100% rename from packages/verser/src/lib/tecemux/tecemux.ts rename to packages/tecemux/src/tecemux.ts diff --git a/packages/verser/src/lib/tecemux/types.ts b/packages/tecemux/src/types.ts similarity index 100% rename from packages/verser/src/lib/tecemux/types.ts rename to packages/tecemux/src/types.ts diff --git a/packages/verser/src/lib/tecemux/utils/index.ts b/packages/tecemux/src/utils/index.ts similarity index 100% rename from packages/verser/src/lib/tecemux/utils/index.ts rename to packages/tecemux/src/utils/index.ts diff --git a/packages/tecemux/test/.keep b/packages/tecemux/test/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/packages/tecemux/test/cert/cleanup-localhost-cert.sh b/packages/tecemux/test/cert/cleanup-localhost-cert.sh new file mode 100755 index 000000000..ca6e7a044 --- /dev/null +++ b/packages/tecemux/test/cert/cleanup-localhost-cert.sh @@ -0,0 +1,2 @@ +#!/bin/bash +rm {localhost,myCA}.* \ No newline at end of file diff --git a/packages/tecemux/test/cert/gen-localhost-cert.sh b/packages/tecemux/test/cert/gen-localhost-cert.sh new file mode 100755 index 000000000..b66eb005f --- /dev/null +++ b/packages/tecemux/test/cert/gen-localhost-cert.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Private key for root CA +openssl genrsa -des3 -out myCA.key -passout pass:test 2048 + +# root CA +openssl req -x509 -new -nodes -key myCA.key -sha256 -days 825 -out myCA.pem -passin pass:test -subj '/CN=www.mydom.com/O=My Company Name LTD./C=US' + +# key for cert +openssl genrsa -out localhost.key 2048 + +# cert +openssl req -new -key localhost.key -out localhost.csr -subj '/CN=localhost/O=My Company Name LTD./C=US' + +>localhost.ext cat <<-EOF +authorityKeyIdentifier = keyid,issuer +basicConstraints = CA:FALSE +keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment +subjectAltName = @alt_names + +[alt_names] +DNS.1 = localhost +IP.1 = 127.0.0.1 +EOF + +# sign cert +openssl x509 -req -in localhost.csr -CA myCA.pem -CAkey myCA.key -passin pass:test -CAcreateserial -out localhost.crt -days 825 -sha256 -extfile localhost.ext + diff --git a/packages/tecemux/test/http-connection.spec.ts b/packages/tecemux/test/http-connection.spec.ts new file mode 100644 index 000000000..c0c4b0a71 --- /dev/null +++ b/packages/tecemux/test/http-connection.spec.ts @@ -0,0 +1,135 @@ +import test from "ava"; +import * as http from "http"; +import * as https from "https"; +import { createServer } from "@scramjet/api-server"; +import { Verser, VerserClient, VerserConnection } from "../src"; +import path from "path"; +import { readFileSync } from "fs"; +import { APIExpose } from "@scramjet/types"; +import { spawnSync } from "child_process"; + +async function connectVerserClientAToVerserB( + apiB: APIExpose, + verserClientA: VerserClient, +): Promise { + const verserB = new Verser(apiB.server); + + const connectionResolver = { res: (_conn: VerserConnection) => {} }; + const connectionPromised = new Promise(res => { connectionResolver.res = res; }); + + verserB.on("connect", (verserConnection) => { + verserConnection.respond(200); + // @TODO we have to do that, which means that verser api is a bit off + verserConnection.reconnect(); + connectionResolver.res(verserConnection); + }); + + await verserClientA.connect(); + + return connectionPromised; +} + +function getJSONResponseFromRequest(request: http.ClientRequest): Promise { + return new Promise(resolve => { + request.on("response", async (response) => { + let responseBody = ""; + + for await (const chunk of response) { + responseBody += chunk; + } + + resolve(JSON.parse(responseBody)); + }); + }); +} + +test("Connect VerserClient A to Verser B and send HTTP GET Request to VerserClient A", async (t) => { + const SERVER_B_PORT = 1999; + const apiB = createServer({ }); + + apiB.server.listen(SERVER_B_PORT); + + const apiA = createServer(); + + // @TODO since this is alway needed, maybe we should be setting that in VerserClient? + (apiA.server as http.Server & { httpAllowHalfOpen?: boolean }).httpAllowHalfOpen = true; + + const verserClientA = new VerserClient({ + headers: { city: "Valencia" }, + verserUrl: `http://0.0.0.0:${SERVER_B_PORT}`, + server: apiA.server + }); + + const verserConnectionB = await connectVerserClientAToVerserB(apiB, verserClientA); + + t.is(verserConnectionB.getHeaders().city, "Valencia"); + + // Forward any request to B to A + apiB.use("*", (req, res) => verserConnectionB.forward(req, res)); + + apiA.get("*", () => ({ greeting: "Bye" })); + + // Make request to B that should be forwarded to A + const requestToAThroughB = http.request({ + port: SERVER_B_PORT, + method: "GET", + }); + + requestToAThroughB.end(); + + const responseFromAToB = await getJSONResponseFromRequest(requestToAThroughB); + + // Verify that response from A got back + t.deepEqual(responseFromAToB, { greeting: "Bye" }); +}); + +test("Connect VerserClient A to Verser B over SSL and send HTTPS GET Request to VerserClient A", async (t) => { + const certDir = path.join(__dirname, "cert"); + + spawnSync("./gen-localhost-cert.sh", { cwd: certDir }); + + const SERVER_B_PORT = 2000; + const apiB = createServer({ + sslKeyPath: path.join(certDir, "localhost.key"), + sslCertPath: path.join(certDir, "localhost.crt"), + }); + + apiB.server.listen(SERVER_B_PORT); + + const apiA = createServer(); + + (apiA.server as http.Server & { httpAllowHalfOpen?: boolean }).httpAllowHalfOpen = true; + + const verserClientA = new VerserClient({ + headers: { city: "Valencia" }, + verserUrl: `https://127.0.0.1:${SERVER_B_PORT}`, + server: apiA.server, + https: { ca: [readFileSync(path.join(certDir, "myCA.pem"))] } + }); + + const verserConnectionB = await connectVerserClientAToVerserB(apiB, verserClientA); + + t.is(verserConnectionB.getHeaders().city, "Valencia"); + + // Forward any request to B to A + apiB.use("*", (req, res) => verserConnectionB.forward(req, res)); + + apiA.get("*", () => ({ greeting: "Bye" })); + + // Make request to B that should be forwarded to A + const requestToAThroughB = https.request({ + port: SERVER_B_PORT, + method: "GET", + ca: [readFileSync(path.join(certDir, "myCA.pem"))] + }); + + requestToAThroughB.flushHeaders(); + + const responseFromAToB = await getJSONResponseFromRequest(requestToAThroughB); + + // Verify that response from A got back + t.deepEqual(responseFromAToB, { greeting: "Bye" }); + + spawnSync("./cleanup-localhost-cert.sh", { cwd: certDir }); +}); + diff --git a/packages/tecemux/test/playgrounds/playground-tecemux.ts b/packages/tecemux/test/playgrounds/playground-tecemux.ts new file mode 100644 index 000000000..8667c18f6 --- /dev/null +++ b/packages/tecemux/test/playgrounds/playground-tecemux.ts @@ -0,0 +1,144 @@ +/*eslint no-unused-vars: ["error", { "args": "none" }]*/ +/* eslint-disable @typescript-eslint/no-floating-promises */ +/* eslint-disable no-console */ + +import { ObjLogger } from "@scramjet/obj-logger"; +import { IncomingMessage, createServer } from "http"; + +import { Socket, createConnection } from "net"; +import { TeceMux } from "../../src/tecemux"; +import { TeceMuxChannel } from "../../src/types"; +import { createReadStream, createWriteStream } from "fs"; +import path from "path"; + +(async () => { + const logger = new ObjLogger("Sandbox"); + + logger.pipe(process.stdout); + + /**********************************************/ + /* SERVER + /**********************************************/ + + const PORT = 6660; + const server = createServer({}); + + server.on("timeout", (_socket) => { + logger.warn("Server on timeout"); + }); + server.requestTimeout = 0; + + server.on("connect", async (req: IncomingMessage, socket: Socket) => { + socket.setNoDelay(true); + + logger.info("Incoming request", req.method, req.headers); + + socket + .on("pipe", () => { + logger.info("Carrier Socket piped"); + }) + .on("unpipe", (c: any) => { + logger.info("Carrier Socket unpiped", c); + }) + .on("pause", () => { + logger.fatal("Carrier Socket paused"); + }) + .on("resume", () => { + logger.info("Carrier Socket resumed"); + }) + .on("error", (error) => { + logger.error("Carrier Socket error", error); + }) + .on("close", () => { + logger.info("Carrier Socket closed"); + }) + .on("timeout", () => { + logger.info("Carrier Socket timeout"); + }); + + const tcmux = new TeceMux(socket, "Server") + .on("error", (err) => { + logger.error("TCMUX err", err); + }); + + tcmux.logger.pipe(logger); + + const channel1 = tcmux.multiplex(); + + channel1.pipe(createWriteStream(path.join(__dirname, "output-server.tar.gz"))); + + createReadStream(path.join(__dirname, "../../../forever.tar.gz")) + .on("end", () => { + logger.info("FILE END"); + }) + .pipe(channel1); + }); + + server.listen(PORT, "0.0.0.0"); + + /**********************************************/ + /* CLIENT + /**********************************************/ + + const socket = createConnection({ port: PORT, allowHalfOpen: true, host: "0.0.0.0" }, () => {}); + + await new Promise((resolve, reject) => { + socket + .on("connect", resolve) + .on("error", reject); + }); + + socket.setNoDelay(true); + + const reqLogger = new ObjLogger("Req", { id: "Client" }); + + reqLogger.pipe(logger); + + socket.write("CONNECT HTTP/1.1\r\n\r\n\r\n"); + + socket.on("error", (error) => { + reqLogger.error("ERROR", error); + }); + + reqLogger.info("connected to server!"); + + const tcmux = new TeceMux(socket, "Request"); + + tcmux.logger.updateBaseLog({ id: reqLogger.baseLog.id }); + + tcmux.logger.pipe(logger); + + tcmux.on("channel", async (channel: TeceMuxChannel) => { + reqLogger.debug("New channel", channel._id); + + let total = 0; + + channel + .on("finish", () => { + tcmux.logger.info("Channel finish", channel._id); + }) + .on("end", () => { + tcmux.logger.info("Channel readable end [id, readableEnded, writableEnded]", channel._id, channel.readableEnded, channel.writableEnded); + }) + .on("data", (d) => { + total += d.length; + tcmux.logger.info("-------------------- data", channel._id, d.length, total); + }) + .on("pause", () => { + tcmux.logger.info("-------------------- paused", channel._id); + }) + .on("resume", () => { + tcmux.logger.info("-------------------- resumed", channel._id); + }) + .pause(); + + createReadStream(path.join(__dirname, "../../../forever.tar.gz")).pipe(channel); + channel.pipe(createWriteStream(path.join(__dirname, "output-client.tar.gz"))); + }); + + socket.on("error", (err) => { + console.error(err); + }); + + await new Promise((_res, _rej) => {}); +})(); diff --git a/packages/tecemux/test/playgrounds/playground.ts b/packages/tecemux/test/playgrounds/playground.ts new file mode 100644 index 000000000..4dc03db15 --- /dev/null +++ b/packages/tecemux/test/playgrounds/playground.ts @@ -0,0 +1,69 @@ +import { FrameDecoder, FrameEncoder } from "../../src/codecs"; +import { ObjLogger } from "@scramjet/obj-logger"; +import { createReadStream, createWriteStream } from "fs"; +import path from "path"; +import { PassThrough, Transform } from "stream"; +import { ITeCeMux } from "../../src/types"; +import { FramesKeeper } from "../../src/frames-keeper"; + +const tcm = { + sequenceNumber: 0, + framesKeeper: new FramesKeeper(), + framesSent: 0 +} as ITeCeMux; + +const logger = new ObjLogger("Sandbox"); + +logger.pipe(process.stdout); + +const encoder = new FrameEncoder(0, tcm); +const decoder = new FrameDecoder(); + +encoder.logger.pipe(logger); +decoder.logger.pipe(logger); + +const ws = createWriteStream(path.join(__dirname, "output.tar.gz")); + +const delayedPassThrough = () => new PassThrough({ + async transform(chunk, encoding, callback) { + await new Promise((res, _rej) => { + setTimeout(res, 1000); + }); + + this.push(chunk, encoding); + callback(null); + }, +}); + +//let t = 0; + +const dh = new Transform({ + writableObjectMode: true, + transform: (chunk, encoding, callback) => { + try { + if (chunk && chunk.length) { + //t += chunk.length; + + if (!ws.write(new Uint8Array(JSON.parse(chunk.toString()).chunk.data))) { + dh.pause(); + ws.once("drain", () => { + callback(); + dh.resume(); + }); + } else { + callback(null); + } + } + } catch (e) { + logger.error("dh error", e); + } + } +}); + +createReadStream(path.join(__dirname, "../../../../forever.tar.gz"), { encoding: undefined }) + .pipe(encoder, { end: false }).out + .pipe(delayedPassThrough()) + .pipe(decoder) + .pipe(delayedPassThrough()) + .pipe(dh); + diff --git a/packages/tecemux/test/tecemux-transfer.spec.ts b/packages/tecemux/test/tecemux-transfer.spec.ts new file mode 100644 index 000000000..35b129b7e --- /dev/null +++ b/packages/tecemux/test/tecemux-transfer.spec.ts @@ -0,0 +1,86 @@ +/* eslint-disable no-console */ +import test from "ava"; +import { IncomingMessage, Server, createServer } from "http"; +import { TeceMux, TeceMuxChannel } from "../src/tecemux"; +import * as crypto from "crypto"; +import { Socket, createConnection } from "net"; +import { Readable } from "stream"; + +let serverTeceMux: TeceMux; + +const PORT = 6660; + +async function startServer() { + const server = createServer({}); + + return new Promise((resolve) => { + server.listen(PORT, () => { resolve(server); }); + }); +} + +test.serial("Protocol send file over http connection", async (t) => { + const server = await startServer(); + + const hashReceived = crypto.createHash("md5"); + const hashSent = crypto.createHash("md5"); + + const serverSideChannelPromise = new Promise((resolve) => { + server.on("connect", async (req: IncomingMessage, socket: Socket) => { + console.log("server on connect!"); + socket.setNoDelay(true); + + serverTeceMux = new TeceMux(socket, "Server") + .on("error", (error) => { + console.error("TeceMux error", error.code); + }); + + resolve(serverTeceMux.multiplex({ channel: 1 })); + }); + }); + + const socket = createConnection({ port: PORT, allowHalfOpen: true, host: "0.0.0.0" }, () => {}); + + await new Promise((resolve, reject) => { + socket + .on("connect", resolve) + .on("error", reject); + }); + + socket.write("CONNECT HTTP/1.1\r\n\r\n\r\n"); + + const clientTeceMux = new TeceMux(socket, "Request"); + + await new Promise(resolve => { + clientTeceMux.on("channel", async (clientSideChannel: TeceMuxChannel) => { + console.log("TEST: on channel", clientSideChannel._id); + function* gen() { + for (let i = 0; i < 1e3; i++) { + const str = crypto.randomBytes(1024).toString("hex"); + + hashSent.update(str); + yield str; + } + } + + Readable.from(gen()).pipe(clientSideChannel); + resolve(clientSideChannel); + }); + }); + + const serverSideChannel = await serverSideChannelPromise; + + console.log("Serverside channel", serverSideChannel._id); + + for await (const d of serverSideChannel) { + hashReceived.update(d); + } + + const res = { + txHash: hashSent.digest("hex"), + rxHash: hashReceived.digest("hex") + }; + + console.dir(res); + + t.assert(res.txHash === res.rxHash, "Unequal hashes"); +}); diff --git a/packages/tecemux/tsconfig.build.json b/packages/tecemux/tsconfig.build.json new file mode 100644 index 000000000..e0848624e --- /dev/null +++ b/packages/tecemux/tsconfig.build.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist" + }, + "include": [ + "src/**/*" + ] +} diff --git a/packages/tecemux/tsconfig.json b/packages/tecemux/tsconfig.json new file mode 100644 index 000000000..9c4e6d593 --- /dev/null +++ b/packages/tecemux/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "outDir": "./dist" + }, + "extends": "../../tsconfig.base.json", + "include": [ + "./src/**/*.ts" + ], + "exclude": [ + "node_modules" + ], + "typedocOptions": { + "entryPoints": [ + "src/index.ts" + ], + "out": "../../docs/tecemux", + "plugin": "typedoc-plugin-markdown", + "excludePrivate": "true", + "gitRevision": "HEAD", + "sort": "alphabetical" + } +} diff --git a/packages/verser/package.json b/packages/verser/package.json index a944978bc..95e62f1b4 100644 --- a/packages/verser/package.json +++ b/packages/verser/package.json @@ -16,7 +16,7 @@ "dependencies": { "@scramjet/obj-logger": "^0.35.3", "@scramjet/utility": "^0.35.3", - "bpmux": "^8.2.1" + "@scramjet/tecemux": "^0.35.3" }, "devDependencies": { "@scramjet/api-server": "^0.35.3", diff --git a/packages/verser/src/index.ts b/packages/verser/src/index.ts index c56cde24d..19bca6084 100644 --- a/packages/verser/src/index.ts +++ b/packages/verser/src/index.ts @@ -1,4 +1,3 @@ export * from "./lib/verser"; export * from "./lib/verser-client"; export * from "./lib/verser-connection"; -export * from "./lib/tecemux"; diff --git a/packages/verser/src/lib/verser-client.ts b/packages/verser/src/lib/verser-client.ts index b010fe07e..226b1997b 100644 --- a/packages/verser/src/lib/verser-client.ts +++ b/packages/verser/src/lib/verser-client.ts @@ -7,7 +7,7 @@ import { createConnection, Socket } from "net"; import { ObjLogger } from "@scramjet/obj-logger"; import { defaultVerserClientOptions } from "./verser-client-default-config"; import { URL } from "url"; -import { TeceMux, TeceMuxChannel } from "./tecemux/tecemux"; +import { TeceMux, TeceMuxChannel } from "@scramjet/tecemux"; type Events = { error: (err: Error) => void; diff --git a/packages/verser/src/lib/verser-connection.ts b/packages/verser/src/lib/verser-connection.ts index 0705774ae..146b524c2 100644 --- a/packages/verser/src/lib/verser-connection.ts +++ b/packages/verser/src/lib/verser-connection.ts @@ -10,7 +10,7 @@ import { import { ObjLogger } from "@scramjet/obj-logger"; import { createConnection, Socket } from "net"; import { VerserRequestResult } from "../types"; -import { TeceMux } from "./tecemux/tecemux"; +import { TeceMux } from "@scramjet/tecemux"; /** * VerserConnection class. From e1c4d3b6b8e8e1c6245bca52f18313ed686730e9 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Fri, 3 Nov 2023 14:26:55 +0000 Subject: [PATCH 226/231] Update deps --- .../sequences/args-to-output/package.json | 2 +- bdd/data/sequences/deploy-app/package.json | 2 +- .../sequences/infinite/package.json | 2 +- bdd/package.json | 10 +- packages/adapters/package.json | 18 +- packages/api-client/package.json | 8 +- packages/api-server/package.json | 10 +- packages/cli/package.json | 12 +- packages/client-utils/package.json | 12 +- packages/host/package.json | 24 +- packages/load-check/package.json | 6 +- packages/logger/package.json | 2 +- packages/manager-api-client/package.json | 6 +- packages/middleware-api-client/package.json | 10 +- packages/model/package.json | 6 +- .../multi-manager-api-client/package.json | 8 +- packages/obj-logger/package.json | 4 +- packages/runner/package.json | 20 +- packages/sth-config/package.json | 4 +- packages/sth/package.json | 12 +- packages/tecemux/package.json | 10 +- packages/telemetry/package.json | 6 +- packages/types/package.json | 2 +- packages/utility/package.json | 2 +- packages/verser/package.json | 12 +- yarn.lock | 211 ++++++++++++++++++ 26 files changed, 316 insertions(+), 105 deletions(-) diff --git a/bdd/data/sequences/args-to-output/package.json b/bdd/data/sequences/args-to-output/package.json index fae3da163..17559d2a0 100644 --- a/bdd/data/sequences/args-to-output/package.json +++ b/bdd/data/sequences/args-to-output/package.json @@ -12,7 +12,7 @@ "author": "Scramjet ", "license": "ISC", "devDependencies": { - "@scramjet/types": "^0.35.3", + "@scramjet/types": "^0.36.1", "@types/node": "15.12.5" }, "repository": { diff --git a/bdd/data/sequences/deploy-app/package.json b/bdd/data/sequences/deploy-app/package.json index fd324c81a..977486414 100644 --- a/bdd/data/sequences/deploy-app/package.json +++ b/bdd/data/sequences/deploy-app/package.json @@ -10,7 +10,7 @@ "author": "Scramjet ", "license": "ISC", "devDependencies": { - "@scramjet/types": "^0.35.3", + "@scramjet/types": "^0.36.1", "@types/node": "15.12.5" }, "repository": { diff --git a/bdd/iac-test-data/sequences/infinite/package.json b/bdd/iac-test-data/sequences/infinite/package.json index d500789ac..e4e93264e 100644 --- a/bdd/iac-test-data/sequences/infinite/package.json +++ b/bdd/iac-test-data/sequences/infinite/package.json @@ -9,6 +9,6 @@ "build": "si pack -o ../hello-2.tar.gz ." }, "devDependencies": { - "@scramjet/types": "^0.35.3" + "@scramjet/types": "^0.36.1" } } diff --git a/bdd/package.json b/bdd/package.json index 03b948db3..7dc9a9297 100644 --- a/bdd/package.json +++ b/bdd/package.json @@ -4,10 +4,10 @@ "description": "As the \"problem scope\" of the business problem that our technology solves is quite complex, we decided to use the BDD practice to support the development process. BDD is a methodology of high automation and agility. It describes a cycle of interactions with well-defined outcomes. As a result of these activities, we obtain working, tested software that has a real value.", "main": "_cucumber.js", "dependencies": { - "@scramjet/api-client": "^0.35.3", - "@scramjet/logger": "^0.35.3", - "@scramjet/obj-logger": "^0.35.3", - "@scramjet/sth-config": "^0.35.3", + "@scramjet/api-client": "^0.36.1", + "@scramjet/logger": "^0.36.1", + "@scramjet/obj-logger": "^0.36.1", + "@scramjet/sth-config": "^0.36.1", "dockerode": "^3.3.4", "find-package-json": "^1.2.0", "freeport": "^1.0.5", @@ -17,7 +17,7 @@ "devDependencies": { "@cucumber/cucumber": "^7.3.2", "@cucumber/pretty-formatter": "^1.0.0", - "@scramjet/types": "^0.35.3" + "@scramjet/types": "^0.36.1" }, "scripts": { "build:bdd": "tsc -p tsconfig.json", diff --git a/packages/adapters/package.json b/packages/adapters/package.json index cd790bc21..f24037023 100644 --- a/packages/adapters/package.json +++ b/packages/adapters/package.json @@ -16,14 +16,14 @@ "license": "AGPL-3.0", "dependencies": { "@kubernetes/client-node": "^0.17.1", - "@scramjet/model": "^0.35.3", - "@scramjet/obj-logger": "^0.35.3", - "@scramjet/pre-runner": "^0.35.3", - "@scramjet/python-runner": "^0.35.3", - "@scramjet/runner": "^0.35.3", - "@scramjet/sth-config": "^0.35.3", - "@scramjet/symbols": "^0.35.3", - "@scramjet/utility": "^0.35.3", + "@scramjet/model": "^0.36.1", + "@scramjet/obj-logger": "^0.36.1", + "@scramjet/pre-runner": "^0.36.1", + "@scramjet/python-runner": "^0.36.1", + "@scramjet/runner": "^0.36.1", + "@scramjet/sth-config": "^0.36.1", + "@scramjet/symbols": "^0.36.1", + "@scramjet/utility": "^0.36.1", "dockerode": "^3.3.4", "scramjet": "^4.36.9", "shell-escape": "^0.2.0", @@ -31,7 +31,7 @@ "ts.data.json": "^2.2.0" }, "devDependencies": { - "@scramjet/types": "^0.35.3", + "@scramjet/types": "^0.36.1", "@types/dockerode": "<=3.3.3", "@types/js-yaml": "4.0.5", "@types/node": "15.12.5", diff --git a/packages/api-client/package.json b/packages/api-client/package.json index 6c0d459a1..5f17d507b 100644 --- a/packages/api-client/package.json +++ b/packages/api-client/package.json @@ -13,14 +13,14 @@ "author": "Scramjet ", "license": "MIT", "dependencies": { - "@scramjet/client-utils": "^0.35.3", - "@scramjet/sth-config": "^0.35.3", - "@scramjet/symbols": "^0.35.3", + "@scramjet/client-utils": "^0.36.1", + "@scramjet/sth-config": "^0.36.1", + "@scramjet/symbols": "^0.36.1", "n-readlines": "^1.0.1", "scramjet": "^4.36.9" }, "devDependencies": { - "@scramjet/types": "^0.35.3", + "@scramjet/types": "^0.36.1", "@types/node": "15.12.5", "ava": "^3.15.0", "ts-node": "^10.9.1", diff --git a/packages/api-server/package.json b/packages/api-server/package.json index 7a653565f..3afc2afc1 100644 --- a/packages/api-server/package.json +++ b/packages/api-server/package.json @@ -15,15 +15,15 @@ "license": "AGPL-3.0", "dependencies": { "0http": "^3.4.1", - "@scramjet/model": "^0.35.3", - "@scramjet/obj-logger": "^0.35.3", - "@scramjet/symbols": "^0.35.3", - "@scramjet/utility": "^0.35.3", + "@scramjet/model": "^0.36.1", + "@scramjet/obj-logger": "^0.36.1", + "@scramjet/symbols": "^0.36.1", + "@scramjet/utility": "^0.36.1", "http-status-codes": "^2.2.0", "scramjet": "^4.36.9" }, "devDependencies": { - "@scramjet/types": "^0.35.3", + "@scramjet/types": "^0.36.1", "@types/node": "15.12.5", "@types/sinon": "^10.0.13", "@types/trouter": "^3.1.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 92c87f7a0..3bf831bd7 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -20,11 +20,11 @@ "author": "Scramjet ", "license": "AGPL-3.0", "dependencies": { - "@scramjet/api-client": "^0.35.3", - "@scramjet/client-utils": "^0.35.3", - "@scramjet/middleware-api-client": "^0.35.3", - "@scramjet/obj-logger": "^0.35.3", - "@scramjet/utility": "^0.35.3", + "@scramjet/api-client": "^0.36.1", + "@scramjet/client-utils": "^0.36.1", + "@scramjet/middleware-api-client": "^0.36.1", + "@scramjet/obj-logger": "^0.36.1", + "@scramjet/utility": "^0.36.1", "chalk": "^4.1.2", "commander": "^9.5.0", "commander-completion": "^1.0.1", @@ -35,7 +35,7 @@ "validator": "^13.7.0" }, "devDependencies": { - "@scramjet/types": "^0.35.3", + "@scramjet/types": "^0.36.1", "@types/find-package-json": "^1.2.3", "@types/minimatch": "^3.0.5", "@types/node": "15.12.5", diff --git a/packages/client-utils/package.json b/packages/client-utils/package.json index faa72db08..07cfe717a 100644 --- a/packages/client-utils/package.json +++ b/packages/client-utils/package.json @@ -14,11 +14,11 @@ "author": "Scramjet ", "license": "MIT", "dependencies": { - "@scramjet/model": "^0.35.3", - "@scramjet/obj-logger": "^0.35.3", - "@scramjet/sth-config": "^0.35.3", - "@scramjet/symbols": "^0.35.3", - "@scramjet/utility": "^0.35.3", + "@scramjet/model": "^0.36.1", + "@scramjet/obj-logger": "^0.36.1", + "@scramjet/sth-config": "^0.36.1", + "@scramjet/symbols": "^0.36.1", + "@scramjet/utility": "^0.36.1", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "n-readlines": "^1.0.1", @@ -27,7 +27,7 @@ "scramjet": "^4.36.9" }, "devDependencies": { - "@scramjet/types": "^0.35.3", + "@scramjet/types": "^0.36.1", "@types/node": "15.12.5", "@types/node-fetch": "^2.6.2", "ava": "^3.15.0", diff --git a/packages/host/package.json b/packages/host/package.json index aee6df2e7..299bde8be 100644 --- a/packages/host/package.json +++ b/packages/host/package.json @@ -18,17 +18,17 @@ "author": "Scramjet ", "license": "AGPL-3.0", "dependencies": { - "@scramjet/adapters": "^0.35.3", - "@scramjet/api-server": "^0.35.3", - "@scramjet/load-check": "^0.35.3", - "@scramjet/model": "^0.35.3", - "@scramjet/obj-logger": "^0.35.3", - "@scramjet/sth-config": "^0.35.3", - "@scramjet/symbols": "^0.35.3", - "@scramjet/telemetry": "^0.35.3", - "@scramjet/utility": "^0.35.3", - "@scramjet/verser": "^0.35.3", - "@scramjet/tecemux": "^0.35.3", + "@scramjet/adapters": "^0.36.1", + "@scramjet/api-server": "^0.36.1", + "@scramjet/load-check": "^0.36.1", + "@scramjet/model": "^0.36.1", + "@scramjet/obj-logger": "^0.36.1", + "@scramjet/sth-config": "^0.36.1", + "@scramjet/symbols": "^0.36.1", + "@scramjet/telemetry": "^0.36.1", + "@scramjet/utility": "^0.36.1", + "@scramjet/verser": "^0.36.1", + "@scramjet/tecemux": "^0.36.1", "bpmux": "^8.2.1", "ext-ip": "^0.3.9", "find-package-json": "^1.2.0", @@ -40,7 +40,7 @@ "systeminformation": "^5.12.7" }, "devDependencies": { - "@scramjet/types": "^0.35.3", + "@scramjet/types": "^0.36.1", "@types/find-package-json": "^1.2.3", "@types/jest": "^29.4.4", "@types/node": "15.12.5", diff --git a/packages/load-check/package.json b/packages/load-check/package.json index 3f759d8a8..0fa1e7d89 100644 --- a/packages/load-check/package.json +++ b/packages/load-check/package.json @@ -14,14 +14,14 @@ "author": "Scramjet ", "license": "AGPL-3.0", "dependencies": { - "@scramjet/obj-logger": "^0.35.3", - "@scramjet/utility": "^0.35.3", + "@scramjet/obj-logger": "^0.36.1", + "@scramjet/utility": "^0.36.1", "scramjet": "^4.36.9", "systeminformation": "^5.12.7", "uuid": "^8.3.2" }, "devDependencies": { - "@scramjet/types": "^0.35.3", + "@scramjet/types": "^0.36.1", "@types/node": "15.12.5", "@types/uuid": "^8.3.4", "ava": "^3.15.0", diff --git a/packages/logger/package.json b/packages/logger/package.json index c312453d6..4e15c9ac6 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -14,7 +14,7 @@ "author": "Scramjet ", "license": "AGPL-3.0", "devDependencies": { - "@scramjet/types": "^0.35.3", + "@scramjet/types": "^0.36.1", "@types/node": "15.12.5", "ava": "^3.15.0", "nyc": "^15.1.0", diff --git a/packages/manager-api-client/package.json b/packages/manager-api-client/package.json index 2cf9a2ae7..d92b5a91b 100644 --- a/packages/manager-api-client/package.json +++ b/packages/manager-api-client/package.json @@ -12,11 +12,11 @@ "test:ava": "ava" }, "dependencies": { - "@scramjet/api-client": "^0.35.3", - "@scramjet/client-utils": "^0.35.3" + "@scramjet/api-client": "^0.36.1", + "@scramjet/client-utils": "^0.36.1" }, "devDependencies": { - "@scramjet/types": "^0.35.3", + "@scramjet/types": "^0.36.1", "@types/node": "15.12.5", "ava": "^3.15.0", "ts-node": "^10.9.1", diff --git a/packages/middleware-api-client/package.json b/packages/middleware-api-client/package.json index 77e3ccc73..126f11ec1 100644 --- a/packages/middleware-api-client/package.json +++ b/packages/middleware-api-client/package.json @@ -14,13 +14,13 @@ "test:ava": "ava" }, "dependencies": { - "@scramjet/api-client": "^0.35.3", - "@scramjet/client-utils": "^0.35.3", - "@scramjet/manager-api-client": "^0.35.3", - "@scramjet/multi-manager-api-client": "^0.35.3" + "@scramjet/api-client": "^0.36.1", + "@scramjet/client-utils": "^0.36.1", + "@scramjet/manager-api-client": "^0.36.1", + "@scramjet/multi-manager-api-client": "^0.36.1" }, "devDependencies": { - "@scramjet/types": "^0.35.3", + "@scramjet/types": "^0.36.1", "ava": "^3.15.0", "esbuild": "^0.14.54", "ts-node": "^10.9.1", diff --git a/packages/model/package.json b/packages/model/package.json index eb04c4012..de5550a4c 100644 --- a/packages/model/package.json +++ b/packages/model/package.json @@ -13,13 +13,13 @@ "author": "Scramjet ", "license": "AGPL-3.0", "dependencies": { - "@scramjet/obj-logger": "^0.35.3", - "@scramjet/symbols": "^0.35.3", + "@scramjet/obj-logger": "^0.36.1", + "@scramjet/symbols": "^0.36.1", "scramjet": "^4.36.9", "uuid": "^8.3.2" }, "devDependencies": { - "@scramjet/types": "^0.35.3", + "@scramjet/types": "^0.36.1", "@types/node": "15.12.5", "@types/uuid": "^8.3.4", "ava": "^3.15.0", diff --git a/packages/multi-manager-api-client/package.json b/packages/multi-manager-api-client/package.json index e27b4cc2b..1f3adef9e 100644 --- a/packages/multi-manager-api-client/package.json +++ b/packages/multi-manager-api-client/package.json @@ -12,12 +12,12 @@ "test:ava": "ava" }, "dependencies": { - "@scramjet/api-client": "^0.35.3", - "@scramjet/client-utils": "^0.35.3", - "@scramjet/manager-api-client": "^0.35.3" + "@scramjet/api-client": "^0.36.1", + "@scramjet/client-utils": "^0.36.1", + "@scramjet/manager-api-client": "^0.36.1" }, "devDependencies": { - "@scramjet/types": "^0.35.3", + "@scramjet/types": "^0.36.1", "@types/node": "15.12.5", "ava": "^3.15.0", "ts-node": "^10.9.1", diff --git a/packages/obj-logger/package.json b/packages/obj-logger/package.json index 06ddb0d93..077bcddfb 100644 --- a/packages/obj-logger/package.json +++ b/packages/obj-logger/package.json @@ -14,7 +14,7 @@ "author": "Scramjet ", "license": "AGPL-3.0", "devDependencies": { - "@scramjet/types": "^0.35.3", + "@scramjet/types": "^0.36.1", "@types/node": "15.12.5", "ava": "^3.15.0", "nyc": "^15.1.0", @@ -24,7 +24,7 @@ "typescript": "~4.7.4" }, "dependencies": { - "@scramjet/utility": "^0.35.3", + "@scramjet/utility": "^0.36.1", "scramjet": "^4.36.9" }, "ava": { diff --git a/packages/runner/package.json b/packages/runner/package.json index 7e466c917..baca334e1 100644 --- a/packages/runner/package.json +++ b/packages/runner/package.json @@ -1,6 +1,6 @@ { "name": "@scramjet/runner", - "version": "0.35.3", + "version": "0.36.1", "description": "This package is part of Scramjet Transform Hub. The package executes the remote runners and provides communication with them through abstraction layer provided by adapters.", "main": "./src/bin/start-runner.ts", "scripts": { @@ -16,18 +16,18 @@ "author": "Scramjet ", "license": "MIT", "dependencies": { - "@scramjet/api-client": "^0.35.2", - "@scramjet/client-utils": "^0.35.2", - "@scramjet/manager-api-client": "^0.35.2", - "@scramjet/model": "^0.35.2", - "@scramjet/obj-logger": "^0.35.2", - "@scramjet/symbols": "^0.35.2", - "@scramjet/utility": "^0.35.2", - "@scramjet/tecemux": "^0.35.2", + "@scramjet/api-client": "^0.36.1", + "@scramjet/client-utils": "^0.36.1", + "@scramjet/manager-api-client": "^0.36.1", + "@scramjet/model": "^0.36.1", + "@scramjet/obj-logger": "^0.36.1", + "@scramjet/symbols": "^0.36.1", + "@scramjet/utility": "^0.36.1", + "@scramjet/tecemux": "^0.36.1", "scramjet": "^4.36.9" }, "devDependencies": { - "@scramjet/types": "^0.35.3", + "@scramjet/types": "^0.36.1", "@types/node": "15.12.5", "@types/sinon": "^10.0.13", "ava": "^3.15.0", diff --git a/packages/sth-config/package.json b/packages/sth-config/package.json index 8e948dc6f..a016866bc 100644 --- a/packages/sth-config/package.json +++ b/packages/sth-config/package.json @@ -39,10 +39,10 @@ "url": "https://github.com/scramjetorg/transform-hub/issues" }, "dependencies": { - "@scramjet/utility": "^0.35.3" + "@scramjet/utility": "^0.36.1" }, "devDependencies": { - "@scramjet/types": "^0.35.3", + "@scramjet/types": "^0.36.1", "ava": "^3.15.0", "typedoc": "0.23.17", "typedoc-plugin-markdown": "3.13.6" diff --git a/packages/sth/package.json b/packages/sth/package.json index ceffa0012..c5e22c7ce 100644 --- a/packages/sth/package.json +++ b/packages/sth/package.json @@ -20,15 +20,15 @@ "author": "Scramjet ", "license": "AGPL-3.0", "dependencies": { - "@scramjet/host": "^0.35.3", - "@scramjet/obj-logger": "^0.35.3", - "@scramjet/sth-config": "^0.35.3", - "@scramjet/utility": "^0.35.3", + "@scramjet/host": "^0.36.1", + "@scramjet/obj-logger": "^0.36.1", + "@scramjet/sth-config": "^0.36.1", + "@scramjet/utility": "^0.36.1", "commander": "^8.3.0" }, "devDependencies": { - "@scramjet/model": "^0.35.3", - "@scramjet/types": "^0.35.3", + "@scramjet/model": "^0.36.1", + "@scramjet/types": "^0.36.1", "@types/node": "15.12.5", "ava": "^3.15.0", "ts-node": "^10.9.1", diff --git a/packages/tecemux/package.json b/packages/tecemux/package.json index eacfbb3e4..f684c1c95 100644 --- a/packages/tecemux/package.json +++ b/packages/tecemux/package.json @@ -1,6 +1,6 @@ { "name": "@scramjet/tecemux", - "version": "0.35.3", + "version": "0.36.1", "description": "This package is part of Scramjet Transform Hub. The package provides a communication protocol used among Scramjet modules.", "main": "./src/index.ts", "scripts": { @@ -14,13 +14,13 @@ "author": "Scramjet ", "license": "AGPL-3.0", "dependencies": { - "@scramjet/obj-logger": "^0.35.3", - "@scramjet/utility": "^0.35.3", + "@scramjet/obj-logger": "^0.36.1", + "@scramjet/utility": "^0.36.1", "bpmux": "^8.2.1" }, "devDependencies": { - "@scramjet/api-server": "^0.35.3", - "@scramjet/types": "^0.35.3", + "@scramjet/api-server": "^0.36.1", + "@scramjet/types": "^0.36.1", "@types/node": "15.12.5", "ava": "^3.15.0", "ts-node": "^10.9.1", diff --git a/packages/telemetry/package.json b/packages/telemetry/package.json index e152e719b..cd49967b3 100644 --- a/packages/telemetry/package.json +++ b/packages/telemetry/package.json @@ -32,13 +32,13 @@ "url": "https://github.com/scramjetorg/transform-hub/issues" }, "dependencies": { - "@scramjet/obj-logger": "^0.35.3", - "@scramjet/utility": "^0.35.3", + "@scramjet/obj-logger": "^0.36.1", + "@scramjet/utility": "^0.36.1", "winston": "^3.8.2", "winston-loki": "^6.0.6" }, "devDependencies": { - "@scramjet/types": "^0.35.3", + "@scramjet/types": "^0.36.1", "ava": "^3.15.0", "typedoc": "0.23.17", "typedoc-plugin-markdown": "3.13.6" diff --git a/packages/types/package.json b/packages/types/package.json index 5454bdad3..aebaf4c28 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -14,7 +14,7 @@ "author": "Scramjet ", "license": "AGPL-3.0", "dependencies": { - "@scramjet/symbols": "^0.35.3", + "@scramjet/symbols": "^0.36.1", "http-status-codes": "^2.2.0" }, "devDependencies": { diff --git a/packages/utility/package.json b/packages/utility/package.json index 4ea964fbd..67afd24b4 100644 --- a/packages/utility/package.json +++ b/packages/utility/package.json @@ -33,7 +33,7 @@ "url": "https://github.com/scramjetorg/transform-hub/issues" }, "devDependencies": { - "@scramjet/types": "^0.35.3", + "@scramjet/types": "^0.36.1", "ava": "^4.3.3", "typed-emitter": "^1.4.0", "typedoc": "0.23.17", diff --git a/packages/verser/package.json b/packages/verser/package.json index 95e62f1b4..71d5c7bfe 100644 --- a/packages/verser/package.json +++ b/packages/verser/package.json @@ -1,6 +1,6 @@ { "name": "@scramjet/verser", - "version": "0.35.3", + "version": "0.36.1", "description": "This package is part of Scramjet Transform Hub. The package provides a reverse server functionality used among Scramjet modules.", "main": "./src/index.ts", "scripts": { @@ -14,13 +14,13 @@ "author": "Scramjet ", "license": "AGPL-3.0", "dependencies": { - "@scramjet/obj-logger": "^0.35.3", - "@scramjet/utility": "^0.35.3", - "@scramjet/tecemux": "^0.35.3" + "@scramjet/obj-logger": "^0.36.1", + "@scramjet/utility": "^0.36.1", + "@scramjet/tecemux": "^0.36.1" }, "devDependencies": { - "@scramjet/api-server": "^0.35.3", - "@scramjet/types": "^0.35.3", + "@scramjet/api-server": "^0.36.1", + "@scramjet/types": "^0.36.1", "@types/node": "15.12.5", "ava": "^3.15.0", "ts-node": "^10.9.1", diff --git a/yarn.lock b/yarn.lock index 216327140..79f5c9cf6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1145,6 +1145,211 @@ resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== +"@scramjet/adapters@^0.36.1": + version "0.36.1" + resolved "https://registry.yarnpkg.com/@scramjet/adapters/-/adapters-0.36.1.tgz#022c2ddc3cdc5594f80802e72e08bc250c179ae1" + integrity sha512-GTj9boNlyy7E39mtag1lsyZGRrORUf4JmFGVtrJYNRK5uzLyKBE+X4nCQQhUGlIame/DsZHQqbK6jQIC6AuWdQ== + dependencies: + "@kubernetes/client-node" "^0.17.1" + "@scramjet/model" "^0.36.1" + "@scramjet/obj-logger" "^0.36.1" + "@scramjet/pre-runner" "^0.36.1" + "@scramjet/python-runner" "^0.36.1" + "@scramjet/runner" "^0.36.1" + "@scramjet/sth-config" "^0.36.1" + "@scramjet/symbols" "^0.36.1" + "@scramjet/utility" "^0.36.1" + dockerode "^3.3.4" + scramjet "^4.36.9" + shell-escape "^0.2.0" + systeminformation "^5.21.7" + ts.data.json "^2.2.0" + +"@scramjet/api-client@^0.36.1": + version "0.36.1" + resolved "https://registry.yarnpkg.com/@scramjet/api-client/-/api-client-0.36.1.tgz#c039e2fa0269839385866679f95e5d5f42efeb3f" + integrity sha512-hRnYi39ovMbnvN0GwAma8D2ISnqxg8VKnixb3lbUse5DyIC3qUwwH70UH+DXxheE5nuqfyU9BPA1FEY1InRD0A== + dependencies: + "@scramjet/client-utils" "^0.36.1" + "@scramjet/sth-config" "^0.36.1" + "@scramjet/symbols" "^0.36.1" + n-readlines "^1.0.1" + scramjet "^4.36.9" + +"@scramjet/api-server@^0.36.1": + version "0.36.1" + resolved "https://registry.yarnpkg.com/@scramjet/api-server/-/api-server-0.36.1.tgz#1ea88b4d8827dc57221968b35e7247fe1052b2c8" + integrity sha512-s8R2gnWBw66ZVmKcTVOaylIwAcLDvU7qoWft2TWHyhmqoh1kgb0/PdcjcOsVG5uK8D+/RXtMqNBoAB/isNjpiQ== + dependencies: + "0http" "^3.4.1" + "@scramjet/model" "^0.36.1" + "@scramjet/obj-logger" "^0.36.1" + "@scramjet/symbols" "^0.36.1" + "@scramjet/utility" "^0.36.1" + http-status-codes "^2.2.0" + scramjet "^4.36.9" + +"@scramjet/client-utils@^0.36.1": + version "0.36.1" + resolved "https://registry.yarnpkg.com/@scramjet/client-utils/-/client-utils-0.36.1.tgz#2c68d1786a1508f1658bd1eda2e6d7f56207933e" + integrity sha512-06DG2Q316OVUN7gXzLRvk4PCtCSXWVK0wL2iU3d77Ks54Jvw7v9N307p0mfQ0M3AE/c1gT7iaI2lwfMb0vrwuw== + dependencies: + "@scramjet/model" "^0.36.1" + "@scramjet/obj-logger" "^0.36.1" + "@scramjet/sth-config" "^0.36.1" + "@scramjet/symbols" "^0.36.1" + "@scramjet/utility" "^0.36.1" + "@types/node-fetch" "^2.6.4" + abort-controller "^3.0.0" + n-readlines "^1.0.1" + node-fetch "^2.6.7" + normalize-url "4" + scramjet "^4.36.9" + +"@scramjet/host@^0.36.1": + version "0.36.1" + resolved "https://registry.yarnpkg.com/@scramjet/host/-/host-0.36.1.tgz#961306c78057edc57a2f2ef1fedbd8620890d803" + integrity sha512-55Et6PiezYB2AktDuBlNtzGna/9FTRGTnRedgJmjIRtQUKVhBFUHBzg2WsL5hPO/OMjxtdsnTES9YYcXRawaTA== + dependencies: + "@scramjet/adapters" "^0.36.1" + "@scramjet/api-server" "^0.36.1" + "@scramjet/load-check" "^0.36.1" + "@scramjet/model" "^0.36.1" + "@scramjet/module-loader" "^0.36.1" + "@scramjet/obj-logger" "^0.36.1" + "@scramjet/sth-config" "^0.36.1" + "@scramjet/symbols" "^0.36.1" + "@scramjet/telemetry" "^0.36.1" + "@scramjet/utility" "^0.36.1" + "@scramjet/verser" "^0.36.1" + bpmux "^8.2.1" + ext-ip "^0.3.9" + find-package-json "^1.2.0" + http-status-codes "^2.2.0" + minimist "^1.2.6" + pico-s3 "^2.0.0" + rereadable-stream "^1.4.14" + scramjet "^4.36.9" + systeminformation "^5.21.7" + +"@scramjet/load-check@^0.36.1": + version "0.36.1" + resolved "https://registry.yarnpkg.com/@scramjet/load-check/-/load-check-0.36.1.tgz#211374c6776a1b0f94d36be750c3fad9105b9fe3" + integrity sha512-R3hhICDItTpHjAsWsjF7C6ko/gm3NBx2JmwB++Spb4zs3xeu9Sz6mK4WmJSKbMAmfMaS/dTYJyOImee1ZkQSzA== + dependencies: + "@scramjet/obj-logger" "^0.36.1" + "@scramjet/utility" "^0.36.1" + scramjet "^4.36.9" + systeminformation "^5.21.7" + uuid "^8.3.2" + +"@scramjet/logger@^0.36.1": + version "0.36.1" + resolved "https://registry.yarnpkg.com/@scramjet/logger/-/logger-0.36.1.tgz#933e844115dae3863ba8ad8982499f53184141be" + integrity sha512-HX6cgSJtDOy+wOAyCKLUdqCOUSoI2qVhmWDrXnhPNescWzcYQ9LuinhRrqGb85qIQPIGYa1i/QVaKheSFk8s3A== + dependencies: + scramjet "^4.36.9" + +"@scramjet/manager-api-client@^0.36.1": + version "0.36.1" + resolved "https://registry.yarnpkg.com/@scramjet/manager-api-client/-/manager-api-client-0.36.1.tgz#7eb8be766a4b0f285b532ffd6e90d95838a69712" + integrity sha512-B9HiBbckoWszT/w5nO68h8PUF29Ry73oECCYmBsVtZgRJG3nR2NKhBg7iu9AGM1DlG1RANSLFjloG50ZolUNGQ== + dependencies: + "@scramjet/api-client" "^0.36.1" + "@scramjet/client-utils" "^0.36.1" + +"@scramjet/middleware-api-client@^0.36.1": + version "0.36.1" + resolved "https://registry.yarnpkg.com/@scramjet/middleware-api-client/-/middleware-api-client-0.36.1.tgz#988018d741f003f7e760d73c6cdbc08b414bcb93" + integrity sha512-slIV+9n7Lrda6a5BR89qezR9WfmfvRNUGm9acP1z7I5PW9cS9zNq8NlWo63tuFJ8q5OOdp5MfBaqbj1irKl+yg== + dependencies: + "@scramjet/api-client" "^0.36.1" + "@scramjet/client-utils" "^0.36.1" + "@scramjet/manager-api-client" "^0.36.1" + "@scramjet/multi-manager-api-client" "^0.36.1" + +"@scramjet/model@^0.36.1": + version "0.36.1" + resolved "https://registry.yarnpkg.com/@scramjet/model/-/model-0.36.1.tgz#89bbad5e324d6fcb7db7bbb685f1b4e2c1f74bc9" + integrity sha512-HRSsiunjmXIFiNbHr55c6fT3u3WMLdY+mKGepINHDA6F/D0y4MfTHwt0xirUlmFxMzX6iXvTGtgvknI8EZemcQ== + dependencies: + "@scramjet/obj-logger" "^0.36.1" + "@scramjet/symbols" "^0.36.1" + scramjet "^4.36.9" + uuid "^8.3.2" + +"@scramjet/module-loader@^0.36.1": + version "0.36.1" + resolved "https://registry.yarnpkg.com/@scramjet/module-loader/-/module-loader-0.36.1.tgz#b73883c0ffc95133e01cf50a241a20a0fcda805d" + integrity sha512-R3PTA2XjSjHFAI7DHnykKvChO+FYqjsFIRv+1TdzHXLKHlVWtfERgYx3IWlQSIAcIJL0+AtSM/0RaZMjVQ3OoQ== + dependencies: + "@scramjet/obj-logger" "^0.36.1" + +"@scramjet/multi-manager-api-client@^0.36.1": + version "0.36.1" + resolved "https://registry.yarnpkg.com/@scramjet/multi-manager-api-client/-/multi-manager-api-client-0.36.1.tgz#bf6d62a8d754f1ed07601ef17e79b76a701a3938" + integrity sha512-9ZsMRJUXav3d8shX3zKL3pY7tZTOrucgqL5KFwLJFtSkoFf/egLKjaifl9PE6Tj0zufRtXi3HTWfXtPE/IFlHw== + dependencies: + "@scramjet/api-client" "^0.36.1" + "@scramjet/client-utils" "^0.36.1" + "@scramjet/manager-api-client" "^0.36.1" + +"@scramjet/obj-logger@^0.36.1": + version "0.36.1" + resolved "https://registry.yarnpkg.com/@scramjet/obj-logger/-/obj-logger-0.36.1.tgz#e7d15c33f22444f3871e62cd5ce3964c2f1d0fce" + integrity sha512-3yDell8/QKf+W4lGCgm74tf+bNS/S0U0//xO/7FyZbiE69ajJhd3f9C70Q2Xf9xYBMyhpJuK+vTteGAJiJCtdQ== + dependencies: + "@scramjet/utility" "^0.36.1" + scramjet "^4.36.9" + +"@scramjet/pre-runner@^0.36.1": + version "0.36.1" + resolved "https://registry.yarnpkg.com/@scramjet/pre-runner/-/pre-runner-0.36.1.tgz#eb4dd97510a7603c005f75d583ea4778d21a484f" + integrity sha512-0KJv85S7w21YjLNbpunWlx4wXE2J1oPI5iOmJ8LinyAa/t87iuQA+4BTVWIdRRP8F8/Z9puI+GQbZ2oYBMuRag== + +"@scramjet/python-runner@^0.36.1": + version "0.36.1" + resolved "https://registry.yarnpkg.com/@scramjet/python-runner/-/python-runner-0.36.1.tgz#9c82bebe661523aa4faea7b907ac5b611359c29e" + integrity sha512-4gQrbzXQlYWNT1ms7RhpRlSyfuZjqTZT9oAkcFE52mtYbif6MmHbGt89mChnuab7RwgtdcjWqO6S/0EE+79a+A== + +"@scramjet/sth-config@^0.36.1": + version "0.36.1" + resolved "https://registry.yarnpkg.com/@scramjet/sth-config/-/sth-config-0.36.1.tgz#73c939b6771df48500fcc3cd7df79cfeea199efa" + integrity sha512-xV5pAx8Ulp0onotBke4ZozP1xBNsF8u0stuxw0ONNBl7v0ZIpcSiZF6ASUVCSwQ7Cv7yoSMwRCcT+vI4QRKAog== + dependencies: + "@scramjet/utility" "^0.36.1" + +"@scramjet/symbols@^0.36.1": + version "0.36.1" + resolved "https://registry.yarnpkg.com/@scramjet/symbols/-/symbols-0.36.1.tgz#9777a3c33d6228572ea053bddac32c652a0d6229" + integrity sha512-gDjEOuKoYn/Tl0MdbkliHnJzMOFlHuf3HKMERVnptdF6oqU3VR+azm0nQ2cnZUjv/VTuI51QMXmgk1HKnq9i5g== + +"@scramjet/telemetry@^0.36.1": + version "0.36.1" + resolved "https://registry.yarnpkg.com/@scramjet/telemetry/-/telemetry-0.36.1.tgz#3b35aab94a4035273293018e5ea5d2156d2af718" + integrity sha512-gt/uS0BvzJFpUGKrUZd+6nJcTJmO+GZBc0arc93xqnADK73/4Qlh1eTSSmtCo36/JO3oZNRFQzRQd1ffL7hBtw== + dependencies: + "@scramjet/obj-logger" "^0.36.1" + "@scramjet/utility" "^0.36.1" + winston "^3.8.2" + winston-loki "^6.0.6" + +"@scramjet/types@^0.36.1": + version "0.36.1" + resolved "https://registry.yarnpkg.com/@scramjet/types/-/types-0.36.1.tgz#659aeb6e129faed2190cd528d9109b740e8968da" + integrity sha512-K40buwGRHSVfDy464YxbDLrYTrczpkMWxbskUtTYAUsgKbiKiQEELmjyUP3yS9AUt4UQZDwKWn2eUmhUWl1sGg== + dependencies: + "@scramjet/symbols" "^0.36.1" + http-status-codes "^2.2.0" + +"@scramjet/utility@^0.36.1": + version "0.36.1" + resolved "https://registry.yarnpkg.com/@scramjet/utility/-/utility-0.36.1.tgz#a92b25093afa6d68904aac5cb9a1f541a7c146eb" + integrity sha512-sDanZi5evXaTs+LsR/O1znCZuAa7sJv5EDwGLEF2RpW5fBFOWGhtcCJJ3yJN4qQeyMtjTu/J6c4wnBiZNK4JHQ== + dependencies: + normalize-url "4" + yaml "^2.2.2" + "@sinclair/typebox@^0.27.8": version "0.27.8" resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.27.8.tgz#6667fac16c436b5434a387a34dedb013198f6e6e" @@ -8226,6 +8431,11 @@ systeminformation@^5.12.7: resolved "https://registry.yarnpkg.com/systeminformation/-/systeminformation-5.21.4.tgz#046cd9c7e8c365c742a4726abbbe93de267308c8" integrity sha512-fLW6j47UoAJDlZPEqykkWTKxubxb8IFuow6pMQlqf4irZ2lBgCrCReavMkH2t8VxxjOcg6wBlZ2EPQcluAT6xg== +systeminformation@^5.21.7: + version "5.21.15" + resolved "https://registry.yarnpkg.com/systeminformation/-/systeminformation-5.21.15.tgz#d40fb97462580e99522efa585b1c643cf930e788" + integrity sha512-vMLwsGgJZW6GvoBXVWNZuRQG0MPxlfQnIIIY9ZxoogWftUpJ9C33qD+32e1meFlXuWpN0moNApPFLpbsSi4OaQ== + tar-fs@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.0.1.tgz#e44086c1c60d31a4f0cf893b1c4e155dabfae9e2" @@ -8943,6 +9153,7 @@ wordwrap@^1.0.0: integrity sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q== "wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: + name wrap-ansi-cjs version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== From e71a8e30846d3a342d2c61e902910d7cea7b3ceb Mon Sep 17 00:00:00 2001 From: patuwwy Date: Fri, 3 Nov 2023 15:13:45 +0000 Subject: [PATCH 227/231] Fix lint issues --- packages/host/src/lib/cpm-connector.ts | 4 +- packages/tecemux/src/frames-keeper.ts | 4 +- packages/tecemux/src/tecemux.ts | 2 +- packages/tecemux/test/.keep | 0 packages/tecemux/test/http-connection.spec.ts | 135 ---------------- .../test/playgrounds/playground-tecemux.ts | 2 +- .../tecemux/test/playgrounds/playground.ts | 69 --------- packages/tecemux/tsconfig.json | 3 +- packages/verser/test/.keep | 0 .../test/playgrounds/playground-tecemux.ts | 144 ------------------ .../verser/test/playgrounds/playground.ts | 68 --------- packages/verser/test/tecemux-transfer.spec.ts | 86 ----------- 12 files changed, 8 insertions(+), 509 deletions(-) delete mode 100644 packages/tecemux/test/.keep delete mode 100644 packages/tecemux/test/http-connection.spec.ts delete mode 100644 packages/tecemux/test/playgrounds/playground.ts delete mode 100644 packages/verser/test/.keep delete mode 100644 packages/verser/test/playgrounds/playground-tecemux.ts delete mode 100644 packages/verser/test/playgrounds/playground.ts delete mode 100644 packages/verser/test/tecemux-transfer.spec.ts diff --git a/packages/host/src/lib/cpm-connector.ts b/packages/host/src/lib/cpm-connector.ts index 442a897cc..d15af2f77 100644 --- a/packages/host/src/lib/cpm-connector.ts +++ b/packages/host/src/lib/cpm-connector.ts @@ -174,8 +174,8 @@ export class CPMConnector extends TypedEmitter { "x-sth-tags": JSON.stringify(typeof this.config.tags !== "undefined" ? this.config.tags : []), "x-sth-id": this.config.id || "", "x-manager-id": cpmId, - ...(orgId ? { "x-org-id": orgId } : {}), - ...(sthKey ? { "Authorization": `Digest cnonce="${sthKey}"` } : {}) + ...orgId ? { "x-org-id": orgId } : {}, + ...sthKey ? { Authorization: `Digest cnonce="${sthKey}"` } : {} }, server, https: this.isHttps diff --git a/packages/tecemux/src/frames-keeper.ts b/packages/tecemux/src/frames-keeper.ts index 06d385fc1..142c974bc 100644 --- a/packages/tecemux/src/frames-keeper.ts +++ b/packages/tecemux/src/frames-keeper.ts @@ -12,7 +12,7 @@ export class FramesKeeper extends TypedEmitter implements IF logger = new ObjLogger(this); - generator: AsyncGenerator = (async function* (this: FramesKeeper) { + generator: AsyncGenerator = async function* (this: FramesKeeper) { while (true) { if (this.lastSequenceSent - this.lastSequenceReceived < this.#MAX_FRAMES_DIFFERENCE) { this.logger.debug("Write allowed"); @@ -30,7 +30,7 @@ export class FramesKeeper extends TypedEmitter implements IF }); }); } - }).apply(this); + }.apply(this); handlePSH(sequenceNumber: number) { this.lastSequenceSent = sequenceNumber; diff --git a/packages/tecemux/src/tecemux.ts b/packages/tecemux/src/tecemux.ts index 8980954fc..8098834fd 100644 --- a/packages/tecemux/src/tecemux.ts +++ b/packages/tecemux/src/tecemux.ts @@ -16,7 +16,7 @@ export class TeceMux extends TypedEmitter { framesSent = 0; carrierDecoder: FrameDecoder; framesKeeper = new FramesKeeper(); - sequenceNumber = Math.abs((Math.random() * (2 ** 32)) / 2 | 0); + sequenceNumber = Math.abs(Math.random() * 2 ** 32 / 2 | 0); channels = new Map(); logger: ObjLogger; diff --git a/packages/tecemux/test/.keep b/packages/tecemux/test/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/packages/tecemux/test/http-connection.spec.ts b/packages/tecemux/test/http-connection.spec.ts deleted file mode 100644 index c0c4b0a71..000000000 --- a/packages/tecemux/test/http-connection.spec.ts +++ /dev/null @@ -1,135 +0,0 @@ -import test from "ava"; -import * as http from "http"; -import * as https from "https"; -import { createServer } from "@scramjet/api-server"; -import { Verser, VerserClient, VerserConnection } from "../src"; -import path from "path"; -import { readFileSync } from "fs"; -import { APIExpose } from "@scramjet/types"; -import { spawnSync } from "child_process"; - -async function connectVerserClientAToVerserB( - apiB: APIExpose, - verserClientA: VerserClient, -): Promise { - const verserB = new Verser(apiB.server); - - const connectionResolver = { res: (_conn: VerserConnection) => {} }; - const connectionPromised = new Promise(res => { connectionResolver.res = res; }); - - verserB.on("connect", (verserConnection) => { - verserConnection.respond(200); - // @TODO we have to do that, which means that verser api is a bit off - verserConnection.reconnect(); - connectionResolver.res(verserConnection); - }); - - await verserClientA.connect(); - - return connectionPromised; -} - -function getJSONResponseFromRequest(request: http.ClientRequest): Promise { - return new Promise(resolve => { - request.on("response", async (response) => { - let responseBody = ""; - - for await (const chunk of response) { - responseBody += chunk; - } - - resolve(JSON.parse(responseBody)); - }); - }); -} - -test("Connect VerserClient A to Verser B and send HTTP GET Request to VerserClient A", async (t) => { - const SERVER_B_PORT = 1999; - const apiB = createServer({ }); - - apiB.server.listen(SERVER_B_PORT); - - const apiA = createServer(); - - // @TODO since this is alway needed, maybe we should be setting that in VerserClient? - (apiA.server as http.Server & { httpAllowHalfOpen?: boolean }).httpAllowHalfOpen = true; - - const verserClientA = new VerserClient({ - headers: { city: "Valencia" }, - verserUrl: `http://0.0.0.0:${SERVER_B_PORT}`, - server: apiA.server - }); - - const verserConnectionB = await connectVerserClientAToVerserB(apiB, verserClientA); - - t.is(verserConnectionB.getHeaders().city, "Valencia"); - - // Forward any request to B to A - apiB.use("*", (req, res) => verserConnectionB.forward(req, res)); - - apiA.get("*", () => ({ greeting: "Bye" })); - - // Make request to B that should be forwarded to A - const requestToAThroughB = http.request({ - port: SERVER_B_PORT, - method: "GET", - }); - - requestToAThroughB.end(); - - const responseFromAToB = await getJSONResponseFromRequest(requestToAThroughB); - - // Verify that response from A got back - t.deepEqual(responseFromAToB, { greeting: "Bye" }); -}); - -test("Connect VerserClient A to Verser B over SSL and send HTTPS GET Request to VerserClient A", async (t) => { - const certDir = path.join(__dirname, "cert"); - - spawnSync("./gen-localhost-cert.sh", { cwd: certDir }); - - const SERVER_B_PORT = 2000; - const apiB = createServer({ - sslKeyPath: path.join(certDir, "localhost.key"), - sslCertPath: path.join(certDir, "localhost.crt"), - }); - - apiB.server.listen(SERVER_B_PORT); - - const apiA = createServer(); - - (apiA.server as http.Server & { httpAllowHalfOpen?: boolean }).httpAllowHalfOpen = true; - - const verserClientA = new VerserClient({ - headers: { city: "Valencia" }, - verserUrl: `https://127.0.0.1:${SERVER_B_PORT}`, - server: apiA.server, - https: { ca: [readFileSync(path.join(certDir, "myCA.pem"))] } - }); - - const verserConnectionB = await connectVerserClientAToVerserB(apiB, verserClientA); - - t.is(verserConnectionB.getHeaders().city, "Valencia"); - - // Forward any request to B to A - apiB.use("*", (req, res) => verserConnectionB.forward(req, res)); - - apiA.get("*", () => ({ greeting: "Bye" })); - - // Make request to B that should be forwarded to A - const requestToAThroughB = https.request({ - port: SERVER_B_PORT, - method: "GET", - ca: [readFileSync(path.join(certDir, "myCA.pem"))] - }); - - requestToAThroughB.flushHeaders(); - - const responseFromAToB = await getJSONResponseFromRequest(requestToAThroughB); - - // Verify that response from A got back - t.deepEqual(responseFromAToB, { greeting: "Bye" }); - - spawnSync("./cleanup-localhost-cert.sh", { cwd: certDir }); -}); - diff --git a/packages/tecemux/test/playgrounds/playground-tecemux.ts b/packages/tecemux/test/playgrounds/playground-tecemux.ts index 8667c18f6..95cbced9f 100644 --- a/packages/tecemux/test/playgrounds/playground-tecemux.ts +++ b/packages/tecemux/test/playgrounds/playground-tecemux.ts @@ -6,7 +6,7 @@ import { ObjLogger } from "@scramjet/obj-logger"; import { IncomingMessage, createServer } from "http"; import { Socket, createConnection } from "net"; -import { TeceMux } from "../../src/tecemux"; +import { TeceMux } from "../.."; import { TeceMuxChannel } from "../../src/types"; import { createReadStream, createWriteStream } from "fs"; import path from "path"; diff --git a/packages/tecemux/test/playgrounds/playground.ts b/packages/tecemux/test/playgrounds/playground.ts deleted file mode 100644 index 4dc03db15..000000000 --- a/packages/tecemux/test/playgrounds/playground.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { FrameDecoder, FrameEncoder } from "../../src/codecs"; -import { ObjLogger } from "@scramjet/obj-logger"; -import { createReadStream, createWriteStream } from "fs"; -import path from "path"; -import { PassThrough, Transform } from "stream"; -import { ITeCeMux } from "../../src/types"; -import { FramesKeeper } from "../../src/frames-keeper"; - -const tcm = { - sequenceNumber: 0, - framesKeeper: new FramesKeeper(), - framesSent: 0 -} as ITeCeMux; - -const logger = new ObjLogger("Sandbox"); - -logger.pipe(process.stdout); - -const encoder = new FrameEncoder(0, tcm); -const decoder = new FrameDecoder(); - -encoder.logger.pipe(logger); -decoder.logger.pipe(logger); - -const ws = createWriteStream(path.join(__dirname, "output.tar.gz")); - -const delayedPassThrough = () => new PassThrough({ - async transform(chunk, encoding, callback) { - await new Promise((res, _rej) => { - setTimeout(res, 1000); - }); - - this.push(chunk, encoding); - callback(null); - }, -}); - -//let t = 0; - -const dh = new Transform({ - writableObjectMode: true, - transform: (chunk, encoding, callback) => { - try { - if (chunk && chunk.length) { - //t += chunk.length; - - if (!ws.write(new Uint8Array(JSON.parse(chunk.toString()).chunk.data))) { - dh.pause(); - ws.once("drain", () => { - callback(); - dh.resume(); - }); - } else { - callback(null); - } - } - } catch (e) { - logger.error("dh error", e); - } - } -}); - -createReadStream(path.join(__dirname, "../../../../forever.tar.gz"), { encoding: undefined }) - .pipe(encoder, { end: false }).out - .pipe(delayedPassThrough()) - .pipe(decoder) - .pipe(delayedPassThrough()) - .pipe(dh); - diff --git a/packages/tecemux/tsconfig.json b/packages/tecemux/tsconfig.json index 9c4e6d593..2891ea005 100644 --- a/packages/tecemux/tsconfig.json +++ b/packages/tecemux/tsconfig.json @@ -4,7 +4,8 @@ }, "extends": "../../tsconfig.base.json", "include": [ - "./src/**/*.ts" + "./src/**/*.ts", + "./test" ], "exclude": [ "node_modules" diff --git a/packages/verser/test/.keep b/packages/verser/test/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/packages/verser/test/playgrounds/playground-tecemux.ts b/packages/verser/test/playgrounds/playground-tecemux.ts deleted file mode 100644 index 5c028b26a..000000000 --- a/packages/verser/test/playgrounds/playground-tecemux.ts +++ /dev/null @@ -1,144 +0,0 @@ -/*eslint no-unused-vars: ["error", { "args": "none" }]*/ -/* eslint-disable @typescript-eslint/no-floating-promises */ -/* eslint-disable no-console */ - -import { ObjLogger } from "@scramjet/obj-logger"; -import { IncomingMessage, createServer } from "http"; - -import { Socket, createConnection } from "net"; -import { TeceMux } from "../../src/lib/tecemux/tecemux"; -import { TeceMuxChannel } from "../../src/lib/tecemux/types"; -import { createReadStream, createWriteStream } from "fs"; -import path from "path"; - -(async () => { - const logger = new ObjLogger("Sandbox"); - - logger.pipe(process.stdout); - - /**********************************************/ - /* SERVER - /**********************************************/ - - const PORT = 6660; - const server = createServer({}); - - server.on("timeout", (_socket) => { - logger.warn("Server on timeout"); - }); - server.requestTimeout = 0; - - server.on("connect", async (req: IncomingMessage, socket: Socket) => { - socket.setNoDelay(true); - - logger.info("Incoming request", req.method, req.headers); - - socket - .on("pipe", () => { - logger.info("Carrier Socket piped"); - }) - .on("unpipe", (c: any) => { - logger.info("Carrier Socket unpiped", c); - }) - .on("pause", () => { - logger.fatal("Carrier Socket paused"); - }) - .on("resume", () => { - logger.info("Carrier Socket resumed"); - }) - .on("error", (error) => { - logger.error("Carrier Socket error", error); - }) - .on("close", () => { - logger.info("Carrier Socket closed"); - }) - .on("timeout", () => { - logger.info("Carrier Socket timeout"); - }); - - const tcmux = new TeceMux(socket, "Server") - .on("error", (err) => { - logger.error("TCMUX err", err); - }); - - tcmux.logger.pipe(logger); - - const channel1 = tcmux.multiplex(); - - channel1.pipe(createWriteStream(path.join(__dirname, "output-server.tar.gz"))); - - createReadStream(path.join(__dirname, "../../../forever.tar.gz")) - .on("end", () => { - logger.info("FILE END"); - }) - .pipe(channel1); - }); - - server.listen(PORT, "0.0.0.0"); - - /**********************************************/ - /* CLIENT - /**********************************************/ - - const socket = createConnection({ port: PORT, allowHalfOpen: true, host: "0.0.0.0" }, () => {}); - - await new Promise((resolve, reject) => { - socket - .on("connect", resolve) - .on("error", reject); - }); - - socket.setNoDelay(true); - - const reqLogger = new ObjLogger("Req", { id: "Client" }); - - reqLogger.pipe(logger); - - socket.write("CONNECT HTTP/1.1\r\n\r\n\r\n"); - - socket.on("error", (error) => { - reqLogger.error("ERROR", error); - }); - - reqLogger.info("connected to server!"); - - const tcmux = new TeceMux(socket, "Request"); - - tcmux.logger.updateBaseLog({ id: reqLogger.baseLog.id }); - - tcmux.logger.pipe(logger); - - tcmux.on("channel", async (channel: TeceMuxChannel) => { - reqLogger.debug("New channel", channel._id); - - let total = 0; - - channel - .on("finish", () => { - tcmux.logger.info("Channel finish", channel._id); - }) - .on("end", () => { - tcmux.logger.info("Channel readable end [id, readableEnded, writableEnded]", channel._id, channel.readableEnded, channel.writableEnded); - }) - .on("data", (d) => { - total += d.length; - tcmux.logger.info("-------------------- data", channel._id, d.length, total); - }) - .on("pause", () => { - tcmux.logger.info("-------------------- paused", channel._id); - }) - .on("resume", () => { - tcmux.logger.info("-------------------- resumed", channel._id); - }) - .pause(); - - createReadStream(path.join(__dirname, "../../../forever.tar.gz")).pipe(channel); - channel.pipe(createWriteStream(path.join(__dirname, "output-client.tar.gz"))); - }); - - socket.on("error", (err) => { - console.error(err); - }); - - await new Promise((_res, _rej) => {}); -})(); diff --git a/packages/verser/test/playgrounds/playground.ts b/packages/verser/test/playgrounds/playground.ts deleted file mode 100644 index 897abfa43..000000000 --- a/packages/verser/test/playgrounds/playground.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { FrameDecoder, FrameEncoder } from "../../src/lib/tecemux/codecs"; -import { ObjLogger } from "@scramjet/obj-logger"; -import { createReadStream, createWriteStream } from "fs"; -import path from "path"; -import { PassThrough, Transform } from "stream"; -import { ITeCeMux } from "../../src/lib/tecemux/types"; -import { FramesKeeper } from "../../src/lib/tecemux/frames-keeper"; - -const tcm = { - sequenceNumber: 0, - framesKeeper: new FramesKeeper(), - framesSent: 0 -} as ITeCeMux; - -const logger = new ObjLogger("Sandbox"); - -logger.pipe(process.stdout); - -const encoder = new FrameEncoder(0, tcm); -const decoder = new FrameDecoder(); - -encoder.logger.pipe(logger); -decoder.logger.pipe(logger); - -const ws = createWriteStream(path.join(__dirname, "output.tar.gz")); - -const delayedPassThrough = () => new PassThrough({ - async transform(chunk, encoding, callback) { - await new Promise((res, _rej) => { - setTimeout(res, 1000); - }); - - this.push(chunk, encoding); - callback(null); - }, -}); - -let t = 0; - -const dh = new Transform({ - writableObjectMode: true, - transform: (chunk, encoding, callback) => { - try { - if (chunk && chunk.length) { - t = t + chunk.length; - - if (!ws.write(new Uint8Array(JSON.parse(chunk.toString()).chunk.data))) { - dh.pause(); - ws.once("drain", () => { - callback(); - dh.resume(); - }); - } else { - callback(null); - } - } - } catch (e) { - logger.error("dh error", e); - } - } -}); - -createReadStream(path.join(__dirname, "../../../../forever.tar.gz"), { encoding: undefined }) - .pipe(encoder, { end: false }).out - .pipe(delayedPassThrough()) - .pipe(decoder) - .pipe(delayedPassThrough()) - .pipe(dh); diff --git a/packages/verser/test/tecemux-transfer.spec.ts b/packages/verser/test/tecemux-transfer.spec.ts deleted file mode 100644 index 326818ed7..000000000 --- a/packages/verser/test/tecemux-transfer.spec.ts +++ /dev/null @@ -1,86 +0,0 @@ -/* eslint-disable no-console */ -import test from "ava"; -import { IncomingMessage, Server, createServer } from "http"; -import { TeceMux, TeceMuxChannel } from "../src/lib/tecemux/tecemux"; -import * as crypto from "crypto"; -import { Socket, createConnection } from "net"; -import { Readable } from "stream"; - -let serverTeceMux: TeceMux; - -const PORT = 6660; - -async function startServer() { - const server = createServer({}); - - return new Promise((resolve) => { - server.listen(PORT, () => { resolve(server); }); - }); -} - -test.serial("Protocol send file over http connection", async (t) => { - const server = await startServer(); - - const hashReceived = crypto.createHash("md5"); - const hashSent = crypto.createHash("md5"); - - const serverSideChannelPromise = new Promise((resolve) => { - server.on("connect", async (req: IncomingMessage, socket: Socket) => { - console.log("server on connect!"); - socket.setNoDelay(true); - - serverTeceMux = new TeceMux(socket, "Server") - .on("error", (error) => { - console.error("TeceMux error", error.code); - }); - - resolve(serverTeceMux.multiplex({ channel: 1 })); - }); - }); - - const socket = createConnection({ port: PORT, allowHalfOpen: true, host: "0.0.0.0" }, () => {}); - - await new Promise((resolve, reject) => { - socket - .on("connect", resolve) - .on("error", reject); - }); - - socket.write("CONNECT HTTP/1.1\r\n\r\n\r\n"); - - const clientTeceMux = new TeceMux(socket, "Request"); - - await new Promise(resolve => { - clientTeceMux.on("channel", async (clientSideChannel: TeceMuxChannel) => { - console.log("TEST: on channel", clientSideChannel._id); - function* gen() { - for (let i = 0; i < 1e3; i++) { - const str = crypto.randomBytes(1024).toString("hex"); - - hashSent.update(str); - yield str; - } - } - - Readable.from(gen()).pipe(clientSideChannel); - resolve(clientSideChannel); - }); - }); - - const serverSideChannel = await serverSideChannelPromise; - - console.log("Serverside channel", serverSideChannel._id); - - for await (const d of serverSideChannel) { - hashReceived.update(d); - } - - const res = { - txHash: hashSent.digest("hex"), - rxHash: hashReceived.digest("hex") - }; - - console.dir(res); - - t.assert(res.txHash === res.rxHash, "Unequal hashes"); -}); From cda80d77969c9dbe35e149f5091f1c4fcbd4b887 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Tue, 14 Nov 2023 12:28:24 +0000 Subject: [PATCH 228/231] Fix max chunk length, remove console.logs --- package.json | 2 +- packages/tecemux/src/codecs/frame-encoder.ts | 2 +- packages/tecemux/src/tecemux.ts | 8 +- yarn.lock | 210 ------------------- 4 files changed, 6 insertions(+), 216 deletions(-) diff --git a/package.json b/package.json index 39884dad1..b488d8efa 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "clean:modules": "find -name node_modules -or -name __pypackages__ -prune -exec rm -rf {} ';' 2> /dev/null", "lint": "TIMING=1 NODE_OPTIONS=\"--max-old-space-size=3072\" eslint . --ext .ts --ext .js --cache --cache-strategy=content --cache-location=.eslintcache_scramjet-csi", "lint:uncached": "rm .eslintcache_scramjet-csi && yarn lint", - "start": "node dist/sth/bin/hub.js", + "start": "node --trace-warnings dist/sth/bin/hub.js", "start:dev": "ts-node packages/sth/src/bin/hub.ts", "start:dev:cli": "ts-node packages/cli/src/bin/index.ts", "install:clean": "yarn clean && yarn clean:modules && yarn install", diff --git a/packages/tecemux/src/codecs/frame-encoder.ts b/packages/tecemux/src/codecs/frame-encoder.ts index 042f0bfbe..91191bf00 100644 --- a/packages/tecemux/src/codecs/frame-encoder.ts +++ b/packages/tecemux/src/codecs/frame-encoder.ts @@ -8,7 +8,7 @@ import { FrameData, ITeCeMux } from "../types"; import { calculateChecksum } from "./utils"; export class FrameEncoder extends Transform { - MAX_CHUNK_SIZE = 64 * 1024 - HEADER_LENGTH; + MAX_CHUNK_SIZE = 64 * 1024 - HEADER_LENGTH - 1; tecemux: ITeCeMux; total = 0; diff --git a/packages/tecemux/src/tecemux.ts b/packages/tecemux/src/tecemux.ts index 8098834fd..279341108 100644 --- a/packages/tecemux/src/tecemux.ts +++ b/packages/tecemux/src/tecemux.ts @@ -100,13 +100,13 @@ export class TeceMux extends TypedEmitter { if (flags.ACK && flags.SYN) { channel?.encoder.pause(); this.framesKeeper.handleACK(acknowledgeNumber); - //console.log("Pause channel command received", channel?._id); + //console.log("Pause channel command received", channel?._id); return 0; } if (flags.ACK) { if (channel?.encoder.isPaused()) { - console.log("Pause and receive ack", channel._id); + //console.log("Pause and receive ack", channel._id); } this.logger.trace("Received ACK flag for sequenceNumber", acknowledgeNumber); this.framesKeeper.handleACK(acknowledgeNumber); @@ -150,12 +150,12 @@ export class TeceMux extends TypedEmitter { channel.sendPauseACK(sequenceNumber); channel.once("resume", () => { - console.log("resumed, send ack to resume"); + //console.log("resumed, send ack to resume"); //channel?.encoder.resume(); channel?.sendACK(sequenceNumber); }); channel.once("drain", () => { - console.log("drained, send ack to resume"); + //console.log("drained, send ack to resume"); //channel?.encoder.resume(); channel?.sendACK(sequenceNumber); }); diff --git a/yarn.lock b/yarn.lock index defdfaf9d..bcc871d49 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1179,211 +1179,6 @@ resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== -"@scramjet/adapters@^0.36.1": - version "0.36.1" - resolved "https://registry.yarnpkg.com/@scramjet/adapters/-/adapters-0.36.1.tgz#022c2ddc3cdc5594f80802e72e08bc250c179ae1" - integrity sha512-GTj9boNlyy7E39mtag1lsyZGRrORUf4JmFGVtrJYNRK5uzLyKBE+X4nCQQhUGlIame/DsZHQqbK6jQIC6AuWdQ== - dependencies: - "@kubernetes/client-node" "^0.17.1" - "@scramjet/model" "^0.36.1" - "@scramjet/obj-logger" "^0.36.1" - "@scramjet/pre-runner" "^0.36.1" - "@scramjet/python-runner" "^0.36.1" - "@scramjet/runner" "^0.36.1" - "@scramjet/sth-config" "^0.36.1" - "@scramjet/symbols" "^0.36.1" - "@scramjet/utility" "^0.36.1" - dockerode "^3.3.4" - scramjet "^4.36.9" - shell-escape "^0.2.0" - systeminformation "^5.21.7" - ts.data.json "^2.2.0" - -"@scramjet/api-client@^0.36.1": - version "0.36.1" - resolved "https://registry.yarnpkg.com/@scramjet/api-client/-/api-client-0.36.1.tgz#c039e2fa0269839385866679f95e5d5f42efeb3f" - integrity sha512-hRnYi39ovMbnvN0GwAma8D2ISnqxg8VKnixb3lbUse5DyIC3qUwwH70UH+DXxheE5nuqfyU9BPA1FEY1InRD0A== - dependencies: - "@scramjet/client-utils" "^0.36.1" - "@scramjet/sth-config" "^0.36.1" - "@scramjet/symbols" "^0.36.1" - n-readlines "^1.0.1" - scramjet "^4.36.9" - -"@scramjet/api-server@^0.36.1": - version "0.36.1" - resolved "https://registry.yarnpkg.com/@scramjet/api-server/-/api-server-0.36.1.tgz#1ea88b4d8827dc57221968b35e7247fe1052b2c8" - integrity sha512-s8R2gnWBw66ZVmKcTVOaylIwAcLDvU7qoWft2TWHyhmqoh1kgb0/PdcjcOsVG5uK8D+/RXtMqNBoAB/isNjpiQ== - dependencies: - "0http" "^3.4.1" - "@scramjet/model" "^0.36.1" - "@scramjet/obj-logger" "^0.36.1" - "@scramjet/symbols" "^0.36.1" - "@scramjet/utility" "^0.36.1" - http-status-codes "^2.2.0" - scramjet "^4.36.9" - -"@scramjet/client-utils@^0.36.1": - version "0.36.1" - resolved "https://registry.yarnpkg.com/@scramjet/client-utils/-/client-utils-0.36.1.tgz#2c68d1786a1508f1658bd1eda2e6d7f56207933e" - integrity sha512-06DG2Q316OVUN7gXzLRvk4PCtCSXWVK0wL2iU3d77Ks54Jvw7v9N307p0mfQ0M3AE/c1gT7iaI2lwfMb0vrwuw== - dependencies: - "@scramjet/model" "^0.36.1" - "@scramjet/obj-logger" "^0.36.1" - "@scramjet/sth-config" "^0.36.1" - "@scramjet/symbols" "^0.36.1" - "@scramjet/utility" "^0.36.1" - "@types/node-fetch" "^2.6.4" - abort-controller "^3.0.0" - n-readlines "^1.0.1" - node-fetch "^2.6.7" - normalize-url "4" - scramjet "^4.36.9" - -"@scramjet/host@^0.36.1": - version "0.36.1" - resolved "https://registry.yarnpkg.com/@scramjet/host/-/host-0.36.1.tgz#961306c78057edc57a2f2ef1fedbd8620890d803" - integrity sha512-55Et6PiezYB2AktDuBlNtzGna/9FTRGTnRedgJmjIRtQUKVhBFUHBzg2WsL5hPO/OMjxtdsnTES9YYcXRawaTA== - dependencies: - "@scramjet/adapters" "^0.36.1" - "@scramjet/api-server" "^0.36.1" - "@scramjet/load-check" "^0.36.1" - "@scramjet/model" "^0.36.1" - "@scramjet/module-loader" "^0.36.1" - "@scramjet/obj-logger" "^0.36.1" - "@scramjet/sth-config" "^0.36.1" - "@scramjet/symbols" "^0.36.1" - "@scramjet/telemetry" "^0.36.1" - "@scramjet/utility" "^0.36.1" - "@scramjet/verser" "^0.36.1" - bpmux "^8.2.1" - ext-ip "^0.3.9" - find-package-json "^1.2.0" - http-status-codes "^2.2.0" - minimist "^1.2.6" - pico-s3 "^2.0.0" - rereadable-stream "^1.4.14" - scramjet "^4.36.9" - systeminformation "^5.21.7" - -"@scramjet/load-check@^0.36.1": - version "0.36.1" - resolved "https://registry.yarnpkg.com/@scramjet/load-check/-/load-check-0.36.1.tgz#211374c6776a1b0f94d36be750c3fad9105b9fe3" - integrity sha512-R3hhICDItTpHjAsWsjF7C6ko/gm3NBx2JmwB++Spb4zs3xeu9Sz6mK4WmJSKbMAmfMaS/dTYJyOImee1ZkQSzA== - dependencies: - "@scramjet/obj-logger" "^0.36.1" - "@scramjet/utility" "^0.36.1" - scramjet "^4.36.9" - systeminformation "^5.21.7" - uuid "^8.3.2" - -"@scramjet/logger@^0.36.1": - version "0.36.1" - resolved "https://registry.yarnpkg.com/@scramjet/logger/-/logger-0.36.1.tgz#933e844115dae3863ba8ad8982499f53184141be" - integrity sha512-HX6cgSJtDOy+wOAyCKLUdqCOUSoI2qVhmWDrXnhPNescWzcYQ9LuinhRrqGb85qIQPIGYa1i/QVaKheSFk8s3A== - dependencies: - scramjet "^4.36.9" - -"@scramjet/manager-api-client@^0.36.1": - version "0.36.1" - resolved "https://registry.yarnpkg.com/@scramjet/manager-api-client/-/manager-api-client-0.36.1.tgz#7eb8be766a4b0f285b532ffd6e90d95838a69712" - integrity sha512-B9HiBbckoWszT/w5nO68h8PUF29Ry73oECCYmBsVtZgRJG3nR2NKhBg7iu9AGM1DlG1RANSLFjloG50ZolUNGQ== - dependencies: - "@scramjet/api-client" "^0.36.1" - "@scramjet/client-utils" "^0.36.1" - -"@scramjet/middleware-api-client@^0.36.1": - version "0.36.1" - resolved "https://registry.yarnpkg.com/@scramjet/middleware-api-client/-/middleware-api-client-0.36.1.tgz#988018d741f003f7e760d73c6cdbc08b414bcb93" - integrity sha512-slIV+9n7Lrda6a5BR89qezR9WfmfvRNUGm9acP1z7I5PW9cS9zNq8NlWo63tuFJ8q5OOdp5MfBaqbj1irKl+yg== - dependencies: - "@scramjet/api-client" "^0.36.1" - "@scramjet/client-utils" "^0.36.1" - "@scramjet/manager-api-client" "^0.36.1" - "@scramjet/multi-manager-api-client" "^0.36.1" - -"@scramjet/model@^0.36.1": - version "0.36.1" - resolved "https://registry.yarnpkg.com/@scramjet/model/-/model-0.36.1.tgz#89bbad5e324d6fcb7db7bbb685f1b4e2c1f74bc9" - integrity sha512-HRSsiunjmXIFiNbHr55c6fT3u3WMLdY+mKGepINHDA6F/D0y4MfTHwt0xirUlmFxMzX6iXvTGtgvknI8EZemcQ== - dependencies: - "@scramjet/obj-logger" "^0.36.1" - "@scramjet/symbols" "^0.36.1" - scramjet "^4.36.9" - uuid "^8.3.2" - -"@scramjet/module-loader@^0.36.1": - version "0.36.1" - resolved "https://registry.yarnpkg.com/@scramjet/module-loader/-/module-loader-0.36.1.tgz#b73883c0ffc95133e01cf50a241a20a0fcda805d" - integrity sha512-R3PTA2XjSjHFAI7DHnykKvChO+FYqjsFIRv+1TdzHXLKHlVWtfERgYx3IWlQSIAcIJL0+AtSM/0RaZMjVQ3OoQ== - dependencies: - "@scramjet/obj-logger" "^0.36.1" - -"@scramjet/multi-manager-api-client@^0.36.1": - version "0.36.1" - resolved "https://registry.yarnpkg.com/@scramjet/multi-manager-api-client/-/multi-manager-api-client-0.36.1.tgz#bf6d62a8d754f1ed07601ef17e79b76a701a3938" - integrity sha512-9ZsMRJUXav3d8shX3zKL3pY7tZTOrucgqL5KFwLJFtSkoFf/egLKjaifl9PE6Tj0zufRtXi3HTWfXtPE/IFlHw== - dependencies: - "@scramjet/api-client" "^0.36.1" - "@scramjet/client-utils" "^0.36.1" - "@scramjet/manager-api-client" "^0.36.1" - -"@scramjet/obj-logger@^0.36.1": - version "0.36.1" - resolved "https://registry.yarnpkg.com/@scramjet/obj-logger/-/obj-logger-0.36.1.tgz#e7d15c33f22444f3871e62cd5ce3964c2f1d0fce" - integrity sha512-3yDell8/QKf+W4lGCgm74tf+bNS/S0U0//xO/7FyZbiE69ajJhd3f9C70Q2Xf9xYBMyhpJuK+vTteGAJiJCtdQ== - dependencies: - "@scramjet/utility" "^0.36.1" - scramjet "^4.36.9" - -"@scramjet/pre-runner@^0.36.1": - version "0.36.1" - resolved "https://registry.yarnpkg.com/@scramjet/pre-runner/-/pre-runner-0.36.1.tgz#eb4dd97510a7603c005f75d583ea4778d21a484f" - integrity sha512-0KJv85S7w21YjLNbpunWlx4wXE2J1oPI5iOmJ8LinyAa/t87iuQA+4BTVWIdRRP8F8/Z9puI+GQbZ2oYBMuRag== - -"@scramjet/python-runner@^0.36.1": - version "0.36.1" - resolved "https://registry.yarnpkg.com/@scramjet/python-runner/-/python-runner-0.36.1.tgz#9c82bebe661523aa4faea7b907ac5b611359c29e" - integrity sha512-4gQrbzXQlYWNT1ms7RhpRlSyfuZjqTZT9oAkcFE52mtYbif6MmHbGt89mChnuab7RwgtdcjWqO6S/0EE+79a+A== - -"@scramjet/sth-config@^0.36.1": - version "0.36.1" - resolved "https://registry.yarnpkg.com/@scramjet/sth-config/-/sth-config-0.36.1.tgz#73c939b6771df48500fcc3cd7df79cfeea199efa" - integrity sha512-xV5pAx8Ulp0onotBke4ZozP1xBNsF8u0stuxw0ONNBl7v0ZIpcSiZF6ASUVCSwQ7Cv7yoSMwRCcT+vI4QRKAog== - dependencies: - "@scramjet/utility" "^0.36.1" - -"@scramjet/symbols@^0.36.1": - version "0.36.1" - resolved "https://registry.yarnpkg.com/@scramjet/symbols/-/symbols-0.36.1.tgz#9777a3c33d6228572ea053bddac32c652a0d6229" - integrity sha512-gDjEOuKoYn/Tl0MdbkliHnJzMOFlHuf3HKMERVnptdF6oqU3VR+azm0nQ2cnZUjv/VTuI51QMXmgk1HKnq9i5g== - -"@scramjet/telemetry@^0.36.1": - version "0.36.1" - resolved "https://registry.yarnpkg.com/@scramjet/telemetry/-/telemetry-0.36.1.tgz#3b35aab94a4035273293018e5ea5d2156d2af718" - integrity sha512-gt/uS0BvzJFpUGKrUZd+6nJcTJmO+GZBc0arc93xqnADK73/4Qlh1eTSSmtCo36/JO3oZNRFQzRQd1ffL7hBtw== - dependencies: - "@scramjet/obj-logger" "^0.36.1" - "@scramjet/utility" "^0.36.1" - winston "^3.8.2" - winston-loki "^6.0.6" - -"@scramjet/types@^0.36.1": - version "0.36.1" - resolved "https://registry.yarnpkg.com/@scramjet/types/-/types-0.36.1.tgz#659aeb6e129faed2190cd528d9109b740e8968da" - integrity sha512-K40buwGRHSVfDy464YxbDLrYTrczpkMWxbskUtTYAUsgKbiKiQEELmjyUP3yS9AUt4UQZDwKWn2eUmhUWl1sGg== - dependencies: - "@scramjet/symbols" "^0.36.1" - http-status-codes "^2.2.0" - -"@scramjet/utility@^0.36.1": - version "0.36.1" - resolved "https://registry.yarnpkg.com/@scramjet/utility/-/utility-0.36.1.tgz#a92b25093afa6d68904aac5cb9a1f541a7c146eb" - integrity sha512-sDanZi5evXaTs+LsR/O1znCZuAa7sJv5EDwGLEF2RpW5fBFOWGhtcCJJ3yJN4qQeyMtjTu/J6c4wnBiZNK4JHQ== - dependencies: - normalize-url "4" - yaml "^2.2.2" - "@sinclair/typebox@^0.27.8": version "0.27.8" resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.27.8.tgz#6667fac16c436b5434a387a34dedb013198f6e6e" @@ -8460,11 +8255,6 @@ syncpack@^5.8.15: read-yaml-file "2.1.0" semver "7.3.5" -systeminformation@^5.21.7: - version "5.21.7" - resolved "https://registry.yarnpkg.com/systeminformation/-/systeminformation-5.21.7.tgz#53ef75daaf5d756d015f4bb02e059126ccac74f2" - integrity sha512-K3LjnajrazTLTD61+87DFg8IXFk5ljx6nSBqB8pQLtC1UPivAjDtTYGPZ8jaBFxcesPaCOkvLRtBq+RFscrsLw== - systeminformation@^5.21.7: version "5.21.15" resolved "https://registry.yarnpkg.com/systeminformation/-/systeminformation-5.21.15.tgz#d40fb97462580e99522efa585b1c643cf930e788" From 20ac958e558f5a49decedf19f257dedf56aa7254 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 16 Nov 2023 10:00:25 +0000 Subject: [PATCH 229/231] Tecemux test server playground --- .../test/playgrounds/start-test-server.ts | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 packages/tecemux/test/playgrounds/start-test-server.ts diff --git a/packages/tecemux/test/playgrounds/start-test-server.ts b/packages/tecemux/test/playgrounds/start-test-server.ts new file mode 100644 index 000000000..163f484c9 --- /dev/null +++ b/packages/tecemux/test/playgrounds/start-test-server.ts @@ -0,0 +1,86 @@ +#!/usr/bin/env ts-node +/* eslint-disable no-console */ + +import net, { Socket, createConnection } from "net"; + +import { TeceMux, TeceMuxChannel } from "@scramjet/tecemux"; +import { Agent, createServer, request } from "http"; + +const httpServer = createServer((req, res) => { + console.log("Headers", req.headers); + + res.writeHead(200); + res.write("OK"); + res.end(); +}); + +const connectServer = net.createServer() + .on("connection", async (conn: net.Socket) => { + conn.setNoDelay(true); + conn.on("error", (err) => { + console.error("Error on connection from runner", err); + }); + + const protocol = new TeceMux(conn); + + protocol.on("channel", async (channel: TeceMuxChannel) => { + httpServer.emit("connection", channel); + }); + }) + .listen(9000, "0.0.0.0", () => { + console.log("SocketServer on", connectServer.address()); + }) + .on("error", (error) => { console.error(error); }); + +const connectionSocket = createConnection({ + host: "0.0.0.0", + port: 9000 +}); + +const multiplexer = new TeceMux(connectionSocket); + +class CustomAgent extends Agent { + createConnection = () => { + try { + const socket = multiplexer.multiplex() as unknown as Socket; + + socket.on("error", () => { + console.error("Muxed stream error"); + }); + + socket.setKeepAlive ||= (_enable?: boolean, _initialDelay?: number | undefined) => socket; + socket.unref ||= () => socket; + socket.setTimeout ||= (_timeout: number, _callback?: () => void) => socket; + + console.log("Creating muxed channel in verser connection"); + + return socket; + } catch (error) { + const ret = new Socket(); + + setImmediate(() => ret.emit("error", error)); + return ret; + } + }; +} + +request({ + method: "get", + host: "scramjet", + port: "9999", + agent: new CustomAgent(), + path: "/get" +}).on("response", (response) => { + response.on("data", (d) => { console.log("1 Req response", d.toString()); }); +}).flushHeaders(); + +request({ + method: "post", + host: "scramjet", + port: "9999", + agent: new CustomAgent(), + path: "/get" +}).on("response", (response) => { + response.on("data", (d) => { console.log("2 Req response", d.toString()); }); +}).flushHeaders(); + From 2dbd2d14e87ab931af001341f9967c240a414c8f Mon Sep 17 00:00:00 2001 From: S4Adam Date: Mon, 4 Dec 2023 11:07:31 +0100 Subject: [PATCH 230/231] PY: prepare runner&Tecemux for Hub custom connector --- packages/python-runner/requirements.txt | 1 + packages/python-runner/runner.py | 5 ++++- packages/python-runner/tecemux/connector.py | 18 ++++++++++++++++++ packages/python-runner/tecemux/multiplexer.py | 9 +++++++++ 4 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 packages/python-runner/tecemux/connector.py diff --git a/packages/python-runner/requirements.txt b/packages/python-runner/requirements.txt index 60d3e7921..c370aa528 100644 --- a/packages/python-runner/requirements.txt +++ b/packages/python-runner/requirements.txt @@ -1,4 +1,5 @@ attrs==23.1.0 pyee==9.0.4 scramjet-framework-py +scramjet-api-client tecemux/ \ No newline at end of file diff --git a/packages/python-runner/runner.py b/packages/python-runner/runner.py index 9870375f9..6e12131c9 100644 --- a/packages/python-runner/runner.py +++ b/packages/python-runner/runner.py @@ -13,7 +13,7 @@ from tecemux.multiplexer import Tecemux from tecemux.hardcoded_magic_values import CommunicationChannels as CC from tecemux.hardcoded_magic_values import RunnerMessageCodes as msg_codes - +from client.host_client import HostClient SEQUENCE_PATH = os.getenv('SEQUENCE_PATH') SERVER_PORT = os.getenv('INSTANCES_SERVER_PORT') @@ -216,6 +216,8 @@ def load_sequence(self): async def run_instance(self, config, input, args): context = AppContext(self, config) + custom_conn = self.multiplexer.setup_connector() + context.hub = HostClient('http://scramjet-host/api/v1') # (url, connector=custom_conn) self.logger.info('Running instance...') try: result = self.sequence.run(context, input, *args) @@ -325,6 +327,7 @@ def __init__(self, runner, config) -> None: self.monitoring = runner.multiplexer.get_channel(CC.MONITORING) self.runner = runner self.emitter = runner.emitter + self.hub = None def set_stop_handler(self, handler, *args): self.runner.stop_handlers.append(handler) diff --git a/packages/python-runner/tecemux/connector.py b/packages/python-runner/tecemux/connector.py new file mode 100644 index 000000000..c427e4e09 --- /dev/null +++ b/packages/python-runner/tecemux/connector.py @@ -0,0 +1,18 @@ +import aiohttp + + +class Connector(aiohttp.BaseConnector): + """Custom connector class so that the Tecemux can be used in aiohttp requests. + """ + def __init__(self, tecemux: "Tecemux"): + self.tecemux = tecemux + super().__init__() + + async def _create_connection(self, req, traces, timeout) -> "ChannelContext": + # TODO: adjust duplex_channel to be an equivalent to aiohttp's 'ResponseHandler' + channel = await self.tecemux.open_channel(force_open=True) + channel_name = channel._channel_enum + await self.tecemux.sync() + duplex_channel = self.tecemux.get_channel(channel_name) + return duplex_channel # should be an equivalent to aiohttp's 'ResponseHandler' + diff --git a/packages/python-runner/tecemux/multiplexer.py b/packages/python-runner/tecemux/multiplexer.py index e775835cf..ff38cb257 100644 --- a/packages/python-runner/tecemux/multiplexer.py +++ b/packages/python-runner/tecemux/multiplexer.py @@ -11,6 +11,7 @@ from .inet import IPPacket from .hardcoded_magic_values import CommunicationChannels as CC from .channel import ChannelContext, ChannelGuard, ChannelState +from .connector import Connector from .proxy import HTTPProxy TECEMUX_INTERNAL_VERBOSE_DEBUG = False @@ -19,6 +20,7 @@ class Tecemux: """Tecemux protocol implementation for Scramjet Transform Hub Python Runner """ + _connector = field(default=None) _queue: asyncio.Queue = field(default=None) _reader: asyncio.StreamReader = field(default=None) @@ -431,3 +433,10 @@ async def outcoming_data_forward(self) -> None: break self._debug('Tecemux/MAIN: Outcoming data forwarder finished') + + def setup_connector(self) -> Connector: + """Sets connector for aiohttp + """ + self._connector = Connector(self) + return self._connector + From e6a8f366290a313300c503ded6b854784f134571 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Mon, 15 Jan 2024 10:50:57 +0000 Subject: [PATCH 231/231] After devel merge fix --- package-lock.json | 5086 +++++++++++------ packages/runner/package.json | 2 +- packages/tecemux/package.json | 6 +- .../test/playgrounds/start-test-server.ts | 2 +- yarn.lock | 1213 ++-- 5 files changed, 4070 insertions(+), 2239 deletions(-) diff --git a/package-lock.json b/package-lock.json index ca9d196be..a96f5ef62 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2274,27 +2274,17 @@ "link": true }, "node_modules/@scramjet/client-utils": { - "version": "0.38.0", - "resolved": "https://registry.npmjs.org/@scramjet/client-utils/-/client-utils-0.38.0.tgz", - "integrity": "sha512-DzxwgcSCEtIxdG+GsIReL1Zhkh9zKnPDIq3zbVt9Svvsq/UuP94KGg25KVelvr0OTM/yEEovJw9/yOMDWLiXbQ==", - "dependencies": { - "@scramjet/model": "^0.38.0", - "@scramjet/obj-logger": "^0.38.0", - "@scramjet/sth-config": "^0.38.0", - "@scramjet/symbols": "^0.38.0", - "@scramjet/utility": "^0.38.0", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "n-readlines": "^1.0.1", - "node-fetch": "^2.6.7", - "normalize-url": "4", - "scramjet": "^4.36.9" - } + "resolved": "packages/client-utils", + "link": true }, "node_modules/@scramjet/host": { "resolved": "packages/host", "link": true }, + "node_modules/@scramjet/load-check": { + "resolved": "packages/load-check", + "link": true + }, "node_modules/@scramjet/logger": { "resolved": "packages/logger", "link": true @@ -2303,16 +2293,17 @@ "resolved": "packages/manager-api-client", "link": true }, + "node_modules/@scramjet/middleware-api-client": { + "resolved": "packages/middleware-api-client", + "link": true + }, "node_modules/@scramjet/model": { - "version": "0.38.0", - "resolved": "https://registry.npmjs.org/@scramjet/model/-/model-0.38.0.tgz", - "integrity": "sha512-KaIqZi6tbNnBZcc7TmvJhki+oO3k+EKUuH7h044LGatVcS5tbMrIki3w5BX41OCi5neigDCoOla/nJraod6IcQ==", - "dependencies": { - "@scramjet/obj-logger": "^0.38.0", - "@scramjet/symbols": "^0.38.0", - "scramjet": "^4.36.9", - "uuid": "^8.3.2" - } + "resolved": "packages/model", + "link": true + }, + "node_modules/@scramjet/module-loader": { + "resolved": "packages/module-loader", + "link": true }, "node_modules/@scramjet/monitoring-server": { "resolved": "packages/monitoring-server", @@ -2323,13 +2314,20 @@ "link": true }, "node_modules/@scramjet/obj-logger": { - "version": "0.38.0", - "resolved": "https://registry.npmjs.org/@scramjet/obj-logger/-/obj-logger-0.38.0.tgz", - "integrity": "sha512-2RLYiGnVbt1rGN74yvhqocz3PDxjYdO2Rb0GKlxYZJ2JQB/hIVAJSE47wUjKmPkG39anH6UrfGV8T73paH9qgg==", - "dependencies": { - "@scramjet/utility": "^0.38.0", - "scramjet": "^4.36.9" - } + "resolved": "packages/obj-logger", + "link": true + }, + "node_modules/@scramjet/pre-runner": { + "resolved": "packages/pre-runner", + "link": true + }, + "node_modules/@scramjet/python-runner": { + "resolved": "packages/python-runner", + "link": true + }, + "node_modules/@scramjet/runner": { + "resolved": "packages/runner", + "link": true }, "node_modules/@scramjet/stdio-sequence": { "resolved": "packages/stdio-sequence", @@ -2340,35 +2338,32 @@ "link": true }, "node_modules/@scramjet/sth-config": { - "version": "0.38.0", - "resolved": "https://registry.npmjs.org/@scramjet/sth-config/-/sth-config-0.38.0.tgz", - "integrity": "sha512-Glm/lXMK3UsdAwnWnINlxGm7UFaJfGCnDgjwtI8DVHOL3+52SujFbO0fNb/ye0htR77vNg6GTydbK8sEFdTbkw==", - "dependencies": { - "@scramjet/utility": "^0.38.0" - } + "resolved": "packages/sth-config", + "link": true }, "node_modules/@scramjet/symbols": { - "version": "0.38.0", - "resolved": "https://registry.npmjs.org/@scramjet/symbols/-/symbols-0.38.0.tgz", - "integrity": "sha512-Wjg0JUFybp++ngDUovkoBb3IrfVAjo3OeK6d7JodCOVlgrPsOs2jvmINhGhTfbB/WZx0/sRHeG6Yr9zmpDY3Cg==" + "resolved": "packages/symbols", + "link": true + }, + "node_modules/@scramjet/tecemux": { + "resolved": "packages/tecemux", + "link": true + }, + "node_modules/@scramjet/telemetry": { + "resolved": "packages/telemetry", + "link": true }, "node_modules/@scramjet/types": { - "version": "0.38.0", - "resolved": "https://registry.npmjs.org/@scramjet/types/-/types-0.38.0.tgz", - "integrity": "sha512-6PlWPgdP5ot65Zr17d01rf9m6rgL9xgsliFw2yhikEPflLV8c2ohSGSsptR+faM+x+pNyN/O9TnxbSzwwWBNVA==", - "dependencies": { - "@scramjet/symbols": "^0.38.0", - "http-status-codes": "^2.2.0" - } + "resolved": "packages/types", + "link": true }, "node_modules/@scramjet/utility": { - "version": "0.38.0", - "resolved": "https://registry.npmjs.org/@scramjet/utility/-/utility-0.38.0.tgz", - "integrity": "sha512-jSxeqm1BJ/ZczZAlcXP8tiU4k0cRTF4oVFsrczSw07sxliHbKSh7FaYsRfDKrO53xOlyD5FGLER/3IJckFlPpQ==", - "dependencies": { - "normalize-url": "4", - "yaml": "^2.2.2" - } + "resolved": "packages/utility", + "link": true + }, + "node_modules/@scramjet/verser": { + "resolved": "packages/verser", + "link": true }, "node_modules/@sinclair/typebox": { "version": "0.27.8", @@ -2636,6 +2631,7 @@ "version": "2.6.10", "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.10.tgz", "integrity": "sha512-PPpPK6F9ALFTn59Ka3BaL+qGuipRfxNE8qVgkp0bVixeiR2c2/L+IVOiBdu9JhhT22sWnQEp6YyHGI2b2+CMcA==", + "dev": true, "dependencies": { "@types/node": "*", "form-data": "^4.0.0" @@ -4331,6 +4327,18 @@ "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==" }, + "node_modules/cbor": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/cbor/-/cbor-8.1.0.tgz", + "integrity": "sha512-DwGjNW9omn6EwP70aXsn7FQJx5kO12tX0bZkaTjzdVFM6/7nhA4t0EENocKGx6D2Bch9PE2KzCUf5SceBdeijg==", + "dev": true, + "dependencies": { + "nofilter": "^3.1.0" + }, + "engines": { + "node": ">=12.19" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -5223,6 +5231,12 @@ "node": ">=0.10.0" } }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true + }, "node_modules/ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -6307,9 +6321,9 @@ "dev": true }, "node_modules/fast-glob": { - "version": "3.2.12", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", - "integrity": "sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==", + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", + "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", "dev": true, "dependencies": { "@nodelib/fs.stat": "^2.0.2", @@ -6560,6 +6574,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dev": true, "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -9650,6 +9665,15 @@ "integrity": "sha512-dFSmB8fFHEH/s81Xi+Y/15DQY6VHW81nXRj86EMSL3lmuTmK1e+aT4wrFCkTbm+gSwkw4KpX+rT/pMM2c1mF+A==", "dev": true }, + "node_modules/nofilter": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/nofilter/-/nofilter-3.1.0.tgz", + "integrity": "sha512-l2NNj07e9afPnhAhvgVrCD/oy2Ai1yfLpuo3EpiO1jFTsB4sFz6oIfAfSZyQzVpkZQ9xS8ZS5g1jCBgq4Hwo0g==", + "dev": true, + "engines": { + "node": ">=12.19" + } + }, "node_modules/nopt": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/nopt/-/nopt-6.0.0.tgz", @@ -10066,6 +10090,8 @@ }, "node_modules/npm/node_modules/@npmcli/installed-package-contents/node_modules/npm-bundled": { "version": "1.1.2", + "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.2.tgz", + "integrity": "sha512-x5DHup0SuyQcmL3s7Rx/YQ8sbw/Hzg0rj48eN0dV7hf5cmQq5PXIeioroH3raV1QC1yh3uTYuMThvEQF3iKgGQ==", "inBundle": true, "license": "ISC", "dependencies": { @@ -10225,6 +10251,8 @@ }, "node_modules/npm/node_modules/agentkeepalive": { "version": "4.2.1", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.2.1.tgz", + "integrity": "sha512-Zn4cw2NEqd+9fiSVWMscnjyQ1a8Yfoc5oBajLeo5w+YBHgDUcEBY2hS4YpTz6iN5f/2zQiktcuM6tS8x1p9dpA==", "inBundle": true, "license": "MIT", "dependencies": { @@ -10338,6 +10366,8 @@ }, "node_modules/npm/node_modules/bin-links/node_modules/npm-normalize-package-bin": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-2.0.0.tgz", + "integrity": "sha512-awzfKUO7v0FscrSpRoogyNm0sajikhBWpU0QMrW09AMi9n1PoKU6WaIqUzuJSQnpciZZmJ/jMZ2Egfmb/9LiWQ==", "inBundle": true, "license": "ISC", "engines": { @@ -10469,7 +10499,9 @@ } }, "node_modules/npm/node_modules/cli-table3": { - "version": "0.6.2", + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/cli-table3/-/cli-table3-0.6.3.tgz", + "integrity": "sha512-w5Jac5SykAeZJKntOxJCrm63Eg5/4dhMWIcuTbo9rpE+brgaSZo0RuNJZeOyMgsUdhDeojvgyQLmjI+K50ZGyg==", "inBundle": true, "license": "MIT", "dependencies": { @@ -10603,6 +10635,8 @@ }, "node_modules/npm/node_modules/debug/node_modules/ms": { "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "inBundle": true, "license": "MIT" }, @@ -10633,6 +10667,8 @@ }, "node_modules/npm/node_modules/depd": { "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", "inBundle": true, "license": "MIT", "engines": { @@ -10748,7 +10784,9 @@ } }, "node_modules/npm/node_modules/glob": { - "version": "8.0.3", + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", "inBundle": true, "license": "ISC", "dependencies": { @@ -11280,7 +11318,9 @@ } }, "node_modules/npm/node_modules/minimatch": { - "version": "5.1.0", + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", "inBundle": true, "license": "ISC", "dependencies": { @@ -11447,15 +11487,18 @@ } }, "node_modules/npm/node_modules/node-gyp": { - "version": "9.1.0", + "version": "9.4.1", + "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-9.4.1.tgz", + "integrity": "sha512-OQkWKbjQKbGkMf/xqI1jjy3oCTgMKJac58G2+bjZb3fza6gW2YrCSdMQYaoTb70crvE//Gngr4f0AgVHmqHvBQ==", "inBundle": true, "license": "MIT", "dependencies": { "env-paths": "^2.2.0", + "exponential-backoff": "^3.1.1", "glob": "^7.1.4", "graceful-fs": "^4.2.6", "make-fetch-happen": "^10.0.3", - "nopt": "^5.0.0", + "nopt": "^6.0.0", "npmlog": "^6.0.0", "rimraf": "^3.0.2", "semver": "^7.3.5", @@ -11466,7 +11509,7 @@ "node-gyp": "bin/node-gyp.js" }, "engines": { - "node": "^12.22 || ^14.13 || >=16" + "node": "^12.13 || ^14.13 || >=16" } }, "node_modules/npm/node_modules/node-gyp/node_modules/brace-expansion": { @@ -11482,6 +11525,8 @@ }, "node_modules/npm/node_modules/node-gyp/node_modules/glob": { "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "inBundle": true, "license": "ISC", "dependencies": { @@ -11526,6 +11571,22 @@ "node": ">=6" } }, + "node_modules/npm/node_modules/node-gyp/node_modules/semver": { + "version": "7.5.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.2.tgz", + "integrity": "sha512-SoftuTROv/cRjCze/scjGyiDtcUyxw1rgYQSZY7XTmtR5hX+dm76iDbTH8TkLPHCQmlbQVSSbNZCPM2hb0knnQ==", + "inBundle": true, + "license": "ISC", + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/npm/node_modules/nopt": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/nopt/-/nopt-6.0.0.tgz", @@ -11586,6 +11647,8 @@ }, "node_modules/npm/node_modules/npm-bundled/node_modules/npm-normalize-package-bin": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-2.0.0.tgz", + "integrity": "sha512-awzfKUO7v0FscrSpRoogyNm0sajikhBWpU0QMrW09AMi9n1PoKU6WaIqUzuJSQnpciZZmJ/jMZ2Egfmb/9LiWQ==", "inBundle": true, "license": "ISC", "engines": { @@ -11647,6 +11710,8 @@ }, "node_modules/npm/node_modules/npm-packlist/node_modules/npm-normalize-package-bin": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-2.0.0.tgz", + "integrity": "sha512-awzfKUO7v0FscrSpRoogyNm0sajikhBWpU0QMrW09AMi9n1PoKU6WaIqUzuJSQnpciZZmJ/jMZ2Egfmb/9LiWQ==", "inBundle": true, "license": "ISC", "engines": { @@ -11671,6 +11736,8 @@ }, "node_modules/npm/node_modules/npm-pick-manifest/node_modules/npm-normalize-package-bin": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-2.0.0.tgz", + "integrity": "sha512-awzfKUO7v0FscrSpRoogyNm0sajikhBWpU0QMrW09AMi9n1PoKU6WaIqUzuJSQnpciZZmJ/jMZ2Egfmb/9LiWQ==", "inBundle": true, "license": "ISC", "engines": { @@ -11963,6 +12030,8 @@ }, "node_modules/npm/node_modules/read-package-json/node_modules/npm-normalize-package-bin": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-2.0.0.tgz", + "integrity": "sha512-awzfKUO7v0FscrSpRoogyNm0sajikhBWpU0QMrW09AMi9n1PoKU6WaIqUzuJSQnpciZZmJ/jMZ2Egfmb/9LiWQ==", "inBundle": true, "license": "ISC", "engines": { @@ -12036,6 +12105,8 @@ }, "node_modules/npm/node_modules/rimraf/node_modules/glob": { "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "inBundle": true, "license": "ISC", "dependencies": { @@ -12096,7 +12167,9 @@ "optional": true }, "node_modules/npm/node_modules/semver": { - "version": "7.3.7", + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", "inBundle": true, "license": "ISC", "dependencies": { @@ -12111,6 +12184,8 @@ }, "node_modules/npm/node_modules/semver/node_modules/lru-cache": { "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", "inBundle": true, "license": "ISC", "dependencies": { @@ -14395,6 +14470,46 @@ "node": ">=8" } }, + "node_modules/slice-ansi": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-5.0.0.tgz", + "integrity": "sha512-FC+lgizVPfie0kkhqUScwRu1O/lF6NOgJmlCgK+/LYxDCTk8sGelYaHDhFcDN+Sn3Cv+3VSa4Byeo+IMCzpMgQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^6.0.0", + "is-fullwidth-code-point": "^4.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/slice-ansi?sponsor=1" + } + }, + "node_modules/slice-ansi/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/slice-ansi/node_modules/is-fullwidth-code-point": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-4.0.0.tgz", + "integrity": "sha512-O4L094N2/dZ7xqVdrXhh9r1KODPJpFms8B5sGdJLPy664AgvXsreZUyCQQNItZRDlYug4xStLjNp/sz3HvBowQ==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/smart-buffer": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", @@ -15451,6 +15566,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/typed-emitter": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/typed-emitter/-/typed-emitter-1.4.0.tgz", + "integrity": "sha512-weBmoo3HhpKGgLBOYwe8EB31CzDFuaK7CCL+axXhUYhn4jo6DSkHnbefboCF5i4DQ2aMFe0C/FdTWcPdObgHyg==", + "dev": true + }, "node_modules/typedarray-to-buffer": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", @@ -16276,18 +16397,6 @@ "typescript": "~4.7.4" } }, - "packages/adapters/node_modules/@scramjet/pre-runner": { - "resolved": "packages/pre-runner", - "link": true - }, - "packages/adapters/node_modules/@scramjet/python-runner": { - "resolved": "packages/python-runner", - "link": true - }, - "packages/adapters/node_modules/@scramjet/runner": { - "resolved": "packages/runner", - "link": true - }, "packages/api-client": { "name": "@scramjet/api-client", "version": "0.38.0", @@ -16372,10 +16481,6 @@ "typescript": "~4.7.4" } }, - "packages/cli/node_modules/@scramjet/middleware-api-client": { - "resolved": "packages/middleware-api-client", - "link": true - }, "packages/cli/node_modules/commander": { "version": "9.5.0", "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", @@ -16388,7 +16493,6 @@ "packages/client-utils": { "name": "@scramjet/client-utils", "version": "0.38.0", - "extraneous": true, "license": "MIT", "dependencies": { "@scramjet/model": "^0.38.0", @@ -16427,10 +16531,10 @@ "@scramjet/obj-logger": "^0.38.0", "@scramjet/sth-config": "^0.38.0", "@scramjet/symbols": "^0.38.0", + "@scramjet/tecemux": "^0.38.0", "@scramjet/telemetry": "^0.38.0", "@scramjet/utility": "^0.38.0", "@scramjet/verser": "^0.38.0", - "bpmux": "^8.2.1", "ext-ip": "^0.3.9", "find-package-json": "^1.2.0", "http-status-codes": "^2.2.0", @@ -16453,22 +16557,6 @@ "typescript": "~4.7.4" } }, - "packages/host/node_modules/@scramjet/load-check": { - "resolved": "packages/load-check", - "link": true - }, - "packages/host/node_modules/@scramjet/module-loader": { - "resolved": "packages/module-loader", - "link": true - }, - "packages/host/node_modules/@scramjet/telemetry": { - "resolved": "packages/telemetry", - "link": true - }, - "packages/host/node_modules/@scramjet/verser": { - "resolved": "packages/verser", - "link": true - }, "packages/load-check": { "name": "@scramjet/load-check", "version": "0.38.0", @@ -16552,7 +16640,6 @@ "packages/model": { "name": "@scramjet/model", "version": "0.38.0", - "extraneous": true, "license": "AGPL-3.0", "dependencies": { "@scramjet/obj-logger": "^0.38.0", @@ -16628,7 +16715,6 @@ "packages/obj-logger": { "name": "@scramjet/obj-logger", "version": "0.38.0", - "extraneous": true, "license": "AGPL-3.0", "dependencies": { "@scramjet/utility": "^0.38.0", @@ -16666,8 +16752,8 @@ "@scramjet/model": "^0.38.0", "@scramjet/obj-logger": "^0.38.0", "@scramjet/symbols": "^0.38.0", + "@scramjet/tecemux": "^0.38.0", "@scramjet/utility": "^0.38.0", - "bpmux": "^8.2.1", "scramjet": "^4.36.9" }, "devDependencies": { @@ -16685,6 +16771,7 @@ } }, "packages/stdio-sequence": { + "name": "@scramjet/stdio-sequence", "version": "0.22.0", "license": "ISC", "dependencies": { @@ -16730,7 +16817,6 @@ "packages/sth-config": { "name": "@scramjet/sth-config", "version": "0.38.0", - "extraneous": true, "license": "ISC", "dependencies": { "@scramjet/utility": "^0.38.0" @@ -16754,7 +16840,6 @@ "packages/symbols": { "name": "@scramjet/symbols", "version": "0.38.0", - "extraneous": true, "license": "AGPL-3.0", "devDependencies": { "@types/node": "15.12.5", @@ -16763,6 +16848,43 @@ "typescript": "~4.7.4" } }, + "packages/tecemux": { + "version": "0.38.0", + "license": "AGPL-3.0", + "dependencies": { + "@scramjet/obj-logger": "^0.36.1", + "@scramjet/utility": "^0.36.1", + "bpmux": "^8.2.1" + }, + "devDependencies": { + "@scramjet/api-server": "^0.38.0", + "@scramjet/types": "^0.38.0", + "@types/node": "15.12.5", + "ava": "^3.15.0", + "ts-node": "^10.9.1", + "typedoc": "0.23.17", + "typedoc-plugin-markdown": "3.13.6", + "typescript": "~4.7.4" + } + }, + "packages/tecemux/node_modules/@scramjet/obj-logger": { + "version": "0.36.1", + "resolved": "https://registry.npmjs.org/@scramjet/obj-logger/-/obj-logger-0.36.1.tgz", + "integrity": "sha512-3yDell8/QKf+W4lGCgm74tf+bNS/S0U0//xO/7FyZbiE69ajJhd3f9C70Q2Xf9xYBMyhpJuK+vTteGAJiJCtdQ==", + "dependencies": { + "@scramjet/utility": "^0.36.1", + "scramjet": "^4.36.9" + } + }, + "packages/tecemux/node_modules/@scramjet/utility": { + "version": "0.36.1", + "resolved": "https://registry.npmjs.org/@scramjet/utility/-/utility-0.36.1.tgz", + "integrity": "sha512-sDanZi5evXaTs+LsR/O1znCZuAa7sJv5EDwGLEF2RpW5fBFOWGhtcCJJ3yJN4qQeyMtjTu/J6c4wnBiZNK4JHQ==", + "dependencies": { + "normalize-url": "4", + "yaml": "^2.2.2" + } + }, "packages/telemetry": { "name": "@scramjet/telemetry", "version": "0.38.0", @@ -16783,7 +16905,6 @@ "packages/types": { "name": "@scramjet/types", "version": "0.38.0", - "extraneous": true, "license": "AGPL-3.0", "dependencies": { "@scramjet/symbols": "^0.38.0", @@ -16800,7 +16921,6 @@ "packages/utility": { "name": "@scramjet/utility", "version": "0.38.0", - "extraneous": true, "license": "ISC", "dependencies": { "normalize-url": "4", @@ -16814,1787 +16934,2750 @@ "typedoc-plugin-markdown": "3.13.6" } }, - "packages/verser": { - "name": "@scramjet/verser", - "version": "0.38.0", - "license": "AGPL-3.0", + "packages/utility/node_modules/aggregate-error": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-4.0.1.tgz", + "integrity": "sha512-0poP0T7el6Vq3rstR8Mn4V/IQrpBLO6POkUSrN7RhyY+GF/InCFShQzsQ39T25gkHhLgSLByyAz+Kjb+c2L98w==", + "dev": true, "dependencies": { - "@scramjet/obj-logger": "^0.38.0", - "@scramjet/utility": "^0.38.0", - "bpmux": "^8.2.1" + "clean-stack": "^4.0.0", + "indent-string": "^5.0.0" }, - "devDependencies": { - "@scramjet/api-server": "^0.38.0", - "@scramjet/types": "^0.38.0", - "@types/node": "15.12.5", - "ava": "^3.15.0", - "ts-node": "^10.9.1", - "typedoc": "0.23.17", - "typedoc-plugin-markdown": "3.13.6", - "typescript": "~4.7.4" + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } - } - }, - "dependencies": { - "@aashutoshrathi/word-wrap": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz", - "integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==", - "dev": true }, - "@ampproject/remapping": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz", - "integrity": "sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w==", + "packages/utility/node_modules/ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", "dev": true, - "requires": { - "@jridgewell/gen-mapping": "^0.1.0", - "@jridgewell/trace-mapping": "^0.3.9" - } - }, - "@babel/code-frame": { - "version": "7.22.13", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.13.tgz", - "integrity": "sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w==", - "dev": true, - "requires": { - "@babel/highlight": "^7.22.13", - "chalk": "^2.4.2" + "engines": { + "node": ">=12" }, - "dependencies": { - "ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "requires": { - "color-convert": "^1.9.0" - } - }, - "chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "requires": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - } - }, - "color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "requires": { - "color-name": "1.1.3" - } - }, - "color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "dev": true - }, - "escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true - }, - "has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", - "dev": true - }, - "supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "requires": { - "has-flag": "^3.0.0" - } - } + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" } }, - "@babel/compat-data": { - "version": "7.20.10", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.20.10.tgz", - "integrity": "sha512-sEnuDPpOJR/fcafHMjpcpGN5M2jbUGUHwmuWKM/YdPzeEDJg8bgmbcWQFUfE32MQjti1koACvoPVsDe8Uq+idg==", - "dev": true - }, - "@babel/core": { - "version": "7.20.12", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.20.12.tgz", - "integrity": "sha512-XsMfHovsUYHFMdrIHkZphTN/2Hzzi78R08NuHfDBehym2VsPDL6Zn/JAD/JQdnRvbSsbQc4mVaU1m6JgtTEElg==", + "packages/utility/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", "dev": true, - "requires": { - "@ampproject/remapping": "^2.1.0", - "@babel/code-frame": "^7.18.6", - "@babel/generator": "^7.20.7", - "@babel/helper-compilation-targets": "^7.20.7", - "@babel/helper-module-transforms": "^7.20.11", - "@babel/helpers": "^7.20.7", - "@babel/parser": "^7.20.7", - "@babel/template": "^7.20.7", - "@babel/traverse": "^7.20.12", - "@babel/types": "^7.20.7", - "convert-source-map": "^1.7.0", - "debug": "^4.1.0", - "gensync": "^1.0.0-beta.2", - "json5": "^2.2.2", - "semver": "^6.3.0" + "engines": { + "node": ">=12" }, - "dependencies": { - "semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", - "dev": true - } + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "@babel/generator": { - "version": "7.23.0", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.23.0.tgz", - "integrity": "sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g==", + "packages/utility/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", "dev": true, - "requires": { - "@babel/types": "^7.23.0", - "@jridgewell/gen-mapping": "^0.3.2", - "@jridgewell/trace-mapping": "^0.3.17", - "jsesc": "^2.5.1" - }, "dependencies": { - "@jridgewell/gen-mapping": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz", - "integrity": "sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==", - "dev": true, - "requires": { - "@jridgewell/set-array": "^1.0.1", - "@jridgewell/sourcemap-codec": "^1.4.10", - "@jridgewell/trace-mapping": "^0.3.9" - } - } + "sprintf-js": "~1.0.2" } }, - "@babel/helper-compilation-targets": { - "version": "7.20.7", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.20.7.tgz", - "integrity": "sha512-4tGORmfQcrc+bvrjb5y3dG9Mx1IOZjsHqQVUz7XCNHO+iTmqxWnVg3KRygjGmpRLJGdQSKuvFinbIb0CnZwHAQ==", + "packages/utility/node_modules/arrify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-3.0.0.tgz", + "integrity": "sha512-tLkvA81vQG/XqE2mjDkGQHoOINtMHtysSnemrmoGe6PydDPMRbVugqyk4A6V/WDWEfm3l+0d8anA9r8cv/5Jaw==", "dev": true, - "requires": { - "@babel/compat-data": "^7.20.5", - "@babel/helper-validator-option": "^7.18.6", - "browserslist": "^4.21.3", - "lru-cache": "^5.1.1", - "semver": "^6.3.0" + "engines": { + "node": ">=12" }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "packages/utility/node_modules/ava": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/ava/-/ava-4.3.3.tgz", + "integrity": "sha512-9Egq/d9R74ExrWohHeqUlexjDbgZJX5jA1Wq4KCTqc3wIfpGEK79zVy4rBtofJ9YKIxs4PzhJ8BgbW5PlAYe6w==", + "dev": true, "dependencies": { - "lru-cache": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", - "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", - "dev": true, - "requires": { - "yallist": "^3.0.2" - } - }, - "semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", - "dev": true - }, - "yallist": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", - "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", - "dev": true + "acorn": "^8.7.1", + "acorn-walk": "^8.2.0", + "ansi-styles": "^6.1.0", + "arrgv": "^1.0.2", + "arrify": "^3.0.0", + "callsites": "^4.0.0", + "cbor": "^8.1.0", + "chalk": "^5.0.1", + "chokidar": "^3.5.3", + "chunkd": "^2.0.1", + "ci-info": "^3.3.1", + "ci-parallel-vars": "^1.0.1", + "clean-yaml-object": "^0.1.0", + "cli-truncate": "^3.1.0", + "code-excerpt": "^4.0.0", + "common-path-prefix": "^3.0.0", + "concordance": "^5.0.4", + "currently-unhandled": "^0.4.1", + "debug": "^4.3.4", + "del": "^6.1.1", + "emittery": "^0.11.0", + "figures": "^4.0.1", + "globby": "^13.1.1", + "ignore-by-default": "^2.1.0", + "indent-string": "^5.0.0", + "is-error": "^2.2.2", + "is-plain-object": "^5.0.0", + "is-promise": "^4.0.0", + "matcher": "^5.0.0", + "mem": "^9.0.2", + "ms": "^2.1.3", + "p-event": "^5.0.1", + "p-map": "^5.4.0", + "picomatch": "^2.3.1", + "pkg-conf": "^4.0.0", + "plur": "^5.1.0", + "pretty-ms": "^7.0.1", + "resolve-cwd": "^3.0.0", + "slash": "^3.0.0", + "stack-utils": "^2.0.5", + "strip-ansi": "^7.0.1", + "supertap": "^3.0.1", + "temp-dir": "^2.0.0", + "write-file-atomic": "^4.0.1", + "yargs": "^17.5.1" + }, + "bin": { + "ava": "entrypoints/cli.mjs" + }, + "engines": { + "node": ">=12.22 <13 || >=14.17 <15 || >=16.4 <17 || >=18" + }, + "peerDependencies": { + "@ava/typescript": "*" + }, + "peerDependenciesMeta": { + "@ava/typescript": { + "optional": true } } }, - "@babel/helper-environment-visitor": { - "version": "7.22.20", - "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz", - "integrity": "sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==", - "dev": true + "packages/utility/node_modules/callsites": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-4.1.0.tgz", + "integrity": "sha512-aBMbD1Xxay75ViYezwT40aQONfr+pSXTHwNKvIXhXD6+LY3F1dLIcceoC5OZKBVHbXcysz1hL9D2w0JJIMXpUw==", + "dev": true, + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } }, - "@babel/helper-function-name": { - "version": "7.23.0", - "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz", - "integrity": "sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==", + "packages/utility/node_modules/chalk": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz", + "integrity": "sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==", "dev": true, - "requires": { - "@babel/template": "^7.22.15", - "@babel/types": "^7.23.0" + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" } }, - "@babel/helper-hoist-variables": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz", - "integrity": "sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==", + "packages/utility/node_modules/ci-info": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", + "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==", "dev": true, - "requires": { - "@babel/types": "^7.22.5" + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "engines": { + "node": ">=8" } }, - "@babel/helper-module-imports": { - "version": "7.18.6", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.18.6.tgz", - "integrity": "sha512-0NFvs3VkuSYbFi1x2Vd6tKrywq+z/cLeYC/RJNFrIX/30Bf5aiGYbtvGXolEktzJH8o5E5KJ3tT+nkxuuZFVlA==", + "packages/utility/node_modules/clean-stack": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-4.2.0.tgz", + "integrity": "sha512-LYv6XPxoyODi36Dp976riBtSY27VmFo+MKqEU9QCCWyTrdEPDog+RWA7xQWHi6Vbp61j5c4cdzzX1NidnwtUWg==", "dev": true, - "requires": { - "@babel/types": "^7.18.6" + "dependencies": { + "escape-string-regexp": "5.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "@babel/helper-module-transforms": { - "version": "7.20.11", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.20.11.tgz", - "integrity": "sha512-uRy78kN4psmji1s2QtbtcCSaj/LILFDp0f/ymhpQH5QY3nljUZCaNWz9X1dEj/8MBdBEFECs7yRhKn8i7NjZgg==", + "packages/utility/node_modules/cli-truncate": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cli-truncate/-/cli-truncate-3.1.0.tgz", + "integrity": "sha512-wfOBkjXteqSnI59oPcJkcPl/ZmwvMMOj340qUIY1SKZCv0B9Cf4D4fAucRkIKQmsIuYK3x1rrgU7MeGRruiuiA==", "dev": true, - "requires": { - "@babel/helper-environment-visitor": "^7.18.9", - "@babel/helper-module-imports": "^7.18.6", - "@babel/helper-simple-access": "^7.20.2", - "@babel/helper-split-export-declaration": "^7.18.6", - "@babel/helper-validator-identifier": "^7.19.1", - "@babel/template": "^7.20.7", - "@babel/traverse": "^7.20.10", - "@babel/types": "^7.20.7" + "dependencies": { + "slice-ansi": "^5.0.0", + "string-width": "^5.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "@babel/helper-plugin-utils": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.22.5.tgz", - "integrity": "sha512-uLls06UVKgFG9QD4OeFYLEGteMIAa5kpTPcFL28yuCIIzsf6ZyKZMllKVOCZFhiZ5ptnwX4mtKdWCBE/uT4amg==", - "dev": true - }, - "@babel/helper-simple-access": { - "version": "7.20.2", - "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.20.2.tgz", - "integrity": "sha512-+0woI/WPq59IrqDYbVGfshjT5Dmk/nnbdpcF8SnMhhXObpTq2KNBdLFRFrkVdbDOyUmHBCxzm5FHV1rACIkIbA==", + "packages/utility/node_modules/cli-truncate/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", "dev": true, - "requires": { - "@babel/types": "^7.20.2" + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "@babel/helper-split-export-declaration": { - "version": "7.22.6", - "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz", - "integrity": "sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==", + "packages/utility/node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", "dev": true, - "requires": { - "@babel/types": "^7.22.5" + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" } }, - "@babel/helper-string-parser": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz", - "integrity": "sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==", - "dev": true + "packages/utility/node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } }, - "@babel/helper-validator-identifier": { - "version": "7.22.20", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz", - "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==", - "dev": true + "packages/utility/node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } }, - "@babel/helper-validator-option": { - "version": "7.18.6", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.18.6.tgz", - "integrity": "sha512-XO7gESt5ouv/LRJdrVjkShckw6STTaB7l9BrpBaAHDeF5YZT+01PCwmR0SJHnkW6i8OwW/EVWRShfi4j2x+KQw==", - "dev": true + "packages/utility/node_modules/code-excerpt": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/code-excerpt/-/code-excerpt-4.0.0.tgz", + "integrity": "sha512-xxodCmBen3iy2i0WtAK8FlFNrRzjUqjRsMfho58xT/wvZU1YTM3fCnRjcy1gJPMepaRlgm/0e6w8SpWHpn3/cA==", + "dev": true, + "dependencies": { + "convert-to-spaces": "^2.0.1" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + } }, - "@babel/helpers": { - "version": "7.20.13", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.20.13.tgz", - "integrity": "sha512-nzJ0DWCL3gB5RCXbUO3KIMMsBY2Eqbx8mBpKGE/02PgyRQFcPQLbkQ1vyy596mZLaP+dAfD+R4ckASzNVmW3jg==", + "packages/utility/node_modules/convert-to-spaces": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/convert-to-spaces/-/convert-to-spaces-2.0.1.tgz", + "integrity": "sha512-rcQ1bsQO9799wq24uE5AM2tAILy4gXGIK/njFWcVQkGNZ96edlpY+A7bjwvzjYvLDyzmG1MmMLZhpcsb+klNMQ==", "dev": true, - "requires": { - "@babel/template": "^7.20.7", - "@babel/traverse": "^7.20.13", - "@babel/types": "^7.20.7" + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" } }, - "@babel/highlight": { - "version": "7.22.20", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.22.20.tgz", - "integrity": "sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg==", + "packages/utility/node_modules/emittery": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.11.0.tgz", + "integrity": "sha512-S/7tzL6v5i+4iJd627Nhv9cLFIo5weAIlGccqJFpnBoDB8U1TF2k5tez4J/QNuxyyhWuFqHg1L84Kd3m7iXg6g==", "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.22.20", - "chalk": "^2.4.2", - "js-tokens": "^4.0.0" + "engines": { + "node": ">=12" }, - "dependencies": { - "ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "requires": { - "color-convert": "^1.9.0" - } - }, - "chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "requires": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - } - }, - "color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "requires": { - "color-name": "1.1.3" - } - }, - "color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "dev": true - }, - "escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true - }, - "has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", - "dev": true - }, - "supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "requires": { - "has-flag": "^3.0.0" - } - } + "funding": { + "url": "https://github.com/sindresorhus/emittery?sponsor=1" } }, - "@babel/parser": { - "version": "7.23.0", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.0.tgz", - "integrity": "sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw==", + "packages/utility/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", "dev": true }, - "@babel/plugin-syntax-async-generators": { - "version": "7.8.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", - "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", + "packages/utility/node_modules/escape-string-regexp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", + "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", "dev": true, - "requires": { - "@babel/helper-plugin-utils": "^7.8.0" + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "@babel/plugin-syntax-bigint": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz", - "integrity": "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==", + "packages/utility/node_modules/figures": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/figures/-/figures-4.0.1.tgz", + "integrity": "sha512-rElJwkA/xS04Vfg+CaZodpso7VqBknOYbzi6I76hI4X80RUjkSxO2oAyPmGbuXUppywjqndOrQDl817hDnI++w==", "dev": true, - "requires": { - "@babel/helper-plugin-utils": "^7.8.0" + "dependencies": { + "escape-string-regexp": "^5.0.0", + "is-unicode-supported": "^1.2.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "@babel/plugin-syntax-class-properties": { - "version": "7.12.13", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", - "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", + "packages/utility/node_modules/find-up": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-6.3.0.tgz", + "integrity": "sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw==", "dev": true, - "requires": { - "@babel/helper-plugin-utils": "^7.12.13" + "dependencies": { + "locate-path": "^7.1.0", + "path-exists": "^5.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "@babel/plugin-syntax-import-meta": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz", - "integrity": "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==", + "packages/utility/node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", "dev": true, - "requires": { - "@babel/helper-plugin-utils": "^7.10.4" + "engines": { + "node": "6.* || 8.* || >= 10.*" } }, - "@babel/plugin-syntax-json-strings": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", - "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", + "packages/utility/node_modules/globby": { + "version": "13.2.2", + "resolved": "https://registry.npmjs.org/globby/-/globby-13.2.2.tgz", + "integrity": "sha512-Y1zNGV+pzQdh7H39l9zgB4PJqjRNqydvdYCDG4HFXM4XuvSaQQlEc91IU1yALL8gUTDomgBAfz3XJdmUS+oo0w==", "dev": true, - "requires": { - "@babel/helper-plugin-utils": "^7.8.0" + "dependencies": { + "dir-glob": "^3.0.1", + "fast-glob": "^3.3.0", + "ignore": "^5.2.4", + "merge2": "^1.4.1", + "slash": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "@babel/plugin-syntax-jsx": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.22.5.tgz", - "integrity": "sha512-gvyP4hZrgrs/wWMaocvxZ44Hw0b3W8Pe+cMxc8V1ULQ07oh8VNbIRaoD1LRZVTvD+0nieDKjfgKg89sD7rrKrg==", + "packages/utility/node_modules/globby/node_modules/slash": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", + "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==", "dev": true, - "requires": { - "@babel/helper-plugin-utils": "^7.22.5" + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "@babel/plugin-syntax-logical-assignment-operators": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", - "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", + "packages/utility/node_modules/indent-string": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-5.0.0.tgz", + "integrity": "sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==", "dev": true, - "requires": { - "@babel/helper-plugin-utils": "^7.10.4" + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "@babel/plugin-syntax-nullish-coalescing-operator": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", - "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", + "packages/utility/node_modules/is-unicode-supported": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-1.3.0.tgz", + "integrity": "sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ==", "dev": true, - "requires": { - "@babel/helper-plugin-utils": "^7.8.0" + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "@babel/plugin-syntax-numeric-separator": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", - "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", + "packages/utility/node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", "dev": true, - "requires": { - "@babel/helper-plugin-utils": "^7.10.4" + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" } }, - "@babel/plugin-syntax-object-rest-spread": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", - "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", + "packages/utility/node_modules/load-json-file": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-7.0.1.tgz", + "integrity": "sha512-Gnxj3ev3mB5TkVBGad0JM6dmLiQL+o0t23JPBZ9sd+yvSLk05mFoqKBw5N8gbbkU4TNXyqCgIrl/VM17OgUIgQ==", "dev": true, - "requires": { - "@babel/helper-plugin-utils": "^7.8.0" + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "@babel/plugin-syntax-optional-catch-binding": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", - "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", + "packages/utility/node_modules/locate-path": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-7.2.0.tgz", + "integrity": "sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==", "dev": true, - "requires": { - "@babel/helper-plugin-utils": "^7.8.0" + "dependencies": { + "p-locate": "^6.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "@babel/plugin-syntax-optional-chaining": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", - "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", + "packages/utility/node_modules/matcher": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-5.0.0.tgz", + "integrity": "sha512-s2EMBOWtXFc8dgqvoAzKJXxNHibcdJMV0gwqKUaw9E2JBJuGUK7DrNKrA6g/i+v72TT16+6sVm5mS3thaMLQUw==", "dev": true, - "requires": { - "@babel/helper-plugin-utils": "^7.8.0" + "dependencies": { + "escape-string-regexp": "^5.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "@babel/plugin-syntax-top-level-await": { - "version": "7.14.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", - "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", + "packages/utility/node_modules/mem": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/mem/-/mem-9.0.2.tgz", + "integrity": "sha512-F2t4YIv9XQUBHt6AOJ0y7lSmP1+cY7Fm1DRh9GClTGzKST7UWLMx6ly9WZdLH/G/ppM5RL4MlQfRT71ri9t19A==", "dev": true, - "requires": { - "@babel/helper-plugin-utils": "^7.14.5" + "dependencies": { + "map-age-cleaner": "^0.1.3", + "mimic-fn": "^4.0.0" + }, + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sindresorhus/mem?sponsor=1" } }, - "@babel/plugin-syntax-typescript": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.22.5.tgz", - "integrity": "sha512-1mS2o03i7t1c6VzH6fdQ3OA8tcEIxwG18zIPRp+UY1Ihv6W+XZzBCVxExF9upussPXJ0xE9XRHwMoNs1ep/nRQ==", + "packages/utility/node_modules/mimic-fn": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", + "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==", "dev": true, - "requires": { - "@babel/helper-plugin-utils": "^7.22.5" + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "@babel/template": { - "version": "7.22.15", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz", - "integrity": "sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==", + "packages/utility/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "packages/utility/node_modules/p-event": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/p-event/-/p-event-5.0.1.tgz", + "integrity": "sha512-dd589iCQ7m1L0bmC5NLlVYfy3TbBEsMUfWx9PyAgPeIcFZ/E2yaTZ4Rz4MiBmmJShviiftHVXOqfnfzJ6kyMrQ==", "dev": true, - "requires": { - "@babel/code-frame": "^7.22.13", - "@babel/parser": "^7.22.15", - "@babel/types": "^7.22.15" + "dependencies": { + "p-timeout": "^5.0.2" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "@babel/traverse": { - "version": "7.23.2", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.2.tgz", - "integrity": "sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw==", + "packages/utility/node_modules/p-limit": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-4.0.0.tgz", + "integrity": "sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==", "dev": true, - "requires": { - "@babel/code-frame": "^7.22.13", - "@babel/generator": "^7.23.0", - "@babel/helper-environment-visitor": "^7.22.20", - "@babel/helper-function-name": "^7.23.0", - "@babel/helper-hoist-variables": "^7.22.5", - "@babel/helper-split-export-declaration": "^7.22.6", - "@babel/parser": "^7.23.0", - "@babel/types": "^7.23.0", - "debug": "^4.1.0", - "globals": "^11.1.0" - }, "dependencies": { - "globals": { - "version": "11.12.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", - "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", - "dev": true - } + "yocto-queue": "^1.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "@babel/types": { - "version": "7.23.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.23.0.tgz", - "integrity": "sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg==", + "packages/utility/node_modules/p-locate": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-6.0.0.tgz", + "integrity": "sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==", "dev": true, - "requires": { - "@babel/helper-string-parser": "^7.22.5", - "@babel/helper-validator-identifier": "^7.22.20", - "to-fast-properties": "^2.0.0" + "dependencies": { + "p-limit": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "@balena/dockerignore": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", - "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==" - }, - "@bcoe/v8-coverage": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", - "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", - "dev": true + "packages/utility/node_modules/p-map": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-5.5.0.tgz", + "integrity": "sha512-VFqfGDHlx87K66yZrNdI4YGtD70IRyd+zSvgks6mzHPRNkoKy+9EKP4SFC77/vTTQYmRmti7dvqC+m5jBrBAcg==", + "dev": true, + "dependencies": { + "aggregate-error": "^4.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } }, - "@brillout/import": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/@brillout/import/-/import-0.2.1.tgz", - "integrity": "sha512-4qA9sNtQZCY02MeLjTP+O9LfY9T6d2ajYS7whyvgBklJzWzLUNbJp940ClxK64hbDPqag9AQZt45beEAGcSYYA==" + "packages/utility/node_modules/p-timeout": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-5.1.0.tgz", + "integrity": "sha512-auFDyzzzGZZZdHz3BtET9VEz0SE/uMEAx7uWfGPucfzEwwe/xH0iVeZibQmANYE/hp9T2+UUZT5m+BKyrDp3Ew==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } }, - "@colors/colors": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz", - "integrity": "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==" + "packages/utility/node_modules/path-exists": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-5.0.0.tgz", + "integrity": "sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==", + "dev": true, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + } }, - "@concordance/react": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@concordance/react/-/react-2.0.0.tgz", - "integrity": "sha512-huLSkUuM2/P+U0uy2WwlKuixMsTODD8p4JVQBI4VKeopkiN0C7M3N9XYVawb4M+4spN5RrO/eLhk7KoQX6nsfA==", + "packages/utility/node_modules/pkg-conf": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/pkg-conf/-/pkg-conf-4.0.0.tgz", + "integrity": "sha512-7dmgi4UY4qk+4mj5Cd8v/GExPo0K+SlY+hulOSdfZ/T6jVH6//y7NtzZo5WrfhDBxuQ0jCa7fLZmNaNh7EWL/w==", "dev": true, - "requires": { - "arrify": "^1.0.1" - }, "dependencies": { - "arrify": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", - "integrity": "sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==", - "dev": true - } + "find-up": "^6.0.0", + "load-json-file": "^7.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "@cspotcode/source-map-support": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", - "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "packages/utility/node_modules/plur": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/plur/-/plur-5.1.0.tgz", + "integrity": "sha512-VP/72JeXqak2KiOzjgKtQen5y3IZHn+9GOuLDafPv0eXa47xq0At93XahYBs26MsifCQ4enGKwbjBTKgb9QJXg==", "dev": true, - "requires": { - "@jridgewell/trace-mapping": "0.3.9" + "dependencies": { + "irregular-plurals": "^3.3.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "packages/utility/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, "dependencies": { - "@jridgewell/trace-mapping": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", - "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", - "dev": true, - "requires": { - "@jridgewell/resolve-uri": "^3.0.3", - "@jridgewell/sourcemap-codec": "^1.4.10" - } - } + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, - "@cucumber/create-meta": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@cucumber/create-meta/-/create-meta-5.0.0.tgz", - "integrity": "sha512-Z5kMZkUff00S3/KSnKzB/KOm2UIxMXY1xXmj2dQMlD49lV6v/W8EEvgDMNtQotQNSOQU5bDupmWQpk+o16tXIw==", + "packages/utility/node_modules/supertap": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/supertap/-/supertap-3.0.1.tgz", + "integrity": "sha512-u1ZpIBCawJnO+0QePsEiOknOfCRq0yERxiAchT0i4li0WHNUJbf0evXXSXOcCAR4M8iMDoajXYmstm/qO81Isw==", "dev": true, - "requires": { - "@cucumber/messages": "^16.0.0" + "dependencies": { + "indent-string": "^5.0.0", + "js-yaml": "^3.14.1", + "serialize-error": "^7.0.1", + "strip-ansi": "^7.0.1" }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + } + }, + "packages/utility/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, "dependencies": { - "@cucumber/messages": { - "version": "16.0.1", - "resolved": "https://registry.npmjs.org/@cucumber/messages/-/messages-16.0.1.tgz", - "integrity": "sha512-80JcaAfQragFqR1rMhRwiqWL9HcR6Z4LDD2mfF0Lxg/lFkCNvmWa9Jl10NUNfFXYD555NKPzP/8xFo55abw8TQ==", - "dev": true, - "requires": { - "@types/uuid": "8.3.0", - "class-transformer": "0.4.0", - "reflect-metadata": "0.1.13", - "uuid": "8.3.2" - } - }, - "@types/uuid": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.0.tgz", - "integrity": "sha512-eQ9qFW/fhfGJF8WKHGEHZEyVWfZxrT+6CLIJGBcZPfxUh/+BnEj+UCGYMlr9qZuX/2AltsvwrGqp0LhEW8D0zQ==", - "dev": true - }, - "class-transformer": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.4.0.tgz", - "integrity": "sha512-ETWD/H2TbWbKEi7m9N4Km5+cw1hNcqJSxlSYhsLsNjQzWWiZIYA1zafxpK9PwVfaZ6AqR5rrjPVUBGESm5tQUA==", - "dev": true - } + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, - "@cucumber/cucumber": { - "version": "7.3.2", - "resolved": "https://registry.npmjs.org/@cucumber/cucumber/-/cucumber-7.3.2.tgz", - "integrity": "sha512-qqptM9w+UqXEYBAkrIGpIVPXDWv+zp0LrS89LiwHZwBp0cJg00su/iPMZ4j8TvCJiKfAwJXsAI1yjrd1POtU+w==", + "packages/utility/node_modules/wrap-ansi/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true, - "requires": { - "@cucumber/create-meta": "^5.0.0", - "@cucumber/cucumber-expressions": "^12.1.1", - "@cucumber/gherkin": "^19.0.3", - "@cucumber/gherkin-streams": "^2.0.2", - "@cucumber/html-formatter": "^15.0.2", - "@cucumber/messages": "^16.0.1", - "@cucumber/tag-expressions": "^3.0.1", - "assertion-error-formatter": "^3.0.0", - "bluebird": "^3.7.2", - "capital-case": "^1.0.4", - "cli-table3": "0.6.1", - "colors": "1.4.0", - "commander": "^7.0.0", - "create-require": "^1.1.1", - "duration": "^0.2.2", - "durations": "^3.4.2", - "figures": "^3.2.0", - "glob": "^7.1.6", - "indent-string": "^4.0.0", - "is-generator": "^1.0.3", - "is-stream": "^2.0.0", - "knuth-shuffle-seeded": "^1.0.6", - "lodash": "^4.17.21", - "mz": "^2.7.0", - "progress": "^2.0.3", - "resolve": "^1.19.0", - "resolve-pkg": "^2.0.0", - "stack-chain": "^2.0.0", - "stacktrace-js": "^2.0.2", - "string-argv": "^0.3.1", - "tmp": "^0.2.1", - "util-arity": "^1.1.0", - "verror": "^1.10.0" + "engines": { + "node": ">=8" + } + }, + "packages/utility/node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "packages/utility/node_modules/wrap-ansi/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, "dependencies": { - "@cucumber/messages": { - "version": "16.0.1", - "resolved": "https://registry.npmjs.org/@cucumber/messages/-/messages-16.0.1.tgz", - "integrity": "sha512-80JcaAfQragFqR1rMhRwiqWL9HcR6Z4LDD2mfF0Lxg/lFkCNvmWa9Jl10NUNfFXYD555NKPzP/8xFo55abw8TQ==", - "dev": true, - "requires": { - "@types/uuid": "8.3.0", - "class-transformer": "0.4.0", - "reflect-metadata": "0.1.13", - "uuid": "8.3.2" - } - }, - "@types/uuid": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.0.tgz", - "integrity": "sha512-eQ9qFW/fhfGJF8WKHGEHZEyVWfZxrT+6CLIJGBcZPfxUh/+BnEj+UCGYMlr9qZuX/2AltsvwrGqp0LhEW8D0zQ==", - "dev": true - }, - "class-transformer": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.4.0.tgz", - "integrity": "sha512-ETWD/H2TbWbKEi7m9N4Km5+cw1hNcqJSxlSYhsLsNjQzWWiZIYA1zafxpK9PwVfaZ6AqR5rrjPVUBGESm5tQUA==", - "dev": true - }, - "commander": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", - "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", - "dev": true - }, - "tmp": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", - "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", - "dev": true, - "requires": { - "rimraf": "^3.0.0" - } - } + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" } }, - "@cucumber/cucumber-expressions": { - "version": "12.1.3", - "resolved": "https://registry.npmjs.org/@cucumber/cucumber-expressions/-/cucumber-expressions-12.1.3.tgz", - "integrity": "sha512-LB8MAzE4F/t2KIgsDEz4gZH0xSI4aG0/LmYUPyISPPjUS1pI/yGWWyeX2WsiUQxpSs765WcNIq5Bggt7gGGO3Q==", + "packages/utility/node_modules/write-file-atomic": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz", + "integrity": "sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg==", + "dev": true, + "dependencies": { + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.7" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "packages/utility/node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "packages/utility/node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "packages/utility/node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "packages/utility/node_modules/yocto-queue": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz", + "integrity": "sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==", + "dev": true, + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "packages/verser": { + "name": "@scramjet/verser", + "version": "0.38.0", + "license": "AGPL-3.0", + "dependencies": { + "@scramjet/obj-logger": "^0.38.0", + "@scramjet/tecemux": "^0.38.0", + "@scramjet/utility": "^0.38.0" + }, + "devDependencies": { + "@scramjet/api-server": "^0.38.0", + "@scramjet/types": "^0.38.0", + "@types/node": "15.12.5", + "ava": "^3.15.0", + "ts-node": "^10.9.1", + "typedoc": "0.23.17", + "typedoc-plugin-markdown": "3.13.6", + "typescript": "~4.7.4" + } + } + }, + "dependencies": { + "@aashutoshrathi/word-wrap": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz", + "integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==", + "dev": true + }, + "@ampproject/remapping": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz", + "integrity": "sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w==", "dev": true, "requires": { - "regexp-match-indices": "1.0.2" + "@jridgewell/gen-mapping": "^0.1.0", + "@jridgewell/trace-mapping": "^0.3.9" } }, - "@cucumber/gherkin": { - "version": "19.0.3", - "resolved": "https://registry.npmjs.org/@cucumber/gherkin/-/gherkin-19.0.3.tgz", - "integrity": "sha512-gWdMm8mfRk3P+VugJWvNALaQV5QnT+5RkqWy3tO+4NsMSQZPo5p4V4vXwriQZ/sZR1Wni5TDRztuRsKLgZ3XHA==", + "@babel/code-frame": { + "version": "7.22.13", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.13.tgz", + "integrity": "sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w==", "dev": true, "requires": { - "@cucumber/message-streams": "^2.0.0", - "@cucumber/messages": "^16.0.1" + "@babel/highlight": "^7.22.13", + "chalk": "^2.4.2" }, "dependencies": { - "@cucumber/messages": { - "version": "16.0.1", - "resolved": "https://registry.npmjs.org/@cucumber/messages/-/messages-16.0.1.tgz", - "integrity": "sha512-80JcaAfQragFqR1rMhRwiqWL9HcR6Z4LDD2mfF0Lxg/lFkCNvmWa9Jl10NUNfFXYD555NKPzP/8xFo55abw8TQ==", + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", "dev": true, "requires": { - "@types/uuid": "8.3.0", - "class-transformer": "0.4.0", - "reflect-metadata": "0.1.13", - "uuid": "8.3.2" + "color-convert": "^1.9.0" } }, - "@types/uuid": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.0.tgz", - "integrity": "sha512-eQ9qFW/fhfGJF8WKHGEHZEyVWfZxrT+6CLIJGBcZPfxUh/+BnEj+UCGYMlr9qZuX/2AltsvwrGqp0LhEW8D0zQ==", + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", "dev": true }, - "class-transformer": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.4.0.tgz", - "integrity": "sha512-ETWD/H2TbWbKEi7m9N4Km5+cw1hNcqJSxlSYhsLsNjQzWWiZIYA1zafxpK9PwVfaZ6AqR5rrjPVUBGESm5tQUA==", + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", "dev": true + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } } } }, - "@cucumber/gherkin-streams": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@cucumber/gherkin-streams/-/gherkin-streams-2.0.2.tgz", - "integrity": "sha512-cKmXOBz4OwGlrHMBCc4qCC3KzLaqcEZ11nWWskIbv6jyfvlIRuM2OgEF6VLcNVewczifW1p6DrDj0OO+BeXocA==", + "@babel/compat-data": { + "version": "7.20.10", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.20.10.tgz", + "integrity": "sha512-sEnuDPpOJR/fcafHMjpcpGN5M2jbUGUHwmuWKM/YdPzeEDJg8bgmbcWQFUfE32MQjti1koACvoPVsDe8Uq+idg==", + "dev": true + }, + "@babel/core": { + "version": "7.20.12", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.20.12.tgz", + "integrity": "sha512-XsMfHovsUYHFMdrIHkZphTN/2Hzzi78R08NuHfDBehym2VsPDL6Zn/JAD/JQdnRvbSsbQc4mVaU1m6JgtTEElg==", "dev": true, "requires": { - "@cucumber/gherkin": "^19.0.1", - "@cucumber/message-streams": "^2.0.0", - "@cucumber/messages": "^16.0.0", - "commander": "7.2.0", - "source-map-support": "0.5.19" + "@ampproject/remapping": "^2.1.0", + "@babel/code-frame": "^7.18.6", + "@babel/generator": "^7.20.7", + "@babel/helper-compilation-targets": "^7.20.7", + "@babel/helper-module-transforms": "^7.20.11", + "@babel/helpers": "^7.20.7", + "@babel/parser": "^7.20.7", + "@babel/template": "^7.20.7", + "@babel/traverse": "^7.20.12", + "@babel/types": "^7.20.7", + "convert-source-map": "^1.7.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.2", + "semver": "^6.3.0" }, "dependencies": { - "@cucumber/messages": { - "version": "16.0.1", - "resolved": "https://registry.npmjs.org/@cucumber/messages/-/messages-16.0.1.tgz", - "integrity": "sha512-80JcaAfQragFqR1rMhRwiqWL9HcR6Z4LDD2mfF0Lxg/lFkCNvmWa9Jl10NUNfFXYD555NKPzP/8xFo55abw8TQ==", - "dev": true, - "requires": { - "@types/uuid": "8.3.0", - "class-transformer": "0.4.0", - "reflect-metadata": "0.1.13", - "uuid": "8.3.2" - } - }, - "@types/uuid": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.0.tgz", - "integrity": "sha512-eQ9qFW/fhfGJF8WKHGEHZEyVWfZxrT+6CLIJGBcZPfxUh/+BnEj+UCGYMlr9qZuX/2AltsvwrGqp0LhEW8D0zQ==", - "dev": true - }, - "class-transformer": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.4.0.tgz", - "integrity": "sha512-ETWD/H2TbWbKEi7m9N4Km5+cw1hNcqJSxlSYhsLsNjQzWWiZIYA1zafxpK9PwVfaZ6AqR5rrjPVUBGESm5tQUA==", - "dev": true - }, - "commander": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", - "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", "dev": true } } }, - "@cucumber/html-formatter": { - "version": "15.0.2", - "resolved": "https://registry.npmjs.org/@cucumber/html-formatter/-/html-formatter-15.0.2.tgz", - "integrity": "sha512-j+YGY4ytj78G/v1gZo53D+vuKXlTg/oxNwSCCGvRQo75+AqYDJSkm/vexXJQ5lY1rXAvlbZ9KI6jhg6LDs0YdQ==", + "@babel/generator": { + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.23.0.tgz", + "integrity": "sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g==", "dev": true, "requires": { - "@cucumber/messages": "^16.0.1", - "commander": "7.2.0", - "source-map-support": "0.5.19" + "@babel/types": "^7.23.0", + "@jridgewell/gen-mapping": "^0.3.2", + "@jridgewell/trace-mapping": "^0.3.17", + "jsesc": "^2.5.1" }, "dependencies": { - "@cucumber/messages": { - "version": "16.0.1", - "resolved": "https://registry.npmjs.org/@cucumber/messages/-/messages-16.0.1.tgz", - "integrity": "sha512-80JcaAfQragFqR1rMhRwiqWL9HcR6Z4LDD2mfF0Lxg/lFkCNvmWa9Jl10NUNfFXYD555NKPzP/8xFo55abw8TQ==", + "@jridgewell/gen-mapping": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz", + "integrity": "sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==", "dev": true, "requires": { - "@types/uuid": "8.3.0", - "class-transformer": "0.4.0", - "reflect-metadata": "0.1.13", - "uuid": "8.3.2" + "@jridgewell/set-array": "^1.0.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.9" } - }, - "@types/uuid": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.0.tgz", - "integrity": "sha512-eQ9qFW/fhfGJF8WKHGEHZEyVWfZxrT+6CLIJGBcZPfxUh/+BnEj+UCGYMlr9qZuX/2AltsvwrGqp0LhEW8D0zQ==", - "dev": true - }, - "class-transformer": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.4.0.tgz", - "integrity": "sha512-ETWD/H2TbWbKEi7m9N4Km5+cw1hNcqJSxlSYhsLsNjQzWWiZIYA1zafxpK9PwVfaZ6AqR5rrjPVUBGESm5tQUA==", - "dev": true - }, - "commander": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", - "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", - "dev": true } } }, - "@cucumber/message-streams": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@cucumber/message-streams/-/message-streams-2.1.0.tgz", - "integrity": "sha512-Yh3mw3qv6QL9NI/ihkZF8V9MX2GbnR6oktv34kC3uAbrQy9d/b2SZ3HNjG3J9JQqpV4B7Om3SPElJYIeo66TrA==", + "@babel/helper-compilation-targets": { + "version": "7.20.7", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.20.7.tgz", + "integrity": "sha512-4tGORmfQcrc+bvrjb5y3dG9Mx1IOZjsHqQVUz7XCNHO+iTmqxWnVg3KRygjGmpRLJGdQSKuvFinbIb0CnZwHAQ==", "dev": true, "requires": { - "@cucumber/messages": "^16.0.1" + "@babel/compat-data": "^7.20.5", + "@babel/helper-validator-option": "^7.18.6", + "browserslist": "^4.21.3", + "lru-cache": "^5.1.1", + "semver": "^6.3.0" }, "dependencies": { - "@cucumber/messages": { - "version": "16.0.1", - "resolved": "https://registry.npmjs.org/@cucumber/messages/-/messages-16.0.1.tgz", - "integrity": "sha512-80JcaAfQragFqR1rMhRwiqWL9HcR6Z4LDD2mfF0Lxg/lFkCNvmWa9Jl10NUNfFXYD555NKPzP/8xFo55abw8TQ==", + "lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", "dev": true, "requires": { - "@types/uuid": "8.3.0", - "class-transformer": "0.4.0", - "reflect-metadata": "0.1.13", - "uuid": "8.3.2" + "yallist": "^3.0.2" } }, - "@types/uuid": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.0.tgz", - "integrity": "sha512-eQ9qFW/fhfGJF8WKHGEHZEyVWfZxrT+6CLIJGBcZPfxUh/+BnEj+UCGYMlr9qZuX/2AltsvwrGqp0LhEW8D0zQ==", + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", "dev": true }, - "class-transformer": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.4.0.tgz", - "integrity": "sha512-ETWD/H2TbWbKEi7m9N4Km5+cw1hNcqJSxlSYhsLsNjQzWWiZIYA1zafxpK9PwVfaZ6AqR5rrjPVUBGESm5tQUA==", + "yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", "dev": true } } }, - "@cucumber/messages": { - "version": "21.0.1", - "resolved": "https://registry.npmjs.org/@cucumber/messages/-/messages-21.0.1.tgz", - "integrity": "sha512-pGR7iURM4SF9Qp1IIpNiVQ77J9kfxMkPOEbyy+zRmGABnWWCsqMpJdfHeh9Mb3VskemVw85++e15JT0PYdcR3g==", + "@babel/helper-environment-visitor": { + "version": "7.22.20", + "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz", + "integrity": "sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==", + "dev": true + }, + "@babel/helper-function-name": { + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz", + "integrity": "sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==", "dev": true, - "peer": true, "requires": { - "@types/uuid": "8.3.4", - "class-transformer": "0.5.1", - "reflect-metadata": "0.1.13", - "uuid": "9.0.0" - }, - "dependencies": { - "uuid": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", - "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", - "dev": true, - "peer": true - } + "@babel/template": "^7.22.15", + "@babel/types": "^7.23.0" } }, - "@cucumber/pretty-formatter": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@cucumber/pretty-formatter/-/pretty-formatter-1.0.0.tgz", - "integrity": "sha512-wcnIMN94HyaHGsfq72dgCvr1d8q6VGH4Y6Gl5weJ2TNZw1qn2UY85Iki4c9VdaLUONYnyYH3+178YB+9RFe/Hw==", + "@babel/helper-hoist-variables": { + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz", + "integrity": "sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==", "dev": true, "requires": { - "ansi-styles": "^5.0.0", - "cli-table3": "^0.6.0", - "figures": "^3.2.0", - "ts-dedent": "^2.0.0" - }, - "dependencies": { - "ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true - } + "@babel/types": "^7.22.5" } }, - "@cucumber/tag-expressions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@cucumber/tag-expressions/-/tag-expressions-3.0.1.tgz", - "integrity": "sha512-OGCXaJ1BQXmQ5b9pw+JYsBGumK2/LPZiLmbj1o1JFVeSNs2PY8WPQFSyXrskhrHz5Nd/6lYg7lvGMtFHOncC4w==", - "dev": true - }, - "@dabh/diagnostics": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.3.tgz", - "integrity": "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==", + "@babel/helper-module-imports": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.18.6.tgz", + "integrity": "sha512-0NFvs3VkuSYbFi1x2Vd6tKrywq+z/cLeYC/RJNFrIX/30Bf5aiGYbtvGXolEktzJH8o5E5KJ3tT+nkxuuZFVlA==", + "dev": true, "requires": { - "colorspace": "1.1.x", - "enabled": "2.0.x", - "kuler": "^2.0.0" + "@babel/types": "^7.18.6" } }, - "@esbuild/linux-loong64": { - "version": "0.14.54", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.14.54.tgz", - "integrity": "sha512-bZBrLAIX1kpWelV0XemxBZllyRmM6vgFQQG2GdNb+r3Fkp0FOh1NJSvekXDs7jq70k4euu1cryLMfU+mTXlEpw==", - "dev": true, - "optional": true - }, - "@eslint-community/eslint-utils": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", - "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==", + "@babel/helper-module-transforms": { + "version": "7.20.11", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.20.11.tgz", + "integrity": "sha512-uRy78kN4psmji1s2QtbtcCSaj/LILFDp0f/ymhpQH5QY3nljUZCaNWz9X1dEj/8MBdBEFECs7yRhKn8i7NjZgg==", "dev": true, "requires": { - "eslint-visitor-keys": "^3.3.0" + "@babel/helper-environment-visitor": "^7.18.9", + "@babel/helper-module-imports": "^7.18.6", + "@babel/helper-simple-access": "^7.20.2", + "@babel/helper-split-export-declaration": "^7.18.6", + "@babel/helper-validator-identifier": "^7.19.1", + "@babel/template": "^7.20.7", + "@babel/traverse": "^7.20.10", + "@babel/types": "^7.20.7" } }, - "@eslint-community/regexpp": { - "version": "4.10.0", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.10.0.tgz", - "integrity": "sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA==", + "@babel/helper-plugin-utils": { + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.22.5.tgz", + "integrity": "sha512-uLls06UVKgFG9QD4OeFYLEGteMIAa5kpTPcFL28yuCIIzsf6ZyKZMllKVOCZFhiZ5ptnwX4mtKdWCBE/uT4amg==", "dev": true }, - "@eslint/eslintrc": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", - "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", + "@babel/helper-simple-access": { + "version": "7.20.2", + "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.20.2.tgz", + "integrity": "sha512-+0woI/WPq59IrqDYbVGfshjT5Dmk/nnbdpcF8SnMhhXObpTq2KNBdLFRFrkVdbDOyUmHBCxzm5FHV1rACIkIbA==", "dev": true, "requires": { - "ajv": "^6.12.4", - "debug": "^4.3.2", - "espree": "^9.6.0", - "globals": "^13.19.0", - "ignore": "^5.2.0", - "import-fresh": "^3.2.1", - "js-yaml": "^4.1.0", - "minimatch": "^3.1.2", - "strip-json-comments": "^3.1.1" + "@babel/types": "^7.20.2" } }, - "@eslint/js": { - "version": "8.56.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.56.0.tgz", - "integrity": "sha512-gMsVel9D7f2HLkBma9VbtzZRehRogVRfbr++f06nL2vnCGCNlzOD+/MUov/F4p8myyAHspEhVobgjpX64q5m6A==", - "dev": true - }, - "@gar/promisify": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz", - "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==", - "dev": true - }, - "@humanwhocodes/config-array": { - "version": "0.11.13", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.13.tgz", - "integrity": "sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ==", + "@babel/helper-split-export-declaration": { + "version": "7.22.6", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz", + "integrity": "sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==", "dev": true, "requires": { - "@humanwhocodes/object-schema": "^2.0.1", - "debug": "^4.1.1", - "minimatch": "^3.0.5" + "@babel/types": "^7.22.5" } }, - "@humanwhocodes/module-importer": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", - "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "@babel/helper-string-parser": { + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz", + "integrity": "sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==", "dev": true }, - "@humanwhocodes/object-schema": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.1.tgz", - "integrity": "sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw==", + "@babel/helper-validator-identifier": { + "version": "7.22.20", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz", + "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==", "dev": true }, - "@istanbuljs/load-nyc-config": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", - "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==", + "@babel/helper-validator-option": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.18.6.tgz", + "integrity": "sha512-XO7gESt5ouv/LRJdrVjkShckw6STTaB7l9BrpBaAHDeF5YZT+01PCwmR0SJHnkW6i8OwW/EVWRShfi4j2x+KQw==", + "dev": true + }, + "@babel/helpers": { + "version": "7.20.13", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.20.13.tgz", + "integrity": "sha512-nzJ0DWCL3gB5RCXbUO3KIMMsBY2Eqbx8mBpKGE/02PgyRQFcPQLbkQ1vyy596mZLaP+dAfD+R4ckASzNVmW3jg==", "dev": true, "requires": { - "camelcase": "^5.3.1", - "find-up": "^4.1.0", - "get-package-type": "^0.1.0", - "js-yaml": "^3.13.1", - "resolve-from": "^5.0.0" + "@babel/template": "^7.20.7", + "@babel/traverse": "^7.20.13", + "@babel/types": "^7.20.7" + } + }, + "@babel/highlight": { + "version": "7.22.20", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.22.20.tgz", + "integrity": "sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg==", + "dev": true, + "requires": { + "@babel/helper-validator-identifier": "^7.22.20", + "chalk": "^2.4.2", + "js-tokens": "^4.0.0" }, "dependencies": { - "argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", "dev": true, "requires": { - "sprintf-js": "~1.0.2" + "color-convert": "^1.9.0" } }, - "find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", "dev": true, "requires": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" } }, - "js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", "dev": true, "requires": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" + "color-name": "1.1.3" } }, - "locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dev": true, - "requires": { - "p-locate": "^4.1.0" - } + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true }, - "p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dev": true, - "requires": { - "p-try": "^2.0.0" - } + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true }, - "p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", "dev": true, "requires": { - "p-limit": "^2.2.0" + "has-flag": "^3.0.0" } - }, - "resolve-from": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", - "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", - "dev": true } } }, - "@istanbuljs/nyc-config-typescript": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@istanbuljs/nyc-config-typescript/-/nyc-config-typescript-1.0.2.tgz", - "integrity": "sha512-iKGIyMoyJuFnJRSVTZ78POIRvNnwZaWIf8vG4ZS3rQq58MMDrqEX2nnzx0R28V2X8JvmKYiqY9FP2hlJsm8A0w==", + "@babel/parser": { + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.0.tgz", + "integrity": "sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw==", + "dev": true + }, + "@babel/plugin-syntax-async-generators": { + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", + "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", "dev": true, "requires": { - "@istanbuljs/schema": "^0.1.2" + "@babel/helper-plugin-utils": "^7.8.0" } }, - "@istanbuljs/schema": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", - "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", - "dev": true - }, - "@jest/console": { - "version": "29.6.2", - "resolved": "https://registry.npmjs.org/@jest/console/-/console-29.6.2.tgz", - "integrity": "sha512-0N0yZof5hi44HAR2pPS+ikJ3nzKNoZdVu8FffRf3wy47I7Dm7etk/3KetMdRUqzVd16V4O2m2ISpNTbnIuqy1w==", + "@babel/plugin-syntax-bigint": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz", + "integrity": "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==", "dev": true, "requires": { - "@jest/types": "^29.6.1", - "@types/node": "*", - "chalk": "^4.0.0", - "jest-message-util": "^29.6.2", - "jest-util": "^29.6.2", - "slash": "^3.0.0" + "@babel/helper-plugin-utils": "^7.8.0" } }, - "@jest/core": { - "version": "29.6.2", - "resolved": "https://registry.npmjs.org/@jest/core/-/core-29.6.2.tgz", - "integrity": "sha512-Oj+5B+sDMiMWLhPFF+4/DvHOf+U10rgvCLGPHP8Xlsy/7QxS51aU/eBngudHlJXnaWD5EohAgJ4js+T6pa+zOg==", + "@babel/plugin-syntax-class-properties": { + "version": "7.12.13", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", + "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", "dev": true, "requires": { - "@jest/console": "^29.6.2", - "@jest/reporters": "^29.6.2", - "@jest/test-result": "^29.6.2", - "@jest/transform": "^29.6.2", - "@jest/types": "^29.6.1", - "@types/node": "*", - "ansi-escapes": "^4.2.1", - "chalk": "^4.0.0", - "ci-info": "^3.2.0", - "exit": "^0.1.2", - "graceful-fs": "^4.2.9", - "jest-changed-files": "^29.5.0", - "jest-config": "^29.6.2", - "jest-haste-map": "^29.6.2", - "jest-message-util": "^29.6.2", - "jest-regex-util": "^29.4.3", - "jest-resolve": "^29.6.2", - "jest-resolve-dependencies": "^29.6.2", - "jest-runner": "^29.6.2", - "jest-runtime": "^29.6.2", - "jest-snapshot": "^29.6.2", - "jest-util": "^29.6.2", - "jest-validate": "^29.6.2", - "jest-watcher": "^29.6.2", - "micromatch": "^4.0.4", - "pretty-format": "^29.6.2", - "slash": "^3.0.0", - "strip-ansi": "^6.0.0" - }, - "dependencies": { - "ci-info": { - "version": "3.8.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.8.0.tgz", - "integrity": "sha512-eXTggHWSooYhq49F2opQhuHWgzucfF2YgODK4e1566GQs5BIfP30B0oenwBJHfWxAs2fyPB1s7Mg949zLf61Yw==", - "dev": true - } + "@babel/helper-plugin-utils": "^7.12.13" } }, - "@jest/environment": { - "version": "29.6.2", - "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-29.6.2.tgz", - "integrity": "sha512-AEcW43C7huGd/vogTddNNTDRpO6vQ2zaQNrttvWV18ArBx9Z56h7BIsXkNFJVOO4/kblWEQz30ckw0+L3izc+Q==", + "@babel/plugin-syntax-import-meta": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz", + "integrity": "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==", "dev": true, "requires": { - "@jest/fake-timers": "^29.6.2", - "@jest/types": "^29.6.1", - "@types/node": "*", - "jest-mock": "^29.6.2" + "@babel/helper-plugin-utils": "^7.10.4" } }, - "@jest/expect": { - "version": "29.6.2", - "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-29.6.2.tgz", - "integrity": "sha512-m6DrEJxVKjkELTVAztTLyS/7C92Y2b0VYqmDROYKLLALHn8T/04yPs70NADUYPrV3ruI+H3J0iUIuhkjp7vkfg==", + "@babel/plugin-syntax-json-strings": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", + "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", "dev": true, "requires": { - "expect": "^29.6.2", - "jest-snapshot": "^29.6.2" + "@babel/helper-plugin-utils": "^7.8.0" } }, - "@jest/expect-utils": { - "version": "29.6.2", - "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.6.2.tgz", - "integrity": "sha512-6zIhM8go3RV2IG4aIZaZbxwpOzz3ZiM23oxAlkquOIole+G6TrbeXnykxWYlqF7kz2HlBjdKtca20x9atkEQYg==", + "@babel/plugin-syntax-jsx": { + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.22.5.tgz", + "integrity": "sha512-gvyP4hZrgrs/wWMaocvxZ44Hw0b3W8Pe+cMxc8V1ULQ07oh8VNbIRaoD1LRZVTvD+0nieDKjfgKg89sD7rrKrg==", "dev": true, "requires": { - "jest-get-type": "^29.4.3" + "@babel/helper-plugin-utils": "^7.22.5" } }, - "@jest/fake-timers": { - "version": "29.6.2", - "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-29.6.2.tgz", - "integrity": "sha512-euZDmIlWjm1Z0lJ1D0f7a0/y5Kh/koLFMUBE5SUYWrmy8oNhJpbTBDAP6CxKnadcMLDoDf4waRYCe35cH6G6PA==", + "@babel/plugin-syntax-logical-assignment-operators": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", + "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", "dev": true, "requires": { - "@jest/types": "^29.6.1", - "@sinonjs/fake-timers": "^10.0.2", - "@types/node": "*", - "jest-message-util": "^29.6.2", - "jest-mock": "^29.6.2", - "jest-util": "^29.6.2" - }, - "dependencies": { - "@sinonjs/commons": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.0.tgz", - "integrity": "sha512-jXBtWAF4vmdNmZgD5FoKsVLv3rPgDnLgPbU84LIJ3otV44vJlDRokVng5v8NFJdCf/da9legHcKaRuZs4L7faA==", - "dev": true, - "requires": { - "type-detect": "4.0.8" - } - }, - "@sinonjs/fake-timers": { - "version": "10.3.0", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", - "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", - "dev": true, - "requires": { - "@sinonjs/commons": "^3.0.0" - } - } + "@babel/helper-plugin-utils": "^7.10.4" } }, - "@jest/globals": { - "version": "29.6.2", - "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-29.6.2.tgz", - "integrity": "sha512-cjuJmNDjs6aMijCmSa1g2TNG4Lby/AeU7/02VtpW+SLcZXzOLK2GpN2nLqcFjmhy3B3AoPeQVx7BnyOf681bAw==", + "@babel/plugin-syntax-nullish-coalescing-operator": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", + "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", "dev": true, "requires": { - "@jest/environment": "^29.6.2", - "@jest/expect": "^29.6.2", - "@jest/types": "^29.6.1", - "jest-mock": "^29.6.2" + "@babel/helper-plugin-utils": "^7.8.0" } }, - "@jest/reporters": { - "version": "29.6.2", - "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-29.6.2.tgz", - "integrity": "sha512-sWtijrvIav8LgfJZlrGCdN0nP2EWbakglJY49J1Y5QihcQLfy7ovyxxjJBRXMNltgt4uPtEcFmIMbVshEDfFWw==", + "@babel/plugin-syntax-numeric-separator": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", + "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", "dev": true, "requires": { - "@bcoe/v8-coverage": "^0.2.3", - "@jest/console": "^29.6.2", - "@jest/test-result": "^29.6.2", - "@jest/transform": "^29.6.2", - "@jest/types": "^29.6.1", - "@jridgewell/trace-mapping": "^0.3.18", - "@types/node": "*", - "chalk": "^4.0.0", - "collect-v8-coverage": "^1.0.0", - "exit": "^0.1.2", - "glob": "^7.1.3", - "graceful-fs": "^4.2.9", - "istanbul-lib-coverage": "^3.0.0", - "istanbul-lib-instrument": "^5.1.0", - "istanbul-lib-report": "^3.0.0", - "istanbul-lib-source-maps": "^4.0.0", - "istanbul-reports": "^3.1.3", - "jest-message-util": "^29.6.2", - "jest-util": "^29.6.2", - "jest-worker": "^29.6.2", - "slash": "^3.0.0", - "string-length": "^4.0.1", - "strip-ansi": "^6.0.0", - "v8-to-istanbul": "^9.0.1" - }, - "dependencies": { - "istanbul-lib-instrument": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz", - "integrity": "sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg==", - "dev": true, - "requires": { - "@babel/core": "^7.12.3", - "@babel/parser": "^7.14.7", - "@istanbuljs/schema": "^0.1.2", - "istanbul-lib-coverage": "^3.2.0", - "semver": "^6.3.0" - } - }, - "semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true - } + "@babel/helper-plugin-utils": "^7.10.4" } }, - "@jest/schemas": { - "version": "29.6.0", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.0.tgz", - "integrity": "sha512-rxLjXyJBTL4LQeJW3aKo0M/+GkCOXsO+8i9Iu7eDb6KwtP65ayoDsitrdPBtujxQ88k4wI2FNYfa6TOGwSn6cQ==", + "@babel/plugin-syntax-object-rest-spread": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", + "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", "dev": true, "requires": { - "@sinclair/typebox": "^0.27.8" + "@babel/helper-plugin-utils": "^7.8.0" } }, - "@jest/source-map": { - "version": "29.6.0", - "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-29.6.0.tgz", - "integrity": "sha512-oA+I2SHHQGxDCZpbrsCQSoMLb3Bz547JnM+jUr9qEbuw0vQlWZfpPS7CO9J7XiwKicEz9OFn/IYoLkkiUD7bzA==", + "@babel/plugin-syntax-optional-catch-binding": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", + "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", "dev": true, "requires": { - "@jridgewell/trace-mapping": "^0.3.18", - "callsites": "^3.0.0", - "graceful-fs": "^4.2.9" + "@babel/helper-plugin-utils": "^7.8.0" } }, - "@jest/test-result": { - "version": "29.6.2", - "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-29.6.2.tgz", - "integrity": "sha512-3VKFXzcV42EYhMCsJQURptSqnyjqCGbtLuX5Xxb6Pm6gUf1wIRIl+mandIRGJyWKgNKYF9cnstti6Ls5ekduqw==", + "@babel/plugin-syntax-optional-chaining": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", + "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", "dev": true, "requires": { - "@jest/console": "^29.6.2", - "@jest/types": "^29.6.1", - "@types/istanbul-lib-coverage": "^2.0.0", - "collect-v8-coverage": "^1.0.0" + "@babel/helper-plugin-utils": "^7.8.0" } }, - "@jest/test-sequencer": { - "version": "29.6.2", - "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-29.6.2.tgz", - "integrity": "sha512-GVYi6PfPwVejO7slw6IDO0qKVum5jtrJ3KoLGbgBWyr2qr4GaxFV6su+ZAjdTX75Sr1DkMFRk09r2ZVa+wtCGw==", + "@babel/plugin-syntax-top-level-await": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", + "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", "dev": true, "requires": { - "@jest/test-result": "^29.6.2", - "graceful-fs": "^4.2.9", - "jest-haste-map": "^29.6.2", - "slash": "^3.0.0" + "@babel/helper-plugin-utils": "^7.14.5" } }, - "@jest/transform": { - "version": "29.6.2", - "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-29.6.2.tgz", - "integrity": "sha512-ZqCqEISr58Ce3U+buNFJYUktLJZOggfyvR+bZMaiV1e8B1SIvJbwZMrYz3gx/KAPn9EXmOmN+uB08yLCjWkQQg==", + "@babel/plugin-syntax-typescript": { + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.22.5.tgz", + "integrity": "sha512-1mS2o03i7t1c6VzH6fdQ3OA8tcEIxwG18zIPRp+UY1Ihv6W+XZzBCVxExF9upussPXJ0xE9XRHwMoNs1ep/nRQ==", "dev": true, "requires": { - "@babel/core": "^7.11.6", - "@jest/types": "^29.6.1", - "@jridgewell/trace-mapping": "^0.3.18", - "babel-plugin-istanbul": "^6.1.1", - "chalk": "^4.0.0", - "convert-source-map": "^2.0.0", - "fast-json-stable-stringify": "^2.1.0", - "graceful-fs": "^4.2.9", - "jest-haste-map": "^29.6.2", - "jest-regex-util": "^29.4.3", - "jest-util": "^29.6.2", - "micromatch": "^4.0.4", - "pirates": "^4.0.4", - "slash": "^3.0.0", - "write-file-atomic": "^4.0.2" + "@babel/helper-plugin-utils": "^7.22.5" + } + }, + "@babel/template": { + "version": "7.22.15", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz", + "integrity": "sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.22.13", + "@babel/parser": "^7.22.15", + "@babel/types": "^7.22.15" + } + }, + "@babel/traverse": { + "version": "7.23.2", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.2.tgz", + "integrity": "sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.22.13", + "@babel/generator": "^7.23.0", + "@babel/helper-environment-visitor": "^7.22.20", + "@babel/helper-function-name": "^7.23.0", + "@babel/helper-hoist-variables": "^7.22.5", + "@babel/helper-split-export-declaration": "^7.22.6", + "@babel/parser": "^7.23.0", + "@babel/types": "^7.23.0", + "debug": "^4.1.0", + "globals": "^11.1.0" }, "dependencies": { - "convert-source-map": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", - "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", "dev": true - }, - "write-file-atomic": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz", - "integrity": "sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg==", - "dev": true, - "requires": { - "imurmurhash": "^0.1.4", - "signal-exit": "^3.0.7" - } } } }, - "@jest/types": { - "version": "29.6.1", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.1.tgz", - "integrity": "sha512-tPKQNMPuXgvdOn2/Lg9HNfUvjYVGolt04Hp03f5hAk878uwOLikN+JzeLY0HcVgKgFl9Hs3EIqpu3WX27XNhnw==", + "@babel/types": { + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.23.0.tgz", + "integrity": "sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg==", "dev": true, "requires": { - "@jest/schemas": "^29.6.0", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" + "@babel/helper-string-parser": "^7.22.5", + "@babel/helper-validator-identifier": "^7.22.20", + "to-fast-properties": "^2.0.0" } }, - "@jridgewell/gen-mapping": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz", - "integrity": "sha512-sQXCasFk+U8lWYEe66WxRDOE9PjVz4vSM51fTu3Hw+ClTpUSQb718772vH3pyS5pShp6lvQM7SxgIDXXXmOX7w==", - "dev": true, - "requires": { - "@jridgewell/set-array": "^1.0.0", - "@jridgewell/sourcemap-codec": "^1.4.10" - } + "@balena/dockerignore": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", + "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==" }, - "@jridgewell/resolve-uri": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", - "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", + "@bcoe/v8-coverage": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", + "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true }, - "@jridgewell/set-array": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", - "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==", - "dev": true + "@brillout/import": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@brillout/import/-/import-0.2.1.tgz", + "integrity": "sha512-4qA9sNtQZCY02MeLjTP+O9LfY9T6d2ajYS7whyvgBklJzWzLUNbJp940ClxK64hbDPqag9AQZt45beEAGcSYYA==" }, - "@jridgewell/sourcemap-codec": { - "version": "1.4.14", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", - "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", - "dev": true + "@colors/colors": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz", + "integrity": "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==" }, - "@jridgewell/trace-mapping": { - "version": "0.3.19", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.19.tgz", - "integrity": "sha512-kf37QtfW+Hwx/buWGMPcR60iF9ziHa6r/CZJIHbmcm4+0qrXiVdxegAH0F6yddEVQ7zdkjcGCgCzUu+BcbhQxw==", + "@concordance/react": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@concordance/react/-/react-2.0.0.tgz", + "integrity": "sha512-huLSkUuM2/P+U0uy2WwlKuixMsTODD8p4JVQBI4VKeopkiN0C7M3N9XYVawb4M+4spN5RrO/eLhk7KoQX6nsfA==", "dev": true, "requires": { - "@jridgewell/resolve-uri": "^3.1.0", - "@jridgewell/sourcemap-codec": "^1.4.14" + "arrify": "^1.0.1" + }, + "dependencies": { + "arrify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", + "integrity": "sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==", + "dev": true + } } }, - "@kubernetes/client-node": { - "version": "0.17.1", - "resolved": "https://registry.npmjs.org/@kubernetes/client-node/-/client-node-0.17.1.tgz", - "integrity": "sha512-qXANjukuTq/drb1hq1NCYZafpdRTvbyTzbliWO6RwW7eEb2b9qwINbw0DiVHpBQg3e9DeQd8+brI1sR1Fck5kQ==", + "@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, "requires": { - "byline": "^5.0.0", - "execa": "5.0.0", - "isomorphic-ws": "^4.0.1", - "js-yaml": "^4.1.0", - "jsonpath-plus": "^0.19.0", - "openid-client": "^5.1.6", - "request": "^2.88.0", - "rfc4648": "^1.3.0", - "shelljs": "^0.8.5", - "stream-buffers": "^3.0.2", - "tar": "^6.1.11", - "tmp-promise": "^3.0.2", - "tslib": "^1.9.3", - "underscore": "^1.9.1", - "ws": "^7.3.1" + "@jridgewell/trace-mapping": "0.3.9" }, "dependencies": { - "execa": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/execa/-/execa-5.0.0.tgz", - "integrity": "sha512-ov6w/2LCiuyO4RLYGdpFGjkcs0wMTgGE8PrkTHikeUy5iJekXyPIKUjifk5CsE0pt7sMCrMZ3YNqoCj6idQOnQ==", + "@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, "requires": { - "cross-spawn": "^7.0.3", - "get-stream": "^6.0.0", - "human-signals": "^2.1.0", - "is-stream": "^2.0.0", - "merge-stream": "^2.0.0", - "npm-run-path": "^4.0.1", - "onetime": "^5.1.2", - "signal-exit": "^3.0.3", - "strip-final-newline": "^2.0.0" + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" } - }, - "get-stream": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", - "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==" - }, - "npm-run-path": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", - "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + } + } + }, + "@cucumber/create-meta": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@cucumber/create-meta/-/create-meta-5.0.0.tgz", + "integrity": "sha512-Z5kMZkUff00S3/KSnKzB/KOm2UIxMXY1xXmj2dQMlD49lV6v/W8EEvgDMNtQotQNSOQU5bDupmWQpk+o16tXIw==", + "dev": true, + "requires": { + "@cucumber/messages": "^16.0.0" + }, + "dependencies": { + "@cucumber/messages": { + "version": "16.0.1", + "resolved": "https://registry.npmjs.org/@cucumber/messages/-/messages-16.0.1.tgz", + "integrity": "sha512-80JcaAfQragFqR1rMhRwiqWL9HcR6Z4LDD2mfF0Lxg/lFkCNvmWa9Jl10NUNfFXYD555NKPzP/8xFo55abw8TQ==", + "dev": true, "requires": { - "path-key": "^3.0.0" + "@types/uuid": "8.3.0", + "class-transformer": "0.4.0", + "reflect-metadata": "0.1.13", + "uuid": "8.3.2" } + }, + "@types/uuid": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.0.tgz", + "integrity": "sha512-eQ9qFW/fhfGJF8WKHGEHZEyVWfZxrT+6CLIJGBcZPfxUh/+BnEj+UCGYMlr9qZuX/2AltsvwrGqp0LhEW8D0zQ==", + "dev": true + }, + "class-transformer": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.4.0.tgz", + "integrity": "sha512-ETWD/H2TbWbKEi7m9N4Km5+cw1hNcqJSxlSYhsLsNjQzWWiZIYA1zafxpK9PwVfaZ6AqR5rrjPVUBGESm5tQUA==", + "dev": true } } }, - "@napi-rs/snappy-android-arm-eabi": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-android-arm-eabi/-/snappy-android-arm-eabi-7.1.1.tgz", - "integrity": "sha512-NKd/ztuVEgQaAaNVQ5zZaCB9VV+7+uBXBHqhaE5iSapQhLc41szTlT0s68FCee75OoT3vhqdA6Jp5TrzZ2WOaw==", - "optional": true - }, - "@napi-rs/snappy-android-arm64": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-android-arm64/-/snappy-android-arm64-7.1.1.tgz", - "integrity": "sha512-DktruMAO0K0toTnxNHg2GWNIAPJqdvIchCsdsRaKyuEnG101qBg0mYiRCAhxHgbT6RJlOGbUPKkbA9KKRhEJUg==", - "optional": true - }, - "@napi-rs/snappy-darwin-arm64": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-darwin-arm64/-/snappy-darwin-arm64-7.1.1.tgz", - "integrity": "sha512-3LZyoAw3Qa5F7sCCTkSkhmGlydwUKU6L3Jl46eKHO2Ctm8Gcjxww6T7MfwlwGZ6JqAM6d1d++WLzUZPCGXVmag==", - "optional": true - }, - "@napi-rs/snappy-darwin-x64": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-darwin-x64/-/snappy-darwin-x64-7.1.1.tgz", - "integrity": "sha512-X1D2F67bQkPwr5iSR29/RnOrLwAkB55YO6t41toABzla3mflLDpzZcakz6FokIukykf7ey31/t73v/4pbgaBkg==", - "optional": true - }, - "@napi-rs/snappy-freebsd-x64": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-freebsd-x64/-/snappy-freebsd-x64-7.1.1.tgz", - "integrity": "sha512-vSeuf+An8jFVHPAn5IbWE9hTGU9PFAaZLj/X7rKTQQtZstnDsHgWe6u4g7FHLuOdwQ8TvhcxAEpNlYIXIk4AJg==", - "optional": true - }, - "@napi-rs/snappy-linux-arm-gnueabihf": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm-gnueabihf/-/snappy-linux-arm-gnueabihf-7.1.1.tgz", - "integrity": "sha512-/yyN6QsnOs3D1+jI3SfRX+gtnD86rbixdfmgxv9g40+FrDaDTLAu/3VuZIqH02qqq/xiWbDnkO+42RGxXDzTCw==", - "optional": true - }, - "@napi-rs/snappy-linux-arm64-gnu": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm64-gnu/-/snappy-linux-arm64-gnu-7.1.1.tgz", - "integrity": "sha512-StEeUCSwUoajgrBtiCQPTkHu+0Q4QlYndghGZNdbN1zJ1ny70YzPpevaFBUyjI/eJ+FN9uICKtwTPtQNSILS5g==", - "optional": true - }, + "@cucumber/cucumber": { + "version": "7.3.2", + "resolved": "https://registry.npmjs.org/@cucumber/cucumber/-/cucumber-7.3.2.tgz", + "integrity": "sha512-qqptM9w+UqXEYBAkrIGpIVPXDWv+zp0LrS89LiwHZwBp0cJg00su/iPMZ4j8TvCJiKfAwJXsAI1yjrd1POtU+w==", + "dev": true, + "requires": { + "@cucumber/create-meta": "^5.0.0", + "@cucumber/cucumber-expressions": "^12.1.1", + "@cucumber/gherkin": "^19.0.3", + "@cucumber/gherkin-streams": "^2.0.2", + "@cucumber/html-formatter": "^15.0.2", + "@cucumber/messages": "^16.0.1", + "@cucumber/tag-expressions": "^3.0.1", + "assertion-error-formatter": "^3.0.0", + "bluebird": "^3.7.2", + "capital-case": "^1.0.4", + "cli-table3": "0.6.1", + "colors": "1.4.0", + "commander": "^7.0.0", + "create-require": "^1.1.1", + "duration": "^0.2.2", + "durations": "^3.4.2", + "figures": "^3.2.0", + "glob": "^7.1.6", + "indent-string": "^4.0.0", + "is-generator": "^1.0.3", + "is-stream": "^2.0.0", + "knuth-shuffle-seeded": "^1.0.6", + "lodash": "^4.17.21", + "mz": "^2.7.0", + "progress": "^2.0.3", + "resolve": "^1.19.0", + "resolve-pkg": "^2.0.0", + "stack-chain": "^2.0.0", + "stacktrace-js": "^2.0.2", + "string-argv": "^0.3.1", + "tmp": "^0.2.1", + "util-arity": "^1.1.0", + "verror": "^1.10.0" + }, + "dependencies": { + "@cucumber/messages": { + "version": "16.0.1", + "resolved": "https://registry.npmjs.org/@cucumber/messages/-/messages-16.0.1.tgz", + "integrity": "sha512-80JcaAfQragFqR1rMhRwiqWL9HcR6Z4LDD2mfF0Lxg/lFkCNvmWa9Jl10NUNfFXYD555NKPzP/8xFo55abw8TQ==", + "dev": true, + "requires": { + "@types/uuid": "8.3.0", + "class-transformer": "0.4.0", + "reflect-metadata": "0.1.13", + "uuid": "8.3.2" + } + }, + "@types/uuid": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.0.tgz", + "integrity": "sha512-eQ9qFW/fhfGJF8WKHGEHZEyVWfZxrT+6CLIJGBcZPfxUh/+BnEj+UCGYMlr9qZuX/2AltsvwrGqp0LhEW8D0zQ==", + "dev": true + }, + "class-transformer": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.4.0.tgz", + "integrity": "sha512-ETWD/H2TbWbKEi7m9N4Km5+cw1hNcqJSxlSYhsLsNjQzWWiZIYA1zafxpK9PwVfaZ6AqR5rrjPVUBGESm5tQUA==", + "dev": true + }, + "commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "dev": true + }, + "tmp": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", + "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", + "dev": true, + "requires": { + "rimraf": "^3.0.0" + } + } + } + }, + "@cucumber/cucumber-expressions": { + "version": "12.1.3", + "resolved": "https://registry.npmjs.org/@cucumber/cucumber-expressions/-/cucumber-expressions-12.1.3.tgz", + "integrity": "sha512-LB8MAzE4F/t2KIgsDEz4gZH0xSI4aG0/LmYUPyISPPjUS1pI/yGWWyeX2WsiUQxpSs765WcNIq5Bggt7gGGO3Q==", + "dev": true, + "requires": { + "regexp-match-indices": "1.0.2" + } + }, + "@cucumber/gherkin": { + "version": "19.0.3", + "resolved": "https://registry.npmjs.org/@cucumber/gherkin/-/gherkin-19.0.3.tgz", + "integrity": "sha512-gWdMm8mfRk3P+VugJWvNALaQV5QnT+5RkqWy3tO+4NsMSQZPo5p4V4vXwriQZ/sZR1Wni5TDRztuRsKLgZ3XHA==", + "dev": true, + "requires": { + "@cucumber/message-streams": "^2.0.0", + "@cucumber/messages": "^16.0.1" + }, + "dependencies": { + "@cucumber/messages": { + "version": "16.0.1", + "resolved": "https://registry.npmjs.org/@cucumber/messages/-/messages-16.0.1.tgz", + "integrity": "sha512-80JcaAfQragFqR1rMhRwiqWL9HcR6Z4LDD2mfF0Lxg/lFkCNvmWa9Jl10NUNfFXYD555NKPzP/8xFo55abw8TQ==", + "dev": true, + "requires": { + "@types/uuid": "8.3.0", + "class-transformer": "0.4.0", + "reflect-metadata": "0.1.13", + "uuid": "8.3.2" + } + }, + "@types/uuid": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.0.tgz", + "integrity": "sha512-eQ9qFW/fhfGJF8WKHGEHZEyVWfZxrT+6CLIJGBcZPfxUh/+BnEj+UCGYMlr9qZuX/2AltsvwrGqp0LhEW8D0zQ==", + "dev": true + }, + "class-transformer": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.4.0.tgz", + "integrity": "sha512-ETWD/H2TbWbKEi7m9N4Km5+cw1hNcqJSxlSYhsLsNjQzWWiZIYA1zafxpK9PwVfaZ6AqR5rrjPVUBGESm5tQUA==", + "dev": true + } + } + }, + "@cucumber/gherkin-streams": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@cucumber/gherkin-streams/-/gherkin-streams-2.0.2.tgz", + "integrity": "sha512-cKmXOBz4OwGlrHMBCc4qCC3KzLaqcEZ11nWWskIbv6jyfvlIRuM2OgEF6VLcNVewczifW1p6DrDj0OO+BeXocA==", + "dev": true, + "requires": { + "@cucumber/gherkin": "^19.0.1", + "@cucumber/message-streams": "^2.0.0", + "@cucumber/messages": "^16.0.0", + "commander": "7.2.0", + "source-map-support": "0.5.19" + }, + "dependencies": { + "@cucumber/messages": { + "version": "16.0.1", + "resolved": "https://registry.npmjs.org/@cucumber/messages/-/messages-16.0.1.tgz", + "integrity": "sha512-80JcaAfQragFqR1rMhRwiqWL9HcR6Z4LDD2mfF0Lxg/lFkCNvmWa9Jl10NUNfFXYD555NKPzP/8xFo55abw8TQ==", + "dev": true, + "requires": { + "@types/uuid": "8.3.0", + "class-transformer": "0.4.0", + "reflect-metadata": "0.1.13", + "uuid": "8.3.2" + } + }, + "@types/uuid": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.0.tgz", + "integrity": "sha512-eQ9qFW/fhfGJF8WKHGEHZEyVWfZxrT+6CLIJGBcZPfxUh/+BnEj+UCGYMlr9qZuX/2AltsvwrGqp0LhEW8D0zQ==", + "dev": true + }, + "class-transformer": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.4.0.tgz", + "integrity": "sha512-ETWD/H2TbWbKEi7m9N4Km5+cw1hNcqJSxlSYhsLsNjQzWWiZIYA1zafxpK9PwVfaZ6AqR5rrjPVUBGESm5tQUA==", + "dev": true + }, + "commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "dev": true + } + } + }, + "@cucumber/html-formatter": { + "version": "15.0.2", + "resolved": "https://registry.npmjs.org/@cucumber/html-formatter/-/html-formatter-15.0.2.tgz", + "integrity": "sha512-j+YGY4ytj78G/v1gZo53D+vuKXlTg/oxNwSCCGvRQo75+AqYDJSkm/vexXJQ5lY1rXAvlbZ9KI6jhg6LDs0YdQ==", + "dev": true, + "requires": { + "@cucumber/messages": "^16.0.1", + "commander": "7.2.0", + "source-map-support": "0.5.19" + }, + "dependencies": { + "@cucumber/messages": { + "version": "16.0.1", + "resolved": "https://registry.npmjs.org/@cucumber/messages/-/messages-16.0.1.tgz", + "integrity": "sha512-80JcaAfQragFqR1rMhRwiqWL9HcR6Z4LDD2mfF0Lxg/lFkCNvmWa9Jl10NUNfFXYD555NKPzP/8xFo55abw8TQ==", + "dev": true, + "requires": { + "@types/uuid": "8.3.0", + "class-transformer": "0.4.0", + "reflect-metadata": "0.1.13", + "uuid": "8.3.2" + } + }, + "@types/uuid": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.0.tgz", + "integrity": "sha512-eQ9qFW/fhfGJF8WKHGEHZEyVWfZxrT+6CLIJGBcZPfxUh/+BnEj+UCGYMlr9qZuX/2AltsvwrGqp0LhEW8D0zQ==", + "dev": true + }, + "class-transformer": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.4.0.tgz", + "integrity": "sha512-ETWD/H2TbWbKEi7m9N4Km5+cw1hNcqJSxlSYhsLsNjQzWWiZIYA1zafxpK9PwVfaZ6AqR5rrjPVUBGESm5tQUA==", + "dev": true + }, + "commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "dev": true + } + } + }, + "@cucumber/message-streams": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@cucumber/message-streams/-/message-streams-2.1.0.tgz", + "integrity": "sha512-Yh3mw3qv6QL9NI/ihkZF8V9MX2GbnR6oktv34kC3uAbrQy9d/b2SZ3HNjG3J9JQqpV4B7Om3SPElJYIeo66TrA==", + "dev": true, + "requires": { + "@cucumber/messages": "^16.0.1" + }, + "dependencies": { + "@cucumber/messages": { + "version": "16.0.1", + "resolved": "https://registry.npmjs.org/@cucumber/messages/-/messages-16.0.1.tgz", + "integrity": "sha512-80JcaAfQragFqR1rMhRwiqWL9HcR6Z4LDD2mfF0Lxg/lFkCNvmWa9Jl10NUNfFXYD555NKPzP/8xFo55abw8TQ==", + "dev": true, + "requires": { + "@types/uuid": "8.3.0", + "class-transformer": "0.4.0", + "reflect-metadata": "0.1.13", + "uuid": "8.3.2" + } + }, + "@types/uuid": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.0.tgz", + "integrity": "sha512-eQ9qFW/fhfGJF8WKHGEHZEyVWfZxrT+6CLIJGBcZPfxUh/+BnEj+UCGYMlr9qZuX/2AltsvwrGqp0LhEW8D0zQ==", + "dev": true + }, + "class-transformer": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.4.0.tgz", + "integrity": "sha512-ETWD/H2TbWbKEi7m9N4Km5+cw1hNcqJSxlSYhsLsNjQzWWiZIYA1zafxpK9PwVfaZ6AqR5rrjPVUBGESm5tQUA==", + "dev": true + } + } + }, + "@cucumber/messages": { + "version": "21.0.1", + "resolved": "https://registry.npmjs.org/@cucumber/messages/-/messages-21.0.1.tgz", + "integrity": "sha512-pGR7iURM4SF9Qp1IIpNiVQ77J9kfxMkPOEbyy+zRmGABnWWCsqMpJdfHeh9Mb3VskemVw85++e15JT0PYdcR3g==", + "dev": true, + "peer": true, + "requires": { + "@types/uuid": "8.3.4", + "class-transformer": "0.5.1", + "reflect-metadata": "0.1.13", + "uuid": "9.0.0" + }, + "dependencies": { + "uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "dev": true, + "peer": true + } + } + }, + "@cucumber/pretty-formatter": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@cucumber/pretty-formatter/-/pretty-formatter-1.0.0.tgz", + "integrity": "sha512-wcnIMN94HyaHGsfq72dgCvr1d8q6VGH4Y6Gl5weJ2TNZw1qn2UY85Iki4c9VdaLUONYnyYH3+178YB+9RFe/Hw==", + "dev": true, + "requires": { + "ansi-styles": "^5.0.0", + "cli-table3": "^0.6.0", + "figures": "^3.2.0", + "ts-dedent": "^2.0.0" + }, + "dependencies": { + "ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true + } + } + }, + "@cucumber/tag-expressions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@cucumber/tag-expressions/-/tag-expressions-3.0.1.tgz", + "integrity": "sha512-OGCXaJ1BQXmQ5b9pw+JYsBGumK2/LPZiLmbj1o1JFVeSNs2PY8WPQFSyXrskhrHz5Nd/6lYg7lvGMtFHOncC4w==", + "dev": true + }, + "@dabh/diagnostics": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.3.tgz", + "integrity": "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==", + "requires": { + "colorspace": "1.1.x", + "enabled": "2.0.x", + "kuler": "^2.0.0" + } + }, + "@esbuild/linux-loong64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.14.54.tgz", + "integrity": "sha512-bZBrLAIX1kpWelV0XemxBZllyRmM6vgFQQG2GdNb+r3Fkp0FOh1NJSvekXDs7jq70k4euu1cryLMfU+mTXlEpw==", + "dev": true, + "optional": true + }, + "@eslint-community/eslint-utils": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", + "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==", + "dev": true, + "requires": { + "eslint-visitor-keys": "^3.3.0" + } + }, + "@eslint-community/regexpp": { + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.10.0.tgz", + "integrity": "sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA==", + "dev": true + }, + "@eslint/eslintrc": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", + "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", + "dev": true, + "requires": { + "ajv": "^6.12.4", + "debug": "^4.3.2", + "espree": "^9.6.0", + "globals": "^13.19.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.0", + "minimatch": "^3.1.2", + "strip-json-comments": "^3.1.1" + } + }, + "@eslint/js": { + "version": "8.56.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.56.0.tgz", + "integrity": "sha512-gMsVel9D7f2HLkBma9VbtzZRehRogVRfbr++f06nL2vnCGCNlzOD+/MUov/F4p8myyAHspEhVobgjpX64q5m6A==", + "dev": true + }, + "@gar/promisify": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz", + "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==", + "dev": true + }, + "@humanwhocodes/config-array": { + "version": "0.11.13", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.13.tgz", + "integrity": "sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ==", + "dev": true, + "requires": { + "@humanwhocodes/object-schema": "^2.0.1", + "debug": "^4.1.1", + "minimatch": "^3.0.5" + } + }, + "@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true + }, + "@humanwhocodes/object-schema": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.1.tgz", + "integrity": "sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw==", + "dev": true + }, + "@istanbuljs/load-nyc-config": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", + "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==", + "dev": true, + "requires": { + "camelcase": "^5.3.1", + "find-up": "^4.1.0", + "get-package-type": "^0.1.0", + "js-yaml": "^3.13.1", + "resolve-from": "^5.0.0" + }, + "dependencies": { + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "requires": { + "sprintf-js": "~1.0.2" + } + }, + "find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "requires": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, + "js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, + "locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "requires": { + "p-locate": "^4.1.0" + } + }, + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "requires": { + "p-try": "^2.0.0" + } + }, + "p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "requires": { + "p-limit": "^2.2.0" + } + }, + "resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true + } + } + }, + "@istanbuljs/nyc-config-typescript": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@istanbuljs/nyc-config-typescript/-/nyc-config-typescript-1.0.2.tgz", + "integrity": "sha512-iKGIyMoyJuFnJRSVTZ78POIRvNnwZaWIf8vG4ZS3rQq58MMDrqEX2nnzx0R28V2X8JvmKYiqY9FP2hlJsm8A0w==", + "dev": true, + "requires": { + "@istanbuljs/schema": "^0.1.2" + } + }, + "@istanbuljs/schema": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", + "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", + "dev": true + }, + "@jest/console": { + "version": "29.6.2", + "resolved": "https://registry.npmjs.org/@jest/console/-/console-29.6.2.tgz", + "integrity": "sha512-0N0yZof5hi44HAR2pPS+ikJ3nzKNoZdVu8FffRf3wy47I7Dm7etk/3KetMdRUqzVd16V4O2m2ISpNTbnIuqy1w==", + "dev": true, + "requires": { + "@jest/types": "^29.6.1", + "@types/node": "*", + "chalk": "^4.0.0", + "jest-message-util": "^29.6.2", + "jest-util": "^29.6.2", + "slash": "^3.0.0" + } + }, + "@jest/core": { + "version": "29.6.2", + "resolved": "https://registry.npmjs.org/@jest/core/-/core-29.6.2.tgz", + "integrity": "sha512-Oj+5B+sDMiMWLhPFF+4/DvHOf+U10rgvCLGPHP8Xlsy/7QxS51aU/eBngudHlJXnaWD5EohAgJ4js+T6pa+zOg==", + "dev": true, + "requires": { + "@jest/console": "^29.6.2", + "@jest/reporters": "^29.6.2", + "@jest/test-result": "^29.6.2", + "@jest/transform": "^29.6.2", + "@jest/types": "^29.6.1", + "@types/node": "*", + "ansi-escapes": "^4.2.1", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "exit": "^0.1.2", + "graceful-fs": "^4.2.9", + "jest-changed-files": "^29.5.0", + "jest-config": "^29.6.2", + "jest-haste-map": "^29.6.2", + "jest-message-util": "^29.6.2", + "jest-regex-util": "^29.4.3", + "jest-resolve": "^29.6.2", + "jest-resolve-dependencies": "^29.6.2", + "jest-runner": "^29.6.2", + "jest-runtime": "^29.6.2", + "jest-snapshot": "^29.6.2", + "jest-util": "^29.6.2", + "jest-validate": "^29.6.2", + "jest-watcher": "^29.6.2", + "micromatch": "^4.0.4", + "pretty-format": "^29.6.2", + "slash": "^3.0.0", + "strip-ansi": "^6.0.0" + }, + "dependencies": { + "ci-info": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.8.0.tgz", + "integrity": "sha512-eXTggHWSooYhq49F2opQhuHWgzucfF2YgODK4e1566GQs5BIfP30B0oenwBJHfWxAs2fyPB1s7Mg949zLf61Yw==", + "dev": true + } + } + }, + "@jest/environment": { + "version": "29.6.2", + "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-29.6.2.tgz", + "integrity": "sha512-AEcW43C7huGd/vogTddNNTDRpO6vQ2zaQNrttvWV18ArBx9Z56h7BIsXkNFJVOO4/kblWEQz30ckw0+L3izc+Q==", + "dev": true, + "requires": { + "@jest/fake-timers": "^29.6.2", + "@jest/types": "^29.6.1", + "@types/node": "*", + "jest-mock": "^29.6.2" + } + }, + "@jest/expect": { + "version": "29.6.2", + "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-29.6.2.tgz", + "integrity": "sha512-m6DrEJxVKjkELTVAztTLyS/7C92Y2b0VYqmDROYKLLALHn8T/04yPs70NADUYPrV3ruI+H3J0iUIuhkjp7vkfg==", + "dev": true, + "requires": { + "expect": "^29.6.2", + "jest-snapshot": "^29.6.2" + } + }, + "@jest/expect-utils": { + "version": "29.6.2", + "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.6.2.tgz", + "integrity": "sha512-6zIhM8go3RV2IG4aIZaZbxwpOzz3ZiM23oxAlkquOIole+G6TrbeXnykxWYlqF7kz2HlBjdKtca20x9atkEQYg==", + "dev": true, + "requires": { + "jest-get-type": "^29.4.3" + } + }, + "@jest/fake-timers": { + "version": "29.6.2", + "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-29.6.2.tgz", + "integrity": "sha512-euZDmIlWjm1Z0lJ1D0f7a0/y5Kh/koLFMUBE5SUYWrmy8oNhJpbTBDAP6CxKnadcMLDoDf4waRYCe35cH6G6PA==", + "dev": true, + "requires": { + "@jest/types": "^29.6.1", + "@sinonjs/fake-timers": "^10.0.2", + "@types/node": "*", + "jest-message-util": "^29.6.2", + "jest-mock": "^29.6.2", + "jest-util": "^29.6.2" + }, + "dependencies": { + "@sinonjs/commons": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.0.tgz", + "integrity": "sha512-jXBtWAF4vmdNmZgD5FoKsVLv3rPgDnLgPbU84LIJ3otV44vJlDRokVng5v8NFJdCf/da9legHcKaRuZs4L7faA==", + "dev": true, + "requires": { + "type-detect": "4.0.8" + } + }, + "@sinonjs/fake-timers": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", + "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", + "dev": true, + "requires": { + "@sinonjs/commons": "^3.0.0" + } + } + } + }, + "@jest/globals": { + "version": "29.6.2", + "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-29.6.2.tgz", + "integrity": "sha512-cjuJmNDjs6aMijCmSa1g2TNG4Lby/AeU7/02VtpW+SLcZXzOLK2GpN2nLqcFjmhy3B3AoPeQVx7BnyOf681bAw==", + "dev": true, + "requires": { + "@jest/environment": "^29.6.2", + "@jest/expect": "^29.6.2", + "@jest/types": "^29.6.1", + "jest-mock": "^29.6.2" + } + }, + "@jest/reporters": { + "version": "29.6.2", + "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-29.6.2.tgz", + "integrity": "sha512-sWtijrvIav8LgfJZlrGCdN0nP2EWbakglJY49J1Y5QihcQLfy7ovyxxjJBRXMNltgt4uPtEcFmIMbVshEDfFWw==", + "dev": true, + "requires": { + "@bcoe/v8-coverage": "^0.2.3", + "@jest/console": "^29.6.2", + "@jest/test-result": "^29.6.2", + "@jest/transform": "^29.6.2", + "@jest/types": "^29.6.1", + "@jridgewell/trace-mapping": "^0.3.18", + "@types/node": "*", + "chalk": "^4.0.0", + "collect-v8-coverage": "^1.0.0", + "exit": "^0.1.2", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "istanbul-lib-coverage": "^3.0.0", + "istanbul-lib-instrument": "^5.1.0", + "istanbul-lib-report": "^3.0.0", + "istanbul-lib-source-maps": "^4.0.0", + "istanbul-reports": "^3.1.3", + "jest-message-util": "^29.6.2", + "jest-util": "^29.6.2", + "jest-worker": "^29.6.2", + "slash": "^3.0.0", + "string-length": "^4.0.1", + "strip-ansi": "^6.0.0", + "v8-to-istanbul": "^9.0.1" + }, + "dependencies": { + "istanbul-lib-instrument": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz", + "integrity": "sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg==", + "dev": true, + "requires": { + "@babel/core": "^7.12.3", + "@babel/parser": "^7.14.7", + "@istanbuljs/schema": "^0.1.2", + "istanbul-lib-coverage": "^3.2.0", + "semver": "^6.3.0" + } + }, + "semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true + } + } + }, + "@jest/schemas": { + "version": "29.6.0", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.0.tgz", + "integrity": "sha512-rxLjXyJBTL4LQeJW3aKo0M/+GkCOXsO+8i9Iu7eDb6KwtP65ayoDsitrdPBtujxQ88k4wI2FNYfa6TOGwSn6cQ==", + "dev": true, + "requires": { + "@sinclair/typebox": "^0.27.8" + } + }, + "@jest/source-map": { + "version": "29.6.0", + "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-29.6.0.tgz", + "integrity": "sha512-oA+I2SHHQGxDCZpbrsCQSoMLb3Bz547JnM+jUr9qEbuw0vQlWZfpPS7CO9J7XiwKicEz9OFn/IYoLkkiUD7bzA==", + "dev": true, + "requires": { + "@jridgewell/trace-mapping": "^0.3.18", + "callsites": "^3.0.0", + "graceful-fs": "^4.2.9" + } + }, + "@jest/test-result": { + "version": "29.6.2", + "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-29.6.2.tgz", + "integrity": "sha512-3VKFXzcV42EYhMCsJQURptSqnyjqCGbtLuX5Xxb6Pm6gUf1wIRIl+mandIRGJyWKgNKYF9cnstti6Ls5ekduqw==", + "dev": true, + "requires": { + "@jest/console": "^29.6.2", + "@jest/types": "^29.6.1", + "@types/istanbul-lib-coverage": "^2.0.0", + "collect-v8-coverage": "^1.0.0" + } + }, + "@jest/test-sequencer": { + "version": "29.6.2", + "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-29.6.2.tgz", + "integrity": "sha512-GVYi6PfPwVejO7slw6IDO0qKVum5jtrJ3KoLGbgBWyr2qr4GaxFV6su+ZAjdTX75Sr1DkMFRk09r2ZVa+wtCGw==", + "dev": true, + "requires": { + "@jest/test-result": "^29.6.2", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.6.2", + "slash": "^3.0.0" + } + }, + "@jest/transform": { + "version": "29.6.2", + "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-29.6.2.tgz", + "integrity": "sha512-ZqCqEISr58Ce3U+buNFJYUktLJZOggfyvR+bZMaiV1e8B1SIvJbwZMrYz3gx/KAPn9EXmOmN+uB08yLCjWkQQg==", + "dev": true, + "requires": { + "@babel/core": "^7.11.6", + "@jest/types": "^29.6.1", + "@jridgewell/trace-mapping": "^0.3.18", + "babel-plugin-istanbul": "^6.1.1", + "chalk": "^4.0.0", + "convert-source-map": "^2.0.0", + "fast-json-stable-stringify": "^2.1.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.6.2", + "jest-regex-util": "^29.4.3", + "jest-util": "^29.6.2", + "micromatch": "^4.0.4", + "pirates": "^4.0.4", + "slash": "^3.0.0", + "write-file-atomic": "^4.0.2" + }, + "dependencies": { + "convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true + }, + "write-file-atomic": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz", + "integrity": "sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg==", + "dev": true, + "requires": { + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.7" + } + } + } + }, + "@jest/types": { + "version": "29.6.1", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.1.tgz", + "integrity": "sha512-tPKQNMPuXgvdOn2/Lg9HNfUvjYVGolt04Hp03f5hAk878uwOLikN+JzeLY0HcVgKgFl9Hs3EIqpu3WX27XNhnw==", + "dev": true, + "requires": { + "@jest/schemas": "^29.6.0", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + } + }, + "@jridgewell/gen-mapping": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz", + "integrity": "sha512-sQXCasFk+U8lWYEe66WxRDOE9PjVz4vSM51fTu3Hw+ClTpUSQb718772vH3pyS5pShp6lvQM7SxgIDXXXmOX7w==", + "dev": true, + "requires": { + "@jridgewell/set-array": "^1.0.0", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "@jridgewell/resolve-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", + "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", + "dev": true + }, + "@jridgewell/set-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", + "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==", + "dev": true + }, + "@jridgewell/sourcemap-codec": { + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", + "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", + "dev": true + }, + "@jridgewell/trace-mapping": { + "version": "0.3.19", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.19.tgz", + "integrity": "sha512-kf37QtfW+Hwx/buWGMPcR60iF9ziHa6r/CZJIHbmcm4+0qrXiVdxegAH0F6yddEVQ7zdkjcGCgCzUu+BcbhQxw==", + "dev": true, + "requires": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "@kubernetes/client-node": { + "version": "0.17.1", + "resolved": "https://registry.npmjs.org/@kubernetes/client-node/-/client-node-0.17.1.tgz", + "integrity": "sha512-qXANjukuTq/drb1hq1NCYZafpdRTvbyTzbliWO6RwW7eEb2b9qwINbw0DiVHpBQg3e9DeQd8+brI1sR1Fck5kQ==", + "requires": { + "byline": "^5.0.0", + "execa": "5.0.0", + "isomorphic-ws": "^4.0.1", + "js-yaml": "^4.1.0", + "jsonpath-plus": "^0.19.0", + "openid-client": "^5.1.6", + "request": "^2.88.0", + "rfc4648": "^1.3.0", + "shelljs": "^0.8.5", + "stream-buffers": "^3.0.2", + "tar": "^6.1.11", + "tmp-promise": "^3.0.2", + "tslib": "^1.9.3", + "underscore": "^1.9.1", + "ws": "^7.3.1" + }, + "dependencies": { + "execa": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.0.0.tgz", + "integrity": "sha512-ov6w/2LCiuyO4RLYGdpFGjkcs0wMTgGE8PrkTHikeUy5iJekXyPIKUjifk5CsE0pt7sMCrMZ3YNqoCj6idQOnQ==", + "requires": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + } + }, + "get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==" + }, + "npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "requires": { + "path-key": "^3.0.0" + } + } + } + }, + "@napi-rs/snappy-android-arm-eabi": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-android-arm-eabi/-/snappy-android-arm-eabi-7.1.1.tgz", + "integrity": "sha512-NKd/ztuVEgQaAaNVQ5zZaCB9VV+7+uBXBHqhaE5iSapQhLc41szTlT0s68FCee75OoT3vhqdA6Jp5TrzZ2WOaw==", + "optional": true + }, + "@napi-rs/snappy-android-arm64": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-android-arm64/-/snappy-android-arm64-7.1.1.tgz", + "integrity": "sha512-DktruMAO0K0toTnxNHg2GWNIAPJqdvIchCsdsRaKyuEnG101qBg0mYiRCAhxHgbT6RJlOGbUPKkbA9KKRhEJUg==", + "optional": true + }, + "@napi-rs/snappy-darwin-arm64": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-darwin-arm64/-/snappy-darwin-arm64-7.1.1.tgz", + "integrity": "sha512-3LZyoAw3Qa5F7sCCTkSkhmGlydwUKU6L3Jl46eKHO2Ctm8Gcjxww6T7MfwlwGZ6JqAM6d1d++WLzUZPCGXVmag==", + "optional": true + }, + "@napi-rs/snappy-darwin-x64": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-darwin-x64/-/snappy-darwin-x64-7.1.1.tgz", + "integrity": "sha512-X1D2F67bQkPwr5iSR29/RnOrLwAkB55YO6t41toABzla3mflLDpzZcakz6FokIukykf7ey31/t73v/4pbgaBkg==", + "optional": true + }, + "@napi-rs/snappy-freebsd-x64": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-freebsd-x64/-/snappy-freebsd-x64-7.1.1.tgz", + "integrity": "sha512-vSeuf+An8jFVHPAn5IbWE9hTGU9PFAaZLj/X7rKTQQtZstnDsHgWe6u4g7FHLuOdwQ8TvhcxAEpNlYIXIk4AJg==", + "optional": true + }, + "@napi-rs/snappy-linux-arm-gnueabihf": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm-gnueabihf/-/snappy-linux-arm-gnueabihf-7.1.1.tgz", + "integrity": "sha512-/yyN6QsnOs3D1+jI3SfRX+gtnD86rbixdfmgxv9g40+FrDaDTLAu/3VuZIqH02qqq/xiWbDnkO+42RGxXDzTCw==", + "optional": true + }, + "@napi-rs/snappy-linux-arm64-gnu": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm64-gnu/-/snappy-linux-arm64-gnu-7.1.1.tgz", + "integrity": "sha512-StEeUCSwUoajgrBtiCQPTkHu+0Q4QlYndghGZNdbN1zJ1ny70YzPpevaFBUyjI/eJ+FN9uICKtwTPtQNSILS5g==", + "optional": true + }, "@napi-rs/snappy-linux-arm64-musl": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm64-musl/-/snappy-linux-arm64-musl-7.1.1.tgz", "integrity": "sha512-jWEBRzj+lswZVYf0b5eY0fjMlBL9L9yqjmTuv2UIMjJNHPuR282LK/s3Fm9sYIXQtKkiCo5JyhmIcoghZ3v0Eg==", "optional": true }, - "@napi-rs/snappy-linux-x64-gnu": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-gnu/-/snappy-linux-x64-gnu-7.1.1.tgz", - "integrity": "sha512-41DPoAUFAU9VNrj/96qKfStH2Xq88ZYIsSz8BlITDm2ScoeDGOGbmaWguCXU7I+bC2uKWTmUVMXKqk6tVY6LEg==", - "optional": true + "@napi-rs/snappy-linux-x64-gnu": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-gnu/-/snappy-linux-x64-gnu-7.1.1.tgz", + "integrity": "sha512-41DPoAUFAU9VNrj/96qKfStH2Xq88ZYIsSz8BlITDm2ScoeDGOGbmaWguCXU7I+bC2uKWTmUVMXKqk6tVY6LEg==", + "optional": true + }, + "@napi-rs/snappy-linux-x64-musl": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-musl/-/snappy-linux-x64-musl-7.1.1.tgz", + "integrity": "sha512-xR4hzFQqVq6J8Zf6XyUVtFJBaRgDyAQYUoBsCr92tZ7gI/0RlWCV6Q6JMO/wP5CSsvyFJIAtSUXXqlzIpw0GPA==", + "optional": true + }, + "@napi-rs/snappy-win32-arm64-msvc": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-arm64-msvc/-/snappy-win32-arm64-msvc-7.1.1.tgz", + "integrity": "sha512-2mHPadctsaYtrfSV5Na8ooTdI5rflPxP1pceY4us6vbjeWrfgB+KQCuEFOHsGXqFNfsi6L9nWH8nB9swnxnSyw==", + "optional": true + }, + "@napi-rs/snappy-win32-ia32-msvc": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-ia32-msvc/-/snappy-win32-ia32-msvc-7.1.1.tgz", + "integrity": "sha512-FOMgs9W71hdgjyl3T9F7b/WEIuoryfgBqsyhtHjAaa/98R0BUHl0bOoHg8ka0b5GgnhLBHkX2Yd6VD+Si9Q2ww==", + "optional": true + }, + "@napi-rs/snappy-win32-x64-msvc": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-x64-msvc/-/snappy-win32-x64-msvc-7.1.1.tgz", + "integrity": "sha512-Mu3yELySvzhBcNTVCq+hYxVh+lH3/KjoQ5HIEb3DDPoX0AGRTm3XZa+usq8pFWjl91Cgp9nWK+9lVSkCCIRaKA==", + "optional": true + }, + "@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "requires": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + } + }, + "@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true + }, + "@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "requires": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + } + }, + "@npmcli/fs": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-2.1.2.tgz", + "integrity": "sha512-yOJKRvohFOaLqipNtwYB9WugyZKhC/DZC4VYPmpaCzDBrA8YpK3qHZ8/HGscMnE4GqbkLNuVcCnxkeQEdGt6LQ==", + "dev": true, + "requires": { + "@gar/promisify": "^1.1.3", + "semver": "^7.3.5" + } + }, + "@npmcli/move-file": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@npmcli/move-file/-/move-file-2.0.1.tgz", + "integrity": "sha512-mJd2Z5TjYWq/ttPLLGqArdtnC74J6bOzg4rMDnN+p1xTacZ2yPRCk2y0oSWQtygLR9YVQXgOcONrwtnk3JupxQ==", + "dev": true, + "requires": { + "mkdirp": "^1.0.4", + "rimraf": "^3.0.2" + } + }, + "@npmcli/node-gyp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@npmcli/node-gyp/-/node-gyp-2.0.0.tgz", + "integrity": "sha512-doNI35wIe3bBaEgrlPfdJPaCpUR89pJWep4Hq3aRdh6gKazIVWfs0jHttvSSoq47ZXgC7h73kDsUl8AoIQUB+A==", + "dev": true + }, + "@npmcli/promise-spawn": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@npmcli/promise-spawn/-/promise-spawn-3.0.0.tgz", + "integrity": "sha512-s9SgS+p3a9Eohe68cSI3fi+hpcZUmXq5P7w0kMlAsWVtR7XbK3ptkZqKT2cK1zLDObJ3sR+8P59sJE0w/KTL1g==", + "dev": true, + "requires": { + "infer-owner": "^1.0.4" + } + }, + "@npmcli/run-script": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@npmcli/run-script/-/run-script-4.2.1.tgz", + "integrity": "sha512-7dqywvVudPSrRCW5nTHpHgeWnbBtz8cFkOuKrecm6ih+oO9ciydhWt6OF7HlqupRRmB8Q/gECVdB9LMfToJbRg==", + "dev": true, + "requires": { + "@npmcli/node-gyp": "^2.0.0", + "@npmcli/promise-spawn": "^3.0.0", + "node-gyp": "^9.0.0", + "read-package-json-fast": "^2.0.3", + "which": "^2.0.2" + } + }, + "@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" + }, + "@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" + }, + "@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" }, - "@napi-rs/snappy-linux-x64-musl": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-musl/-/snappy-linux-x64-musl-7.1.1.tgz", - "integrity": "sha512-xR4hzFQqVq6J8Zf6XyUVtFJBaRgDyAQYUoBsCr92tZ7gI/0RlWCV6Q6JMO/wP5CSsvyFJIAtSUXXqlzIpw0GPA==", - "optional": true + "@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" }, - "@napi-rs/snappy-win32-arm64-msvc": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-arm64-msvc/-/snappy-win32-arm64-msvc-7.1.1.tgz", - "integrity": "sha512-2mHPadctsaYtrfSV5Na8ooTdI5rflPxP1pceY4us6vbjeWrfgB+KQCuEFOHsGXqFNfsi6L9nWH8nB9swnxnSyw==", - "optional": true + "@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "requires": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } }, - "@napi-rs/snappy-win32-ia32-msvc": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-ia32-msvc/-/snappy-win32-ia32-msvc-7.1.1.tgz", - "integrity": "sha512-FOMgs9W71hdgjyl3T9F7b/WEIuoryfgBqsyhtHjAaa/98R0BUHl0bOoHg8ka0b5GgnhLBHkX2Yd6VD+Si9Q2ww==", - "optional": true + "@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" }, - "@napi-rs/snappy-win32-x64-msvc": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-x64-msvc/-/snappy-win32-x64-msvc-7.1.1.tgz", - "integrity": "sha512-Mu3yELySvzhBcNTVCq+hYxVh+lH3/KjoQ5HIEb3DDPoX0AGRTm3XZa+usq8pFWjl91Cgp9nWK+9lVSkCCIRaKA==", - "optional": true + "@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" }, - "@nodelib/fs.scandir": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", - "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "dev": true, + "@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" + }, + "@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" + }, + "@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" + }, + "@scramjet/adapters": { + "version": "file:packages/adapters", "requires": { - "@nodelib/fs.stat": "2.0.5", - "run-parallel": "^1.1.9" + "@kubernetes/client-node": "^0.17.1", + "@scramjet/model": "^0.38.0", + "@scramjet/obj-logger": "^0.38.0", + "@scramjet/pre-runner": "^0.38.0", + "@scramjet/python-runner": "^0.38.0", + "@scramjet/runner": "^0.38.0", + "@scramjet/sth-config": "^0.38.0", + "@scramjet/symbols": "^0.38.0", + "@scramjet/types": "^0.38.0", + "@scramjet/utility": "^0.38.0", + "@types/dockerode": "<=3.3.3", + "@types/js-yaml": "4.0.5", + "@types/node": "15.12.5", + "@types/request": "2.48.8", + "@types/shell-escape": "^0.2.1", + "@types/ws": "8.5.3", + "ava": "^3.15.0", + "dockerode": "^3.3.4", + "scramjet": "^4.36.9", + "shell-escape": "^0.2.0", + "ts-node": "^10.9.1", + "ts.data.json": "^2.2.0", + "typedoc": "0.23.17", + "typedoc-plugin-markdown": "3.13.6", + "typescript": "~4.7.4" } }, - "@nodelib/fs.stat": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", - "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "dev": true + "@scramjet/api-client": { + "version": "file:packages/api-client", + "requires": { + "@scramjet/client-utils": "^0.38.0", + "@scramjet/sth-config": "^0.38.0", + "@scramjet/symbols": "^0.38.0", + "@scramjet/types": "^0.38.0", + "@types/node": "15.12.5", + "ava": "^3.15.0", + "n-readlines": "^1.0.1", + "scramjet": "^4.36.9", + "ts-node": "^10.9.1", + "typedoc": "0.23.17", + "typedoc-plugin-markdown": "3.13.6", + "typescript": "~4.7.4" + } }, - "@nodelib/fs.walk": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", - "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dev": true, + "@scramjet/api-server": { + "version": "file:packages/api-server", "requires": { - "@nodelib/fs.scandir": "2.1.5", - "fastq": "^1.6.0" + "@scramjet/model": "^0.38.0", + "@scramjet/obj-logger": "^0.38.0", + "@scramjet/symbols": "^0.38.0", + "@scramjet/types": "^0.38.0", + "@scramjet/utility": "^0.38.0", + "@types/node": "15.12.5", + "@types/sinon": "^10.0.13", + "@types/trouter": "^3.1.1", + "0http": "^3.4.1", + "ava": "^3.15.0", + "http-status-codes": "^2.2.0", + "scramjet": "^4.36.9", + "sinon": "^14.0.1", + "trouter": "^3.2.0", + "ts-node": "^10.9.1", + "typedoc": "0.23.17", + "typedoc-plugin-markdown": "3.13.6", + "typescript": "~4.7.4" + } + }, + "@scramjet/cli": { + "version": "file:packages/cli", + "requires": { + "@scramjet/api-client": "^0.38.0", + "@scramjet/client-utils": "^0.38.0", + "@scramjet/middleware-api-client": "^0.38.0", + "@scramjet/obj-logger": "^0.38.0", + "@scramjet/types": "^0.38.0", + "@scramjet/utility": "^0.38.0", + "@types/find-package-json": "^1.2.3", + "@types/minimatch": "^3.0.5", + "@types/node": "15.12.5", + "@types/tar": "^6.1.3", + "@types/validator": "^13.7.8", + "ava": "^3.15.0", + "chalk": "^4.1.2", + "commander": "^9.5.0", + "find-package-json": "^1.2.0", + "minimatch": "^3.1.2", + "scramjet": "^4.36.9", + "tar": "^6.1.11", + "ts-node": "^10.9.1", + "typedoc": "0.23.17", + "typedoc-plugin-markdown": "3.13.6", + "typescript": "~4.7.4", + "validator": "^13.7.0" + }, + "dependencies": { + "commander": { + "version": "9.5.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", + "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==" + } + } + }, + "@scramjet/client-utils": { + "version": "file:packages/client-utils", + "requires": { + "@scramjet/model": "^0.38.0", + "@scramjet/obj-logger": "^0.38.0", + "@scramjet/sth-config": "^0.38.0", + "@scramjet/symbols": "^0.38.0", + "@scramjet/types": "^0.38.0", + "@scramjet/utility": "^0.38.0", + "@types/node": "15.12.5", + "@types/node-fetch": "^2.6.2", + "abort-controller": "^3.0.0", + "ava": "^3.15.0", + "n-readlines": "^1.0.1", + "node-fetch": "^2.6.7", + "normalize-url": "4", + "scramjet": "^4.36.9", + "ts-node": "^10.9.1", + "typedoc": "0.23.17", + "typedoc-plugin-markdown": "3.13.6", + "typescript": "~4.7.4" + } + }, + "@scramjet/host": { + "version": "file:packages/host", + "requires": { + "@scramjet/adapters": "^0.38.0", + "@scramjet/api-server": "^0.38.0", + "@scramjet/load-check": "^0.38.0", + "@scramjet/model": "^0.38.0", + "@scramjet/module-loader": "^0.38.0", + "@scramjet/obj-logger": "^0.38.0", + "@scramjet/sth-config": "^0.38.0", + "@scramjet/symbols": "^0.38.0", + "@scramjet/tecemux": "^0.38.0", + "@scramjet/telemetry": "^0.38.0", + "@scramjet/types": "^0.38.0", + "@scramjet/utility": "^0.38.0", + "@scramjet/verser": "^0.38.0", + "@types/find-package-json": "^1.2.3", + "@types/jest": "^29.4.4", + "@types/node": "15.12.5", + "ava": "^3.15.0", + "ext-ip": "^0.3.9", + "find-package-json": "^1.2.0", + "http-status-codes": "^2.2.0", + "jest": "^29.5.0", + "minimist": "^1.2.6", + "pico-s3": "^2.0.0", + "rereadable-stream": "^1.4.14", + "scramjet": "^4.36.9", + "ts-jest": "^29.0.5", + "ts-node": "^10.9.1", + "typedoc": "0.23.17", + "typedoc-plugin-markdown": "3.13.6", + "typescript": "~4.7.4" + } + }, + "@scramjet/load-check": { + "version": "file:packages/load-check", + "requires": { + "@scramjet/obj-logger": "^0.38.0", + "@scramjet/types": "^0.38.0", + "@scramjet/utility": "^0.38.0", + "@types/node": "15.12.5", + "@types/node-os-utils": "^1.2.0", + "@types/uuid": "^8.3.4", + "ava": "^3.15.0", + "diskusage-ng": "1.0.2", + "node-os-utils": "1.3.7", + "scramjet": "^4.36.9", + "ts-node": "^10.9.1", + "typedoc": "0.23.17", + "typedoc-plugin-markdown": "3.13.6", + "typescript": "~4.7.4", + "uuid": "^8.3.2" } }, - "@npmcli/fs": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-2.1.2.tgz", - "integrity": "sha512-yOJKRvohFOaLqipNtwYB9WugyZKhC/DZC4VYPmpaCzDBrA8YpK3qHZ8/HGscMnE4GqbkLNuVcCnxkeQEdGt6LQ==", - "dev": true, + "@scramjet/logger": { + "version": "file:packages/logger", "requires": { - "@gar/promisify": "^1.1.3", - "semver": "^7.3.5" + "@scramjet/types": "^0.38.0", + "@types/node": "15.12.5", + "ava": "^3.15.0", + "nyc": "^15.1.0", + "scramjet": "^4.36.9", + "ts-node": "^10.9.1", + "typedoc": "0.23.17", + "typedoc-plugin-markdown": "3.13.6", + "typescript": "~4.7.4" } }, - "@npmcli/move-file": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@npmcli/move-file/-/move-file-2.0.1.tgz", - "integrity": "sha512-mJd2Z5TjYWq/ttPLLGqArdtnC74J6bOzg4rMDnN+p1xTacZ2yPRCk2y0oSWQtygLR9YVQXgOcONrwtnk3JupxQ==", - "dev": true, + "@scramjet/manager-api-client": { + "version": "file:packages/manager-api-client", "requires": { - "mkdirp": "^1.0.4", - "rimraf": "^3.0.2" + "@scramjet/api-client": "^0.38.0", + "@scramjet/client-utils": "^0.38.0", + "@scramjet/types": "^0.38.0", + "@types/node": "15.12.5", + "ava": "^3.15.0", + "ts-node": "^10.9.1", + "typedoc": "0.23.17", + "typedoc-plugin-markdown": "3.13.6", + "typescript": "~4.7.4" } }, - "@npmcli/node-gyp": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@npmcli/node-gyp/-/node-gyp-2.0.0.tgz", - "integrity": "sha512-doNI35wIe3bBaEgrlPfdJPaCpUR89pJWep4Hq3aRdh6gKazIVWfs0jHttvSSoq47ZXgC7h73kDsUl8AoIQUB+A==", - "dev": true - }, - "@npmcli/promise-spawn": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@npmcli/promise-spawn/-/promise-spawn-3.0.0.tgz", - "integrity": "sha512-s9SgS+p3a9Eohe68cSI3fi+hpcZUmXq5P7w0kMlAsWVtR7XbK3ptkZqKT2cK1zLDObJ3sR+8P59sJE0w/KTL1g==", - "dev": true, + "@scramjet/middleware-api-client": { + "version": "file:packages/middleware-api-client", "requires": { - "infer-owner": "^1.0.4" + "@scramjet/api-client": "^0.38.0", + "@scramjet/client-utils": "^0.38.0", + "@scramjet/manager-api-client": "^0.38.0", + "@scramjet/multi-manager-api-client": "^0.38.0", + "@scramjet/types": "^0.38.0", + "ava": "^3.15.0", + "esbuild": "^0.14.54", + "ts-node": "^10.9.1", + "typedoc": "0.23.17", + "typedoc-plugin-markdown": "3.13.6", + "typescript": "~4.7.4" } }, - "@npmcli/run-script": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/@npmcli/run-script/-/run-script-4.2.1.tgz", - "integrity": "sha512-7dqywvVudPSrRCW5nTHpHgeWnbBtz8cFkOuKrecm6ih+oO9ciydhWt6OF7HlqupRRmB8Q/gECVdB9LMfToJbRg==", - "dev": true, + "@scramjet/model": { + "version": "file:packages/model", "requires": { - "@npmcli/node-gyp": "^2.0.0", - "@npmcli/promise-spawn": "^3.0.0", - "node-gyp": "^9.0.0", - "read-package-json-fast": "^2.0.3", - "which": "^2.0.2" + "@scramjet/obj-logger": "^0.38.0", + "@scramjet/symbols": "^0.38.0", + "@scramjet/types": "^0.38.0", + "@types/node": "15.12.5", + "@types/uuid": "^8.3.4", + "ava": "^3.15.0", + "scramjet": "^4.36.9", + "ts-node": "^10.9.1", + "typedoc": "0.23.17", + "typedoc-plugin-markdown": "3.13.6", + "typescript": "~4.7.4", + "uuid": "^8.3.2" } }, - "@protobufjs/aspromise": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" - }, - "@protobufjs/base64": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" - }, - "@protobufjs/codegen": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", - "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" - }, - "@protobufjs/eventemitter": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" - }, - "@protobufjs/fetch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", - "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "@scramjet/module-loader": { + "version": "file:packages/module-loader", "requires": { - "@protobufjs/aspromise": "^1.1.1", - "@protobufjs/inquire": "^1.1.0" + "@scramjet/obj-logger": "^0.38.0", + "@scramjet/types": "^0.38.0", + "@types/node": "15.12.5", + "ava": "^3.15.0", + "ts-node": "^10.9.1", + "typedoc": "0.23.17", + "typedoc-plugin-markdown": "3.13.6", + "typescript": "~4.7.4" } }, - "@protobufjs/float": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" - }, - "@protobufjs/inquire": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", - "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" - }, - "@protobufjs/path": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" - }, - "@protobufjs/pool": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" - }, - "@protobufjs/utf8": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", - "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" - }, - "@scramjet/adapters": { - "version": "file:packages/adapters", + "@scramjet/monitoring-server": { + "version": "file:packages/monitoring-server", "requires": { - "@kubernetes/client-node": "^0.17.1", - "@scramjet/model": "^0.38.0", - "@scramjet/obj-logger": "^0.38.0", - "@scramjet/pre-runner": "^0.38.0", - "@scramjet/python-runner": "^0.38.0", - "@scramjet/runner": "^0.38.0", - "@scramjet/sth-config": "^0.38.0", - "@scramjet/symbols": "^0.38.0", "@scramjet/types": "^0.38.0", "@scramjet/utility": "^0.38.0", - "@types/dockerode": "<=3.3.3", - "@types/js-yaml": "4.0.5", "@types/node": "15.12.5", - "@types/request": "2.48.8", - "@types/shell-escape": "^0.2.1", - "@types/ws": "8.5.3", "ava": "^3.15.0", - "dockerode": "^3.3.4", - "scramjet": "^4.36.9", - "shell-escape": "^0.2.0", + "nyc": "^15.1.0", "ts-node": "^10.9.1", - "ts.data.json": "^2.2.0", "typedoc": "0.23.17", "typedoc-plugin-markdown": "3.13.6", "typescript": "~4.7.4" - }, - "dependencies": { - "@scramjet/pre-runner": { - "version": "file:packages/pre-runner" - }, - "@scramjet/python-runner": { - "version": "file:packages/python-runner" - }, - "@scramjet/runner": { - "version": "file:packages/runner", - "requires": { - "@scramjet/api-client": "^0.38.0", - "@scramjet/client-utils": "^0.38.0", - "@scramjet/manager-api-client": "^0.38.0", - "@scramjet/model": "^0.38.0", - "@scramjet/obj-logger": "^0.38.0", - "@scramjet/symbols": "^0.38.0", - "@scramjet/types": "^0.38.0", - "@scramjet/utility": "^0.38.0", - "@types/node": "15.12.5", - "@types/sinon": "^10.0.13", - "ava": "^3.15.0", - "bpmux": "^8.2.1", - "nyc": "^15.1.0", - "proxyquire": "^2.1.3", - "scramjet": "^4.36.9", - "sinon": "^14.0.1", - "ts-node": "^10.9.1", - "typedoc": "0.23.17", - "typedoc-plugin-markdown": "3.13.6", - "typescript": "~4.7.4" - } - } } }, - "@scramjet/api-client": { - "version": "file:packages/api-client", + "@scramjet/multi-manager-api-client": { + "version": "file:packages/multi-manager-api-client", "requires": { + "@scramjet/api-client": "^0.38.0", "@scramjet/client-utils": "^0.38.0", - "@scramjet/sth-config": "^0.38.0", - "@scramjet/symbols": "^0.38.0", + "@scramjet/manager-api-client": "^0.38.0", "@scramjet/types": "^0.38.0", "@types/node": "15.12.5", "ava": "^3.15.0", - "n-readlines": "^1.0.1", + "ts-node": "^10.9.1", + "typedoc": "0.23.17", + "typedoc-plugin-markdown": "3.13.6", + "typescript": "~4.7.4" + } + }, + "@scramjet/obj-logger": { + "version": "file:packages/obj-logger", + "requires": { + "@scramjet/types": "^0.38.0", + "@scramjet/utility": "^0.38.0", + "@types/node": "15.12.5", + "ava": "^3.15.0", + "nyc": "^15.1.0", "scramjet": "^4.36.9", "ts-node": "^10.9.1", "typedoc": "0.23.17", @@ -18602,334 +19685,660 @@ "typescript": "~4.7.4" } }, - "@scramjet/api-server": { - "version": "file:packages/api-server", + "@scramjet/pre-runner": { + "version": "file:packages/pre-runner" + }, + "@scramjet/python-runner": { + "version": "file:packages/python-runner" + }, + "@scramjet/runner": { + "version": "file:packages/runner", "requires": { + "@scramjet/api-client": "^0.38.0", + "@scramjet/client-utils": "^0.38.0", + "@scramjet/manager-api-client": "^0.38.0", "@scramjet/model": "^0.38.0", "@scramjet/obj-logger": "^0.38.0", "@scramjet/symbols": "^0.38.0", + "@scramjet/tecemux": "^0.38.0", "@scramjet/types": "^0.38.0", "@scramjet/utility": "^0.38.0", "@types/node": "15.12.5", "@types/sinon": "^10.0.13", - "@types/trouter": "^3.1.1", - "0http": "^3.4.1", "ava": "^3.15.0", - "http-status-codes": "^2.2.0", + "nyc": "^15.1.0", + "proxyquire": "^2.1.3", "scramjet": "^4.36.9", "sinon": "^14.0.1", - "trouter": "^3.2.0", "ts-node": "^10.9.1", "typedoc": "0.23.17", "typedoc-plugin-markdown": "3.13.6", "typescript": "~4.7.4" } }, - "@scramjet/cli": { - "version": "file:packages/cli", + "@scramjet/stdio-sequence": { + "version": "file:packages/stdio-sequence", "requires": { - "@scramjet/api-client": "^0.38.0", - "@scramjet/client-utils": "^0.38.0", - "@scramjet/middleware-api-client": "^0.38.0", + "scramjet": "^4.36.6" + } + }, + "@scramjet/sth": { + "version": "file:packages/sth", + "requires": { + "@scramjet/host": "^0.38.0", + "@scramjet/model": "^0.38.0", "@scramjet/obj-logger": "^0.38.0", + "@scramjet/sth-config": "^0.38.0", + "@scramjet/types": "^0.38.0", + "@scramjet/utility": "^0.38.0", + "@types/node": "15.12.5", + "ava": "^3.15.0", + "commander": "^8.3.0", + "ts-node": "^10.9.1", + "typedoc": "0.23.17", + "typedoc-plugin-markdown": "3.13.6", + "typescript": "~4.7.4" + }, + "dependencies": { + "commander": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", + "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==" + } + } + }, + "@scramjet/sth-config": { + "version": "file:packages/sth-config", + "requires": { + "@scramjet/types": "^0.38.0", + "@scramjet/utility": "^0.38.0", + "ava": "^3.15.0", + "typedoc": "0.23.17", + "typedoc-plugin-markdown": "3.13.6" + } + }, + "@scramjet/symbols": { + "version": "file:packages/symbols", + "requires": { + "@types/node": "15.12.5", + "typedoc": "0.23.17", + "typedoc-plugin-markdown": "3.13.6", + "typescript": "~4.7.4" + } + }, + "@scramjet/tecemux": { + "version": "file:packages/tecemux", + "requires": { + "@scramjet/api-server": "^0.38.0", + "@scramjet/obj-logger": "^0.36.1", "@scramjet/types": "^0.38.0", - "@scramjet/utility": "^0.38.0", - "@types/find-package-json": "^1.2.3", - "@types/minimatch": "^3.0.5", + "@scramjet/utility": "^0.36.1", "@types/node": "15.12.5", - "@types/tar": "^6.1.3", - "@types/validator": "^13.7.8", "ava": "^3.15.0", - "chalk": "^4.1.2", - "commander": "^9.5.0", - "find-package-json": "^1.2.0", - "minimatch": "^3.1.2", - "scramjet": "^4.36.9", - "tar": "^6.1.11", + "bpmux": "^8.2.1", "ts-node": "^10.9.1", "typedoc": "0.23.17", "typedoc-plugin-markdown": "3.13.6", - "typescript": "~4.7.4", - "validator": "^13.7.0" + "typescript": "~4.7.4" }, "dependencies": { - "@scramjet/middleware-api-client": { - "version": "file:packages/middleware-api-client", + "@scramjet/obj-logger": { + "version": "0.36.1", + "resolved": "https://registry.npmjs.org/@scramjet/obj-logger/-/obj-logger-0.36.1.tgz", + "integrity": "sha512-3yDell8/QKf+W4lGCgm74tf+bNS/S0U0//xO/7FyZbiE69ajJhd3f9C70Q2Xf9xYBMyhpJuK+vTteGAJiJCtdQ==", "requires": { - "@scramjet/api-client": "^0.38.0", - "@scramjet/client-utils": "^0.38.0", - "@scramjet/manager-api-client": "^0.38.0", - "@scramjet/multi-manager-api-client": "^0.38.0", - "@scramjet/types": "^0.38.0", - "ava": "^3.15.0", - "esbuild": "^0.14.54", - "ts-node": "^10.9.1", - "typedoc": "0.23.17", - "typedoc-plugin-markdown": "3.13.6", - "typescript": "~4.7.4" + "@scramjet/utility": "^0.36.1", + "scramjet": "^4.36.9" } }, - "commander": { - "version": "9.5.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", - "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==" + "@scramjet/utility": { + "version": "0.36.1", + "resolved": "https://registry.npmjs.org/@scramjet/utility/-/utility-0.36.1.tgz", + "integrity": "sha512-sDanZi5evXaTs+LsR/O1znCZuAa7sJv5EDwGLEF2RpW5fBFOWGhtcCJJ3yJN4qQeyMtjTu/J6c4wnBiZNK4JHQ==", + "requires": { + "normalize-url": "4", + "yaml": "^2.2.2" + } } } }, - "@scramjet/client-utils": { - "version": "0.38.0", - "resolved": "https://registry.npmjs.org/@scramjet/client-utils/-/client-utils-0.38.0.tgz", - "integrity": "sha512-DzxwgcSCEtIxdG+GsIReL1Zhkh9zKnPDIq3zbVt9Svvsq/UuP94KGg25KVelvr0OTM/yEEovJw9/yOMDWLiXbQ==", + "@scramjet/telemetry": { + "version": "file:packages/telemetry", "requires": { - "@scramjet/model": "^0.38.0", "@scramjet/obj-logger": "^0.38.0", - "@scramjet/sth-config": "^0.38.0", - "@scramjet/symbols": "^0.38.0", + "@scramjet/types": "^0.38.0", "@scramjet/utility": "^0.38.0", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "n-readlines": "^1.0.1", - "node-fetch": "^2.6.7", - "normalize-url": "4", - "scramjet": "^4.36.9" + "ava": "^3.15.0", + "typedoc": "0.23.17", + "typedoc-plugin-markdown": "3.13.6", + "winston": "^3.8.2", + "winston-loki": "^6.0.6" } }, - "@scramjet/host": { - "version": "file:packages/host", + "@scramjet/types": { + "version": "file:packages/types", "requires": { - "@scramjet/adapters": "^0.38.0", - "@scramjet/api-server": "^0.38.0", - "@scramjet/load-check": "^0.38.0", - "@scramjet/model": "^0.38.0", - "@scramjet/module-loader": "^0.38.0", - "@scramjet/obj-logger": "^0.38.0", - "@scramjet/sth-config": "^0.38.0", "@scramjet/symbols": "^0.38.0", - "@scramjet/telemetry": "^0.38.0", - "@scramjet/types": "^0.38.0", - "@scramjet/utility": "^0.38.0", - "@scramjet/verser": "^0.38.0", - "@types/find-package-json": "^1.2.3", - "@types/jest": "^29.4.4", "@types/node": "15.12.5", - "ava": "^3.15.0", - "bpmux": "^8.2.1", - "ext-ip": "^0.3.9", - "find-package-json": "^1.2.0", "http-status-codes": "^2.2.0", - "jest": "^29.5.0", - "minimist": "^1.2.6", - "pico-s3": "^2.0.0", - "rereadable-stream": "^1.4.14", "scramjet": "^4.36.9", - "ts-jest": "^29.0.5", - "ts-node": "^10.9.1", "typedoc": "0.23.17", "typedoc-plugin-markdown": "3.13.6", "typescript": "~4.7.4" + } + }, + "@scramjet/utility": { + "version": "file:packages/utility", + "requires": { + "@scramjet/types": "^0.38.0", + "ava": "^4.3.3", + "normalize-url": "4", + "typed-emitter": "^1.4.0", + "typedoc": "0.23.17", + "typedoc-plugin-markdown": "3.13.6", + "yaml": "^2.2.2" }, "dependencies": { - "@scramjet/load-check": { - "version": "file:packages/load-check", + "aggregate-error": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-4.0.1.tgz", + "integrity": "sha512-0poP0T7el6Vq3rstR8Mn4V/IQrpBLO6POkUSrN7RhyY+GF/InCFShQzsQ39T25gkHhLgSLByyAz+Kjb+c2L98w==", + "dev": true, + "requires": { + "clean-stack": "^4.0.0", + "indent-string": "^5.0.0" + } + }, + "ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "dev": true + }, + "ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "dev": true + }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "requires": { + "sprintf-js": "~1.0.2" + } + }, + "arrify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-3.0.0.tgz", + "integrity": "sha512-tLkvA81vQG/XqE2mjDkGQHoOINtMHtysSnemrmoGe6PydDPMRbVugqyk4A6V/WDWEfm3l+0d8anA9r8cv/5Jaw==", + "dev": true + }, + "ava": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/ava/-/ava-4.3.3.tgz", + "integrity": "sha512-9Egq/d9R74ExrWohHeqUlexjDbgZJX5jA1Wq4KCTqc3wIfpGEK79zVy4rBtofJ9YKIxs4PzhJ8BgbW5PlAYe6w==", + "dev": true, + "requires": { + "acorn": "^8.7.1", + "acorn-walk": "^8.2.0", + "ansi-styles": "^6.1.0", + "arrgv": "^1.0.2", + "arrify": "^3.0.0", + "callsites": "^4.0.0", + "cbor": "^8.1.0", + "chalk": "^5.0.1", + "chokidar": "^3.5.3", + "chunkd": "^2.0.1", + "ci-info": "^3.3.1", + "ci-parallel-vars": "^1.0.1", + "clean-yaml-object": "^0.1.0", + "cli-truncate": "^3.1.0", + "code-excerpt": "^4.0.0", + "common-path-prefix": "^3.0.0", + "concordance": "^5.0.4", + "currently-unhandled": "^0.4.1", + "debug": "^4.3.4", + "del": "^6.1.1", + "emittery": "^0.11.0", + "figures": "^4.0.1", + "globby": "^13.1.1", + "ignore-by-default": "^2.1.0", + "indent-string": "^5.0.0", + "is-error": "^2.2.2", + "is-plain-object": "^5.0.0", + "is-promise": "^4.0.0", + "matcher": "^5.0.0", + "mem": "^9.0.2", + "ms": "^2.1.3", + "p-event": "^5.0.1", + "p-map": "^5.4.0", + "picomatch": "^2.3.1", + "pkg-conf": "^4.0.0", + "plur": "^5.1.0", + "pretty-ms": "^7.0.1", + "resolve-cwd": "^3.0.0", + "slash": "^3.0.0", + "stack-utils": "^2.0.5", + "strip-ansi": "^7.0.1", + "supertap": "^3.0.1", + "temp-dir": "^2.0.0", + "write-file-atomic": "^4.0.1", + "yargs": "^17.5.1" + } + }, + "callsites": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-4.1.0.tgz", + "integrity": "sha512-aBMbD1Xxay75ViYezwT40aQONfr+pSXTHwNKvIXhXD6+LY3F1dLIcceoC5OZKBVHbXcysz1hL9D2w0JJIMXpUw==", + "dev": true + }, + "chalk": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz", + "integrity": "sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==", + "dev": true + }, + "ci-info": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", + "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==", + "dev": true + }, + "clean-stack": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-4.2.0.tgz", + "integrity": "sha512-LYv6XPxoyODi36Dp976riBtSY27VmFo+MKqEU9QCCWyTrdEPDog+RWA7xQWHi6Vbp61j5c4cdzzX1NidnwtUWg==", + "dev": true, + "requires": { + "escape-string-regexp": "5.0.0" + } + }, + "cli-truncate": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cli-truncate/-/cli-truncate-3.1.0.tgz", + "integrity": "sha512-wfOBkjXteqSnI59oPcJkcPl/ZmwvMMOj340qUIY1SKZCv0B9Cf4D4fAucRkIKQmsIuYK3x1rrgU7MeGRruiuiA==", + "dev": true, + "requires": { + "slice-ansi": "^5.0.0", + "string-width": "^5.0.0" + }, + "dependencies": { + "string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "requires": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + } + } + } + }, + "cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.1" + } + } + } + }, + "code-excerpt": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/code-excerpt/-/code-excerpt-4.0.0.tgz", + "integrity": "sha512-xxodCmBen3iy2i0WtAK8FlFNrRzjUqjRsMfho58xT/wvZU1YTM3fCnRjcy1gJPMepaRlgm/0e6w8SpWHpn3/cA==", + "dev": true, + "requires": { + "convert-to-spaces": "^2.0.1" + } + }, + "convert-to-spaces": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/convert-to-spaces/-/convert-to-spaces-2.0.1.tgz", + "integrity": "sha512-rcQ1bsQO9799wq24uE5AM2tAILy4gXGIK/njFWcVQkGNZ96edlpY+A7bjwvzjYvLDyzmG1MmMLZhpcsb+klNMQ==", + "dev": true + }, + "emittery": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.11.0.tgz", + "integrity": "sha512-S/7tzL6v5i+4iJd627Nhv9cLFIo5weAIlGccqJFpnBoDB8U1TF2k5tez4J/QNuxyyhWuFqHg1L84Kd3m7iXg6g==", + "dev": true + }, + "emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true + }, + "escape-string-regexp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", + "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", + "dev": true + }, + "figures": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/figures/-/figures-4.0.1.tgz", + "integrity": "sha512-rElJwkA/xS04Vfg+CaZodpso7VqBknOYbzi6I76hI4X80RUjkSxO2oAyPmGbuXUppywjqndOrQDl817hDnI++w==", + "dev": true, + "requires": { + "escape-string-regexp": "^5.0.0", + "is-unicode-supported": "^1.2.0" + } + }, + "find-up": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-6.3.0.tgz", + "integrity": "sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw==", + "dev": true, + "requires": { + "locate-path": "^7.1.0", + "path-exists": "^5.0.0" + } + }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true + }, + "globby": { + "version": "13.2.2", + "resolved": "https://registry.npmjs.org/globby/-/globby-13.2.2.tgz", + "integrity": "sha512-Y1zNGV+pzQdh7H39l9zgB4PJqjRNqydvdYCDG4HFXM4XuvSaQQlEc91IU1yALL8gUTDomgBAfz3XJdmUS+oo0w==", + "dev": true, + "requires": { + "dir-glob": "^3.0.1", + "fast-glob": "^3.3.0", + "ignore": "^5.2.4", + "merge2": "^1.4.1", + "slash": "^4.0.0" + }, + "dependencies": { + "slash": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", + "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==", + "dev": true + } + } + }, + "indent-string": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-5.0.0.tgz", + "integrity": "sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==", + "dev": true + }, + "is-unicode-supported": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-1.3.0.tgz", + "integrity": "sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ==", + "dev": true + }, + "js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, + "load-json-file": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-7.0.1.tgz", + "integrity": "sha512-Gnxj3ev3mB5TkVBGad0JM6dmLiQL+o0t23JPBZ9sd+yvSLk05mFoqKBw5N8gbbkU4TNXyqCgIrl/VM17OgUIgQ==", + "dev": true + }, + "locate-path": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-7.2.0.tgz", + "integrity": "sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==", + "dev": true, + "requires": { + "p-locate": "^6.0.0" + } + }, + "matcher": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-5.0.0.tgz", + "integrity": "sha512-s2EMBOWtXFc8dgqvoAzKJXxNHibcdJMV0gwqKUaw9E2JBJuGUK7DrNKrA6g/i+v72TT16+6sVm5mS3thaMLQUw==", + "dev": true, + "requires": { + "escape-string-regexp": "^5.0.0" + } + }, + "mem": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/mem/-/mem-9.0.2.tgz", + "integrity": "sha512-F2t4YIv9XQUBHt6AOJ0y7lSmP1+cY7Fm1DRh9GClTGzKST7UWLMx6ly9WZdLH/G/ppM5RL4MlQfRT71ri9t19A==", + "dev": true, + "requires": { + "map-age-cleaner": "^0.1.3", + "mimic-fn": "^4.0.0" + } + }, + "mimic-fn": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", + "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==", + "dev": true + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "p-event": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/p-event/-/p-event-5.0.1.tgz", + "integrity": "sha512-dd589iCQ7m1L0bmC5NLlVYfy3TbBEsMUfWx9PyAgPeIcFZ/E2yaTZ4Rz4MiBmmJShviiftHVXOqfnfzJ6kyMrQ==", + "dev": true, + "requires": { + "p-timeout": "^5.0.2" + } + }, + "p-limit": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-4.0.0.tgz", + "integrity": "sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==", + "dev": true, + "requires": { + "yocto-queue": "^1.0.0" + } + }, + "p-locate": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-6.0.0.tgz", + "integrity": "sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==", + "dev": true, "requires": { - "@scramjet/obj-logger": "^0.38.0", - "@scramjet/types": "^0.38.0", - "@scramjet/utility": "^0.38.0", - "@types/node": "15.12.5", - "@types/node-os-utils": "^1.2.0", - "@types/uuid": "^8.3.4", - "ava": "^3.15.0", - "diskusage-ng": "1.0.2", - "node-os-utils": "1.3.7", - "scramjet": "^4.36.9", - "ts-node": "^10.9.1", - "typedoc": "0.23.17", - "typedoc-plugin-markdown": "3.13.6", - "typescript": "~4.7.4", - "uuid": "^8.3.2" + "p-limit": "^4.0.0" } }, - "@scramjet/module-loader": { - "version": "file:packages/module-loader", + "p-map": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-5.5.0.tgz", + "integrity": "sha512-VFqfGDHlx87K66yZrNdI4YGtD70IRyd+zSvgks6mzHPRNkoKy+9EKP4SFC77/vTTQYmRmti7dvqC+m5jBrBAcg==", + "dev": true, + "requires": { + "aggregate-error": "^4.0.0" + } + }, + "p-timeout": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-5.1.0.tgz", + "integrity": "sha512-auFDyzzzGZZZdHz3BtET9VEz0SE/uMEAx7uWfGPucfzEwwe/xH0iVeZibQmANYE/hp9T2+UUZT5m+BKyrDp3Ew==", + "dev": true + }, + "path-exists": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-5.0.0.tgz", + "integrity": "sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==", + "dev": true + }, + "pkg-conf": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/pkg-conf/-/pkg-conf-4.0.0.tgz", + "integrity": "sha512-7dmgi4UY4qk+4mj5Cd8v/GExPo0K+SlY+hulOSdfZ/T6jVH6//y7NtzZo5WrfhDBxuQ0jCa7fLZmNaNh7EWL/w==", + "dev": true, + "requires": { + "find-up": "^6.0.0", + "load-json-file": "^7.0.0" + } + }, + "plur": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/plur/-/plur-5.1.0.tgz", + "integrity": "sha512-VP/72JeXqak2KiOzjgKtQen5y3IZHn+9GOuLDafPv0eXa47xq0At93XahYBs26MsifCQ4enGKwbjBTKgb9QJXg==", + "dev": true, + "requires": { + "irregular-plurals": "^3.3.0" + } + }, + "strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "requires": { + "ansi-regex": "^6.0.1" + } + }, + "supertap": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/supertap/-/supertap-3.0.1.tgz", + "integrity": "sha512-u1ZpIBCawJnO+0QePsEiOknOfCRq0yERxiAchT0i4li0WHNUJbf0evXXSXOcCAR4M8iMDoajXYmstm/qO81Isw==", + "dev": true, + "requires": { + "indent-string": "^5.0.0", + "js-yaml": "^3.14.1", + "serialize-error": "^7.0.1", + "strip-ansi": "^7.0.1" + } + }, + "wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, "requires": { - "@scramjet/obj-logger": "^0.38.0", - "@scramjet/types": "^0.38.0", - "@types/node": "15.12.5", - "ava": "^3.15.0", - "ts-node": "^10.9.1", - "typedoc": "0.23.17", - "typedoc-plugin-markdown": "3.13.6", - "typescript": "~4.7.4" + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.1" + } + } } }, - "@scramjet/telemetry": { - "version": "file:packages/telemetry", + "write-file-atomic": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz", + "integrity": "sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg==", + "dev": true, "requires": { - "@scramjet/obj-logger": "^0.38.0", - "@scramjet/types": "^0.38.0", - "@scramjet/utility": "^0.38.0", - "ava": "^3.15.0", - "typedoc": "0.23.17", - "typedoc-plugin-markdown": "3.13.6", - "winston": "^3.8.2", - "winston-loki": "^6.0.6" + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.7" } }, - "@scramjet/verser": { - "version": "file:packages/verser", + "y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true + }, + "yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, "requires": { - "@scramjet/api-server": "^0.38.0", - "@scramjet/obj-logger": "^0.38.0", - "@scramjet/types": "^0.38.0", - "@scramjet/utility": "^0.38.0", - "@types/node": "15.12.5", - "ava": "^3.15.0", - "bpmux": "^8.2.1", - "ts-node": "^10.9.1", - "typedoc": "0.23.17", - "typedoc-plugin-markdown": "3.13.6", - "typescript": "~4.7.4" + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" } + }, + "yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true + }, + "yocto-queue": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz", + "integrity": "sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==", + "dev": true } } }, - "@scramjet/logger": { - "version": "file:packages/logger", - "requires": { - "@scramjet/types": "^0.38.0", - "@types/node": "15.12.5", - "ava": "^3.15.0", - "nyc": "^15.1.0", - "scramjet": "^4.36.9", - "ts-node": "^10.9.1", - "typedoc": "0.23.17", - "typedoc-plugin-markdown": "3.13.6", - "typescript": "~4.7.4" - } - }, - "@scramjet/manager-api-client": { - "version": "file:packages/manager-api-client", - "requires": { - "@scramjet/api-client": "^0.38.0", - "@scramjet/client-utils": "^0.38.0", - "@scramjet/types": "^0.38.0", - "@types/node": "15.12.5", - "ava": "^3.15.0", - "ts-node": "^10.9.1", - "typedoc": "0.23.17", - "typedoc-plugin-markdown": "3.13.6", - "typescript": "~4.7.4" - } - }, - "@scramjet/model": { - "version": "0.38.0", - "resolved": "https://registry.npmjs.org/@scramjet/model/-/model-0.38.0.tgz", - "integrity": "sha512-KaIqZi6tbNnBZcc7TmvJhki+oO3k+EKUuH7h044LGatVcS5tbMrIki3w5BX41OCi5neigDCoOla/nJraod6IcQ==", - "requires": { - "@scramjet/obj-logger": "^0.38.0", - "@scramjet/symbols": "^0.38.0", - "scramjet": "^4.36.9", - "uuid": "^8.3.2" - } - }, - "@scramjet/monitoring-server": { - "version": "file:packages/monitoring-server", - "requires": { - "@scramjet/types": "^0.38.0", - "@scramjet/utility": "^0.38.0", - "@types/node": "15.12.5", - "ava": "^3.15.0", - "nyc": "^15.1.0", - "ts-node": "^10.9.1", - "typedoc": "0.23.17", - "typedoc-plugin-markdown": "3.13.6", - "typescript": "~4.7.4" - } - }, - "@scramjet/multi-manager-api-client": { - "version": "file:packages/multi-manager-api-client", - "requires": { - "@scramjet/api-client": "^0.38.0", - "@scramjet/client-utils": "^0.38.0", - "@scramjet/manager-api-client": "^0.38.0", - "@scramjet/types": "^0.38.0", - "@types/node": "15.12.5", - "ava": "^3.15.0", - "ts-node": "^10.9.1", - "typedoc": "0.23.17", - "typedoc-plugin-markdown": "3.13.6", - "typescript": "~4.7.4" - } - }, - "@scramjet/obj-logger": { - "version": "0.38.0", - "resolved": "https://registry.npmjs.org/@scramjet/obj-logger/-/obj-logger-0.38.0.tgz", - "integrity": "sha512-2RLYiGnVbt1rGN74yvhqocz3PDxjYdO2Rb0GKlxYZJ2JQB/hIVAJSE47wUjKmPkG39anH6UrfGV8T73paH9qgg==", - "requires": { - "@scramjet/utility": "^0.38.0", - "scramjet": "^4.36.9" - } - }, - "@scramjet/stdio-sequence": { - "version": "file:packages/stdio-sequence", - "requires": { - "scramjet": "^4.36.6" - } - }, - "@scramjet/sth": { - "version": "file:packages/sth", + "@scramjet/verser": { + "version": "file:packages/verser", "requires": { - "@scramjet/host": "^0.38.0", - "@scramjet/model": "^0.38.0", + "@scramjet/api-server": "^0.38.0", "@scramjet/obj-logger": "^0.38.0", - "@scramjet/sth-config": "^0.38.0", + "@scramjet/tecemux": "^0.38.0", "@scramjet/types": "^0.38.0", "@scramjet/utility": "^0.38.0", "@types/node": "15.12.5", "ava": "^3.15.0", - "commander": "^8.3.0", "ts-node": "^10.9.1", "typedoc": "0.23.17", "typedoc-plugin-markdown": "3.13.6", "typescript": "~4.7.4" - }, - "dependencies": { - "commander": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", - "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==" - } - } - }, - "@scramjet/sth-config": { - "version": "0.38.0", - "resolved": "https://registry.npmjs.org/@scramjet/sth-config/-/sth-config-0.38.0.tgz", - "integrity": "sha512-Glm/lXMK3UsdAwnWnINlxGm7UFaJfGCnDgjwtI8DVHOL3+52SujFbO0fNb/ye0htR77vNg6GTydbK8sEFdTbkw==", - "requires": { - "@scramjet/utility": "^0.38.0" - } - }, - "@scramjet/symbols": { - "version": "0.38.0", - "resolved": "https://registry.npmjs.org/@scramjet/symbols/-/symbols-0.38.0.tgz", - "integrity": "sha512-Wjg0JUFybp++ngDUovkoBb3IrfVAjo3OeK6d7JodCOVlgrPsOs2jvmINhGhTfbB/WZx0/sRHeG6Yr9zmpDY3Cg==" - }, - "@scramjet/types": { - "version": "0.38.0", - "resolved": "https://registry.npmjs.org/@scramjet/types/-/types-0.38.0.tgz", - "integrity": "sha512-6PlWPgdP5ot65Zr17d01rf9m6rgL9xgsliFw2yhikEPflLV8c2ohSGSsptR+faM+x+pNyN/O9TnxbSzwwWBNVA==", - "requires": { - "@scramjet/symbols": "^0.38.0", - "http-status-codes": "^2.2.0" - } - }, - "@scramjet/utility": { - "version": "0.38.0", - "resolved": "https://registry.npmjs.org/@scramjet/utility/-/utility-0.38.0.tgz", - "integrity": "sha512-jSxeqm1BJ/ZczZAlcXP8tiU4k0cRTF4oVFsrczSw07sxliHbKSh7FaYsRfDKrO53xOlyD5FGLER/3IJckFlPpQ==", - "requires": { - "normalize-url": "4", - "yaml": "^2.2.2" } }, "@sinclair/typebox": { @@ -19191,6 +20600,7 @@ "version": "2.6.10", "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.10.tgz", "integrity": "sha512-PPpPK6F9ALFTn59Ka3BaL+qGuipRfxNE8qVgkp0bVixeiR2c2/L+IVOiBdu9JhhT22sWnQEp6YyHGI2b2+CMcA==", + "dev": true, "requires": { "@types/node": "*", "form-data": "^4.0.0" @@ -20457,6 +21867,15 @@ "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==" }, + "cbor": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/cbor/-/cbor-8.1.0.tgz", + "integrity": "sha512-DwGjNW9omn6EwP70aXsn7FQJx5kO12tX0bZkaTjzdVFM6/7nhA4t0EENocKGx6D2Bch9PE2KzCUf5SceBdeijg==", + "dev": true, + "requires": { + "nofilter": "^3.1.0" + } + }, "chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -21137,6 +22556,12 @@ "integrity": "sha512-V/lf7y33dGaypZZetVI1eu7BmvkbC4dItq12OElLRpKuaU5JxQstV2zHwLv8P7cNbQ+KL1WD80zMCTx5dNC4dg==", "dev": true }, + "eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true + }, "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -21889,9 +23314,9 @@ "dev": true }, "fast-glob": { - "version": "3.2.12", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", - "integrity": "sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==", + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", + "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", "dev": true, "requires": { "@nodelib/fs.stat": "^2.0.2", @@ -22081,6 +23506,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dev": true, "requires": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -24390,6 +25816,12 @@ "integrity": "sha512-dFSmB8fFHEH/s81Xi+Y/15DQY6VHW81nXRj86EMSL3lmuTmK1e+aT4wrFCkTbm+gSwkw4KpX+rT/pMM2c1mF+A==", "dev": true }, + "nofilter": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/nofilter/-/nofilter-3.1.0.tgz", + "integrity": "sha512-l2NNj07e9afPnhAhvgVrCD/oy2Ai1yfLpuo3EpiO1jFTsB4sFz6oIfAfSZyQzVpkZQ9xS8ZS5g1jCBgq4Hwo0g==", + "dev": true + }, "nopt": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/nopt/-/nopt-6.0.0.tgz", @@ -24644,6 +26076,8 @@ "dependencies": { "npm-bundled": { "version": "1.1.2", + "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.2.tgz", + "integrity": "sha512-x5DHup0SuyQcmL3s7Rx/YQ8sbw/Hzg0rj48eN0dV7hf5cmQq5PXIeioroH3raV1QC1yh3uTYuMThvEQF3iKgGQ==", "bundled": true, "requires": { "npm-normalize-package-bin": "^1.0.1" @@ -24762,6 +26196,8 @@ }, "agentkeepalive": { "version": "4.2.1", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.2.1.tgz", + "integrity": "sha512-Zn4cw2NEqd+9fiSVWMscnjyQ1a8Yfoc5oBajLeo5w+YBHgDUcEBY2hS4YpTz6iN5f/2zQiktcuM6tS8x1p9dpA==", "bundled": true, "requires": { "debug": "^4.1.0", @@ -24844,6 +26280,8 @@ "dependencies": { "npm-normalize-package-bin": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-2.0.0.tgz", + "integrity": "sha512-awzfKUO7v0FscrSpRoogyNm0sajikhBWpU0QMrW09AMi9n1PoKU6WaIqUzuJSQnpciZZmJ/jMZ2Egfmb/9LiWQ==", "bundled": true } } @@ -24940,7 +26378,9 @@ } }, "cli-table3": { - "version": "0.6.2", + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/cli-table3/-/cli-table3-0.6.3.tgz", + "integrity": "sha512-w5Jac5SykAeZJKntOxJCrm63Eg5/4dhMWIcuTbo9rpE+brgaSZo0RuNJZeOyMgsUdhDeojvgyQLmjI+K50ZGyg==", "bundled": true, "requires": { "@colors/colors": "1.5.0", @@ -25028,6 +26468,8 @@ "dependencies": { "ms": { "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "bundled": true } } @@ -25053,6 +26495,8 @@ }, "depd": { "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", "bundled": true }, "dezalgo": { @@ -25141,7 +26585,9 @@ } }, "glob": { - "version": "8.0.3", + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", "bundled": true, "requires": { "fs.realpath": "^1.0.0", @@ -25533,7 +26979,9 @@ } }, "minimatch": { - "version": "5.1.0", + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", "bundled": true, "requires": { "brace-expansion": "^2.0.1" @@ -25648,14 +27096,17 @@ "bundled": true }, "node-gyp": { - "version": "9.1.0", + "version": "9.4.1", + "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-9.4.1.tgz", + "integrity": "sha512-OQkWKbjQKbGkMf/xqI1jjy3oCTgMKJac58G2+bjZb3fza6gW2YrCSdMQYaoTb70crvE//Gngr4f0AgVHmqHvBQ==", "bundled": true, "requires": { "env-paths": "^2.2.0", + "exponential-backoff": "^3.1.1", "glob": "^7.1.4", "graceful-fs": "^4.2.6", "make-fetch-happen": "^10.0.3", - "nopt": "^5.0.0", + "nopt": "^6.0.0", "npmlog": "^6.0.0", "rimraf": "^3.0.2", "semver": "^7.3.5", @@ -25675,6 +27126,8 @@ }, "glob": { "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "bundled": true, "requires": { "fs.realpath": "^1.0.0", @@ -25700,6 +27153,15 @@ "requires": { "abbrev": "1" } + }, + "semver": { + "version": "7.5.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.2.tgz", + "integrity": "sha512-SoftuTROv/cRjCze/scjGyiDtcUyxw1rgYQSZY7XTmtR5hX+dm76iDbTH8TkLPHCQmlbQVSSbNZCPM2hb0knnQ==", + "bundled": true, + "requires": { + "lru-cache": "^6.0.0" + } } } }, @@ -25744,6 +27206,8 @@ "dependencies": { "npm-normalize-package-bin": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-2.0.0.tgz", + "integrity": "sha512-awzfKUO7v0FscrSpRoogyNm0sajikhBWpU0QMrW09AMi9n1PoKU6WaIqUzuJSQnpciZZmJ/jMZ2Egfmb/9LiWQ==", "bundled": true } } @@ -25787,6 +27251,8 @@ "dependencies": { "npm-normalize-package-bin": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-2.0.0.tgz", + "integrity": "sha512-awzfKUO7v0FscrSpRoogyNm0sajikhBWpU0QMrW09AMi9n1PoKU6WaIqUzuJSQnpciZZmJ/jMZ2Egfmb/9LiWQ==", "bundled": true } } @@ -25805,6 +27271,8 @@ "dependencies": { "npm-normalize-package-bin": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-2.0.0.tgz", + "integrity": "sha512-awzfKUO7v0FscrSpRoogyNm0sajikhBWpU0QMrW09AMi9n1PoKU6WaIqUzuJSQnpciZZmJ/jMZ2Egfmb/9LiWQ==", "bundled": true } } @@ -26004,6 +27472,8 @@ "dependencies": { "npm-normalize-package-bin": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-2.0.0.tgz", + "integrity": "sha512-awzfKUO7v0FscrSpRoogyNm0sajikhBWpU0QMrW09AMi9n1PoKU6WaIqUzuJSQnpciZZmJ/jMZ2Egfmb/9LiWQ==", "bundled": true } } @@ -26068,6 +27538,8 @@ }, "glob": { "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "bundled": true, "requires": { "fs.realpath": "^1.0.0", @@ -26103,7 +27575,9 @@ "optional": true }, "semver": { - "version": "7.3.7", + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", "bundled": true, "requires": { "lru-cache": "^6.0.0" @@ -26111,6 +27585,8 @@ "dependencies": { "lru-cache": { "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", "bundled": true, "requires": { "yallist": "^4.0.0" @@ -27841,6 +29317,30 @@ "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", "dev": true }, + "slice-ansi": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-5.0.0.tgz", + "integrity": "sha512-FC+lgizVPfie0kkhqUScwRu1O/lF6NOgJmlCgK+/LYxDCTk8sGelYaHDhFcDN+Sn3Cv+3VSa4Byeo+IMCzpMgQ==", + "dev": true, + "requires": { + "ansi-styles": "^6.0.0", + "is-fullwidth-code-point": "^4.0.0" + }, + "dependencies": { + "ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-4.0.0.tgz", + "integrity": "sha512-O4L094N2/dZ7xqVdrXhh9r1KODPJpFms8B5sGdJLPy664AgvXsreZUyCQQNItZRDlYug4xStLjNp/sz3HvBowQ==", + "dev": true + } + } + }, "smart-buffer": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", @@ -28638,6 +30138,12 @@ "is-typed-array": "^1.1.9" } }, + "typed-emitter": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/typed-emitter/-/typed-emitter-1.4.0.tgz", + "integrity": "sha512-weBmoo3HhpKGgLBOYwe8EB31CzDFuaK7CCL+axXhUYhn4jo6DSkHnbefboCF5i4DQ2aMFe0C/FdTWcPdObgHyg==", + "dev": true + }, "typedarray-to-buffer": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", diff --git a/packages/runner/package.json b/packages/runner/package.json index 3bace4869..f4c02afaf 100644 --- a/packages/runner/package.json +++ b/packages/runner/package.json @@ -23,7 +23,7 @@ "@scramjet/obj-logger": "^0.38.0", "@scramjet/symbols": "^0.38.0", "@scramjet/utility": "^0.38.0", - "@scramjet/tecemux": "^0.38.0" + "@scramjet/tecemux": "^0.38.0", "scramjet": "^4.36.9" }, "devDependencies": { diff --git a/packages/tecemux/package.json b/packages/tecemux/package.json index f684c1c95..baf5b7237 100644 --- a/packages/tecemux/package.json +++ b/packages/tecemux/package.json @@ -1,6 +1,6 @@ { "name": "@scramjet/tecemux", - "version": "0.36.1", + "version": "0.38.0", "description": "This package is part of Scramjet Transform Hub. The package provides a communication protocol used among Scramjet modules.", "main": "./src/index.ts", "scripts": { @@ -19,8 +19,8 @@ "bpmux": "^8.2.1" }, "devDependencies": { - "@scramjet/api-server": "^0.36.1", - "@scramjet/types": "^0.36.1", + "@scramjet/api-server": "^0.38.0", + "@scramjet/types": "^0.38.0", "@types/node": "15.12.5", "ava": "^3.15.0", "ts-node": "^10.9.1", diff --git a/packages/tecemux/test/playgrounds/start-test-server.ts b/packages/tecemux/test/playgrounds/start-test-server.ts index 163f484c9..65b8e5e1b 100644 --- a/packages/tecemux/test/playgrounds/start-test-server.ts +++ b/packages/tecemux/test/playgrounds/start-test-server.ts @@ -3,7 +3,7 @@ import net, { Socket, createConnection } from "net"; -import { TeceMux, TeceMuxChannel } from "@scramjet/tecemux"; +import { TeceMux, TeceMuxChannel } from "../../src"; import { Agent, createServer, request } from "http"; const httpServer = createServer((req, res) => { diff --git a/yarn.lock b/yarn.lock index cb3a5a8d1..255566256 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,15 +2,6 @@ # yarn lockfile v1 -"0http@^3.4.1": - version "3.4.2" - resolved "https://registry.npmjs.org/0http/-/0http-3.4.2.tgz" - integrity sha512-zDbvsof0hqUdtHjcXE8pCzAmBGhNjwhypDfMKQf4sJz9QB9CMphgQzo/te2bGz7GjEuqiwXss7z7DJMt7IQo3Q== - dependencies: - lru-cache "^7.14.1" - regexparam "^2.0.1" - trouter "^3.2.0" - "@aashutoshrathi/word-wrap@^1.2.3": version "1.2.6" resolved "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz" @@ -37,7 +28,7 @@ resolved "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.20.10.tgz" integrity sha512-sEnuDPpOJR/fcafHMjpcpGN5M2jbUGUHwmuWKM/YdPzeEDJg8bgmbcWQFUfE32MQjti1koACvoPVsDe8Uq+idg== -"@babel/core@^7.11.6", "@babel/core@^7.12.3", "@babel/core@^7.7.5": +"@babel/core@^7.0.0", "@babel/core@^7.0.0-0", "@babel/core@^7.11.6", "@babel/core@^7.12.3", "@babel/core@^7.7.5", "@babel/core@^7.8.0", "@babel/core@>=7.0.0-beta.0 <8": version "7.20.12" resolved "https://registry.npmjs.org/@babel/core/-/core-7.20.12.tgz" integrity sha512-XsMfHovsUYHFMdrIHkZphTN/2Hzzi78R08NuHfDBehym2VsPDL6Zn/JAD/JQdnRvbSsbQc4mVaU1m6JgtTEElg== @@ -357,7 +348,7 @@ dependencies: regexp-match-indices "1.0.2" -"@cucumber/cucumber@^7.3.2": +"@cucumber/cucumber@^7.3.2", "@cucumber/cucumber@>=7.0.0": version "7.3.2" resolved "https://registry.npmjs.org/@cucumber/cucumber/-/cucumber-7.3.2.tgz" integrity sha512-qqptM9w+UqXEYBAkrIGpIVPXDWv+zp0LrS89LiwHZwBp0cJg00su/iPMZ4j8TvCJiKfAwJXsAI1yjrd1POtU+w== @@ -431,7 +422,27 @@ dependencies: "@cucumber/messages" "^16.0.1" -"@cucumber/messages@^16.0.0", "@cucumber/messages@^16.0.1": +"@cucumber/messages@*": + version "21.0.1" + resolved "https://registry.npmjs.org/@cucumber/messages/-/messages-21.0.1.tgz" + integrity sha512-pGR7iURM4SF9Qp1IIpNiVQ77J9kfxMkPOEbyy+zRmGABnWWCsqMpJdfHeh9Mb3VskemVw85++e15JT0PYdcR3g== + dependencies: + "@types/uuid" "8.3.4" + class-transformer "0.5.1" + reflect-metadata "0.1.13" + uuid "9.0.0" + +"@cucumber/messages@^16.0.0": + version "16.0.1" + resolved "https://registry.npmjs.org/@cucumber/messages/-/messages-16.0.1.tgz" + integrity sha512-80JcaAfQragFqR1rMhRwiqWL9HcR6Z4LDD2mfF0Lxg/lFkCNvmWa9Jl10NUNfFXYD555NKPzP/8xFo55abw8TQ== + dependencies: + "@types/uuid" "8.3.0" + class-transformer "0.4.0" + reflect-metadata "0.1.13" + uuid "8.3.2" + +"@cucumber/messages@^16.0.1": version "16.0.1" resolved "https://registry.npmjs.org/@cucumber/messages/-/messages-16.0.1.tgz" integrity sha512-80JcaAfQragFqR1rMhRwiqWL9HcR6Z4LDD2mfF0Lxg/lFkCNvmWa9Jl10NUNfFXYD555NKPzP/8xFo55abw8TQ== @@ -465,11 +476,6 @@ enabled "2.0.x" kuler "^2.0.0" -"@esbuild/linux-loong64@0.14.54": - version "0.14.54" - resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.14.54.tgz#de2a4be678bd4d0d1ffbb86e6de779cde5999028" - integrity sha512-bZBrLAIX1kpWelV0XemxBZllyRmM6vgFQQG2GdNb+r3Fkp0FOh1NJSvekXDs7jq70k4euu1cryLMfU+mTXlEpw== - "@eslint-community/eslint-utils@^4.2.0", "@eslint-community/eslint-utils@^4.4.0": version "4.4.0" resolved "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz" @@ -734,7 +740,7 @@ slash "^3.0.0" write-file-atomic "^4.0.2" -"@jest/types@^29.6.1": +"@jest/types@^29.0.0", "@jest/types@^29.6.1": version "29.6.1" resolved "https://registry.npmjs.org/@jest/types/-/types-29.6.1.tgz" integrity sha512-tPKQNMPuXgvdOn2/Lg9HNfUvjYVGolt04Hp03f5hAk878uwOLikN+JzeLY0HcVgKgFl9Hs3EIqpu3WX27XNhnw== @@ -778,14 +784,6 @@ resolved "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz" integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw== -"@jridgewell/trace-mapping@0.3.9": - version "0.3.9" - resolved "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz" - integrity sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ== - dependencies: - "@jridgewell/resolve-uri" "^3.0.3" - "@jridgewell/sourcemap-codec" "^1.4.10" - "@jridgewell/trace-mapping@^0.3.12", "@jridgewell/trace-mapping@^0.3.17", "@jridgewell/trace-mapping@^0.3.18", "@jridgewell/trace-mapping@^0.3.9": version "0.3.19" resolved "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.19.tgz" @@ -794,6 +792,14 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" +"@jridgewell/trace-mapping@0.3.9": + version "0.3.9" + resolved "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz" + integrity sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ== + dependencies: + "@jridgewell/resolve-uri" "^3.0.3" + "@jridgewell/sourcemap-codec" "^1.4.10" + "@kubernetes/client-node@^0.17.1": version "0.17.1" resolved "https://registry.npmjs.org/@kubernetes/client-node/-/client-node-0.17.1.tgz" @@ -816,46 +822,6 @@ optionalDependencies: openid-client "^5.1.6" -"@napi-rs/snappy-android-arm-eabi@7.1.1": - version "7.1.1" - resolved "https://registry.yarnpkg.com/@napi-rs/snappy-android-arm-eabi/-/snappy-android-arm-eabi-7.1.1.tgz#9e514082e9588da9cf569a9eece3336ba858d903" - integrity sha512-NKd/ztuVEgQaAaNVQ5zZaCB9VV+7+uBXBHqhaE5iSapQhLc41szTlT0s68FCee75OoT3vhqdA6Jp5TrzZ2WOaw== - -"@napi-rs/snappy-android-arm64@7.1.1": - version "7.1.1" - resolved "https://registry.yarnpkg.com/@napi-rs/snappy-android-arm64/-/snappy-android-arm64-7.1.1.tgz#ec9b7b70bd8b3a589511cd0cd903a4acf339747d" - integrity sha512-DktruMAO0K0toTnxNHg2GWNIAPJqdvIchCsdsRaKyuEnG101qBg0mYiRCAhxHgbT6RJlOGbUPKkbA9KKRhEJUg== - -"@napi-rs/snappy-darwin-arm64@7.1.1": - version "7.1.1" - resolved "https://registry.yarnpkg.com/@napi-rs/snappy-darwin-arm64/-/snappy-darwin-arm64-7.1.1.tgz#4b0a95783cea97daef80057d2e060b311dc376a5" - integrity sha512-3LZyoAw3Qa5F7sCCTkSkhmGlydwUKU6L3Jl46eKHO2Ctm8Gcjxww6T7MfwlwGZ6JqAM6d1d++WLzUZPCGXVmag== - -"@napi-rs/snappy-darwin-x64@7.1.1": - version "7.1.1" - resolved "https://registry.yarnpkg.com/@napi-rs/snappy-darwin-x64/-/snappy-darwin-x64-7.1.1.tgz#d9eb124c6169fdbc3a8389c88f1962a4dad728ff" - integrity sha512-X1D2F67bQkPwr5iSR29/RnOrLwAkB55YO6t41toABzla3mflLDpzZcakz6FokIukykf7ey31/t73v/4pbgaBkg== - -"@napi-rs/snappy-freebsd-x64@7.1.1": - version "7.1.1" - resolved "https://registry.yarnpkg.com/@napi-rs/snappy-freebsd-x64/-/snappy-freebsd-x64-7.1.1.tgz#bff06d3c5644d15da18dfaaad8fd33e198b3b7ef" - integrity sha512-vSeuf+An8jFVHPAn5IbWE9hTGU9PFAaZLj/X7rKTQQtZstnDsHgWe6u4g7FHLuOdwQ8TvhcxAEpNlYIXIk4AJg== - -"@napi-rs/snappy-linux-arm-gnueabihf@7.1.1": - version "7.1.1" - resolved "https://registry.yarnpkg.com/@napi-rs/snappy-linux-arm-gnueabihf/-/snappy-linux-arm-gnueabihf-7.1.1.tgz#f8451e9738811a62baa97661a8ef1647c34ae580" - integrity sha512-/yyN6QsnOs3D1+jI3SfRX+gtnD86rbixdfmgxv9g40+FrDaDTLAu/3VuZIqH02qqq/xiWbDnkO+42RGxXDzTCw== - -"@napi-rs/snappy-linux-arm64-gnu@7.1.1": - version "7.1.1" - resolved "https://registry.yarnpkg.com/@napi-rs/snappy-linux-arm64-gnu/-/snappy-linux-arm64-gnu-7.1.1.tgz#24fe19bc4dc3008316645ff52cd42771a501ee13" - integrity sha512-StEeUCSwUoajgrBtiCQPTkHu+0Q4QlYndghGZNdbN1zJ1ny70YzPpevaFBUyjI/eJ+FN9uICKtwTPtQNSILS5g== - -"@napi-rs/snappy-linux-arm64-musl@7.1.1": - version "7.1.1" - resolved "https://registry.yarnpkg.com/@napi-rs/snappy-linux-arm64-musl/-/snappy-linux-arm64-musl-7.1.1.tgz#0b270227725b40be81b8e3aadcf64dd520f37bc6" - integrity sha512-jWEBRzj+lswZVYf0b5eY0fjMlBL9L9yqjmTuv2UIMjJNHPuR282LK/s3Fm9sYIXQtKkiCo5JyhmIcoghZ3v0Eg== - "@napi-rs/snappy-linux-x64-gnu@7.1.1": version "7.1.1" resolved "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-gnu/-/snappy-linux-x64-gnu-7.1.1.tgz" @@ -866,21 +832,6 @@ resolved "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-musl/-/snappy-linux-x64-musl-7.1.1.tgz" integrity sha512-xR4hzFQqVq6J8Zf6XyUVtFJBaRgDyAQYUoBsCr92tZ7gI/0RlWCV6Q6JMO/wP5CSsvyFJIAtSUXXqlzIpw0GPA== -"@napi-rs/snappy-win32-arm64-msvc@7.1.1": - version "7.1.1" - resolved "https://registry.yarnpkg.com/@napi-rs/snappy-win32-arm64-msvc/-/snappy-win32-arm64-msvc-7.1.1.tgz#7c53b6327ca55e6a7998a1b38f11e81563ccdcfc" - integrity sha512-2mHPadctsaYtrfSV5Na8ooTdI5rflPxP1pceY4us6vbjeWrfgB+KQCuEFOHsGXqFNfsi6L9nWH8nB9swnxnSyw== - -"@napi-rs/snappy-win32-ia32-msvc@7.1.1": - version "7.1.1" - resolved "https://registry.yarnpkg.com/@napi-rs/snappy-win32-ia32-msvc/-/snappy-win32-ia32-msvc-7.1.1.tgz#a7b3ace9468e79fc1065309dcdbfff494eaaa6c0" - integrity sha512-FOMgs9W71hdgjyl3T9F7b/WEIuoryfgBqsyhtHjAaa/98R0BUHl0bOoHg8ka0b5GgnhLBHkX2Yd6VD+Si9Q2ww== - -"@napi-rs/snappy-win32-x64-msvc@7.1.1": - version "7.1.1" - resolved "https://registry.yarnpkg.com/@napi-rs/snappy-win32-x64-msvc/-/snappy-win32-x64-msvc-7.1.1.tgz#d30667e62057b8ec60a117722cc0662617330691" - integrity sha512-Mu3yELySvzhBcNTVCq+hYxVh+lH3/KjoQ5HIEb3DDPoX0AGRTm3XZa+usq8pFWjl91Cgp9nWK+9lVSkCCIRaKA== - "@nodelib/fs.scandir@2.1.5": version "2.1.5" resolved "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz" @@ -889,7 +840,7 @@ "@nodelib/fs.stat" "2.0.5" run-parallel "^1.1.9" -"@nodelib/fs.stat@2.0.5", "@nodelib/fs.stat@^2.0.2": +"@nodelib/fs.stat@^2.0.2", "@nodelib/fs.stat@2.0.5": version "2.0.5" resolved "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz" integrity sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A== @@ -1063,7 +1014,18 @@ postcss-selector-parser "^6.0.10" semver "^7.3.7" -"@npmcli/run-script@4.2.1", "@npmcli/run-script@^4.1.0", "@npmcli/run-script@^4.1.3", "@npmcli/run-script@^4.2.0", "@npmcli/run-script@^4.2.1": +"@npmcli/run-script@^4.1.0", "@npmcli/run-script@^4.1.3", "@npmcli/run-script@^4.2.0", "@npmcli/run-script@^4.2.1": + version "4.2.1" + resolved "https://registry.npmjs.org/@npmcli/run-script/-/run-script-4.2.1.tgz" + integrity sha512-7dqywvVudPSrRCW5nTHpHgeWnbBtz8cFkOuKrecm6ih+oO9ciydhWt6OF7HlqupRRmB8Q/gECVdB9LMfToJbRg== + dependencies: + "@npmcli/node-gyp" "^2.0.0" + "@npmcli/promise-spawn" "^3.0.0" + node-gyp "^9.0.0" + read-package-json-fast "^2.0.3" + which "^2.0.2" + +"@npmcli/run-script@4.2.1": version "4.2.1" resolved "https://registry.npmjs.org/@npmcli/run-script/-/run-script-4.2.1.tgz" integrity sha512-7dqywvVudPSrRCW5nTHpHgeWnbBtz8cFkOuKrecm6ih+oO9ciydhWt6OF7HlqupRRmB8Q/gECVdB9LMfToJbRg== @@ -1127,6 +1089,275 @@ resolved "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz" integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== +"@scramjet/adapters@^0.38.0", "@scramjet/adapters@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/adapters": + version "0.38.0" + resolved "file:packages/adapters" + dependencies: + "@kubernetes/client-node" "^0.17.1" + "@scramjet/model" "^0.38.0" + "@scramjet/obj-logger" "^0.38.0" + "@scramjet/pre-runner" "^0.38.0" + "@scramjet/python-runner" "^0.38.0" + "@scramjet/runner" "^0.38.0" + "@scramjet/sth-config" "^0.38.0" + "@scramjet/symbols" "^0.38.0" + "@scramjet/utility" "^0.38.0" + dockerode "^3.3.4" + scramjet "^4.36.9" + shell-escape "^0.2.0" + ts.data.json "^2.2.0" + +"@scramjet/api-client@^0.38.0", "@scramjet/api-client@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/api-client": + version "0.38.0" + resolved "file:packages/api-client" + dependencies: + "@scramjet/client-utils" "^0.38.0" + "@scramjet/sth-config" "^0.38.0" + "@scramjet/symbols" "^0.38.0" + n-readlines "^1.0.1" + scramjet "^4.36.9" + +"@scramjet/api-server@^0.38.0", "@scramjet/api-server@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/api-server": + version "0.38.0" + resolved "file:packages/api-server" + dependencies: + "@scramjet/model" "^0.38.0" + "@scramjet/obj-logger" "^0.38.0" + "@scramjet/symbols" "^0.38.0" + "@scramjet/utility" "^0.38.0" + "0http" "^3.4.1" + http-status-codes "^2.2.0" + scramjet "^4.36.9" + +"@scramjet/cli@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/cli": + version "0.38.0" + resolved "file:packages/cli" + dependencies: + "@scramjet/api-client" "^0.38.0" + "@scramjet/client-utils" "^0.38.0" + "@scramjet/middleware-api-client" "^0.38.0" + "@scramjet/obj-logger" "^0.38.0" + "@scramjet/utility" "^0.38.0" + chalk "^4.1.2" + commander "^9.5.0" + find-package-json "^1.2.0" + minimatch "^3.1.2" + scramjet "^4.36.9" + tar "^6.1.11" + validator "^13.7.0" + +"@scramjet/client-utils@^0.38.0", "@scramjet/client-utils@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/client-utils": + version "0.38.0" + resolved "file:packages/client-utils" + dependencies: + "@scramjet/model" "^0.38.0" + "@scramjet/obj-logger" "^0.38.0" + "@scramjet/sth-config" "^0.38.0" + "@scramjet/symbols" "^0.38.0" + "@scramjet/utility" "^0.38.0" + "@types/node-fetch" "^2.6.4" + abort-controller "^3.0.0" + n-readlines "^1.0.1" + node-fetch "^2.6.7" + normalize-url "4" + scramjet "^4.36.9" + +"@scramjet/host@^0.38.0", "@scramjet/host@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/host": + version "0.38.0" + resolved "file:packages/host" + dependencies: + "@scramjet/adapters" "^0.38.0" + "@scramjet/api-server" "^0.38.0" + "@scramjet/load-check" "^0.38.0" + "@scramjet/model" "^0.38.0" + "@scramjet/module-loader" "^0.38.0" + "@scramjet/obj-logger" "^0.38.0" + "@scramjet/sth-config" "^0.38.0" + "@scramjet/symbols" "^0.38.0" + "@scramjet/telemetry" "^0.38.0" + "@scramjet/utility" "^0.38.0" + "@scramjet/verser" "^0.38.0" + bpmux "^8.2.1" + ext-ip "^0.3.9" + find-package-json "^1.2.0" + http-status-codes "^2.2.0" + minimist "^1.2.6" + pico-s3 "^2.0.0" + rereadable-stream "^1.4.14" + scramjet "^4.36.9" + +"@scramjet/load-check@^0.38.0", "@scramjet/load-check@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/load-check": + version "0.38.0" + resolved "file:packages/load-check" + dependencies: + "@scramjet/obj-logger" "^0.38.0" + "@scramjet/utility" "^0.38.0" + diskusage-ng "1.0.2" + node-os-utils "1.3.7" + scramjet "^4.36.9" + uuid "^8.3.2" + +"@scramjet/logger@^0.38.0", "@scramjet/logger@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/logger": + version "0.38.0" + resolved "file:packages/logger" + dependencies: + scramjet "^4.36.9" + +"@scramjet/manager-api-client@^0.38.0", "@scramjet/manager-api-client@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/manager-api-client": + version "0.38.0" + resolved "file:packages/manager-api-client" + dependencies: + "@scramjet/api-client" "^0.38.0" + "@scramjet/client-utils" "^0.38.0" + +"@scramjet/middleware-api-client@^0.38.0", "@scramjet/middleware-api-client@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/middleware-api-client": + version "0.38.0" + resolved "file:packages/middleware-api-client" + dependencies: + "@scramjet/api-client" "^0.38.0" + "@scramjet/client-utils" "^0.38.0" + "@scramjet/manager-api-client" "^0.38.0" + "@scramjet/multi-manager-api-client" "^0.38.0" + +"@scramjet/model@^0.38.0", "@scramjet/model@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/model": + version "0.38.0" + resolved "file:packages/model" + dependencies: + "@scramjet/obj-logger" "^0.38.0" + "@scramjet/symbols" "^0.38.0" + scramjet "^4.36.9" + uuid "^8.3.2" + +"@scramjet/module-loader@^0.38.0", "@scramjet/module-loader@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/module-loader": + version "0.38.0" + resolved "file:packages/module-loader" + dependencies: + "@scramjet/obj-logger" "^0.38.0" + +"@scramjet/monitoring-server@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/monitoring-server": + version "0.38.0" + resolved "file:packages/monitoring-server" + dependencies: + "@scramjet/utility" "^0.38.0" + +"@scramjet/multi-manager-api-client@^0.38.0", "@scramjet/multi-manager-api-client@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/multi-manager-api-client": + version "0.38.0" + resolved "file:packages/multi-manager-api-client" + dependencies: + "@scramjet/api-client" "^0.38.0" + "@scramjet/client-utils" "^0.38.0" + "@scramjet/manager-api-client" "^0.38.0" + +"@scramjet/obj-logger@^0.36.1": + version "0.36.1" + resolved "https://registry.npmjs.org/@scramjet/obj-logger/-/obj-logger-0.36.1.tgz" + integrity sha512-3yDell8/QKf+W4lGCgm74tf+bNS/S0U0//xO/7FyZbiE69ajJhd3f9C70Q2Xf9xYBMyhpJuK+vTteGAJiJCtdQ== + dependencies: + "@scramjet/utility" "^0.36.1" + scramjet "^4.36.9" + +"@scramjet/obj-logger@^0.38.0", "@scramjet/obj-logger@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/obj-logger": + version "0.38.0" + resolved "file:packages/obj-logger" + dependencies: + "@scramjet/utility" "^0.38.0" + scramjet "^4.36.9" + +"@scramjet/pre-runner@^0.38.0", "@scramjet/pre-runner@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/pre-runner": + version "0.38.0" + resolved "file:packages/pre-runner" + +"@scramjet/python-runner@^0.38.0", "@scramjet/python-runner@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/python-runner": + version "0.38.0" + resolved "file:packages/python-runner" + +"@scramjet/runner@^0.38.0", "@scramjet/runner@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/runner": + version "0.38.0" + resolved "file:packages/runner" + dependencies: + "@scramjet/api-client" "^0.38.0" + "@scramjet/client-utils" "^0.38.0" + "@scramjet/manager-api-client" "^0.38.0" + "@scramjet/model" "^0.38.0" + "@scramjet/obj-logger" "^0.38.0" + "@scramjet/symbols" "^0.38.0" + "@scramjet/tecemux" "^0.38.0" + "@scramjet/utility" "^0.38.0" + scramjet "^4.36.9" + +"@scramjet/stdio-sequence@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/stdio-sequence": + version "0.22.0" + resolved "file:packages/stdio-sequence" + dependencies: + scramjet "^4.36.6" + +"@scramjet/sth-config@^0.38.0", "@scramjet/sth-config@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/sth-config": + version "0.38.0" + resolved "file:packages/sth-config" + dependencies: + "@scramjet/utility" "^0.38.0" + +"@scramjet/sth@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/sth": + version "0.38.0" + resolved "file:packages/sth" + dependencies: + "@scramjet/host" "^0.38.0" + "@scramjet/obj-logger" "^0.38.0" + "@scramjet/sth-config" "^0.38.0" + "@scramjet/utility" "^0.38.0" + commander "^8.3.0" + +"@scramjet/symbols@^0.38.0", "@scramjet/symbols@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/symbols": + version "0.38.0" + resolved "file:packages/symbols" + +"@scramjet/tecemux@^0.38.0", "@scramjet/tecemux@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/tecemux": + version "0.38.0" + resolved "file:packages/tecemux" + dependencies: + "@scramjet/obj-logger" "^0.36.1" + "@scramjet/utility" "^0.36.1" + bpmux "^8.2.1" + +"@scramjet/telemetry@^0.38.0", "@scramjet/telemetry@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/telemetry": + version "0.38.0" + resolved "file:packages/telemetry" + dependencies: + "@scramjet/obj-logger" "^0.38.0" + "@scramjet/utility" "^0.38.0" + winston "^3.8.2" + winston-loki "^6.0.6" + +"@scramjet/types@^0.38.0", "@scramjet/types@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/types": + version "0.38.0" + resolved "file:packages/types" + dependencies: + "@scramjet/symbols" "^0.38.0" + http-status-codes "^2.2.0" + +"@scramjet/utility@^0.36.1": + version "0.36.1" + resolved "https://registry.npmjs.org/@scramjet/utility/-/utility-0.36.1.tgz" + integrity sha512-sDanZi5evXaTs+LsR/O1znCZuAa7sJv5EDwGLEF2RpW5fBFOWGhtcCJJ3yJN4qQeyMtjTu/J6c4wnBiZNK4JHQ== + dependencies: + normalize-url "4" + yaml "^2.2.2" + +"@scramjet/utility@^0.38.0", "@scramjet/utility@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/utility": + version "0.38.0" + resolved "file:packages/utility" + dependencies: + normalize-url "4" + yaml "^2.2.2" + +"@scramjet/verser@^0.38.0", "@scramjet/verser@file:/home/pfalba/github/scramjet-cpm-dev/sth/packages/verser": + version "0.38.0" + resolved "file:packages/verser" + dependencies: + "@scramjet/obj-logger" "^0.38.0" + "@scramjet/tecemux" "^0.38.0" + "@scramjet/utility" "^0.38.0" + "@sinclair/typebox@^0.27.8": version "0.27.8" resolved "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz" @@ -1343,7 +1574,7 @@ resolved "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.5.tgz" integrity sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ== -"@types/node-fetch@^2.6.4": +"@types/node-fetch@^2.6.2": version "2.6.10" resolved "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.10.tgz" integrity sha512-PPpPK6F9ALFTn59Ka3BaL+qGuipRfxNE8qVgkp0bVixeiR2c2/L+IVOiBdu9JhhT22sWnQEp6YyHGI2b2+CMcA== @@ -1356,7 +1587,7 @@ resolved "https://registry.npmjs.org/@types/node-os-utils/-/node-os-utils-1.3.4.tgz" integrity sha512-BCUYrbdoO4FUbx6MB9atLNFnkxdliFaxdiTJMIPPiecXIApc5zf4NIqV5G1jWv/ReZvtYyHLs40RkBjHX+vykA== -"@types/node@*", "@types/node@15.12.5", "@types/node@>=13.7.0": +"@types/node@*", "@types/node@>=13.7.0", "@types/node@15.12.5": version "15.12.5" resolved "https://registry.npmjs.org/@types/node/-/node-15.12.5.tgz" integrity sha512-se3yX7UHv5Bscf8f1ERKvQOD6sTyycH3hdaoozvaLxgUiY5lIGEeH37AD0G0Qi9kPqihPn0HOfd2yaIEN9VwEg== @@ -1433,16 +1664,16 @@ resolved "https://registry.npmjs.org/@types/trouter/-/trouter-3.1.1.tgz" integrity sha512-XTpWPg/bsGt2oHRmNFfq6kjXDJgeX14pPVFq/PutOe83OFiwrSNP9pV0V2j5O+L/r2XiXRlOZPJtyaOAX5fCBQ== +"@types/uuid@^8.3.4", "@types/uuid@8.3.4": + version "8.3.4" + resolved "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.4.tgz" + integrity sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw== + "@types/uuid@8.3.0": version "8.3.0" resolved "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.0.tgz" integrity sha512-eQ9qFW/fhfGJF8WKHGEHZEyVWfZxrT+6CLIJGBcZPfxUh/+BnEj+UCGYMlr9qZuX/2AltsvwrGqp0LhEW8D0zQ== -"@types/uuid@^8.3.4": - version "8.3.4" - resolved "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.4.tgz" - integrity sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw== - "@types/validator@^13.7.8": version "13.7.11" resolved "https://registry.npmjs.org/@types/validator/-/validator-13.7.11.tgz" @@ -1484,7 +1715,7 @@ semver "^7.5.4" ts-api-utils "^1.0.1" -"@typescript-eslint/parser@^6.14.0": +"@typescript-eslint/parser@^6.0.0 || ^6.0.0-alpha", "@typescript-eslint/parser@^6.14.0": version "6.18.1" resolved "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.18.1.tgz" integrity sha512-zct/MdJnVaRRNy9e84XnVtRv9Vf91/qqe+hZJtKanjojud4wAVy/7lXxJmMyX6X6J+xc6c//YEWvpeif8cAhWA== @@ -1558,7 +1789,16 @@ resolved "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz" integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ== -abbrev@^1.0.0, abbrev@~1.1.1: +"0http@^3.4.1": + version "3.4.2" + resolved "https://registry.npmjs.org/0http/-/0http-3.4.2.tgz" + integrity sha512-zDbvsof0hqUdtHjcXE8pCzAmBGhNjwhypDfMKQf4sJz9QB9CMphgQzo/te2bGz7GjEuqiwXss7z7DJMt7IQo3Q== + dependencies: + lru-cache "^7.14.1" + regexparam "^2.0.1" + trouter "^3.2.0" + +abbrev@^1.0.0, abbrev@~1.1.1, abbrev@1: version "1.1.1" resolved "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz" integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== @@ -1575,22 +1815,17 @@ acorn-jsx@^5.3.2: resolved "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz" integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== -acorn-walk@^8.0.0, acorn-walk@^8.1.1: +acorn-walk@^8.0.0, acorn-walk@^8.1.1, acorn-walk@^8.2.0: version "8.2.0" resolved "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz" integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA== -acorn-walk@^8.2.0: - version "8.3.1" - resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.3.1.tgz#2f10f5b69329d90ae18c58bf1fa8fccd8b959a43" - integrity sha512-TgUZgYvqZprrl7YldZNoa9OciCAyZR+Ejm9eXzKCmjsF5IKp/wgQ7Z/ZpjpGTIUPwrHQIcYeI8qDh4PsEwxMbw== - -acorn@^8.0.4, acorn@^8.4.1, acorn@^8.7.1, acorn@^8.9.0: +"acorn@^6.0.0 || ^7.0.0 || ^8.0.0", acorn@^8.0.4, acorn@^8.4.1, acorn@^8.7.1, acorn@^8.9.0: version "8.11.3" resolved "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz" integrity sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg== -agent-base@6, agent-base@^6.0.2: +agent-base@^6.0.2, agent-base@6: version "6.0.2" resolved "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz" integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ== @@ -1616,7 +1851,7 @@ aggregate-error@^3.0.0: aggregate-error@^4.0.0: version "4.0.1" - resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-4.0.1.tgz#25091fe1573b9e0be892aeda15c7c66a545f758e" + resolved "https://registry.npmjs.org/aggregate-error/-/aggregate-error-4.0.1.tgz" integrity sha512-0poP0T7el6Vq3rstR8Mn4V/IQrpBLO6POkUSrN7RhyY+GF/InCFShQzsQ39T25gkHhLgSLByyAz+Kjb+c2L98w== dependencies: clean-stack "^4.0.0" @@ -1663,7 +1898,7 @@ ansi-regex@^5.0.1: ansi-regex@^6.0.1: version "6.0.1" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-6.0.1.tgz#3183e38fae9a65d7cb5e53945cd5897d0260a06a" + resolved "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz" integrity sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA== ansi-styles@^3.2.1: @@ -1685,9 +1920,14 @@ ansi-styles@^5.0.0: resolved "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz" integrity sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA== -ansi-styles@^6.0.0, ansi-styles@^6.1.0: +ansi-styles@^6.0.0: version "6.2.1" - resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-6.2.1.tgz#0e62320cf99c21afff3b3012192546aacbfb05c5" + resolved "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz" + integrity sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug== + +ansi-styles@^6.1.0: + version "6.2.1" + resolved "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz" integrity sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug== any-promise@^1.0.0: @@ -1715,7 +1955,12 @@ append-transform@^2.0.0: resolved "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz" integrity sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ== -archy@^1.0.0, archy@~1.0.0: +archy@^1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz" + integrity sha512-Xg+9RwCg/0p32teKdGMPTPnVXKD0w3DfHnFTficozsAgsvq2XenPJq/MYpzzQ/v8zrOyJn6Ds39VA4JIDwFfqw== + +archy@~1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz" integrity sha512-Xg+9RwCg/0p32teKdGMPTPnVXKD0w3DfHnFTficozsAgsvq2XenPJq/MYpzzQ/v8zrOyJn6Ds39VA4JIDwFfqw== @@ -1835,7 +2080,7 @@ arrify@^2.0.1: arrify@^3.0.0: version "3.0.0" - resolved "https://registry.yarnpkg.com/arrify/-/arrify-3.0.0.tgz#ccdefb8eaf2a1d2ab0da1ca2ce53118759fd46bc" + resolved "https://registry.npmjs.org/arrify/-/arrify-3.0.0.tgz" integrity sha512-tLkvA81vQG/XqE2mjDkGQHoOINtMHtysSnemrmoGe6PydDPMRbVugqyk4A6V/WDWEfm3l+0d8anA9r8cv/5Jaw== asap@^2.0.0: @@ -1850,7 +2095,7 @@ asn1@^0.2.4, asn1@~0.2.3: dependencies: safer-buffer "~2.1.0" -assert-plus@1.0.0, assert-plus@^1.0.0: +assert-plus@^1.0.0, assert-plus@1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" integrity sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw== @@ -1953,7 +2198,7 @@ ava@^3.15.0: ava@^4.3.3: version "4.3.3" - resolved "https://registry.yarnpkg.com/ava/-/ava-4.3.3.tgz#4ddcb650ed059e4e999a4b640de53ff00f4a008b" + resolved "https://registry.npmjs.org/ava/-/ava-4.3.3.tgz" integrity sha512-9Egq/d9R74ExrWohHeqUlexjDbgZJX5jA1Wq4KCTqc3wIfpGEK79zVy4rBtofJ9YKIxs4PzhJ8BgbW5PlAYe6w== dependencies: acorn "^8.7.1" @@ -2024,7 +2269,7 @@ axios@^0.26.1: dependencies: follow-redirects "^1.14.8" -babel-jest@^29.6.2: +babel-jest@^29.0.0, babel-jest@^29.6.2: version "29.6.2" resolved "https://registry.npmjs.org/babel-jest/-/babel-jest-29.6.2.tgz" integrity sha512-BYCzImLos6J3BH/+HvUCHG1dTf2MzmAB4jaVxHV+29RZLjR29XuYTmsf2sdDwkrb+FczkGo3kOhE7ga6sI0P4A== @@ -2113,7 +2358,12 @@ bin-links@^3.0.3: rimraf "^3.0.0" write-file-atomic "^4.0.0" -binary-extensions@^2.0.0, binary-extensions@^2.2.0: +binary-extensions@^2.0.0: + version "2.2.0" + resolved "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz" + integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== + +binary-extensions@^2.2.0: version "2.2.0" resolved "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz" integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== @@ -2180,7 +2430,7 @@ braces@^3.0.2, braces@~3.0.2: dependencies: fill-range "^7.0.1" -browserslist@^4.21.3: +browserslist@^4.21.3, "browserslist@>= 4.21.0": version "4.21.4" resolved "https://registry.npmjs.org/browserslist/-/browserslist-4.21.4.tgz" integrity sha512-CBHJJdDmgjl3daYjN5Cp5kbTf1mUhZoS+beLklHIvkOWscs83YAhLlF3Wsh/lciQYAcbBJgTOD44VtG31ZM4Hw== @@ -2302,7 +2552,7 @@ callsites@^3.0.0, callsites@^3.1.0: callsites@^4.0.0: version "4.1.0" - resolved "https://registry.yarnpkg.com/callsites/-/callsites-4.1.0.tgz#de72b98612eed4e1e2564c952498677faa9d86c2" + resolved "https://registry.npmjs.org/callsites/-/callsites-4.1.0.tgz" integrity sha512-aBMbD1Xxay75ViYezwT40aQONfr+pSXTHwNKvIXhXD6+LY3F1dLIcceoC5OZKBVHbXcysz1hL9D2w0JJIMXpUw== camelcase@^5.0.0, camelcase@^5.3.1: @@ -2336,7 +2586,7 @@ caseless@~0.12.0: cbor@^8.1.0: version "8.1.0" - resolved "https://registry.yarnpkg.com/cbor/-/cbor-8.1.0.tgz#cfc56437e770b73417a2ecbfc9caf6b771af60d5" + resolved "https://registry.npmjs.org/cbor/-/cbor-8.1.0.tgz" integrity sha512-DwGjNW9omn6EwP70aXsn7FQJx5kO12tX0bZkaTjzdVFM6/7nhA4t0EENocKGx6D2Bch9PE2KzCUf5SceBdeijg== dependencies: nofilter "^3.1.0" @@ -2360,7 +2610,7 @@ chalk@^4.0.0, chalk@^4.1.0, chalk@^4.1.2: chalk@^5.0.1: version "5.3.0" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-5.3.0.tgz#67c20a7ebef70e7f3970a01f90fa210cb6860385" + resolved "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz" integrity sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w== char-regex@^1.0.2: @@ -2410,7 +2660,7 @@ ci-info@^3.2.0: ci-info@^3.3.1: version "3.9.0" - resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.9.0.tgz#4279a62028a7b1f262f3473fc9605f5e218c59b4" + resolved "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz" integrity sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ== ci-parallel-vars@^1.0.1: @@ -2435,6 +2685,11 @@ class-transformer@0.4.0: resolved "https://registry.npmjs.org/class-transformer/-/class-transformer-0.4.0.tgz" integrity sha512-ETWD/H2TbWbKEi7m9N4Km5+cw1hNcqJSxlSYhsLsNjQzWWiZIYA1zafxpK9PwVfaZ6AqR5rrjPVUBGESm5tQUA== +class-transformer@0.5.1: + version "0.5.1" + resolved "https://registry.npmjs.org/class-transformer/-/class-transformer-0.5.1.tgz" + integrity sha512-SQa1Ws6hUbfC98vKGxZH3KFY0Y1lm5Zm0SY8XX9zbK7FJCyVEac3ATW0RIpwzW+oOfmHE5PMPufDG9hCfoEOMw== + clean-stack@^2.0.0: version "2.2.0" resolved "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz" @@ -2442,7 +2697,7 @@ clean-stack@^2.0.0: clean-stack@^4.0.0: version "4.2.0" - resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-4.2.0.tgz#c464e4cde4ac789f4e0735c5d75beb49d7b30b31" + resolved "https://registry.npmjs.org/clean-stack/-/clean-stack-4.2.0.tgz" integrity sha512-LYv6XPxoyODi36Dp976riBtSY27VmFo+MKqEU9QCCWyTrdEPDog+RWA7xQWHi6Vbp61j5c4cdzzX1NidnwtUWg== dependencies: escape-string-regexp "5.0.0" @@ -2477,7 +2732,7 @@ cli-spinners@^2.5.0: resolved "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.7.0.tgz" integrity sha512-qu3pN8Y3qHNgE2AFweciB1IfMnmZ/fsNTEE+NOFjmGB2F/7rLhnhzppvpCnN4FovtP26k8lHyy9ptEbNwWFLzw== -cli-table3@0.6.1, cli-table3@^0.6.0: +cli-table3@^0.6.0, cli-table3@0.6.1: version "0.6.1" resolved "https://registry.npmjs.org/cli-table3/-/cli-table3-0.6.1.tgz" integrity sha512-w0q/enDHhPLq44ovMGdQeeDLvwxwavsJX7oQGYt/LrBlYsyaxyDnp6z3QzFut/6kLLKnlcUVJLrpB7KBfgG/RA== @@ -2488,7 +2743,7 @@ cli-table3@0.6.1, cli-table3@^0.6.0: cli-table3@^0.6.2: version "0.6.3" - resolved "https://registry.yarnpkg.com/cli-table3/-/cli-table3-0.6.3.tgz#61ab765aac156b52f222954ffc607a6f01dbeeb2" + resolved "https://registry.npmjs.org/cli-table3/-/cli-table3-0.6.3.tgz" integrity sha512-w5Jac5SykAeZJKntOxJCrm63Eg5/4dhMWIcuTbo9rpE+brgaSZo0RuNJZeOyMgsUdhDeojvgyQLmjI+K50ZGyg== dependencies: string-width "^4.2.0" @@ -2505,7 +2760,7 @@ cli-truncate@^2.1.0: cli-truncate@^3.1.0: version "3.1.0" - resolved "https://registry.yarnpkg.com/cli-truncate/-/cli-truncate-3.1.0.tgz#3f23ab12535e3d73e839bb43e73c9de487db1389" + resolved "https://registry.npmjs.org/cli-truncate/-/cli-truncate-3.1.0.tgz" integrity sha512-wfOBkjXteqSnI59oPcJkcPl/ZmwvMMOj340qUIY1SKZCv0B9Cf4D4fAucRkIKQmsIuYK3x1rrgU7MeGRruiuiA== dependencies: slice-ansi "^5.0.0" @@ -2585,7 +2840,7 @@ code-excerpt@^3.0.0: code-excerpt@^4.0.0: version "4.0.0" - resolved "https://registry.yarnpkg.com/code-excerpt/-/code-excerpt-4.0.0.tgz#2de7d46e98514385cb01f7b3b741320115f4c95e" + resolved "https://registry.npmjs.org/code-excerpt/-/code-excerpt-4.0.0.tgz" integrity sha512-xxodCmBen3iy2i0WtAK8FlFNrRzjUqjRsMfho58xT/wvZU1YTM3fCnRjcy1gJPMepaRlgm/0e6w8SpWHpn3/cA== dependencies: convert-to-spaces "^2.0.1" @@ -2600,7 +2855,14 @@ collect-v8-coverage@^1.0.0: resolved "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.2.tgz" integrity sha512-lHl4d5/ONEbLlJvaJNtsF/Lz+WvB07u2ycqTYbdrq7UypDXailES4valYb2eWiJFxZlVmpGekfqoxQhzyFdT4Q== -color-convert@^1.9.0, color-convert@^1.9.3: +color-convert@^1.9.0: + version "1.9.3" + resolved "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz" + integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== + dependencies: + color-name "1.1.3" + +color-convert@^1.9.3: version "1.9.3" resolved "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz" integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== @@ -2614,16 +2876,16 @@ color-convert@^2.0.1: dependencies: color-name "~1.1.4" -color-name@1.1.3: - version "1.1.3" - resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz" - integrity sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw== - color-name@^1.0.0, color-name@~1.1.4: version "1.1.4" resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== +color-name@1.1.3: + version "1.1.3" + resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz" + integrity sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw== + color-string@^1.6.0: version "1.9.1" resolved "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz" @@ -2673,16 +2935,16 @@ combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6: dependencies: delayed-stream "~1.0.0" -commander@7.2.0, commander@^7.0.0: - version "7.2.0" - resolved "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz" - integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw== - commander@^2.9.0: version "2.20.3" resolved "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz" integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== +commander@^7.0.0: + version "7.2.0" + resolved "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz" + integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw== + commander@^8.3.0: version "8.3.0" resolved "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz" @@ -2693,6 +2955,11 @@ commander@^9.5.0: resolved "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz" integrity sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ== +commander@7.2.0: + version "7.2.0" + resolved "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz" + integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw== + common-ancestor-path@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/common-ancestor-path/-/common-ancestor-path-1.0.1.tgz" @@ -2761,7 +3028,7 @@ convert-to-spaces@^1.0.1: convert-to-spaces@^2.0.1: version "2.0.1" - resolved "https://registry.yarnpkg.com/convert-to-spaces/-/convert-to-spaces-2.0.1.tgz#61a6c98f8aa626c16b296b862a91412a33bceb6b" + resolved "https://registry.npmjs.org/convert-to-spaces/-/convert-to-spaces-2.0.1.tgz" integrity sha512-rcQ1bsQO9799wq24uE5AM2tAILy4gXGIK/njFWcVQkGNZ96edlpY+A7bjwvzjYvLDyzmG1MmMLZhpcsb+klNMQ== core-util-is@1.0.2: @@ -2819,7 +3086,7 @@ currently-unhandled@^0.4.1: dependencies: array-find-index "^1.0.1" -d@1, d@^1.0.1: +d@^1.0.1, d@1: version "1.0.1" resolved "https://registry.npmjs.org/d/-/d-1.0.1.tgz" integrity sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA== @@ -2849,13 +3116,6 @@ date-time@^3.1.0: dependencies: time-zone "^1.0.0" -debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.3, debug@^4.3.4: - version "4.3.4" - resolved "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz" - integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== - dependencies: - ms "2.1.2" - debug@^3.2.7: version "3.2.7" resolved "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz" @@ -2863,6 +3123,13 @@ debug@^3.2.7: dependencies: ms "^2.1.1" +debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.3, debug@^4.3.4, debug@4: + version "4.3.4" + resolved "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz" + integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== + dependencies: + ms "2.1.2" + debuglog@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz" @@ -2989,7 +3256,12 @@ diff@^4.0.1: resolved "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz" integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A== -diff@^5.0.0, diff@^5.1.0: +diff@^5.0.0: + version "5.1.0" + resolved "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz" + integrity sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw== + +diff@^5.1.0: version "5.1.0" resolved "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz" integrity sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw== @@ -3066,7 +3338,7 @@ durations@^3.4.2: eastasianwidth@^0.2.0: version "0.2.0" - resolved "https://registry.yarnpkg.com/eastasianwidth/-/eastasianwidth-0.2.0.tgz#696ce2ec0aa0e6ea93a397ffcf24aa7840c827cb" + resolved "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz" integrity sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA== ecc-jsbn@~0.1.1: @@ -3084,7 +3356,7 @@ electron-to-chromium@^1.4.251: emittery@^0.11.0: version "0.11.0" - resolved "https://registry.yarnpkg.com/emittery/-/emittery-0.11.0.tgz#eb5f756a200d3431de2c6e850cb2d8afd97a03b9" + resolved "https://registry.npmjs.org/emittery/-/emittery-0.11.0.tgz" integrity sha512-S/7tzL6v5i+4iJd627Nhv9cLFIo5weAIlGccqJFpnBoDB8U1TF2k5tez4J/QNuxyyhWuFqHg1L84Kd3m7iXg6g== emittery@^0.13.1: @@ -3104,7 +3376,7 @@ emoji-regex@^8.0.0: emoji-regex@^9.2.2: version "9.2.2" - resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-9.2.2.tgz#840c8803b0d8047f4ff0cf963176b32d4ef3ed72" + resolved "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz" integrity sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg== enabled@2.0.x: @@ -3112,7 +3384,7 @@ enabled@2.0.x: resolved "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz" integrity sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ== -encoding@^0.1.13: +encoding@^0.1.0, encoding@^0.1.13: version "0.1.13" resolved "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz" integrity sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A== @@ -3256,106 +3528,11 @@ es6-symbol@^3.1.1, es6-symbol@^3.1.3: d "^1.0.1" ext "^1.1.2" -esbuild-android-64@0.14.54: - version "0.14.54" - resolved "https://registry.yarnpkg.com/esbuild-android-64/-/esbuild-android-64-0.14.54.tgz#505f41832884313bbaffb27704b8bcaa2d8616be" - integrity sha512-Tz2++Aqqz0rJ7kYBfz+iqyE3QMycD4vk7LBRyWaAVFgFtQ/O8EJOnVmTOiDWYZ/uYzB4kvP+bqejYdVKzE5lAQ== - -esbuild-android-arm64@0.14.54: - version "0.14.54" - resolved "https://registry.yarnpkg.com/esbuild-android-arm64/-/esbuild-android-arm64-0.14.54.tgz#8ce69d7caba49646e009968fe5754a21a9871771" - integrity sha512-F9E+/QDi9sSkLaClO8SOV6etqPd+5DgJje1F9lOWoNncDdOBL2YF59IhsWATSt0TLZbYCf3pNlTHvVV5VfHdvg== - -esbuild-darwin-64@0.14.54: - version "0.14.54" - resolved "https://registry.yarnpkg.com/esbuild-darwin-64/-/esbuild-darwin-64-0.14.54.tgz#24ba67b9a8cb890a3c08d9018f887cc221cdda25" - integrity sha512-jtdKWV3nBviOd5v4hOpkVmpxsBy90CGzebpbO9beiqUYVMBtSc0AL9zGftFuBon7PNDcdvNCEuQqw2x0wP9yug== - -esbuild-darwin-arm64@0.14.54: - version "0.14.54" - resolved "https://registry.yarnpkg.com/esbuild-darwin-arm64/-/esbuild-darwin-arm64-0.14.54.tgz#3f7cdb78888ee05e488d250a2bdaab1fa671bf73" - integrity sha512-OPafJHD2oUPyvJMrsCvDGkRrVCar5aVyHfWGQzY1dWnzErjrDuSETxwA2HSsyg2jORLY8yBfzc1MIpUkXlctmw== - -esbuild-freebsd-64@0.14.54: - version "0.14.54" - resolved "https://registry.yarnpkg.com/esbuild-freebsd-64/-/esbuild-freebsd-64-0.14.54.tgz#09250f997a56ed4650f3e1979c905ffc40bbe94d" - integrity sha512-OKwd4gmwHqOTp4mOGZKe/XUlbDJ4Q9TjX0hMPIDBUWWu/kwhBAudJdBoxnjNf9ocIB6GN6CPowYpR/hRCbSYAg== - -esbuild-freebsd-arm64@0.14.54: - version "0.14.54" - resolved "https://registry.yarnpkg.com/esbuild-freebsd-arm64/-/esbuild-freebsd-arm64-0.14.54.tgz#bafb46ed04fc5f97cbdb016d86947a79579f8e48" - integrity sha512-sFwueGr7OvIFiQT6WeG0jRLjkjdqWWSrfbVwZp8iMP+8UHEHRBvlaxL6IuKNDwAozNUmbb8nIMXa7oAOARGs1Q== - -esbuild-linux-32@0.14.54: - version "0.14.54" - resolved "https://registry.yarnpkg.com/esbuild-linux-32/-/esbuild-linux-32-0.14.54.tgz#e2a8c4a8efdc355405325033fcebeb941f781fe5" - integrity sha512-1ZuY+JDI//WmklKlBgJnglpUL1owm2OX+8E1syCD6UAxcMM/XoWd76OHSjl/0MR0LisSAXDqgjT3uJqT67O3qw== - esbuild-linux-64@0.14.54: version "0.14.54" resolved "https://registry.npmjs.org/esbuild-linux-64/-/esbuild-linux-64-0.14.54.tgz" integrity sha512-EgjAgH5HwTbtNsTqQOXWApBaPVdDn7XcK+/PtJwZLT1UmpLoznPd8c5CxqsH2dQK3j05YsB3L17T8vE7cp4cCg== -esbuild-linux-arm64@0.14.54: - version "0.14.54" - resolved "https://registry.yarnpkg.com/esbuild-linux-arm64/-/esbuild-linux-arm64-0.14.54.tgz#dae4cd42ae9787468b6a5c158da4c84e83b0ce8b" - integrity sha512-WL71L+0Rwv+Gv/HTmxTEmpv0UgmxYa5ftZILVi2QmZBgX3q7+tDeOQNqGtdXSdsL8TQi1vIaVFHUPDe0O0kdig== - -esbuild-linux-arm@0.14.54: - version "0.14.54" - resolved "https://registry.yarnpkg.com/esbuild-linux-arm/-/esbuild-linux-arm-0.14.54.tgz#a2c1dff6d0f21dbe8fc6998a122675533ddfcd59" - integrity sha512-qqz/SjemQhVMTnvcLGoLOdFpCYbz4v4fUo+TfsWG+1aOu70/80RV6bgNpR2JCrppV2moUQkww+6bWxXRL9YMGw== - -esbuild-linux-mips64le@0.14.54: - version "0.14.54" - resolved "https://registry.yarnpkg.com/esbuild-linux-mips64le/-/esbuild-linux-mips64le-0.14.54.tgz#d9918e9e4cb972f8d6dae8e8655bf9ee131eda34" - integrity sha512-qTHGQB8D1etd0u1+sB6p0ikLKRVuCWhYQhAHRPkO+OF3I/iSlTKNNS0Lh2Oc0g0UFGguaFZZiPJdJey3AGpAlw== - -esbuild-linux-ppc64le@0.14.54: - version "0.14.54" - resolved "https://registry.yarnpkg.com/esbuild-linux-ppc64le/-/esbuild-linux-ppc64le-0.14.54.tgz#3f9a0f6d41073fb1a640680845c7de52995f137e" - integrity sha512-j3OMlzHiqwZBDPRCDFKcx595XVfOfOnv68Ax3U4UKZ3MTYQB5Yz3X1mn5GnodEVYzhtZgxEBidLWeIs8FDSfrQ== - -esbuild-linux-riscv64@0.14.54: - version "0.14.54" - resolved "https://registry.yarnpkg.com/esbuild-linux-riscv64/-/esbuild-linux-riscv64-0.14.54.tgz#618853c028178a61837bc799d2013d4695e451c8" - integrity sha512-y7Vt7Wl9dkOGZjxQZnDAqqn+XOqFD7IMWiewY5SPlNlzMX39ocPQlOaoxvT4FllA5viyV26/QzHtvTjVNOxHZg== - -esbuild-linux-s390x@0.14.54: - version "0.14.54" - resolved "https://registry.yarnpkg.com/esbuild-linux-s390x/-/esbuild-linux-s390x-0.14.54.tgz#d1885c4c5a76bbb5a0fe182e2c8c60eb9e29f2a6" - integrity sha512-zaHpW9dziAsi7lRcyV4r8dhfG1qBidQWUXweUjnw+lliChJqQr+6XD71K41oEIC3Mx1KStovEmlzm+MkGZHnHA== - -esbuild-netbsd-64@0.14.54: - version "0.14.54" - resolved "https://registry.yarnpkg.com/esbuild-netbsd-64/-/esbuild-netbsd-64-0.14.54.tgz#69ae917a2ff241b7df1dbf22baf04bd330349e81" - integrity sha512-PR01lmIMnfJTgeU9VJTDY9ZerDWVFIUzAtJuDHwwceppW7cQWjBBqP48NdeRtoP04/AtO9a7w3viI+PIDr6d+w== - -esbuild-openbsd-64@0.14.54: - version "0.14.54" - resolved "https://registry.yarnpkg.com/esbuild-openbsd-64/-/esbuild-openbsd-64-0.14.54.tgz#db4c8495287a350a6790de22edea247a57c5d47b" - integrity sha512-Qyk7ikT2o7Wu76UsvvDS5q0amJvmRzDyVlL0qf5VLsLchjCa1+IAvd8kTBgUxD7VBUUVgItLkk609ZHUc1oCaw== - -esbuild-sunos-64@0.14.54: - version "0.14.54" - resolved "https://registry.yarnpkg.com/esbuild-sunos-64/-/esbuild-sunos-64-0.14.54.tgz#54287ee3da73d3844b721c21bc80c1dc7e1bf7da" - integrity sha512-28GZ24KmMSeKi5ueWzMcco6EBHStL3B6ubM7M51RmPwXQGLe0teBGJocmWhgwccA1GeFXqxzILIxXpHbl9Q/Kw== - -esbuild-windows-32@0.14.54: - version "0.14.54" - resolved "https://registry.yarnpkg.com/esbuild-windows-32/-/esbuild-windows-32-0.14.54.tgz#f8aaf9a5667630b40f0fb3aa37bf01bbd340ce31" - integrity sha512-T+rdZW19ql9MjS7pixmZYVObd9G7kcaZo+sETqNH4RCkuuYSuv9AGHUVnPoP9hhuE1WM1ZimHz1CIBHBboLU7w== - -esbuild-windows-64@0.14.54: - version "0.14.54" - resolved "https://registry.yarnpkg.com/esbuild-windows-64/-/esbuild-windows-64-0.14.54.tgz#bf54b51bd3e9b0f1886ffdb224a4176031ea0af4" - integrity sha512-AoHTRBUuYwXtZhjXZbA1pGfTo8cJo3vZIcWGLiUcTNgHpJJMC1rVA44ZereBHMJtotyN71S8Qw0npiCIkW96cQ== - -esbuild-windows-arm64@0.14.54: - version "0.14.54" - resolved "https://registry.yarnpkg.com/esbuild-windows-arm64/-/esbuild-windows-arm64-0.14.54.tgz#937d15675a15e4b0e4fafdbaa3a01a776a2be982" - integrity sha512-M0kuUvXhot1zOISQGXwWn6YtS+Y/1RT9WrVIOywZnJHo3jCDyewAc79aKNQWFCQm+xNHVTq9h8dZKvygoXQQRg== - esbuild@^0.14.54: version "0.14.54" resolved "https://registry.npmjs.org/esbuild/-/esbuild-0.14.54.tgz" @@ -3393,11 +3570,6 @@ escape-goat@^2.0.0: resolved "https://registry.npmjs.org/escape-goat/-/escape-goat-2.1.1.tgz" integrity sha512-8/uIhbG12Csjy2JEW7D9pHbreaVaS/OpN3ycnyvElTdwM5n6GY6W6e2IPemfvGZeUMqZ9A/3GqIZMgKnBhAw/Q== -escape-string-regexp@5.0.0, escape-string-regexp@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz#4683126b500b61762f2dbebace1806e8be31b1c8" - integrity sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw== - escape-string-regexp@^1.0.5: version "1.0.5" resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz" @@ -3413,6 +3585,11 @@ escape-string-regexp@^4.0.0: resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz" integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== +escape-string-regexp@^5.0.0, escape-string-regexp@5.0.0: + version "5.0.0" + resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz" + integrity sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw== + eslint-import-resolver-node@^0.3.9: version "0.3.9" resolved "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.9.tgz" @@ -3473,7 +3650,7 @@ eslint-visitor-keys@^3.3.0, eslint-visitor-keys@^3.4.1, eslint-visitor-keys@^3.4 resolved "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz" integrity sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag== -eslint@^8.55.0: +"eslint@^2 || ^3 || ^4 || ^5 || ^6 || ^7.2.0 || ^8", "eslint@^6.0.0 || ^7.0.0 || >=8.0.0", "eslint@^7.0.0 || ^8.0.0", eslint@^8.55.0, eslint@>=2.0.0: version "8.56.0" resolved "https://registry.npmjs.org/eslint/-/eslint-8.56.0.tgz" integrity sha512-Go19xM6T9puCOWntie1/P997aXxFsOi37JIHRWI514Hc6ZnaHGKY9xFhrU65RT6CcBEzZoGG1e6Nq+DT04ZtZQ== @@ -3560,21 +3737,6 @@ event-target-shim@^5.0.0: resolved "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz" integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ== -execa@5.0.0: - version "5.0.0" - resolved "https://registry.npmjs.org/execa/-/execa-5.0.0.tgz" - integrity sha512-ov6w/2LCiuyO4RLYGdpFGjkcs0wMTgGE8PrkTHikeUy5iJekXyPIKUjifk5CsE0pt7sMCrMZ3YNqoCj6idQOnQ== - dependencies: - cross-spawn "^7.0.3" - get-stream "^6.0.0" - human-signals "^2.1.0" - is-stream "^2.0.0" - merge-stream "^2.0.0" - npm-run-path "^4.0.1" - onetime "^5.1.2" - signal-exit "^3.0.3" - strip-final-newline "^2.0.0" - execa@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz" @@ -3603,6 +3765,21 @@ execa@^5.0.0: signal-exit "^3.0.3" strip-final-newline "^2.0.0" +execa@5.0.0: + version "5.0.0" + resolved "https://registry.npmjs.org/execa/-/execa-5.0.0.tgz" + integrity sha512-ov6w/2LCiuyO4RLYGdpFGjkcs0wMTgGE8PrkTHikeUy5iJekXyPIKUjifk5CsE0pt7sMCrMZ3YNqoCj6idQOnQ== + dependencies: + cross-spawn "^7.0.3" + get-stream "^6.0.0" + human-signals "^2.1.0" + is-stream "^2.0.0" + merge-stream "^2.0.0" + npm-run-path "^4.0.1" + onetime "^5.1.2" + signal-exit "^3.0.3" + strip-final-newline "^2.0.0" + exit@^0.1.2: version "0.1.2" resolved "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz" @@ -3620,11 +3797,6 @@ expect@^29.0.0, expect@^29.6.2: jest-message-util "^29.6.2" jest-util "^29.6.2" -exponential-backoff@^3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/exponential-backoff/-/exponential-backoff-3.1.1.tgz#64ac7526fe341ab18a39016cd22c787d01e00bf6" - integrity sha512-dX7e/LHVJ6W3DE1MHWi9S1EYzDESENfLrYohG2G++ovZrYOkm4Knwa0mc1cn84xJOR4KEU0WSchhLbd0UklbHw== - ext-ip@^0.3.9: version "0.3.9" resolved "https://registry.npmjs.org/ext-ip/-/ext-ip-0.3.9.tgz" @@ -3645,7 +3817,7 @@ extend@~3.0.2: resolved "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz" integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== -extsprintf@1.3.0, extsprintf@^1.2.0: +extsprintf@^1.2.0, extsprintf@1.3.0: version "1.3.0" resolved "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz" integrity sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g== @@ -3660,20 +3832,9 @@ fast-diff@^1.2.0: resolved "https://registry.npmjs.org/fast-diff/-/fast-diff-1.2.0.tgz" integrity sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w== -fast-glob@^3.2.9: - version "3.2.12" - resolved "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz" - integrity sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w== - dependencies: - "@nodelib/fs.stat" "^2.0.2" - "@nodelib/fs.walk" "^1.2.3" - glob-parent "^5.1.2" - merge2 "^1.3.0" - micromatch "^4.0.4" - -fast-glob@^3.3.0: +fast-glob@^3.2.9, fast-glob@^3.3.0: version "3.3.2" - resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.3.2.tgz#a904501e57cfdd2ffcded45e99a54fef55e46129" + resolved "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz" integrity sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow== dependencies: "@nodelib/fs.stat" "^2.0.2" @@ -3682,7 +3843,7 @@ fast-glob@^3.3.0: merge2 "^1.3.0" micromatch "^4.0.4" -fast-json-stable-stringify@2.x, fast-json-stable-stringify@^2.0.0, fast-json-stable-stringify@^2.1.0: +fast-json-stable-stringify@^2.0.0, fast-json-stable-stringify@^2.1.0, fast-json-stable-stringify@2.x: version "2.1.0" resolved "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz" integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== @@ -3693,9 +3854,7 @@ fast-levenshtein@^2.0.6: integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw== fastest-levenshtein@^1.0.12: - version "1.0.16" - resolved "https://registry.yarnpkg.com/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz#210e61b6ff181de91ea9b3d1b84fdedd47e034e5" - integrity sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg== + version "1.0.12" fastq@^1.6.0: version "1.15.0" @@ -3725,7 +3884,7 @@ figures@^3.2.0: figures@^4.0.1: version "4.0.1" - resolved "https://registry.yarnpkg.com/figures/-/figures-4.0.1.tgz#27b26609907bc888b3e3b0ef5403643f80aa2518" + resolved "https://registry.npmjs.org/figures/-/figures-4.0.1.tgz" integrity sha512-rElJwkA/xS04Vfg+CaZodpso7VqBknOYbzi6I76hI4X80RUjkSxO2oAyPmGbuXUppywjqndOrQDl817hDnI++w== dependencies: escape-string-regexp "^5.0.0" @@ -3783,7 +3942,15 @@ find-up@^3.0.0: dependencies: locate-path "^3.0.0" -find-up@^4.0.0, find-up@^4.1.0: +find-up@^4.0.0: + version "4.1.0" + resolved "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz" + integrity sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw== + dependencies: + locate-path "^5.0.0" + path-exists "^4.0.0" + +find-up@^4.1.0: version "4.1.0" resolved "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz" integrity sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw== @@ -3801,7 +3968,7 @@ find-up@^5.0.0: find-up@^6.0.0: version "6.3.0" - resolved "https://registry.yarnpkg.com/find-up/-/find-up-6.3.0.tgz#2abab3d3280b2dc7ac10199ef324c4e002c8c790" + resolved "https://registry.npmjs.org/find-up/-/find-up-6.3.0.tgz" integrity sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw== dependencies: locate-path "^7.1.0" @@ -3919,10 +4086,10 @@ fs.realpath@^1.0.0: resolved "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz" integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== -fsevents@^2.3.2, fsevents@~2.3.2: - version "2.3.3" - resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" - integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== +function-bind@^1.1.1: + version "1.1.1" + resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz" + integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== function-bind@^1.1.2: version "1.1.2" @@ -3968,7 +4135,12 @@ get-caller-file@^1.0.1: resolved "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz" integrity sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w== -get-caller-file@^2.0.1, get-caller-file@^2.0.5: +get-caller-file@^2.0.1: + version "2.0.5" + resolved "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz" + integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== + +get-caller-file@^2.0.5: version "2.0.5" resolved "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== @@ -4022,7 +4194,7 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob-parent@^5.1.2, glob-parent@~5.1.2: +glob-parent@^5.1.2: version "5.1.2" resolved "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz" integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow== @@ -4036,6 +4208,13 @@ glob-parent@^6.0.2: dependencies: is-glob "^4.0.3" +glob-parent@~5.1.2: + version "5.1.2" + resolved "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz" + integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow== + dependencies: + is-glob "^4.0.1" + glob@^7.0.0, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glob@^7.2.3: version "7.2.3" resolved "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz" @@ -4099,7 +4278,7 @@ globby@^11.0.1, globby@^11.1.0: globby@^13.1.1: version "13.2.2" - resolved "https://registry.yarnpkg.com/globby/-/globby-13.2.2.tgz#63b90b1bf68619c2135475cbd4e71e66aa090592" + resolved "https://registry.npmjs.org/globby/-/globby-13.2.2.tgz" integrity sha512-Y1zNGV+pzQdh7H39l9zgB4PJqjRNqydvdYCDG4HFXM4XuvSaQQlEc91IU1yALL8gUTDomgBAfz3XJdmUS+oo0w== dependencies: dir-glob "^3.0.1" @@ -4221,6 +4400,13 @@ has-yarn@^2.1.0: resolved "https://registry.npmjs.org/has-yarn/-/has-yarn-2.1.0.tgz" integrity sha512-UqBRqi4ju7T+TqGNdqAO0PaSVGsDGJUBQvk9eUWNGRY1CFGDzYhLWoM7JQEemnlvVcv/YEmc2wNW8BC24EnUsw== +has@^1.0.3: + version "1.0.3" + resolved "https://registry.npmjs.org/has/-/has-1.0.3.tgz" + integrity sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw== + dependencies: + function-bind "^1.1.1" + hasha@^5.0.0: version "5.2.2" resolved "https://registry.npmjs.org/hasha/-/hasha-5.2.2.tgz" @@ -4382,7 +4568,7 @@ indent-string@^4.0.0: indent-string@^5.0.0: version "5.0.0" - resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-5.0.0.tgz#4fd2980fccaf8622d14c64d694f4cf33c81951a5" + resolved "https://registry.npmjs.org/indent-string/-/indent-string-5.0.0.tgz" integrity sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg== infer-owner@^1.0.4: @@ -4398,16 +4584,11 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.3: +inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.3, inherits@2: version "2.0.4" resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== -ini@2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz" - integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA== - ini@^3.0.0, ini@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/ini/-/ini-3.0.1.tgz" @@ -4418,6 +4599,11 @@ ini@~1.3.0: resolved "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz" integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew== +ini@2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz" + integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA== + init-package-json@^3.0.2: version "3.0.2" resolved "https://registry.npmjs.org/init-package-json/-/init-package-json-3.0.2.tgz" @@ -4460,16 +4646,11 @@ ip@^2.0.0: resolved "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz" integrity sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ== -irregular-plurals@^3.2.0: +irregular-plurals@^3.2.0, irregular-plurals@^3.3.0: version "3.4.0" resolved "https://registry.npmjs.org/irregular-plurals/-/irregular-plurals-3.4.0.tgz" integrity sha512-YXxECO/W6N9aMBVKMKKZ8TXESgq7EFrp3emCGGUcrYY1cgJIeZjoB75MTu8qi+NAKntS9NwPU8VdcQ3r6E6aWQ== -irregular-plurals@^3.3.0: - version "3.5.0" - resolved "https://registry.yarnpkg.com/irregular-plurals/-/irregular-plurals-3.5.0.tgz#0835e6639aa8425bdc8b0d33d0dc4e89d9c01d2b" - integrity sha512-1ANGLZ+Nkv1ptFb2pa8oG8Lem4krflKuX/gINiHJHjJUKaJHk/SXk5x6K3J+39/p0h1RQ2saROclJJ+QLvETCQ== - is-array-buffer@^3.0.1, is-array-buffer@^3.0.2: version "3.0.2" resolved "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.2.tgz" @@ -4530,13 +4711,18 @@ is-cidr@^4.0.2: dependencies: cidr-regex "^3.1.1" -is-core-module@^2.13.0, is-core-module@^2.13.1, is-core-module@^2.8.1: +is-core-module@^2.13.0, is-core-module@^2.13.1: version "2.13.1" - resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.13.1.tgz#ad0d7532c6fea9da1ebdc82742d74525c6273384" + resolved "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz" integrity sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw== dependencies: hasown "^2.0.0" +is-core-module@^2.8.1: + version "2.10.0" + dependencies: + has "^1.0.3" + is-date-object@^1.0.1: version "1.0.5" resolved "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz" @@ -4573,7 +4759,7 @@ is-fullwidth-code-point@^3.0.0: is-fullwidth-code-point@^4.0.0: version "4.0.0" - resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-4.0.0.tgz#fae3167c729e7463f8461ce512b080a49268aa88" + resolved "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-4.0.0.tgz" integrity sha512-O4L094N2/dZ7xqVdrXhh9r1KODPJpFms8B5sGdJLPy664AgvXsreZUyCQQNItZRDlYug4xStLjNp/sz3HvBowQ== is-generator-fn@^2.0.0: @@ -4721,7 +4907,7 @@ is-unicode-supported@^0.1.0: is-unicode-supported@^1.2.0: version "1.3.0" - resolved "https://registry.yarnpkg.com/is-unicode-supported/-/is-unicode-supported-1.3.0.tgz#d824984b616c292a2e198207d4a609983842f714" + resolved "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-1.3.0.tgz" integrity sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ== is-weakref@^1.0.2: @@ -4741,16 +4927,16 @@ is-yarn-global@^0.3.0: resolved "https://registry.npmjs.org/is-yarn-global/-/is-yarn-global-0.3.0.tgz" integrity sha512-VjSeb/lHmkoyd8ryPVIKvOCn4D1koMqY+vqyjjUfc3xyKtP4dYOxM44sZrnqQSzSds3xyOrUTLTC9LVCVgLngw== -isarray@0.0.1: - version "0.0.1" - resolved "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" - integrity sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ== - isarray@^2.0.5: version "2.0.5" resolved "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz" integrity sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw== +isarray@0.0.1: + version "0.0.1" + resolved "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" + integrity sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ== + isexe@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz" @@ -4788,7 +4974,18 @@ istanbul-lib-instrument@^4.0.0: istanbul-lib-coverage "^3.0.0" semver "^6.3.0" -istanbul-lib-instrument@^5.0.4, istanbul-lib-instrument@^5.1.0: +istanbul-lib-instrument@^5.0.4: + version "5.2.1" + resolved "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz" + integrity sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg== + dependencies: + "@babel/core" "^7.12.3" + "@babel/parser" "^7.14.7" + "@istanbuljs/schema" "^0.1.2" + istanbul-lib-coverage "^3.2.0" + semver "^6.3.0" + +istanbul-lib-instrument@^5.1.0: version "5.2.1" resolved "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz" integrity sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg== @@ -5041,7 +5238,7 @@ jest-resolve-dependencies@^29.6.2: jest-regex-util "^29.4.3" jest-snapshot "^29.6.2" -jest-resolve@^29.6.2: +jest-resolve@*, jest-resolve@^29.6.2: version "29.6.2" resolved "https://registry.npmjs.org/jest-resolve/-/jest-resolve-29.6.2.tgz" integrity sha512-G/iQUvZWI5e3SMFssc4ug4dH0aZiZpsDq9o1PtXTV1210Ztyb2+w+ZgQkB3iOiC5SmAEzJBOHWz6Hvrd+QnNPw== @@ -5185,7 +5382,7 @@ jest-worker@^29.6.2: merge-stream "^2.0.0" supports-color "^8.0.0" -jest@^29.5.0: +jest@^29.0.0, jest@^29.5.0: version "29.6.2" resolved "https://registry.npmjs.org/jest/-/jest-29.6.2.tgz" integrity sha512-8eQg2mqFbaP7CwfsTpCxQ+sHzw1WuNWL5UUvjnWP4hx2riGz9fPSzYOaU5q8/GqWn1TfgZIVTqYJygbGbWAANg== @@ -5210,7 +5407,23 @@ js-tokens@^4.0.0: resolved "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz" integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== -js-yaml@^3.13.1, js-yaml@^3.14.0, js-yaml@^3.14.1: +js-yaml@^3.13.1: + version "3.14.1" + resolved "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz" + integrity sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g== + dependencies: + argparse "^1.0.7" + esprima "^4.0.0" + +js-yaml@^3.14.0: + version "3.14.1" + resolved "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz" + integrity sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g== + dependencies: + argparse "^1.0.7" + esprima "^4.0.0" + +js-yaml@^3.14.1: version "3.14.1" resolved "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz" integrity sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g== @@ -5322,14 +5535,10 @@ jsprim@^1.2.2: verror "1.10.0" just-diff-apply@^5.2.0: - version "5.5.0" - resolved "https://registry.yarnpkg.com/just-diff-apply/-/just-diff-apply-5.5.0.tgz#771c2ca9fa69f3d2b54e7c3f5c1dfcbcc47f9f0f" - integrity sha512-OYTthRfSh55WOItVqwpefPtNt2VdKsq5AnAK6apdtR6yCH8pr0CmSr710J0Mf+WdQy7K/OzMy7K2MgAfdQURDw== + version "5.4.1" just-diff@^5.0.1: - version "5.2.0" - resolved "https://registry.yarnpkg.com/just-diff/-/just-diff-5.2.0.tgz#60dca55891cf24cd4a094e33504660692348a241" - integrity sha512-6ufhP9SHjb7jibNFrNxyFZ6od3g+An6Ai9mhGRvcYe8UJlH0prseN64M+6ZBBUoKYHZsitDP42gAJ8+eVWr3lw== + version "5.1.1" just-extend@^4.0.2: version "4.2.1" @@ -5518,7 +5727,7 @@ load-json-file@^5.2.0: load-json-file@^7.0.0: version "7.0.1" - resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-7.0.1.tgz#a3c9fde6beffb6bedb5acf104fad6bb1604e1b00" + resolved "https://registry.npmjs.org/load-json-file/-/load-json-file-7.0.1.tgz" integrity sha512-Gnxj3ev3mB5TkVBGad0JM6dmLiQL+o0t23JPBZ9sd+yvSLk05mFoqKBw5N8gbbkU4TNXyqCgIrl/VM17OgUIgQ== locate-path@^3.0.0: @@ -5545,7 +5754,7 @@ locate-path@^6.0.0: locate-path@^7.1.0: version "7.2.0" - resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-7.2.0.tgz#69cb1779bd90b35ab1e771e1f2f89a202c2a8a8a" + resolved "https://registry.npmjs.org/locate-path/-/locate-path-7.2.0.tgz" integrity sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA== dependencies: p-locate "^6.0.0" @@ -5636,9 +5845,7 @@ lru-cache@^7.14.1, lru-cache@^7.7.1: integrity sha512-ysxwsnTKdAx96aTRdhDOCQfDgbHnt8SK0KY8SEjO0wHinhWOFTESbjVCMPbU1uGXg/ch4lifqx0wfjOawU2+WA== lru-cache@^7.4.4, lru-cache@^7.5.1: - version "7.18.3" - resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.18.3.tgz#f793896e0fd0e954a59dfdd82f0773808df6aa89" - integrity sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA== + version "7.13.2" lunr@^2.3.9: version "2.3.9" @@ -5652,7 +5859,7 @@ make-dir@^3.0.0, make-dir@^3.0.2: dependencies: semver "^6.0.0" -make-error@1.x, make-error@^1.1.1: +make-error@^1.1.1, make-error@1.x: version "1.3.6" resolved "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz" integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw== @@ -5707,7 +5914,7 @@ matcher@^3.0.0: matcher@^5.0.0: version "5.0.0" - resolved "https://registry.yarnpkg.com/matcher/-/matcher-5.0.0.tgz#cd82f1c7ae7ee472a9eeaf8ec7cac45e0fe0da62" + resolved "https://registry.npmjs.org/matcher/-/matcher-5.0.0.tgz" integrity sha512-s2EMBOWtXFc8dgqvoAzKJXxNHibcdJMV0gwqKUaw9E2JBJuGUK7DrNKrA6g/i+v72TT16+6sVm5mS3thaMLQUw== dependencies: escape-string-regexp "^5.0.0" @@ -5738,7 +5945,7 @@ mem@^8.0.0: mem@^9.0.2: version "9.0.2" - resolved "https://registry.yarnpkg.com/mem/-/mem-9.0.2.tgz#bbc2d40be045afe30749681e8f5d554cee0c0354" + resolved "https://registry.npmjs.org/mem/-/mem-9.0.2.tgz" integrity sha512-F2t4YIv9XQUBHt6AOJ0y7lSmP1+cY7Fm1DRh9GClTGzKST7UWLMx6ly9WZdLH/G/ppM5RL4MlQfRT71ri9t19A== dependencies: map-age-cleaner "^0.1.3" @@ -5796,7 +6003,7 @@ mimic-fn@^3.1.0: mimic-fn@^4.0.0: version "4.0.0" - resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-4.0.0.tgz#60a90550d5cb0b239cca65d893b1a53b29871ecc" + resolved "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz" integrity sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw== mimic-response@^1.0.0, mimic-response@^1.0.1: @@ -5804,13 +6011,6 @@ mimic-response@^1.0.0, mimic-response@^1.0.1: resolved "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz" integrity sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ== -minimatch@9.0.3: - version "9.0.3" - resolved "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz" - integrity sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg== - dependencies: - brace-expansion "^2.0.1" - minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2: version "3.1.2" resolved "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz" @@ -5825,6 +6025,20 @@ minimatch@^5.0.1, minimatch@^5.1.0: dependencies: brace-expansion "^2.0.1" +minimatch@^5.1.0: + version "5.1.6" + resolved "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz" + integrity sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g== + dependencies: + brace-expansion "^2.0.1" + +minimatch@9.0.3: + version "9.0.3" + resolved "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz" + integrity sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg== + dependencies: + brace-expansion "^2.0.1" + minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6, minimist@^1.2.7: version "1.2.7" resolved "https://registry.npmjs.org/minimist/-/minimist-1.2.7.tgz" @@ -5891,11 +6105,6 @@ minipass@^4.0.0: dependencies: yallist "^4.0.0" -minipass@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/minipass/-/minipass-5.0.0.tgz#3e9788ffb90b694a5d0ec94479a45b5d8738133d" - integrity sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ== - minizlib@^2.1.1, minizlib@^2.1.2: version "2.1.2" resolved "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz" @@ -5928,12 +6137,17 @@ module-not-found-error@^1.0.1: resolved "https://registry.npmjs.org/module-not-found-error/-/module-not-found-error-1.0.1.tgz" integrity sha512-pEk4ECWQXV6z2zjhRZUongnLJNUeGQJ3w6OQ5ctGwD+i5o93qjRQUk2Rt6VdNeu3sEP0AB4LcfvdebpxBRVr4g== -ms@2.1.2, ms@^2.0.0, ms@^2.1.1: +ms@^2.0.0, ms@^2.1.1, ms@2.1.2: version "2.1.2" resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@^2.1.2, ms@^2.1.3: +ms@^2.1.2: + version "2.1.3" + resolved "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== + +ms@^2.1.3: version "2.1.3" resolved "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== @@ -6031,7 +6245,7 @@ node-gyp@^9.0.0: node-gyp@^9.1.0: version "9.4.1" - resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-9.4.1.tgz#8a1023e0d6766ecb52764cc3a734b36ff275e185" + resolved "https://registry.npmjs.org/node-gyp/-/node-gyp-9.4.1.tgz" integrity sha512-OQkWKbjQKbGkMf/xqI1jjy3oCTgMKJac58G2+bjZb3fza6gW2YrCSdMQYaoTb70crvE//Gngr4f0AgVHmqHvBQ== dependencies: env-paths "^2.2.0" @@ -6070,7 +6284,7 @@ node-releases@^2.0.6: nofilter@^3.1.0: version "3.1.0" - resolved "https://registry.yarnpkg.com/nofilter/-/nofilter-3.1.0.tgz#c757ba68801d41ff930ba2ec55bab52ca184aa66" + resolved "https://registry.npmjs.org/nofilter/-/nofilter-3.1.0.tgz" integrity sha512-l2NNj07e9afPnhAhvgVrCD/oy2Ai1yfLpuo3EpiO1jFTsB4sFz6oIfAfSZyQzVpkZQ9xS8ZS5g1jCBgq4Hwo0g== nopt@^6.0.0: @@ -6105,7 +6319,7 @@ normalize-path@^3.0.0, normalize-path@~3.0.0: resolved "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz" integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA== -normalize-url@4, normalize-url@^4.1.0: +normalize-url@^4.1.0, normalize-url@4: version "4.5.1" resolved "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.1.tgz" integrity sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA== @@ -6119,7 +6333,7 @@ npm-audit-report@^3.0.0: npm-bundled@^1.1.1: version "1.1.2" - resolved "https://registry.yarnpkg.com/npm-bundled/-/npm-bundled-1.1.2.tgz#944c78789bd739035b70baa2ca5cc32b8d860bc1" + resolved "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.2.tgz" integrity sha512-x5DHup0SuyQcmL3s7Rx/YQ8sbw/Hzg0rj48eN0dV7hf5cmQq5PXIeioroH3raV1QC1yh3uTYuMThvEQF3iKgGQ== dependencies: npm-normalize-package-bin "^1.0.1" @@ -6145,13 +6359,11 @@ npm-normalize-package-bin@^1.0.1: npm-normalize-package-bin@^2.0.0: version "2.0.0" - resolved "https://registry.yarnpkg.com/npm-normalize-package-bin/-/npm-normalize-package-bin-2.0.0.tgz#9447a1adaaf89d8ad0abe24c6c84ad614a675fff" + resolved "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-2.0.0.tgz" integrity sha512-awzfKUO7v0FscrSpRoogyNm0sajikhBWpU0QMrW09AMi9n1PoKU6WaIqUzuJSQnpciZZmJ/jMZ2Egfmb/9LiWQ== npm-package-arg@^9.0.0, npm-package-arg@^9.0.1, npm-package-arg@^9.1.0: - version "9.1.2" - resolved "https://registry.yarnpkg.com/npm-package-arg/-/npm-package-arg-9.1.2.tgz#fc8acecb00235f42270dda446f36926ddd9ac2bc" - integrity sha512-pzd9rLEx4TfNJkovvlBSLGhq31gGu2QDexFPWT19yCDh0JgnRhlBLNo5759N0AJmBk+kQ9Y/hXoLnlgFD+ukmg== + version "9.1.0" dependencies: hosted-git-info "^5.0.0" proc-log "^2.0.1" @@ -6312,7 +6524,7 @@ number-is-nan@^1.0.0: resolved "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz" integrity sha512-4jbtZXNAsfZbAHiiqjLPBiCl16dES1zI4Hpzzxw61Tk+loF+sBDBKx1ICKKKwIqQ7M0mFn1TmkN7euSncWgHiQ== -nyc@^15.1.0: +nyc@^15.1.0, nyc@>=15: version "15.1.0" resolved "https://registry.npmjs.org/nyc/-/nyc-15.1.0.tgz" integrity sha512-jMW04n9SxKdKi1ZMGhvUTHBN0EICCRkHemEoE5jm6mTYcqcdas0ATzgUgejlQUHMvpnOZqGB5Xxsv9KxJW1j8A== @@ -6504,7 +6716,7 @@ p-event@^4.2.0: p-event@^5.0.1: version "5.0.1" - resolved "https://registry.yarnpkg.com/p-event/-/p-event-5.0.1.tgz#614624ec02ae7f4f13d09a721c90586184af5b0c" + resolved "https://registry.npmjs.org/p-event/-/p-event-5.0.1.tgz" integrity sha512-dd589iCQ7m1L0bmC5NLlVYfy3TbBEsMUfWx9PyAgPeIcFZ/E2yaTZ4Rz4MiBmmJShviiftHVXOqfnfzJ6kyMrQ== dependencies: p-timeout "^5.0.2" @@ -6519,7 +6731,14 @@ p-is-promise@^2.0.0: resolved "https://registry.npmjs.org/p-is-promise/-/p-is-promise-2.1.0.tgz" integrity sha512-Y3W0wlRPK8ZMRbNq97l4M5otioeA5lm1z7bkNkxCka8HSPjR0xRWmpCmc9utiaLP9Jb1eD8BgeIxTW4AIF45Pg== -p-limit@^2.0.0, p-limit@^2.2.0: +p-limit@^2.0.0: + version "2.3.0" + resolved "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz" + integrity sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w== + dependencies: + p-try "^2.0.0" + +p-limit@^2.2.0: version "2.3.0" resolved "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz" integrity sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w== @@ -6535,7 +6754,7 @@ p-limit@^3.0.2, p-limit@^3.1.0: p-limit@^4.0.0: version "4.0.0" - resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-4.0.0.tgz#914af6544ed32bfa54670b061cafcbd04984b644" + resolved "https://registry.npmjs.org/p-limit/-/p-limit-4.0.0.tgz" integrity sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ== dependencies: yocto-queue "^1.0.0" @@ -6563,7 +6782,7 @@ p-locate@^5.0.0: p-locate@^6.0.0: version "6.0.0" - resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-6.0.0.tgz#3da9a49d4934b901089dca3302fa65dc5a05c04f" + resolved "https://registry.npmjs.org/p-locate/-/p-locate-6.0.0.tgz" integrity sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw== dependencies: p-limit "^4.0.0" @@ -6584,7 +6803,7 @@ p-map@^4.0.0: p-map@^5.4.0: version "5.5.0" - resolved "https://registry.yarnpkg.com/p-map/-/p-map-5.5.0.tgz#054ca8ca778dfa4cf3f8db6638ccb5b937266715" + resolved "https://registry.npmjs.org/p-map/-/p-map-5.5.0.tgz" integrity sha512-VFqfGDHlx87K66yZrNdI4YGtD70IRyd+zSvgks6mzHPRNkoKy+9EKP4SFC77/vTTQYmRmti7dvqC+m5jBrBAcg== dependencies: aggregate-error "^4.0.0" @@ -6598,7 +6817,7 @@ p-timeout@^3.1.0: p-timeout@^5.0.2: version "5.1.0" - resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-5.1.0.tgz#b3c691cf4415138ce2d9cfe071dba11f0fee085b" + resolved "https://registry.npmjs.org/p-timeout/-/p-timeout-5.1.0.tgz" integrity sha512-auFDyzzzGZZZdHz3BtET9VEz0SE/uMEAx7uWfGPucfzEwwe/xH0iVeZibQmANYE/hp9T2+UUZT5m+BKyrDp3Ew== p-try@^2.0.0: @@ -6716,7 +6935,7 @@ path-exists@^4.0.0: path-exists@^5.0.0: version "5.0.0" - resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-5.0.0.tgz#a6aad9489200b21fab31e49cf09277e5116fb9e7" + resolved "https://registry.npmjs.org/path-exists/-/path-exists-5.0.0.tgz" integrity sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ== path-is-absolute@^1.0.0: @@ -6724,7 +6943,12 @@ path-is-absolute@^1.0.0: resolved "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz" integrity sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg== -path-key@^2.0.0, path-key@^2.0.1: +path-key@^2.0.0: + version "2.0.1" + resolved "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz" + integrity sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw== + +path-key@^2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz" integrity sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw== @@ -6803,7 +7027,7 @@ pkg-conf@^3.1.0: pkg-conf@^4.0.0: version "4.0.0" - resolved "https://registry.yarnpkg.com/pkg-conf/-/pkg-conf-4.0.0.tgz#63ace00cbacfa94c2226aee133800802d3e3b80c" + resolved "https://registry.npmjs.org/pkg-conf/-/pkg-conf-4.0.0.tgz" integrity sha512-7dmgi4UY4qk+4mj5Cd8v/GExPo0K+SlY+hulOSdfZ/T6jVH6//y7NtzZo5WrfhDBxuQ0jCa7fLZmNaNh7EWL/w== dependencies: find-up "^6.0.0" @@ -6825,15 +7049,13 @@ plur@^4.0.0: plur@^5.1.0: version "5.1.0" - resolved "https://registry.yarnpkg.com/plur/-/plur-5.1.0.tgz#bff58c9f557b9061d60d8ebf93959cf4b08594ae" + resolved "https://registry.npmjs.org/plur/-/plur-5.1.0.tgz" integrity sha512-VP/72JeXqak2KiOzjgKtQen5y3IZHn+9GOuLDafPv0eXa47xq0At93XahYBs26MsifCQ4enGKwbjBTKgb9QJXg== dependencies: irregular-plurals "^3.3.0" postcss-selector-parser@^6.0.10: - version "6.0.15" - resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.0.15.tgz#11cc2b21eebc0b99ea374ffb9887174855a01535" - integrity sha512-rEYkQOMUCEMhsKbK66tbEU9QVIxbhN18YiniAwA7XQYTVBqrBy+P2p5JcdqsHgKM2zWylp8d7J6eszocfds5Sw== + version "6.0.10" dependencies: cssesc "^3.0.0" util-deprecate "^1.0.2" @@ -6887,9 +7109,7 @@ promise-all-reject-late@^1.0.0: integrity sha512-vuf0Lf0lOxyQREH7GDIOUMLS7kz+gs8i6B+Yi8dC68a2sychGrHTJYghMBD6k7eUcH0H5P73EckCA48xijWqXw== promise-call-limit@^1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/promise-call-limit/-/promise-call-limit-1.0.2.tgz#f64b8dd9ef7693c9c7613e7dfe8d6d24de3031ea" - integrity sha512-1vTUnfI2hzui8AEIixbdAJlFY4LFDXqQswy/2eOlThAscXCY4It8FdVuI0fMJGAB2aWGbdQf/gv0skKYXmdrHA== + version "1.0.1" promise-inflight@^1.0.1: version "1.0.1" @@ -6999,7 +7219,7 @@ queue@6.0.2: dependencies: inherits "~2.0.3" -rc@1.2.8, rc@^1.2.8: +rc@^1.2.8, rc@1.2.8: version "1.2.8" resolved "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz" integrity sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw== @@ -7015,9 +7235,7 @@ react-is@^18.0.0: integrity sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w== read-cmd-shim@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/read-cmd-shim/-/read-cmd-shim-3.0.1.tgz#868c235ec59d1de2db69e11aec885bc095aea087" - integrity sha512-kEmDUoYf/CDy8yZbLTmhB1X9kkjf9Q80PCNsDMb7ufrGd6zZSQA1+UyjrO+pZm5K/S4OXCWJeiIt1JA8kAsa6g== + version "3.0.0" read-package-json-fast@^2.0.2, read-package-json-fast@^2.0.3: version "2.0.3" @@ -7047,7 +7265,7 @@ read-pkg@^5.2.0: parse-json "^5.0.0" type-fest "^0.6.0" -read@1, read@^1.0.7, read@~1.0.7: +read@^1.0.7, read@~1.0.7, read@1: version "1.0.7" resolved "https://registry.npmjs.org/read/-/read-1.0.7.tgz" integrity sha512-rSOKNYUmaxy0om1BNjMN4ezNT6VKK+2xF4GBhc81mkH7L60i6dp8qPYrkndNLT3QPphoII3maL9PVC9XmhHwVQ== @@ -7318,20 +7536,34 @@ safe-stable-stringify@^2.3.1: resolved "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.2.tgz" integrity sha512-gMxvPJYhP0O9n2pvcfYfIuYgbledAOJFcqRThtPRmjscaipiwcwPPKLytpVzMkG2HAN87Qmo2d4PtGiri1dSLA== -"safer-buffer@>= 2.1.2 < 3.0.0", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: +safer-buffer@^2.0.2, safer-buffer@^2.1.0, "safer-buffer@>= 2.1.2 < 3.0.0", safer-buffer@~2.1.0: version "2.1.2" resolved "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== +"scramjet-bdd@file:/home/pfalba/github/scramjet-cpm-dev/sth/bdd": + version "0.38.0" + resolved "file:bdd" + dependencies: + "@scramjet/api-client" "^0.38.0" + "@scramjet/logger" "^0.38.0" + "@scramjet/obj-logger" "^0.38.0" + "@scramjet/sth-config" "^0.38.0" + dockerode "^3.3.4" + find-package-json "^1.2.0" + freeport "^1.0.5" + n-readlines "^1.0.1" + scramjet "^4.36.9" + scramjet-core@^4.32.10: version "4.32.10" resolved "https://registry.npmjs.org/scramjet-core/-/scramjet-core-4.32.10.tgz" integrity sha512-B4xYXl8+sT3Fy/DAKmGE1TJafHm+U46AUY3w0v4XseNwhy94C+JfzKR/9+F1gmEihn+E7tDPXnqlquGDmQ6o8Q== -scramjet@^4.36.6, scramjet@^4.36.9, scramjet@^4.37.0: - version "4.37.0" - resolved "https://registry.yarnpkg.com/scramjet/-/scramjet-4.37.0.tgz#2e89f07cbaffd1f9cdd5a3da64aba250745aac13" - integrity sha512-Y6b59qGsulkr5MxiVn9CABnL9pE/sPKihCcWSUhzZc6W0YWbfLWRXc1fE1M40QKfOQUBxks81efzJ7WpEuFmlQ== +scramjet@^4.36.6, scramjet@^4.36.9: + version "4.36.9" + resolved "https://registry.npmjs.org/scramjet/-/scramjet-4.36.9.tgz" + integrity sha512-vHRQy3hE1gC3tYDKPLGmvQgEQV4oL3JpOA7tP07eGcez+Py2mX8yFu81OeSBPwSwjHX5BrAe8pXVkgT3CMkC/g== dependencies: papaparse "^5.3.2" rereadable-stream "^1.4.14" @@ -7349,12 +7581,22 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -"semver@2 || 3 || 4 || 5", semver@^5.5.0: +semver@^5.5.0: version "5.7.1" resolved "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz" integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== -semver@^6.0.0, semver@^6.2.0, semver@^6.3.0: +semver@^6.0.0: + version "6.3.0" + resolved "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz" + integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== + +semver@^6.2.0: + version "6.3.0" + resolved "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz" + integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== + +semver@^6.3.0: version "6.3.0" resolved "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== @@ -7364,9 +7606,9 @@ semver@^6.3.1: resolved "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz" integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== -semver@^7.0.0, semver@^7.1.1, semver@^7.3.7, semver@^7.5.3, semver@^7.5.4: +semver@^7.0.0, semver@^7.1.1, semver@^7.3.7: version "7.5.4" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.4.tgz#483986ec4ed38e1c6c48c34894a9182dbff68a6e" + resolved "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz" integrity sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA== dependencies: lru-cache "^6.0.0" @@ -7378,6 +7620,25 @@ semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.5.2: dependencies: lru-cache "^6.0.0" +semver@^7.5.3: + version "7.5.4" + resolved "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz" + integrity sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA== + dependencies: + lru-cache "^6.0.0" + +semver@^7.5.4: + version "7.5.4" + resolved "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz" + integrity sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA== + dependencies: + lru-cache "^6.0.0" + +"semver@2 || 3 || 4 || 5": + version "5.7.1" + resolved "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz" + integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== + serialize-error@^7.0.1: version "7.0.1" resolved "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz" @@ -7501,7 +7762,7 @@ slash@^3.0.0: slash@^4.0.0: version "4.0.0" - resolved "https://registry.yarnpkg.com/slash/-/slash-4.0.0.tgz#2422372176c4c6c5addb5e2ada885af984b396a7" + resolved "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz" integrity sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew== slice-ansi@^3.0.0: @@ -7515,7 +7776,7 @@ slice-ansi@^3.0.0: slice-ansi@^5.0.0: version "5.0.0" - resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-5.0.0.tgz#b73063c57aa96f9cd881654b15294d95d285c42a" + resolved "https://registry.npmjs.org/slice-ansi/-/slice-ansi-5.0.0.tgz" integrity sha512-FC+lgizVPfie0kkhqUScwRu1O/lF6NOgJmlCgK+/LYxDCTk8sGelYaHDhFcDN+Sn3Cv+3VSa4Byeo+IMCzpMgQ== dependencies: ansi-styles "^6.0.0" @@ -7562,6 +7823,14 @@ socks@^2.6.2: ip "^2.0.0" smart-buffer "^4.2.0" +source-map-support@^0.5.19, source-map-support@0.5.19: + version "0.5.19" + resolved "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz" + integrity sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw== + dependencies: + buffer-from "^1.0.0" + source-map "^0.6.0" + source-map-support@0.5.13: version "0.5.13" resolved "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz" @@ -7570,24 +7839,16 @@ source-map-support@0.5.13: buffer-from "^1.0.0" source-map "^0.6.0" -source-map-support@0.5.19, source-map-support@^0.5.19: - version "0.5.19" - resolved "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz" - integrity sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw== - dependencies: - buffer-from "^1.0.0" - source-map "^0.6.0" +source-map@^0.6.0, source-map@^0.6.1: + version "0.6.1" + resolved "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz" + integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== source-map@0.5.6: version "0.5.6" resolved "https://registry.npmjs.org/source-map/-/source-map-0.5.6.tgz" integrity sha512-MjZkVp0NHr5+TPihLcadqnlVoGIoWo4IBHptutGh9wI3ttUYvCG26HkSuDi+K6lsZ25syXJXcctwgyVCt//xqA== -source-map@^0.6.0, source-map@^0.6.1: - version "0.6.1" - resolved "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz" - integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== - spawn-wrap@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/spawn-wrap/-/spawn-wrap-2.0.0.tgz" @@ -7720,6 +7981,13 @@ stream-buffers@^3.0.2: resolved "https://registry.npmjs.org/stream-buffers/-/stream-buffers-3.0.2.tgz" integrity sha512-DQi1h8VEBA/lURbSwFtEHnSTb9s2/pwLEaFuNhXwy1Dx3Sa0lOuYT2yNUr4/j2fs8oCAMANtrZ5OrPZtyVs3MQ== +string_decoder@^1.1.1: + version "1.3.0" + resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz" + integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== + dependencies: + safe-buffer "~5.2.0" + string-argv@^0.3.1: version "0.3.1" resolved "https://registry.npmjs.org/string-argv/-/string-argv-0.3.1.tgz" @@ -7751,7 +8019,15 @@ string-width@^1.0.1: is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.1" -string-width@^2.0.0, string-width@^2.1.1: +string-width@^2.0.0: + version "2.1.1" + resolved "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz" + integrity sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw== + dependencies: + is-fullwidth-code-point "^2.0.0" + strip-ansi "^4.0.0" + +string-width@^2.1.1: version "2.1.1" resolved "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz" integrity sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw== @@ -7761,7 +8037,7 @@ string-width@^2.0.0, string-width@^2.1.1: string-width@^5.0.0: version "5.1.2" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-5.1.2.tgz#14f8daec6d81e7221d2a357e668cab73bdbca794" + resolved "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz" integrity sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA== dependencies: eastasianwidth "^0.2.0" @@ -7795,13 +8071,6 @@ string.prototype.trimstart@^1.0.7: define-properties "^1.2.0" es-abstract "^1.22.1" -string_decoder@^1.1.1: - version "1.3.0" - resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz" - integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== - dependencies: - safe-buffer "~5.2.0" - strip-ansi@^3.0.0, strip-ansi@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz" @@ -7825,7 +8094,7 @@ strip-ansi@^6.0.0, strip-ansi@^6.0.1: strip-ansi@^7.0.1: version "7.1.0" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" + resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz" integrity sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ== dependencies: ansi-regex "^6.0.1" @@ -7881,7 +8150,7 @@ supertap@^2.0.0: supertap@^3.0.1: version "3.0.1" - resolved "https://registry.yarnpkg.com/supertap/-/supertap-3.0.1.tgz#aa89e4522104402c6e8fe470a7d2db6dc4037c6a" + resolved "https://registry.npmjs.org/supertap/-/supertap-3.0.1.tgz" integrity sha512-u1ZpIBCawJnO+0QePsEiOknOfCRq0yERxiAchT0i4li0WHNUJbf0evXXSXOcCAR4M8iMDoajXYmstm/qO81Isw== dependencies: indent-string "^5.0.0" @@ -7937,13 +8206,11 @@ tar-stream@^2.0.0: readable-stream "^3.1.1" tar@^6.1.0: - version "6.2.0" - resolved "https://registry.yarnpkg.com/tar/-/tar-6.2.0.tgz#b14ce49a79cb1cd23bc9b016302dea5474493f73" - integrity sha512-/Wo7DcT0u5HUV486xg675HtjNd3BXZ6xDbzsCUZPt5iw8bTQ63bP0Raut3mvro9u+CUyq7YQd8Cx55fsZXxqLQ== + version "6.1.11" dependencies: chownr "^2.0.0" fs-minipass "^2.0.0" - minipass "^5.0.0" + minipass "^3.0.0" minizlib "^2.1.1" mkdirp "^1.0.3" yallist "^4.0.0" @@ -7979,7 +8246,12 @@ text-hex@1.0.x: resolved "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz" integrity sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg== -text-table@^0.2.0, text-table@~0.2.0: +text-table@^0.2.0: + version "0.2.0" + resolved "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz" + integrity sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw== + +text-table@~0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz" integrity sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw== @@ -8015,7 +8287,14 @@ tmp-promise@^3.0.2: dependencies: tmp "^0.2.0" -tmp@^0.2.0, tmp@^0.2.1: +tmp@^0.2.0: + version "0.2.1" + resolved "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz" + integrity sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ== + dependencies: + rimraf "^3.0.0" + +tmp@^0.2.1: version "0.2.1" resolved "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz" integrity sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ== @@ -8116,7 +8395,7 @@ ts-jest@^29.0.5: semver "^7.5.3" yargs-parser "^21.0.1" -ts-node@^10.9.1: +ts-node@^10.9.1, ts-node@>=9.0.0: version "10.9.1" resolved "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz" integrity sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw== @@ -8179,7 +8458,7 @@ type-check@^0.4.0, type-check@~0.4.0: dependencies: prelude-ls "^1.2.1" -type-detect@4.0.8, type-detect@^4.0.8: +type-detect@^4.0.8, type-detect@4.0.8: version "4.0.8" resolved "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz" integrity sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g== @@ -8265,7 +8544,7 @@ typed-array-length@^1.0.4: typed-emitter@^1.4.0: version "1.4.0" - resolved "https://registry.yarnpkg.com/typed-emitter/-/typed-emitter-1.4.0.tgz#38c6bf1224e764906bb20cb0b458fa914100607c" + resolved "https://registry.npmjs.org/typed-emitter/-/typed-emitter-1.4.0.tgz" integrity sha512-weBmoo3HhpKGgLBOYwe8EB31CzDFuaK7CCL+axXhUYhn4jo6DSkHnbefboCF5i4DQ2aMFe0C/FdTWcPdObgHyg== typedarray-to-buffer@^3.1.5: @@ -8282,7 +8561,7 @@ typedoc-plugin-markdown@3.13.6: dependencies: handlebars "^4.7.7" -typedoc@0.23.17: +typedoc@>=0.23.0, typedoc@0.23.17: version "0.23.17" resolved "https://registry.npmjs.org/typedoc/-/typedoc-0.23.17.tgz" integrity sha512-3rtNubo1dK0pvs6ixpMAq4pESULd5/JNUqJbdyZoeilI14reb1RNVomN4fMgIadd0RMX1aenYjJSSMBOJ+/+0Q== @@ -8292,7 +8571,7 @@ typedoc@0.23.17: minimatch "^5.1.0" shiki "^0.11.1" -typescript@~4.7.4: +typescript@>=2.7, typescript@>=4.2.0, "typescript@>=4.3 <6", typescript@~4.7.4, "typescript@4.6.x || 4.7.x || 4.8.x": version "4.7.4" resolved "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz" integrity sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ== @@ -8402,16 +8681,21 @@ util-deprecate@^1.0.1, util-deprecate@^1.0.2: resolved "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz" integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== -uuid@8.3.2, uuid@^8.3.2: - version "8.3.2" - resolved "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz" - integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== - uuid@^3.3.2: version "3.4.0" resolved "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz" integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== +uuid@^8.3.2, uuid@8.3.2: + version "8.3.2" + resolved "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz" + integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== + +uuid@9.0.0: + version "9.0.0" + resolved "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz" + integrity sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg== + v8-compile-cache-lib@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz" @@ -8426,7 +8710,15 @@ v8-to-istanbul@^9.0.1: "@types/istanbul-lib-coverage" "^2.0.1" convert-source-map "^1.6.0" -validate-npm-package-license@^3.0.1, validate-npm-package-license@^3.0.4: +validate-npm-package-license@^3.0.1: + version "3.0.4" + resolved "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz" + integrity sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew== + dependencies: + spdx-correct "^3.0.0" + spdx-expression-parse "^3.0.0" + +validate-npm-package-license@^3.0.4: version "3.0.4" resolved "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz" integrity sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew== @@ -8446,19 +8738,19 @@ validator@^13.7.0: resolved "https://registry.npmjs.org/validator/-/validator-13.7.0.tgz" integrity sha512-nYXQLCBkpJ8X6ltALua9dRrZDHVYxjJ1wgskNt1lH9fzGjs3tgojGSCBjmEPwkWS1y29+DrizMTW19Pr9uB2nw== -verror@1.10.0: - version "1.10.0" - resolved "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz" - integrity sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw== +verror@^1.10.0: + version "1.10.1" + resolved "https://registry.npmjs.org/verror/-/verror-1.10.1.tgz" + integrity sha512-veufcmxri4e3XSrT0xwfUR7kguIkaxBeosDg00yDWhk49wdwkSUrvvsm7nc75e1PUyvIeZj6nS8VQRYz2/S4Xg== dependencies: assert-plus "^1.0.0" core-util-is "1.0.2" extsprintf "^1.2.0" -verror@^1.10.0: - version "1.10.1" - resolved "https://registry.npmjs.org/verror/-/verror-1.10.1.tgz" - integrity sha512-veufcmxri4e3XSrT0xwfUR7kguIkaxBeosDg00yDWhk49wdwkSUrvvsm7nc75e1PUyvIeZj6nS8VQRYz2/S4Xg== +verror@1.10.0: + version "1.10.0" + resolved "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz" + integrity sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw== dependencies: assert-plus "^1.0.0" core-util-is "1.0.2" @@ -8486,7 +8778,14 @@ walker@^1.0.8: dependencies: makeerror "1.0.12" -wcwidth@^1.0.0, wcwidth@^1.0.1: +wcwidth@^1.0.0: + version "1.0.1" + resolved "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz" + integrity sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg== + dependencies: + defaults "^1.0.3" + +wcwidth@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz" integrity sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg== @@ -8650,7 +8949,15 @@ write-file-atomic@^3.0.0, write-file-atomic@^3.0.3: signal-exit "^3.0.2" typedarray-to-buffer "^3.1.5" -write-file-atomic@^4.0.0, write-file-atomic@^4.0.1, write-file-atomic@^4.0.2: +write-file-atomic@^4.0.0, write-file-atomic@^4.0.1: + version "4.0.2" + resolved "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz" + integrity sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg== + dependencies: + imurmurhash "^0.1.4" + signal-exit "^3.0.7" + +write-file-atomic@^4.0.2: version "4.0.2" resolved "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz" integrity sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg== @@ -8658,7 +8965,7 @@ write-file-atomic@^4.0.0, write-file-atomic@^4.0.1, write-file-atomic@^4.0.2: imurmurhash "^0.1.4" signal-exit "^3.0.7" -ws@^7.3.1: +ws@*, ws@^7.3.1: version "7.5.9" resolved "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz" integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q== @@ -8714,7 +9021,12 @@ yargs-parser@^20.2.2: resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz" integrity sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w== -yargs-parser@^21.0.1, yargs-parser@^21.1.1: +yargs-parser@^21.0.1: + version "21.1.1" + resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz" + integrity sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw== + +yargs-parser@^21.1.1: version "21.1.1" resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz" integrity sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw== @@ -8767,7 +9079,20 @@ yargs@^16.2.0: y18n "^5.0.5" yargs-parser "^20.2.2" -yargs@^17.3.1, yargs@^17.5.1: +yargs@^17.3.1: + version "17.7.2" + resolved "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz" + integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w== + dependencies: + cliui "^8.0.1" + escalade "^3.1.1" + get-caller-file "^2.0.5" + require-directory "^2.1.1" + string-width "^4.2.3" + y18n "^5.0.5" + yargs-parser "^21.1.1" + +yargs@^17.5.1: version "17.7.2" resolved "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz" integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w== @@ -8797,5 +9122,5 @@ yocto-queue@^0.1.0: yocto-queue@^1.0.0: version "1.0.0" - resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-1.0.0.tgz#7f816433fb2cbc511ec8bf7d263c3b58a1a3c251" + resolved "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz" integrity sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==