From 2b81602bba28de5b1f58dd71b6ee5c026895ff85 Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Sun, 16 Jul 2017 14:28:15 +1000 Subject: [PATCH 01/24] wip --- src/server/dev.ts | 13 +- .../web_socket_proxy_nuclearnet_server.ts | 7 +- src/server/nusight_server.ts | 112 ++++++++++++++++++ src/server/prod.ts | 10 +- src/shared/nuclearnet/nuclearnet_client.ts | 1 + 5 files changed, 129 insertions(+), 14 deletions(-) create mode 100644 src/server/nusight_server.ts diff --git a/src/server/dev.ts b/src/server/dev.ts index 98ceb703..57a3dd07 100644 --- a/src/server/dev.ts +++ b/src/server/dev.ts @@ -9,10 +9,13 @@ import * as webpack from 'webpack' import * as webpackDevMiddleware from 'webpack-dev-middleware' import * as webpackHotMiddleware from 'webpack-hot-middleware' import webpackConfig from '../../webpack.config' -import { VirtualRobots } from '../simulators/virtual_robots' import { SensorDataSimulator } from '../simulators/sensor_data_simulator' +import { VirtualRobots } from '../simulators/virtual_robots' +import { DirectNUClearNetClient } from './nuclearnet/direct_nuclearnet_client' +import { FakeNUClearNetClient } from './nuclearnet/fake_nuclearnet_client' import { WebSocketProxyNUClearNetServer } from './nuclearnet/web_socket_proxy_nuclearnet_server' import { WebSocketServer } from './nuclearnet/web_socket_server' +import { NUsightServer } from './nusight_server' const compiler = webpack(webpackConfig) @@ -58,6 +61,8 @@ if (withSimulators) { virtualRobots.simulateWithFrequency(60) } -WebSocketProxyNUClearNetServer.of(WebSocketServer.of(sioNetwork.of('/nuclearnet')), { - fakeNetworking: withSimulators, -}) +const nuclearnetClient = withSimulators ? FakeNUClearNetClient.of() : DirectNUClearNetClient.of() + +WebSocketProxyNUClearNetServer.of(WebSocketServer.of(sioNetwork.of('/nuclearnet')), nuclearnetClient) + +NUsightServer.of(WebSocketServer.of(sioNetwork.of('/nusight')), nuclearnetClient) diff --git a/src/server/nuclearnet/web_socket_proxy_nuclearnet_server.ts b/src/server/nuclearnet/web_socket_proxy_nuclearnet_server.ts index a4540913..b12dc706 100644 --- a/src/server/nuclearnet/web_socket_proxy_nuclearnet_server.ts +++ b/src/server/nuclearnet/web_socket_proxy_nuclearnet_server.ts @@ -9,10 +9,6 @@ import { WebSocket } from './web_socket_server' import { NodeSystemClock } from '../time/node_clock' import { Clock } from '../time/clock' -type Opts = { - fakeNetworking: boolean -} - /** * The server component of a NUClearNet proxy running over web sockets. Acts as a gateway to the NUClear network. * All clients currently share a single NUClearNet connection, mostly for performance reasons. Could potentially be @@ -23,8 +19,7 @@ export class WebSocketProxyNUClearNetServer { server.onConnection(this.onClientConnection) } - public static of(server: WebSocketServer, { fakeNetworking }: Opts): WebSocketProxyNUClearNetServer { - const nuclearnetClient: NUClearNetClient = fakeNetworking ? FakeNUClearNetClient.of() : DirectNUClearNetClient.of() + public static of(server: WebSocketServer, nuclearnetClient: NUClearNetClient): WebSocketProxyNUClearNetServer { return new WebSocketProxyNUClearNetServer(server, nuclearnetClient) } diff --git a/src/server/nusight_server.ts b/src/server/nusight_server.ts new file mode 100644 index 00000000..a39f87f1 --- /dev/null +++ b/src/server/nusight_server.ts @@ -0,0 +1,112 @@ +import * as fs from 'fs' +import { NUClearNetPeer } from 'nuclearnet.js' +import { NUClearNetPacket } from 'nuclearnet.js' +import { NUClearNetClient } from '../shared/nuclearnet/nuclearnet_client' +import { WebSocketServer } from './nuclearnet/web_socket_server' +import { WebSocket } from './nuclearnet/web_socket_server' +import { WriteStream } from 'fs' +import { NodeSystemClock } from './time/node_clock' +import { Clock } from './time/clock' + +export class NUsightServer { + public constructor(private server: WebSocketServer, private nuclearnetClient: NUClearNetClient) { + server.onConnection(this.onClientConnection) + } + + public static of(server: WebSocketServer, nuclearnetClient: NUClearNetClient): NUsightServer { + return new NUsightServer(server, nuclearnetClient) + } + + private onClientConnection = (socket: WebSocket) => { + NUsightServerClient.of(socket, this.nuclearnetClient) + } +} + +class NUsightServerClient { + private stopRecordingMap: Map void> + + public constructor(private socket: WebSocket, private clock: Clock, private nuclearnetClient: NUClearNetClient) { + this.socket.on('record', this.onRecord) + this.socket.on('unrecord', this.onUnrecord) + } + + public static of(socket: WebSocket, nuclearnetClient: NUClearNetClient): NUsightServerClient { + return new NUsightServerClient(socket, NodeSystemClock, nuclearnetClient) + } + + public onRecord(peer: NUClearNetPeer, requestToken: string) { + const recorder = NbsRecorder.of(peer, this.nuclearnetClient) + const stopRecording = recorder.record(`recordings/${this.clock.now()}.tbs`) + this.stopRecordingMap.set(requestToken, stopRecording) + } + + public onUnrecord(requestToken: string) { + const stopRecording = this.stopRecordingMap.get(requestToken) + if (stopRecording) { + stopRecording() + } + } +} + +class NbsRecorder { + private recording: boolean + private file: WriteStream + + public constructor(private peer: NUClearNetPeer, + private clock: Clock, + private nuclearnetClient: NUClearNetClient) { + this.recording = false + } + + public static of(peer: NUClearNetPeer, nuclearnetClient: NUClearNetClient): NbsRecorder { + return new NbsRecorder(peer, NodeSystemClock, nuclearnetClient) + } + + public record(filename: string): () => void { + this.file = fs.createWriteStream(filename, { defaultEncoding: 'binary' }) + const stopListening = this.nuclearnetClient.onPacket(this.onPacket) + this.recording = true + return () => { + stopListening() + this.recording = false + } + } + + private onPacket = (packet: NUClearNetPacket) => { + if (!this.arePeersEqual(this.peer, packet.peer)) { + return + } + + // NBS File Format: + // 3 Bytes - NUClear radiation symbol header, useful for synchronisation when attaching to a existing stream. + // 4 Bytes - The remaining packet length i.e. 16 bytes + N payload bytes + // 8 Bytes - 64bit timestamp in microseconds. Note: this is not necessarily a unix timestamp. + // 8 Bytes - 64bit bit hash of the message type. + // N bytes - The binary packet data itself. + + const header = Buffer.from([0xE2, 0x98, 0xA2]) // NUClear radiation symbol. + + // 64bit timestamp in microseconds. + const time = this.clock.now() * 1000; + const timeBuffer = new Buffer(8); + const MAX_UINT32 = 0xFFFFFFFF; + const highByte = ~~(time / MAX_UINT32); + const lowByte = (time % MAX_UINT32) - highByte; + timeBuffer.writeUInt32BE(highByte, 0); + timeBuffer.writeUInt32BE(lowByte, 4); + + const remainingByteLength = new Buffer(4); + remainingByteLength.writeUInt32LE(timeBuffer.byteLength + packet.hash.byteLength + packet.payload.byteLength, 0) + + // Write parts to file + this.file.write(header) + this.file.write(remainingByteLength); + this.file.write(timeBuffer); + this.file.write(packet.hash); + + } + + private arePeersEqual(peerA: NUClearNetPeer, peerB: NUClearNetPeer) { + return peerA.name === peerB.name && peerA.address === peerB.address && peerA.port === peerB.port + } +} diff --git a/src/server/prod.ts b/src/server/prod.ts index 53b448f3..c14faf2e 100644 --- a/src/server/prod.ts +++ b/src/server/prod.ts @@ -5,8 +5,10 @@ import * as http from 'http' import * as minimist from 'minimist' import * as favicon from 'serve-favicon' import * as sio from 'socket.io' -import { VirtualRobots } from '../simulators/virtual_robots' import { SensorDataSimulator } from '../simulators/sensor_data_simulator' +import { VirtualRobots } from '../simulators/virtual_robots' +import { DirectNUClearNetClient } from './nuclearnet/direct_nuclearnet_client' +import { FakeNUClearNetClient } from './nuclearnet/fake_nuclearnet_client' import { WebSocketProxyNUClearNetServer } from './nuclearnet/web_socket_proxy_nuclearnet_server' import { WebSocketServer } from './nuclearnet/web_socket_server' @@ -40,6 +42,6 @@ if (withSimulators) { virtualRobots.simulateWithFrequency(60) } -WebSocketProxyNUClearNetServer.of(WebSocketServer.of(sioNetwork.of('/nuclearnet')), { - fakeNetworking: withSimulators, -}) +const nuclearnetClient = withSimulators ? FakeNUClearNetClient.of() : DirectNUClearNetClient.of() + +WebSocketProxyNUClearNetServer.of(WebSocketServer.of(sioNetwork.of('/nuclearnet')), nuclearnetClient) diff --git a/src/shared/nuclearnet/nuclearnet_client.ts b/src/shared/nuclearnet/nuclearnet_client.ts index 033c3e4e..eb18ca6a 100644 --- a/src/shared/nuclearnet/nuclearnet_client.ts +++ b/src/shared/nuclearnet/nuclearnet_client.ts @@ -12,5 +12,6 @@ export interface NUClearNetClient { onJoin(cb: NUClearEventListener): () => void onLeave(cb: NUClearEventListener): () => void on(event: string, cb: NUClearPacketListener): () => void + onPacket(cb: NUClearPacketListener): () => void send(options: NUClearNetSend): void } From a8e48bd9142bc8202327845e68fef9e41b9230fb Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Sun, 16 Jul 2017 14:32:10 +1000 Subject: [PATCH 02/24] Upgrade nuclearnet.js --- package.json | 2 +- src/client/nuclearnet/web_socket_proxy_nuclearnet_client.ts | 4 ++++ src/server/nuclearnet/direct_nuclearnet_client.ts | 4 ++++ src/server/nuclearnet/fake_nuclearnet_client.ts | 4 ++++ src/server/nuclearnet/fake_nuclearnet_server.ts | 1 + yarn.lock | 6 +++--- 6 files changed, 17 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index c45e0df7..e066e4c0 100644 --- a/package.json +++ b/package.json @@ -95,7 +95,7 @@ "minimist": "^1.2.0", "mobx": "^3.2.0", "mobx-react": "^4.2.2", - "nuclearnet.js": "^1.2.0", + "nuclearnet.js": "^1.3.0", "protobufjs": "^6.7.3", "react": "^15.6.1", "react-dom": "^15.6.1", diff --git a/src/client/nuclearnet/web_socket_proxy_nuclearnet_client.ts b/src/client/nuclearnet/web_socket_proxy_nuclearnet_client.ts index 51cd41ac..4541715f 100644 --- a/src/client/nuclearnet/web_socket_proxy_nuclearnet_client.ts +++ b/src/client/nuclearnet/web_socket_proxy_nuclearnet_client.ts @@ -111,6 +111,10 @@ export class WebSocketProxyNUClearNetClient implements NUClearNetClient { } } + onPacket(cb: NUClearPacketListener): () => void { + return this.on('nuclear_packet', cb) + } + public send(options: NUClearNetSend): void { if (typeof options.type === 'string') { this.socket.send(options.type, options) diff --git a/src/server/nuclearnet/direct_nuclearnet_client.ts b/src/server/nuclearnet/direct_nuclearnet_client.ts index 8575584b..14a95726 100644 --- a/src/server/nuclearnet/direct_nuclearnet_client.ts +++ b/src/server/nuclearnet/direct_nuclearnet_client.ts @@ -38,6 +38,10 @@ export class DirectNUClearNetClient implements NUClearNetClient { return () => this.nuclearNetwork.removeListener(event, cb) } + public onPacket(cb: NUClearPacketListener): () => void { + return this.on('nuclear_packet', cb) + } + public send(options: NUClearNetSend): void { this.nuclearNetwork.send(options) } diff --git a/src/server/nuclearnet/fake_nuclearnet_client.ts b/src/server/nuclearnet/fake_nuclearnet_client.ts index 38dcd395..50e255ad 100644 --- a/src/server/nuclearnet/fake_nuclearnet_client.ts +++ b/src/server/nuclearnet/fake_nuclearnet_client.ts @@ -79,6 +79,10 @@ export class FakeNUClearNetClient implements NUClearNetClient { return () => this.events.removeListener(event, listener) } + public onPacket(cb: NUClearPacketListener): () => void { + return this.on('nuclear_packet', cb) + } + public send(options: NUClearNetSend): void { this.server.send(this, options) } diff --git a/src/server/nuclearnet/fake_nuclearnet_server.ts b/src/server/nuclearnet/fake_nuclearnet_server.ts index 0fe39ce4..8a0d70ff 100644 --- a/src/server/nuclearnet/fake_nuclearnet_server.ts +++ b/src/server/nuclearnet/fake_nuclearnet_server.ts @@ -76,6 +76,7 @@ export class FakeNUClearNetServer { for (const client of targetClients) { client.fakePacket(opts.type, packet) + client.fakePacket('nuclear_packet', packet) } } } diff --git a/yarn.lock b/yarn.lock index 3f3f7d6b..019e9d69 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4475,9 +4475,9 @@ nth-check@~1.0.1: dependencies: boolbase "~1.0.0" -nuclearnet.js@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/nuclearnet.js/-/nuclearnet.js-1.2.0.tgz#a26c83d7495974c87dff1817a4eade8d8e012abe" +nuclearnet.js@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/nuclearnet.js/-/nuclearnet.js-1.3.0.tgz#aefae50336b1337ed211edcef4e0774e8bf4ee74" dependencies: bindings "^1.2.1" nan "^2.0.0" From 2bdb34fa5f190b6612dd35acafc9a483a472c583 Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Sun, 16 Jul 2017 15:04:55 +1000 Subject: [PATCH 03/24] . --- src/server/nusight_server.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/server/nusight_server.ts b/src/server/nusight_server.ts index a39f87f1..49006884 100644 --- a/src/server/nusight_server.ts +++ b/src/server/nusight_server.ts @@ -69,6 +69,7 @@ class NbsRecorder { return () => { stopListening() this.recording = false + this.file.end() } } @@ -82,13 +83,14 @@ class NbsRecorder { // 4 Bytes - The remaining packet length i.e. 16 bytes + N payload bytes // 8 Bytes - 64bit timestamp in microseconds. Note: this is not necessarily a unix timestamp. // 8 Bytes - 64bit bit hash of the message type. - // N bytes - The binary packet data itself. + // N bytes - The binary packet payload. const header = Buffer.from([0xE2, 0x98, 0xA2]) // NUClear radiation symbol. // 64bit timestamp in microseconds. const time = this.clock.now() * 1000; const timeBuffer = new Buffer(8); + // Convert double into two 32 bit integers. const MAX_UINT32 = 0xFFFFFFFF; const highByte = ~~(time / MAX_UINT32); const lowByte = (time % MAX_UINT32) - highByte; @@ -103,7 +105,6 @@ class NbsRecorder { this.file.write(remainingByteLength); this.file.write(timeBuffer); this.file.write(packet.hash); - } private arePeersEqual(peerA: NUClearNetPeer, peerB: NUClearNetPeer) { From 7d1892d9d437244054a70ab1da391181b32399e3 Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Sun, 16 Jul 2017 20:25:10 +1000 Subject: [PATCH 04/24] wip --- .gitignore | 3 + package.json | 1 + src/client/base/memoize.ts | 8 +- .../localisation/darwin_robot/model.ts | 1 + .../components/navigation/icons/record.svg | 7 + src/client/components/navigation/view.tsx | 2 + src/client/components/record/controller.ts | 27 ++ src/client/components/record/model.ts | 58 ++++ src/client/components/record/styles.css | 6 + src/client/components/record/view.tsx | 58 ++++ src/client/index.tsx | 6 + src/client/network/nusight_network.ts | 25 +- .../nuclearnet/fake_nuclearnet_client.ts | 8 +- .../nuclearnet/fake_nuclearnet_server.ts | 51 ++- src/server/nusight_server.ts | 15 +- src/tests/networking_integration.tests.ts | 328 +++++++++--------- src/validator/validate.ts | 76 ++++ 17 files changed, 477 insertions(+), 203 deletions(-) create mode 100644 src/client/components/navigation/icons/record.svg create mode 100644 src/client/components/record/controller.ts create mode 100644 src/client/components/record/model.ts create mode 100644 src/client/components/record/styles.css create mode 100644 src/client/components/record/view.tsx create mode 100644 src/validator/validate.ts diff --git a/.gitignore b/.gitignore index 17e895c7..7d259617 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ build # Jest coverage report /coverage + +# NBS recordings +/recordings diff --git a/package.json b/package.json index e066e4c0..97f5f40d 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "start": "nodemon ./src/server/dev.ts", "start:sim": "nodemon ./src/server/dev.ts --with-simulators", "simulate": "nodemon ./src/simulators/simulate.ts", + "validate": "ts-node ./src/validator/validate.ts", "prod": "ts-node -F ./src/server/prod.ts", "prod:sim": "ts-node -F ./src/server/prod.ts --with-simulators", "build": "yarn clean:build && webpack -p --progress --colors", diff --git a/src/client/base/memoize.ts b/src/client/base/memoize.ts index ba156745..b4e54508 100644 --- a/src/client/base/memoize.ts +++ b/src/client/base/memoize.ts @@ -1,10 +1,10 @@ /** * Given a function that takes an object A and returns a B, create a new function which memoizes that A -> B transform. * - * i.e. The first time the memoized function called with an A, it calculates B using fn(A) and stores B in its internal - * map. The second time it is called with the same A, it will not call fn(A) and instead just return the B that was - * created the previous time. Internally the function uses a WeakMap, so B will be automatically garbage collected when - * its corresponding A no longer exists in memory. + * i.e. The first time the memoized function is called with an A, it calculates B using fn(A) and stores B in its + * internal map. The second time it is called with the same A, it will not call fn(A) and instead just return the B that + * was created the previous time. Internally the function uses a WeakMap, so B will be automatically garbage collected + * when its corresponding A no longer exists in memory. * * e.g. * const a = { name: 'Foo' } diff --git a/src/client/components/localisation/darwin_robot/model.ts b/src/client/components/localisation/darwin_robot/model.ts index b05f9c90..dd727775 100644 --- a/src/client/components/localisation/darwin_robot/model.ts +++ b/src/client/components/localisation/darwin_robot/model.ts @@ -4,6 +4,7 @@ import { memoize } from '../../../base/memoize' import { RobotModel } from '../../robot/model' import { Vector3 } from '../model' import { Quaternion } from '../model' +import { memoize } from '../../../base/memoize' export class LocalisationRobotModel { @observable private model: RobotModel diff --git a/src/client/components/navigation/icons/record.svg b/src/client/components/navigation/icons/record.svg new file mode 100644 index 00000000..008df2f3 --- /dev/null +++ b/src/client/components/navigation/icons/record.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/src/client/components/navigation/view.tsx b/src/client/components/navigation/view.tsx index b237c33e..2d4c29ad 100644 --- a/src/client/components/navigation/view.tsx +++ b/src/client/components/navigation/view.tsx @@ -7,6 +7,7 @@ import EyeIcon from './icons/eye.svg' import MapIcon from './icons/map.svg' import NUClearIcon from './icons/nuclear.svg' import OrderingIcon from './icons/ordering.svg' +import RecordIcon from './icons/record.svg' import ScatterIcon from './icons/scatter.svg' import SpeedometerIcon from './icons/speedometer.svg' import * as style from './style.css' @@ -40,6 +41,7 @@ export const NavigationView = () => ( Classifier Subsumption GameState + Record ) diff --git a/src/client/components/record/controller.ts b/src/client/components/record/controller.ts new file mode 100644 index 00000000..61ec6b09 --- /dev/null +++ b/src/client/components/record/controller.ts @@ -0,0 +1,27 @@ +import { action } from 'mobx' +import { NUsightNetwork } from '../../network/nusight_network' +import { RecordRobotModel } from './model' + +export class RecordController { + public constructor(private nusightNetwork: NUsightNetwork) { + } + + public static of(nusightNetwork: NUsightNetwork): RecordController { + return new RecordController(nusightNetwork) + } + + @action + onStartRecordingClick(robot: RecordRobotModel) { + const peer = { name: robot.name, address: robot.address, port: robot.port } + robot.stopRecording = this.nusightNetwork.record(peer) + robot.recording = true + } + + @action + onStopRecordingClick(robot: RecordRobotModel) { + if (robot.stopRecording) { + robot.stopRecording() + } + robot.recording = false + } +} diff --git a/src/client/components/record/model.ts b/src/client/components/record/model.ts new file mode 100644 index 00000000..43dc0ca9 --- /dev/null +++ b/src/client/components/record/model.ts @@ -0,0 +1,58 @@ +import { observable } from 'mobx' +import { computed } from 'mobx' +import { memoize } from '../../base/memoize' +import { AppModel } from '../app/model' +import { RobotModel } from '../robot/model' + +export class RecordModel { + @observable private appModel: AppModel + + public constructor(appModel: AppModel) { + this.appModel = appModel + } + + public static of(appModel: AppModel) { + return new RecordModel(appModel) + } + + @computed + public get robots(): RecordRobotModel[] { + return this.appModel.robots.map(robot => RecordRobotModel.of(robot)) + } +} + +type RecordRobotModelOpts = { + recording: boolean +} + +export class RecordRobotModel { + @observable robotModel: RobotModel + @observable public recording: boolean + public stopRecording?: () => void + + public constructor(robotModel: RobotModel, opts: RecordRobotModelOpts) { + this.robotModel = robotModel + this.recording = opts.recording + } + + public static of = memoize((robot: RobotModel): RecordRobotModel => { + return new RecordRobotModel(robot, { + recording: false // TODO (Annable): get from server? + }) + }) + + @computed + public get name(): string { + return this.robotModel.name + } + + @computed + public get address(): string { + return this.robotModel.address + } + + @computed + public get port(): number { + return this.robotModel.port + } +} diff --git a/src/client/components/record/styles.css b/src/client/components/record/styles.css new file mode 100644 index 00000000..a4375cc5 --- /dev/null +++ b/src/client/components/record/styles.css @@ -0,0 +1,6 @@ +.record { + flex-grow: 1; +} +.recordMenuBar { + flex: 1; +} diff --git a/src/client/components/record/view.tsx b/src/client/components/record/view.tsx new file mode 100644 index 00000000..3ccb56bc --- /dev/null +++ b/src/client/components/record/view.tsx @@ -0,0 +1,58 @@ +import { observer } from 'mobx-react' +import * as React from 'react' +import { ComponentType } from 'react' +import { Component } from 'react' +import { NUsightNetwork } from '../../network/nusight_network' +import { RecordController } from './controller' +import { RecordModel } from './model' +import * as styles from './styles.css' + +type Props = { + menu: ComponentType<{}> + controller: RecordController + model: RecordModel +} + +@observer +export class RecordView extends Component { + render() { + const { menu, controller, model } = this.props + const { robots } = model + return ( +
+ +
+ {robots.map(robot => ( +
+
Name: {robot.name}
+
Record: {robot.recording + ? + : } +
+
+ ))} +
+
+ ) + } + + public static of(menu: ComponentType<{}>, nusightNetwork: NUsightNetwork, model: RecordModel) { + const controller = RecordController.of(nusightNetwork) + return + } +} + +type RecordMenuBarProps = { + menu: ComponentType<{}> +} + +const RecordMenuBar = observer((props: RecordMenuBarProps) => { + const { menu: Menu } = props + return ( + +
    +     + ) +}) diff --git a/src/client/index.tsx b/src/client/index.tsx index 60fb6e44..6e2deedd 100644 --- a/src/client/index.tsx +++ b/src/client/index.tsx @@ -18,10 +18,12 @@ import { LocalisationNetwork } from './components/localisation/network' import { LocalisationView } from './components/localisation/view' import { withRobotSelectorMenuBar } from './components/menu_bar/view' import { NUClear } from './components/nuclear/view' +import { RecordView } from './components/record/view' import { Scatter } from './components/scatter_plot/view' import { Subsumption } from './components/subsumption/view' import { Vision } from './components/vision/view' import { NUsightNetwork } from './network/nusight_network' +import { RecordModel } from './components/record/model' // enable MobX strict mode useStrict(true) @@ -36,6 +38,9 @@ const appController = AppController.of() AppNetwork.of(nusightNetwork, appModel) const menu = withRobotSelectorMenuBar(appModel.robots, appController.toggleRobotEnabled) + +const recordModel = RecordModel.of(appModel) + ReactDOM.render( @@ -54,6 +59,7 @@ ReactDOM.render( + RecordView.of(menu, nusightNetwork, recordModel)}/> , diff --git a/src/client/network/nusight_network.ts b/src/client/network/nusight_network.ts index e44ffe81..1b972cac 100644 --- a/src/client/network/nusight_network.ts +++ b/src/client/network/nusight_network.ts @@ -6,6 +6,7 @@ import { WebSocketProxyNUClearNetClient } from '../nuclearnet/web_socket_proxy_n import { MessageTypePath } from './message_type_names' import { RobotModel } from '../components/robot/model' import { AppModel } from '../components/app/model' +import { WebSocketClient } from '../nuclearnet/web_socket_client' const HEADER_SIZE = 9 @@ -15,15 +16,24 @@ const HEADER_SIZE = 9 * instead create their own ComponentNetwork class which uses the Network helper class. */ export class NUsightNetwork { + private nextRequestTokenId: number + public constructor(private nuclearnetClient: NUClearNetClient, + private socket: WebSocketClient, private appModel: AppModel, private messageTypePath: MessageTypePath) { + this.nextRequestTokenId = 0 } public static of(appModel: AppModel) { const messageTypePath = MessageTypePath.of() const nuclearnetClient: NUClearNetClient = WebSocketProxyNUClearNetClient.of() - return new NUsightNetwork(nuclearnetClient, appModel, messageTypePath) + const uri = `${document.location.origin}/nusight` + const socket = WebSocketClient.of(uri, { + upgrade: false, + transports: ['websocket'], + }) + return new NUsightNetwork(nuclearnetClient, socket, appModel, messageTypePath) } public connect(opts: NUClearNetOptions): () => void { @@ -53,6 +63,19 @@ export class NUsightNetwork { public onNUClearLeave(cb: (peer: NUClearNetPeer) => void) { this.nuclearnetClient.onLeave(cb) } + + public record(peer: NUClearNetPeer): () => void { + const token = this.getNextRequestToken() + this.socket.send('record', peer, token) + return () => { + console.log(`unrecording for ${peer}`) + this.socket.send('unrecord', token) + } + } + + getNextRequestToken() { + return String(this.nextRequestTokenId++) + } } export interface MessageType { diff --git a/src/server/nuclearnet/fake_nuclearnet_client.ts b/src/server/nuclearnet/fake_nuclearnet_client.ts index 50e255ad..aa9b4524 100644 --- a/src/server/nuclearnet/fake_nuclearnet_client.ts +++ b/src/server/nuclearnet/fake_nuclearnet_client.ts @@ -7,6 +7,7 @@ import { NUClearEventListener } from '../../shared/nuclearnet/nuclearnet_client' import { NUClearPacketListener } from '../../shared/nuclearnet/nuclearnet_client' import { NUClearNetClient } from '../../shared/nuclearnet/nuclearnet_client' import { FakeNUClearNetServer } from './fake_nuclearnet_server' +import { hashType } from './fake_nuclearnet_server' /** * A fake NUClearNetClient, which collaborates with FakeNUClearNetServer. Designed to allow completely offline @@ -70,12 +71,13 @@ export class FakeNUClearNetClient implements NUClearNetClient { } public on(event: string, cb: NUClearPacketListener): () => void { + const hash = hashType(event).toString('hex') const listener = (packet: NUClearNetPacket) => { if (this.connected) { cb(packet) } } - this.events.on(event, listener) + this.events.on(hash, listener) return () => this.events.removeListener(event, listener) } @@ -96,7 +98,7 @@ export class FakeNUClearNetClient implements NUClearNetClient { this.events.emit('nuclear_leave', peer) } - public fakePacket(event: string, packet: NUClearNetPacket) { - this.events.emit(event, packet) + public fakePacket(hash: string, packet: NUClearNetPacket) { + this.events.emit(hash, packet) } } diff --git a/src/server/nuclearnet/fake_nuclearnet_server.ts b/src/server/nuclearnet/fake_nuclearnet_server.ts index 8a0d70ff..ad24ad5c 100644 --- a/src/server/nuclearnet/fake_nuclearnet_server.ts +++ b/src/server/nuclearnet/fake_nuclearnet_server.ts @@ -1,8 +1,8 @@ import * as EventEmitter from 'events' import { NUClearNetSend } from 'nuclearnet.js' +import * as XXH from 'xxhashjs' import { createSingletonFactory } from '../../shared/base/create_singleton_factory' import { FakeNUClearNetClient } from './fake_nuclearnet_client' -import * as XXH from 'xxhashjs' /** * A fake in-memory NUClearNet 'server' which routes messages between each FakeNUClearNetClient. @@ -57,34 +57,33 @@ export class FakeNUClearNetServer { } public send(client: FakeNUClearNetClient, opts: NUClearNetSend) { - if (typeof opts.type === 'string') { - const packet = { - peer: client.peer, - type: opts.type, - hash: this.hash(opts.type), - payload: opts.payload, - reliable: !!opts.reliable, - } + const hash: Buffer = typeof opts.type === 'string' ? hashType(opts.type) : opts.type + const packet = { + peer: client.peer, + type: typeof opts.type === 'string' ? opts.type : undefined, + hash, + payload: opts.payload, + reliable: !!opts.reliable, + } - /* - * This list intentially includes the sender unless explicitly targeting another peer. This matches the real - * NUClearNet behaviour. - */ - const targetClients = opts.target === undefined - ? this.clients - : this.clients.filter(otherClient => otherClient.peer.name === opts.target) + /* + * This list intentially includes the sender unless explicitly targeting another peer. This matches the real + * NUClearNet behaviour. + */ + const targetClients = opts.target === undefined + ? this.clients + : this.clients.filter(otherClient => otherClient.peer.name === opts.target) - for (const client of targetClients) { - client.fakePacket(opts.type, packet) - client.fakePacket('nuclear_packet', packet) - } + for (const client of targetClients) { + client.fakePacket(hash.toString('hex'), packet) + client.fakePacket('nuclear_packet', packet) } } +} - private hash(input: string): Buffer { - // Matches hashing implementation from NUClearNet - // See https://goo.gl/6NDPo2 - const hashString: string = XXH.h64(input, 0x4e55436c).toString(16) - return Buffer.from((hashString.match(/../g) as string[]).reverse().join(''), 'hex') - } +export function hashType(type: string): Buffer { + // Matches hashing implementation from NUClearNet + // See https://goo.gl/6NDPo2 + const hashString: string = XXH.h64(type, 0x4e55436c).toString(16) + return Buffer.from((hashString.match(/../g) as string[]).reverse().join(''), 'hex') } diff --git a/src/server/nusight_server.ts b/src/server/nusight_server.ts index 49006884..04ad2a47 100644 --- a/src/server/nusight_server.ts +++ b/src/server/nusight_server.ts @@ -26,6 +26,8 @@ class NUsightServerClient { private stopRecordingMap: Map void> public constructor(private socket: WebSocket, private clock: Clock, private nuclearnetClient: NUClearNetClient) { + this.stopRecordingMap = new Map() + this.socket.on('record', this.onRecord) this.socket.on('unrecord', this.onUnrecord) } @@ -34,15 +36,18 @@ class NUsightServerClient { return new NUsightServerClient(socket, NodeSystemClock, nuclearnetClient) } - public onRecord(peer: NUClearNetPeer, requestToken: string) { + public onRecord = (peer: NUClearNetPeer, requestToken: string) => { const recorder = NbsRecorder.of(peer, this.nuclearnetClient) - const stopRecording = recorder.record(`recordings/${this.clock.now()}.tbs`) + const filename = `${peer.name.replace(/[^A-Za-z0-9]/g, '_')}_${this.clock.now()}.tbs` + console.log('recording', peer, requestToken) + const stopRecording = recorder.record(`recordings/${filename}`) this.stopRecordingMap.set(requestToken, stopRecording) } - public onUnrecord(requestToken: string) { + public onUnrecord = (requestToken: string) => { const stopRecording = this.stopRecordingMap.get(requestToken) if (stopRecording) { + console.log('stop recording', requestToken) stopRecording() } } @@ -94,8 +99,8 @@ class NbsRecorder { const MAX_UINT32 = 0xFFFFFFFF; const highByte = ~~(time / MAX_UINT32); const lowByte = (time % MAX_UINT32) - highByte; - timeBuffer.writeUInt32BE(highByte, 0); - timeBuffer.writeUInt32BE(lowByte, 4); + timeBuffer.writeUInt32LE(highByte, 0); + timeBuffer.writeUInt32LE(lowByte, 4); const remainingByteLength = new Buffer(4); remainingByteLength.writeUInt32LE(timeBuffer.byteLength + packet.hash.byteLength + packet.payload.byteLength, 0) diff --git a/src/tests/networking_integration.tests.ts b/src/tests/networking_integration.tests.ts index 898ea5c5..85fc65cf 100644 --- a/src/tests/networking_integration.tests.ts +++ b/src/tests/networking_integration.tests.ts @@ -14,167 +14,167 @@ import VisionObject = message.vision.VisionObject import Overview = message.support.nubugger.Overview import { AppNetwork } from '../client/components/app/network' -describe('Networking Integration', () => { - let nuclearnetServer: FakeNUClearNetServer - let nusightNetwork: NUsightNetwork - let virtualRobots: VirtualRobots - let disconnectNusightNetwork: () => void - - beforeEach(() => { - nuclearnetServer = new FakeNUClearNetServer() - nusightNetwork = createNUsightNetwork() - disconnectNusightNetwork = nusightNetwork.connect({ name: 'nusight' }) - - virtualRobots = new VirtualRobots({ - robots: [ - new VirtualRobot( - new FakeNUClearNetClient(nuclearnetServer), - NodeSystemClock, - { - name: 'Robot #1', - simulators: [ - // TODO (Annable): Add vision and overview simulators when they exist - new SensorDataSimulator(), - ], - }, - ), - ], - }) - virtualRobots.connect() - }) - - function createNUsightNetwork() { - const appModel = AppModel.of() - const nuclearnetClient = new FakeNUClearNetClient(nuclearnetServer) - const messageTypePath = new MessageTypePath() - const nusightNetwork = new NUsightNetwork(nuclearnetClient, appModel, messageTypePath) - AppNetwork.of(nusightNetwork, appModel) - return nusightNetwork - } - - describe('a single networked component', () => { - let network: Network - - beforeEach(() => { - network = new Network(nusightNetwork) - }) - - it('receives a Sensors message after subscribing and a robot sending it', () => { - const onSensors = jest.fn() - network.on(Sensors, onSensors) - - virtualRobots.simulate() - - expect(onSensors).toHaveBeenCalledWith(expect.objectContaining({ name: 'Robot #1' }), expect.any(Sensors)) - expect(onSensors).toHaveBeenCalledTimes(1) - }) - - it('does not receive any messages after unsubscribing', () => { - const onSensors1 = jest.fn() - const onSensors2 = jest.fn() - network.on(Sensors, onSensors1) - network.on(Sensors, onSensors2) - - network.off() - - virtualRobots.simulate() - - expect(onSensors1).not.toHaveBeenCalled() - expect(onSensors2).not.toHaveBeenCalled() - }) - - it('does not receive message on specific unsubscribed callback', () => { - const onSensors1 = jest.fn() - const onSensors2 = jest.fn() - const off1 = network.on(Sensors, onSensors1) - network.on(Sensors, onSensors2) - - off1() - - virtualRobots.simulate() - - expect(onSensors1).not.toHaveBeenCalled() - expect(onSensors2).toHaveBeenCalledWith(expect.objectContaining({ name: 'Robot #1' }), expect.any(Sensors)) - }) - }) - - describe('sessions', () => { - let network: Network - - beforeEach(() => { - network = new Network(nusightNetwork) - }) - - it('handles reconnects', () => { - const onSensors = jest.fn() - network.on(Sensors, onSensors) - - disconnectNusightNetwork() - - nusightNetwork.connect({ name: 'nusight' }) - - virtualRobots.simulate() - - expect(onSensors).toHaveBeenCalledWith(expect.objectContaining({ name: 'Robot #1' }), expect.any(Sensors)) - }) - - it('handles multiple sessions simutaneously', () => { - const nusightNetwork2 = createNUsightNetwork() - nusightNetwork2.connect({ name: 'nusight' }) - const network2 = new Network(nusightNetwork2) - - const onSensors1 = jest.fn() - network.on(Sensors, onSensors1) - - const onSensors2 = jest.fn() - network2.on(Sensors, onSensors2) - - virtualRobots.simulate() - - expect(onSensors1).toHaveBeenCalledWith(expect.objectContaining({ name: 'Robot #1' }), expect.any(Sensors)) - expect(onSensors2).toHaveBeenCalledWith(expect.objectContaining({ name: 'Robot #1' }), expect.any(Sensors)) - }) - }) - - describe('multiple networked components', () => { - let localisationNetwork: Network - let visionNetwork: Network - let dashboardNetwork: Network - - beforeEach(() => { - localisationNetwork = new Network(nusightNetwork) - visionNetwork = new Network(nusightNetwork) - dashboardNetwork = new Network(nusightNetwork) - }) - - it('subscribes and unsubscribes as expected when switching between components', () => { - const onSensors = jest.fn() - localisationNetwork.on(Sensors, onSensors) - - virtualRobots.simulate() - - expect(onSensors).toHaveBeenCalledTimes(1) - - localisationNetwork.off() - - const onVisionObject = jest.fn() - visionNetwork.on(VisionObject, onVisionObject) - - virtualRobots.simulate() - - expect(onVisionObject).toHaveBeenCalledTimes(0) - expect(onSensors).toHaveBeenCalledTimes(1) - - visionNetwork.off() - - const onOverview = jest.fn() - dashboardNetwork.on(Overview, onOverview) - - expect(onOverview).toHaveBeenCalledTimes(0) - expect(onVisionObject).toHaveBeenCalledTimes(0) - expect(onSensors).toHaveBeenCalledTimes(1) - - dashboardNetwork.off() - }) - }) -}) +// describe('Networking Integration', () => { +// let nuclearnetServer: FakeNUClearNetServer +// let nusightNetwork: NUsightNetwork +// let virtualRobots: VirtualRobots +// let disconnectNusightNetwork: () => void +// +// beforeEach(() => { +// nuclearnetServer = new FakeNUClearNetServer() +// nusightNetwork = createNUsightNetwork() +// disconnectNusightNetwork = nusightNetwork.connect({ name: 'nusight' }) +// +// virtualRobots = new VirtualRobots({ +// robots: [ +// new VirtualRobot( +// new FakeNUClearNetClient(nuclearnetServer), +// NodeSystemClock, +// { +// name: 'Robot #1', +// simulators: [ +// // TODO (Annable): Add vision and overview simulators when they exist +// new SensorDataSimulator(), +// ], +// }, +// ), +// ], +// }) +// virtualRobots.connect() +// }) +// +// function createNUsightNetwork() { +// const appModel = AppModel.of() +// const nuclearnetClient = new FakeNUClearNetClient(nuclearnetServer) +// const messageTypePath = new MessageTypePath() +// const nusightNetwork = new NUsightNetwork(nuclearnetClient, appModel, messageTypePath) +// AppNetwork.of(nusightNetwork, appModel) +// return nusightNetwork +// } +// +// describe('a single networked component', () => { +// let network: Network +// +// beforeEach(() => { +// network = new Network(nusightNetwork) +// }) +// +// it('receives a Sensors message after subscribing and a robot sending it', () => { +// const onSensors = jest.fn() +// network.on(Sensors, onSensors) +// +// virtualRobots.simulate() +// +// expect(onSensors).toHaveBeenCalledWith(expect.objectContaining({ name: 'Robot #1' }), expect.any(Sensors)) +// expect(onSensors).toHaveBeenCalledTimes(1) +// }) +// +// it('does not receive any messages after unsubscribing', () => { +// const onSensors1 = jest.fn() +// const onSensors2 = jest.fn() +// network.on(Sensors, onSensors1) +// network.on(Sensors, onSensors2) +// +// network.off() +// +// virtualRobots.simulate() +// +// expect(onSensors1).not.toHaveBeenCalled() +// expect(onSensors2).not.toHaveBeenCalled() +// }) +// +// it('does not receive message on specific unsubscribed callback', () => { +// const onSensors1 = jest.fn() +// const onSensors2 = jest.fn() +// const off1 = network.on(Sensors, onSensors1) +// network.on(Sensors, onSensors2) +// +// off1() +// +// virtualRobots.simulate() +// +// expect(onSensors1).not.toHaveBeenCalled() +// expect(onSensors2).toHaveBeenCalledWith(expect.objectContaining({ name: 'Robot #1' }), expect.any(Sensors)) +// }) +// }) +// +// describe('sessions', () => { +// let network: Network +// +// beforeEach(() => { +// network = new Network(nusightNetwork) +// }) +// +// it('handles reconnects', () => { +// const onSensors = jest.fn() +// network.on(Sensors, onSensors) +// +// disconnectNusightNetwork() +// +// nusightNetwork.connect({ name: 'nusight' }) +// +// virtualRobots.simulate() +// +// expect(onSensors).toHaveBeenCalledWith(expect.objectContaining({ name: 'Robot #1' }), expect.any(Sensors)) +// }) +// +// it('handles multiple sessions simutaneously', () => { +// const nusightNetwork2 = createNUsightNetwork() +// nusightNetwork2.connect({ name: 'nusight' }) +// const network2 = new Network(nusightNetwork2) +// +// const onSensors1 = jest.fn() +// network.on(Sensors, onSensors1) +// +// const onSensors2 = jest.fn() +// network2.on(Sensors, onSensors2) +// +// virtualRobots.simulate() +// +// expect(onSensors1).toHaveBeenCalledWith(expect.objectContaining({ name: 'Robot #1' }), expect.any(Sensors)) +// expect(onSensors2).toHaveBeenCalledWith(expect.objectContaining({ name: 'Robot #1' }), expect.any(Sensors)) +// }) +// }) +// +// describe('multiple networked components', () => { +// let localisationNetwork: Network +// let visionNetwork: Network +// let dashboardNetwork: Network +// +// beforeEach(() => { +// localisationNetwork = new Network(nusightNetwork) +// visionNetwork = new Network(nusightNetwork) +// dashboardNetwork = new Network(nusightNetwork) +// }) +// +// it('subscribes and unsubscribes as expected when switching between components', () => { +// const onSensors = jest.fn() +// localisationNetwork.on(Sensors, onSensors) +// +// virtualRobots.simulate() +// +// expect(onSensors).toHaveBeenCalledTimes(1) +// +// localisationNetwork.off() +// +// const onVisionObject = jest.fn() +// visionNetwork.on(VisionObject, onVisionObject) +// +// virtualRobots.simulate() +// +// expect(onVisionObject).toHaveBeenCalledTimes(0) +// expect(onSensors).toHaveBeenCalledTimes(1) +// +// visionNetwork.off() +// +// const onOverview = jest.fn() +// dashboardNetwork.on(Overview, onOverview) +// +// expect(onOverview).toHaveBeenCalledTimes(0) +// expect(onVisionObject).toHaveBeenCalledTimes(0) +// expect(onSensors).toHaveBeenCalledTimes(1) +// +// dashboardNetwork.off() +// }) +// }) +// }) diff --git a/src/validator/validate.ts b/src/validator/validate.ts new file mode 100644 index 00000000..ca41b51d --- /dev/null +++ b/src/validator/validate.ts @@ -0,0 +1,76 @@ +import * as fs from 'fs' +import * as minimist from 'minimist' + +const HEADER = Buffer.from([0xE2, 0x98, 0xA2]) +const HEADER_SIZE = HEADER.byteLength +const REMAINING_LENGTH_SIZE = 4 +const TIMESTAMP_SIZE = 8 +const HASH_SIZE = 8 + +function main() { + const args = minimist(process.argv.slice(2)) + const filename = args.filename + const buffer = fs.readFileSync(filename) + + const packets = readPackets(buffer) + const types = packets.reduce((map: Map, packet) => { + map.set(packet.hash, (map.get(packet.hash) || 0) + 1) + return map + }, new Map()) + console.log(types) + + console.log(`Num packets: ${packets.length}`) + console.log('Types:') + console.log(Array.from(types.entries()).map(([hash, occurances]) => `${hash} = ${occurances}`).join('\n')) +} + +function readPackets(buffer: Buffer) { + const packets = [] + let index = 0 + do { + const packet = readPacket(buffer, index) + packets.push(packet) + index = packet.lastIndex + } while (index < buffer.byteLength) + return packets +} + +function readPacket(buffer: Buffer, offset: number) { + let index = findNextHeader(buffer, offset) + // console.log(`Header found at index ${index}`) + index += HEADER_SIZE + + const remainingByteLength = buffer.readUInt32LE(index) + // console.log(`Remaining byte length: ${remainingByteLength}`) + index += REMAINING_LENGTH_SIZE + + const timeHighByte = buffer.readUInt32LE(index) + const timeLowByte = buffer.readUInt32LE(index + 4) + const timestamp = timeHighByte + timeLowByte // TODO (Annable): actually convert + // console.log(`Timestamp: ${timeHighByte} ${timeLowByte} ${timestamp}`) + index += TIMESTAMP_SIZE + + const hash = buffer.slice(index, index + HASH_SIZE).toString('hex') + // console.log(`Hash: ${hash}`) + index += HASH_SIZE + + const payloadByteLength = remainingByteLength - TIMESTAMP_SIZE - HASH_SIZE + // console.log(`Data exists: ${payloadByteLength} ${buffer.byteLength >= index + payloadByteLength}`) + const payload = buffer.slice(index, payloadByteLength) + index += payloadByteLength + + return { remainingByteLength, timestamp, hash, payloadByteLength, payload, lastIndex: index } +} + +function findNextHeader(buffer: Buffer, offset: number) { + for (var i = offset; i < buffer.byteLength; i++) { + if (buffer[i] === HEADER[0] && buffer[i + 1] === HEADER[1] && buffer[i + 2] === HEADER[2]) { + return i + } + } + return -1 +} + +if (require.main === module) { + main() +} From 9767415fe3e4802c763912d5b4c7d91a309f8825 Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Sun, 16 Jul 2017 21:50:57 +1000 Subject: [PATCH 05/24] . --- package.json | 2 +- .../localisation/darwin_robot/model.ts | 1 - src/tests/networking_integration.tests.ts | 328 +++++++++--------- tsconfig.json | 3 +- webpack.config.ts | 18 +- yarn.lock | 6 +- 6 files changed, 179 insertions(+), 179 deletions(-) diff --git a/package.json b/package.json index 97f5f40d..e7d9a98f 100644 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "minimist": "^1.2.0", "mobx": "^3.2.0", "mobx-react": "^4.2.2", - "nuclearnet.js": "^1.3.0", + "nuclearnet.js": "^1.4.0", "protobufjs": "^6.7.3", "react": "^15.6.1", "react-dom": "^15.6.1", diff --git a/src/client/components/localisation/darwin_robot/model.ts b/src/client/components/localisation/darwin_robot/model.ts index dd727775..b05f9c90 100644 --- a/src/client/components/localisation/darwin_robot/model.ts +++ b/src/client/components/localisation/darwin_robot/model.ts @@ -4,7 +4,6 @@ import { memoize } from '../../../base/memoize' import { RobotModel } from '../../robot/model' import { Vector3 } from '../model' import { Quaternion } from '../model' -import { memoize } from '../../../base/memoize' export class LocalisationRobotModel { @observable private model: RobotModel diff --git a/src/tests/networking_integration.tests.ts b/src/tests/networking_integration.tests.ts index 85fc65cf..898ea5c5 100644 --- a/src/tests/networking_integration.tests.ts +++ b/src/tests/networking_integration.tests.ts @@ -14,167 +14,167 @@ import VisionObject = message.vision.VisionObject import Overview = message.support.nubugger.Overview import { AppNetwork } from '../client/components/app/network' -// describe('Networking Integration', () => { -// let nuclearnetServer: FakeNUClearNetServer -// let nusightNetwork: NUsightNetwork -// let virtualRobots: VirtualRobots -// let disconnectNusightNetwork: () => void -// -// beforeEach(() => { -// nuclearnetServer = new FakeNUClearNetServer() -// nusightNetwork = createNUsightNetwork() -// disconnectNusightNetwork = nusightNetwork.connect({ name: 'nusight' }) -// -// virtualRobots = new VirtualRobots({ -// robots: [ -// new VirtualRobot( -// new FakeNUClearNetClient(nuclearnetServer), -// NodeSystemClock, -// { -// name: 'Robot #1', -// simulators: [ -// // TODO (Annable): Add vision and overview simulators when they exist -// new SensorDataSimulator(), -// ], -// }, -// ), -// ], -// }) -// virtualRobots.connect() -// }) -// -// function createNUsightNetwork() { -// const appModel = AppModel.of() -// const nuclearnetClient = new FakeNUClearNetClient(nuclearnetServer) -// const messageTypePath = new MessageTypePath() -// const nusightNetwork = new NUsightNetwork(nuclearnetClient, appModel, messageTypePath) -// AppNetwork.of(nusightNetwork, appModel) -// return nusightNetwork -// } -// -// describe('a single networked component', () => { -// let network: Network -// -// beforeEach(() => { -// network = new Network(nusightNetwork) -// }) -// -// it('receives a Sensors message after subscribing and a robot sending it', () => { -// const onSensors = jest.fn() -// network.on(Sensors, onSensors) -// -// virtualRobots.simulate() -// -// expect(onSensors).toHaveBeenCalledWith(expect.objectContaining({ name: 'Robot #1' }), expect.any(Sensors)) -// expect(onSensors).toHaveBeenCalledTimes(1) -// }) -// -// it('does not receive any messages after unsubscribing', () => { -// const onSensors1 = jest.fn() -// const onSensors2 = jest.fn() -// network.on(Sensors, onSensors1) -// network.on(Sensors, onSensors2) -// -// network.off() -// -// virtualRobots.simulate() -// -// expect(onSensors1).not.toHaveBeenCalled() -// expect(onSensors2).not.toHaveBeenCalled() -// }) -// -// it('does not receive message on specific unsubscribed callback', () => { -// const onSensors1 = jest.fn() -// const onSensors2 = jest.fn() -// const off1 = network.on(Sensors, onSensors1) -// network.on(Sensors, onSensors2) -// -// off1() -// -// virtualRobots.simulate() -// -// expect(onSensors1).not.toHaveBeenCalled() -// expect(onSensors2).toHaveBeenCalledWith(expect.objectContaining({ name: 'Robot #1' }), expect.any(Sensors)) -// }) -// }) -// -// describe('sessions', () => { -// let network: Network -// -// beforeEach(() => { -// network = new Network(nusightNetwork) -// }) -// -// it('handles reconnects', () => { -// const onSensors = jest.fn() -// network.on(Sensors, onSensors) -// -// disconnectNusightNetwork() -// -// nusightNetwork.connect({ name: 'nusight' }) -// -// virtualRobots.simulate() -// -// expect(onSensors).toHaveBeenCalledWith(expect.objectContaining({ name: 'Robot #1' }), expect.any(Sensors)) -// }) -// -// it('handles multiple sessions simutaneously', () => { -// const nusightNetwork2 = createNUsightNetwork() -// nusightNetwork2.connect({ name: 'nusight' }) -// const network2 = new Network(nusightNetwork2) -// -// const onSensors1 = jest.fn() -// network.on(Sensors, onSensors1) -// -// const onSensors2 = jest.fn() -// network2.on(Sensors, onSensors2) -// -// virtualRobots.simulate() -// -// expect(onSensors1).toHaveBeenCalledWith(expect.objectContaining({ name: 'Robot #1' }), expect.any(Sensors)) -// expect(onSensors2).toHaveBeenCalledWith(expect.objectContaining({ name: 'Robot #1' }), expect.any(Sensors)) -// }) -// }) -// -// describe('multiple networked components', () => { -// let localisationNetwork: Network -// let visionNetwork: Network -// let dashboardNetwork: Network -// -// beforeEach(() => { -// localisationNetwork = new Network(nusightNetwork) -// visionNetwork = new Network(nusightNetwork) -// dashboardNetwork = new Network(nusightNetwork) -// }) -// -// it('subscribes and unsubscribes as expected when switching between components', () => { -// const onSensors = jest.fn() -// localisationNetwork.on(Sensors, onSensors) -// -// virtualRobots.simulate() -// -// expect(onSensors).toHaveBeenCalledTimes(1) -// -// localisationNetwork.off() -// -// const onVisionObject = jest.fn() -// visionNetwork.on(VisionObject, onVisionObject) -// -// virtualRobots.simulate() -// -// expect(onVisionObject).toHaveBeenCalledTimes(0) -// expect(onSensors).toHaveBeenCalledTimes(1) -// -// visionNetwork.off() -// -// const onOverview = jest.fn() -// dashboardNetwork.on(Overview, onOverview) -// -// expect(onOverview).toHaveBeenCalledTimes(0) -// expect(onVisionObject).toHaveBeenCalledTimes(0) -// expect(onSensors).toHaveBeenCalledTimes(1) -// -// dashboardNetwork.off() -// }) -// }) -// }) +describe('Networking Integration', () => { + let nuclearnetServer: FakeNUClearNetServer + let nusightNetwork: NUsightNetwork + let virtualRobots: VirtualRobots + let disconnectNusightNetwork: () => void + + beforeEach(() => { + nuclearnetServer = new FakeNUClearNetServer() + nusightNetwork = createNUsightNetwork() + disconnectNusightNetwork = nusightNetwork.connect({ name: 'nusight' }) + + virtualRobots = new VirtualRobots({ + robots: [ + new VirtualRobot( + new FakeNUClearNetClient(nuclearnetServer), + NodeSystemClock, + { + name: 'Robot #1', + simulators: [ + // TODO (Annable): Add vision and overview simulators when they exist + new SensorDataSimulator(), + ], + }, + ), + ], + }) + virtualRobots.connect() + }) + + function createNUsightNetwork() { + const appModel = AppModel.of() + const nuclearnetClient = new FakeNUClearNetClient(nuclearnetServer) + const messageTypePath = new MessageTypePath() + const nusightNetwork = new NUsightNetwork(nuclearnetClient, appModel, messageTypePath) + AppNetwork.of(nusightNetwork, appModel) + return nusightNetwork + } + + describe('a single networked component', () => { + let network: Network + + beforeEach(() => { + network = new Network(nusightNetwork) + }) + + it('receives a Sensors message after subscribing and a robot sending it', () => { + const onSensors = jest.fn() + network.on(Sensors, onSensors) + + virtualRobots.simulate() + + expect(onSensors).toHaveBeenCalledWith(expect.objectContaining({ name: 'Robot #1' }), expect.any(Sensors)) + expect(onSensors).toHaveBeenCalledTimes(1) + }) + + it('does not receive any messages after unsubscribing', () => { + const onSensors1 = jest.fn() + const onSensors2 = jest.fn() + network.on(Sensors, onSensors1) + network.on(Sensors, onSensors2) + + network.off() + + virtualRobots.simulate() + + expect(onSensors1).not.toHaveBeenCalled() + expect(onSensors2).not.toHaveBeenCalled() + }) + + it('does not receive message on specific unsubscribed callback', () => { + const onSensors1 = jest.fn() + const onSensors2 = jest.fn() + const off1 = network.on(Sensors, onSensors1) + network.on(Sensors, onSensors2) + + off1() + + virtualRobots.simulate() + + expect(onSensors1).not.toHaveBeenCalled() + expect(onSensors2).toHaveBeenCalledWith(expect.objectContaining({ name: 'Robot #1' }), expect.any(Sensors)) + }) + }) + + describe('sessions', () => { + let network: Network + + beforeEach(() => { + network = new Network(nusightNetwork) + }) + + it('handles reconnects', () => { + const onSensors = jest.fn() + network.on(Sensors, onSensors) + + disconnectNusightNetwork() + + nusightNetwork.connect({ name: 'nusight' }) + + virtualRobots.simulate() + + expect(onSensors).toHaveBeenCalledWith(expect.objectContaining({ name: 'Robot #1' }), expect.any(Sensors)) + }) + + it('handles multiple sessions simutaneously', () => { + const nusightNetwork2 = createNUsightNetwork() + nusightNetwork2.connect({ name: 'nusight' }) + const network2 = new Network(nusightNetwork2) + + const onSensors1 = jest.fn() + network.on(Sensors, onSensors1) + + const onSensors2 = jest.fn() + network2.on(Sensors, onSensors2) + + virtualRobots.simulate() + + expect(onSensors1).toHaveBeenCalledWith(expect.objectContaining({ name: 'Robot #1' }), expect.any(Sensors)) + expect(onSensors2).toHaveBeenCalledWith(expect.objectContaining({ name: 'Robot #1' }), expect.any(Sensors)) + }) + }) + + describe('multiple networked components', () => { + let localisationNetwork: Network + let visionNetwork: Network + let dashboardNetwork: Network + + beforeEach(() => { + localisationNetwork = new Network(nusightNetwork) + visionNetwork = new Network(nusightNetwork) + dashboardNetwork = new Network(nusightNetwork) + }) + + it('subscribes and unsubscribes as expected when switching between components', () => { + const onSensors = jest.fn() + localisationNetwork.on(Sensors, onSensors) + + virtualRobots.simulate() + + expect(onSensors).toHaveBeenCalledTimes(1) + + localisationNetwork.off() + + const onVisionObject = jest.fn() + visionNetwork.on(VisionObject, onVisionObject) + + virtualRobots.simulate() + + expect(onVisionObject).toHaveBeenCalledTimes(0) + expect(onSensors).toHaveBeenCalledTimes(1) + + visionNetwork.off() + + const onOverview = jest.fn() + dashboardNetwork.on(Overview, onOverview) + + expect(onOverview).toHaveBeenCalledTimes(0) + expect(onVisionObject).toHaveBeenCalledTimes(0) + expect(onSensors).toHaveBeenCalledTimes(1) + + dashboardNetwork.off() + }) + }) +}) diff --git a/tsconfig.json b/tsconfig.json index 5755b861..3b49aa99 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -26,7 +26,8 @@ ], "exclude": [ "build", - "node_modules" + "node_modules", + "**/*.tests.ts" ], "awesomeTypescriptLoaderOptions": { "useWebpackText": true, diff --git a/webpack.config.ts b/webpack.config.ts index a2a1411e..f4d5c6d2 100644 --- a/webpack.config.ts +++ b/webpack.config.ts @@ -47,11 +47,11 @@ export default { { test: /\.tsx?$/, use: isProduction - ? 'awesome-typescript-loader?module=es6' - : [ - 'react-hot-loader', - 'awesome-typescript-loader', - ], + ? 'awesome-typescript-loader?module=es6' + : [ + 'react-hot-loader', + 'awesome-typescript-loader', + ], }, // local css { @@ -78,15 +78,15 @@ export default { }), }, /* - External libraries generally do not support css modules so the selector mangling will break external components. - This separate simplified loader is used for anything within the node_modules folder instead. - */ + External libraries generally do not support css modules so the selector mangling will break external components. + This separate simplified loader is used for anything within the node_modules folder instead. + */ { test: /\.css$/, include: [ path.resolve(__dirname, 'node_modules'), ], - use: [ 'style-loader', 'css-loader' ], + use: ['style-loader', 'css-loader'], }, { test: /\.svg$/, diff --git a/yarn.lock b/yarn.lock index 019e9d69..c22601dc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4475,9 +4475,9 @@ nth-check@~1.0.1: dependencies: boolbase "~1.0.0" -nuclearnet.js@^1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/nuclearnet.js/-/nuclearnet.js-1.3.0.tgz#aefae50336b1337ed211edcef4e0774e8bf4ee74" +nuclearnet.js@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/nuclearnet.js/-/nuclearnet.js-1.4.0.tgz#f0390e72f3657c2b7066b3b807a4232485548ba5" dependencies: bindings "^1.2.1" nan "^2.0.0" From d1e39b95a27467dd969b7e699be579c70e9213c5 Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Sun, 16 Jul 2017 22:43:26 +1000 Subject: [PATCH 06/24] . --- src/client/components/record/view.tsx | 6 +--- .../nuclearnet/fake_nuclearnet_client.ts | 9 ++++- .../nuclearnet/fake_nuclearnet_server.ts | 8 ++--- .../tests/fake_nuclearnet_client.tests.ts | 36 +++++++++++++++++++ 4 files changed, 49 insertions(+), 10 deletions(-) diff --git a/src/client/components/record/view.tsx b/src/client/components/record/view.tsx index 3ccb56bc..38138759 100644 --- a/src/client/components/record/view.tsx +++ b/src/client/components/record/view.tsx @@ -50,9 +50,5 @@ type RecordMenuBarProps = { const RecordMenuBar = observer((props: RecordMenuBarProps) => { const { menu: Menu } = props - return ( - -
      -       - ) + return }) diff --git a/src/server/nuclearnet/fake_nuclearnet_client.ts b/src/server/nuclearnet/fake_nuclearnet_client.ts index aa9b4524..58d3af6a 100644 --- a/src/server/nuclearnet/fake_nuclearnet_client.ts +++ b/src/server/nuclearnet/fake_nuclearnet_client.ts @@ -82,7 +82,13 @@ export class FakeNUClearNetClient implements NUClearNetClient { } public onPacket(cb: NUClearPacketListener): () => void { - return this.on('nuclear_packet', cb) + const listener = (packet: NUClearNetPacket) => { + if (this.connected) { + cb(packet) + } + } + this.events.on('nuclear_packet', listener) + return () => this.events.removeListener('nuclear_packet', listener) } public send(options: NUClearNetSend): void { @@ -100,5 +106,6 @@ export class FakeNUClearNetClient implements NUClearNetClient { public fakePacket(hash: string, packet: NUClearNetPacket) { this.events.emit(hash, packet) + this.events.emit('nuclear_packet', packet) } } diff --git a/src/server/nuclearnet/fake_nuclearnet_server.ts b/src/server/nuclearnet/fake_nuclearnet_server.ts index ad24ad5c..6a15a9a5 100644 --- a/src/server/nuclearnet/fake_nuclearnet_server.ts +++ b/src/server/nuclearnet/fake_nuclearnet_server.ts @@ -8,7 +8,7 @@ import { FakeNUClearNetClient } from './fake_nuclearnet_client' * A fake in-memory NUClearNet 'server' which routes messages between each FakeNUClearNetClient. * * All messages are 'reliable' in that nothing is intentially dropped. - * Targetted messages are supported. + * Targeted messages are supported. */ export class FakeNUClearNetServer { private events: EventEmitter @@ -67,16 +67,16 @@ export class FakeNUClearNetServer { } /* - * This list intentially includes the sender unless explicitly targeting another peer. This matches the real + * This list intentionally includes the sender unless explicitly targeting another peer. This matches the real * NUClearNet behaviour. */ const targetClients = opts.target === undefined ? this.clients : this.clients.filter(otherClient => otherClient.peer.name === opts.target) + const hashString = hash.toString('hex') for (const client of targetClients) { - client.fakePacket(hash.toString('hex'), packet) - client.fakePacket('nuclear_packet', packet) + client.fakePacket(hashString, packet) } } } diff --git a/src/server/nuclearnet/tests/fake_nuclearnet_client.tests.ts b/src/server/nuclearnet/tests/fake_nuclearnet_client.tests.ts index a4d3202f..edfc2e2b 100644 --- a/src/server/nuclearnet/tests/fake_nuclearnet_client.tests.ts +++ b/src/server/nuclearnet/tests/fake_nuclearnet_client.tests.ts @@ -178,6 +178,42 @@ describe('FakeNUClearNetClient', () => { })) }) + it('receives messages sent from other clients', () => { + bob.connect({ name: 'bob' }) + alice.connect({ name: 'alice' }) + eve.connect({ name: 'eve' }) + + const bobOnPacket = jest.fn() + bob.onPacket(bobOnPacket) + + const aliceOnPacket = jest.fn() + alice.onPacket(aliceOnPacket) + + const eveOnSensors = jest.fn() + eve.on('sensors', eveOnSensors) + + const payload = new Buffer(8) + eve.send({ type: 'sensors', payload }) + + expect(bobOnPacket).toHaveBeenCalledTimes(1) + expect(bobOnPacket).toHaveBeenLastCalledWith(expect.objectContaining({ + payload, + peer: expect.objectContaining({ name: 'eve' }), + })) + + expect(aliceOnPacket).toHaveBeenCalledTimes(1) + expect(aliceOnPacket).toHaveBeenLastCalledWith(expect.objectContaining({ + payload, + peer: expect.objectContaining({ name: 'eve' }), + })) + + expect(eveOnSensors).toHaveBeenCalledTimes(1) + expect(eveOnSensors).toHaveBeenLastCalledWith(expect.objectContaining({ + payload, + peer: expect.objectContaining({ name: 'eve' }), + })) + }) + it('only receives targetted messages if they are the target', () => { bob.connect({ name: 'bob' }) alice.connect({ name: 'alice' }) From 5b118fd21fa148fb905532f10cf8130e21d19489 Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Mon, 17 Jul 2017 00:55:02 +1000 Subject: [PATCH 07/24] Actually record the payload lol --- src/server/nusight_server.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/server/nusight_server.ts b/src/server/nusight_server.ts index 04ad2a47..73e0d870 100644 --- a/src/server/nusight_server.ts +++ b/src/server/nusight_server.ts @@ -110,6 +110,7 @@ class NbsRecorder { this.file.write(remainingByteLength); this.file.write(timeBuffer); this.file.write(packet.hash); + this.file.write(packet.payload); } private arePeersEqual(peerA: NUClearNetPeer, peerB: NUClearNetPeer) { From 2eee3439101e2abba7cf1cbb8f35be4eb80b3174 Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Mon, 17 Jul 2017 01:30:51 +1000 Subject: [PATCH 08/24] swap timestamp order --- src/server/nusight_server.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/nusight_server.ts b/src/server/nusight_server.ts index 73e0d870..651078cc 100644 --- a/src/server/nusight_server.ts +++ b/src/server/nusight_server.ts @@ -99,8 +99,8 @@ class NbsRecorder { const MAX_UINT32 = 0xFFFFFFFF; const highByte = ~~(time / MAX_UINT32); const lowByte = (time % MAX_UINT32) - highByte; - timeBuffer.writeUInt32LE(highByte, 0); - timeBuffer.writeUInt32LE(lowByte, 4); + timeBuffer.writeUInt32LE(lowByte, 0); + timeBuffer.writeUInt32LE(highByte, 4); const remainingByteLength = new Buffer(4); remainingByteLength.writeUInt32LE(timeBuffer.byteLength + packet.hash.byteLength + packet.payload.byteLength, 0) From 32e2b5c01d8f0162fb724b6aadf13eab3685ee97 Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Mon, 17 Jul 2017 01:47:15 +1000 Subject: [PATCH 09/24] s/tbs/nbs --- src/server/nusight_server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/nusight_server.ts b/src/server/nusight_server.ts index 651078cc..18ea9768 100644 --- a/src/server/nusight_server.ts +++ b/src/server/nusight_server.ts @@ -38,7 +38,7 @@ class NUsightServerClient { public onRecord = (peer: NUClearNetPeer, requestToken: string) => { const recorder = NbsRecorder.of(peer, this.nuclearnetClient) - const filename = `${peer.name.replace(/[^A-Za-z0-9]/g, '_')}_${this.clock.now()}.tbs` + const filename = `${peer.name.replace(/[^A-Za-z0-9]/g, '_')}_${this.clock.now()}.nbs` console.log('recording', peer, requestToken) const stopRecording = recorder.record(`recordings/${filename}`) this.stopRecordingMap.set(requestToken, stopRecording) From 6c666db82b93cd4b7c7cd6c605cd4ff65fb85094 Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Mon, 17 Jul 2017 10:53:40 +1000 Subject: [PATCH 10/24] time --- .../web_socket_proxy_nuclearnet_server.ts | 4 ++-- src/server/nusight_server.ts | 4 ++-- src/server/time/clock.ts | 5 +++-- src/server/time/node_clock.ts | 17 ++++++++++++----- src/simulators/sensor_data_simulator.ts | 6 +++--- src/simulators/virtual_robot.ts | 2 +- 6 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/server/nuclearnet/web_socket_proxy_nuclearnet_server.ts b/src/server/nuclearnet/web_socket_proxy_nuclearnet_server.ts index b12dc706..71979026 100644 --- a/src/server/nuclearnet/web_socket_proxy_nuclearnet_server.ts +++ b/src/server/nuclearnet/web_socket_proxy_nuclearnet_server.ts @@ -122,7 +122,7 @@ class PacketProcessor { // The maximum number of packets of a unique type to send before receiving acknowledgements. private limit: number - // The number of milliseconds before giving up on an acknowledge + // The number of seconds before giving up on an acknowledge private timeout: number constructor(private socket: WebSocket, @@ -134,7 +134,7 @@ class PacketProcessor { } public static of(socket: WebSocket) { - return new PacketProcessor(socket, NodeSystemClock, { limit: 1, timeout: 5000 }) + return new PacketProcessor(socket, NodeSystemClock, { limit: 1, timeout: 5 }) } public onPacket(event: string, packet: NUClearNetPacket) { diff --git a/src/server/nusight_server.ts b/src/server/nusight_server.ts index 18ea9768..753c8920 100644 --- a/src/server/nusight_server.ts +++ b/src/server/nusight_server.ts @@ -84,7 +84,7 @@ class NbsRecorder { } // NBS File Format: - // 3 Bytes - NUClear radiation symbol header, useful for synchronisation when attaching to a existing stream. + // 3 Bytes - NUClear radiation symbol header, useful for synchronisation when attaching to an existing stream. // 4 Bytes - The remaining packet length i.e. 16 bytes + N payload bytes // 8 Bytes - 64bit timestamp in microseconds. Note: this is not necessarily a unix timestamp. // 8 Bytes - 64bit bit hash of the message type. @@ -93,7 +93,7 @@ class NbsRecorder { const header = Buffer.from([0xE2, 0x98, 0xA2]) // NUClear radiation symbol. // 64bit timestamp in microseconds. - const time = this.clock.now() * 1000; + const time = this.clock.performanceNow() * 1e6; const timeBuffer = new Buffer(8); // Convert double into two 32 bit integers. const MAX_UINT32 = 0xFFFFFFFF; diff --git a/src/server/time/clock.ts b/src/server/time/clock.ts index 04507fd7..db9ee7e1 100644 --- a/src/server/time/clock.ts +++ b/src/server/time/clock.ts @@ -1,6 +1,7 @@ export interface Clock { now(): number - setTimeout(cb: (...args: any[]) => void, ms: number): () => void - setInterval(cb: (...args: any[]) => void, ms: number): () => void + performanceNow(): number + setTimeout(cb: (...args: any[]) => void, seconds: number): () => void + setInterval(cb: (...args: any[]) => void, seconds: number): () => void setImmediate(cb: (...args: any[]) => void): () => void } diff --git a/src/server/time/node_clock.ts b/src/server/time/node_clock.ts index 64238432..787d2cb8 100644 --- a/src/server/time/node_clock.ts +++ b/src/server/time/node_clock.ts @@ -2,13 +2,13 @@ import { Clock } from './clock' export type CancelTimer = () => void -function setTimeout(cb: (...args: any[]) => void, ms: number): CancelTimer { - const handle = global.setTimeout(cb, ms) +function setTimeout(cb: (...args: any[]) => void, seconds: number): CancelTimer { + const handle = global.setTimeout(cb, seconds * 1000) return global.clearTimeout.bind(null, handle) } -function setInterval(cb: (...args: any[]) => void, ms: number): CancelTimer { - const handle = global.setInterval(cb, ms) +function setInterval(cb: (...args: any[]) => void, seconds: number): CancelTimer { + const handle = global.setInterval(cb, seconds * 1000) return global.clearInterval.bind(null, handle) } @@ -17,8 +17,15 @@ function setImmediate(cb: (...args: any[]) => void): CancelTimer { return global.clearImmediate.bind(null, handle) } +function performanceNow() { + const t = process.hrtime() + return t[0] + t[1] * 1e-9 +} + + export const NodeSystemClock: Clock = { - now: () => Date.now(), + now: () => Date.now() / 1000, + performanceNow, setTimeout, setInterval, setImmediate, diff --git a/src/simulators/sensor_data_simulator.ts b/src/simulators/sensor_data_simulator.ts index 6bd363f4..28946002 100644 --- a/src/simulators/sensor_data_simulator.ts +++ b/src/simulators/sensor_data_simulator.ts @@ -16,10 +16,10 @@ export class SensorDataSimulator implements Simulator { const messageType = 'message.input.Sensors' // Simulate a walk - const t = time * 5E-3 + index + const t = time + index - const angle = index * (2 * Math.PI) / numRobots + time / 4E4 - const distance = Math.cos(time / 1E3 + 4 * index) * 0.3 + 1 + const angle = index * (2 * Math.PI) / numRobots + time / 4E1 + const distance = Math.cos(time + 4 * index) * 0.3 + 1 const x = distance * Math.cos(angle) const y = distance * Math.sin(angle) const heading = -angle - Math.PI / 2 diff --git a/src/simulators/virtual_robot.ts b/src/simulators/virtual_robot.ts index e6a4d1b3..feb9cbf1 100644 --- a/src/simulators/virtual_robot.ts +++ b/src/simulators/virtual_robot.ts @@ -32,7 +32,7 @@ export class VirtualRobot { public simulateWithFrequency(frequency: number, index: number, numRobots: number) { const disconnect = this.connect() - const period = 1000 / frequency + const period = 1 / frequency const cancelLoop = this.clock.setInterval(() => this.simulate(index, numRobots), period) return () => { From a4efde4d2318ad9c14078cb6cdeca98eee5de9f7 Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Tue, 18 Jul 2017 14:13:59 +1000 Subject: [PATCH 11/24] stuffs --- jest.config.js | 43 +--- jestconfig.json | 45 ++++ package.json | 2 + src/server/nusight_server.ts | 196 ++++++++++++++++-- .../nbs_nuclear_writeable_stream.tests.ts | 44 ++++ src/validator/validate.ts | 1 - yarn.lock | 10 + 7 files changed, 284 insertions(+), 57 deletions(-) create mode 100644 jestconfig.json create mode 100644 src/server/tests/nbs_nuclear_writeable_stream.tests.ts diff --git a/jest.config.js b/jest.config.js index ccb47896..b4334611 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1,43 +1,2 @@ -module.exports = { - coverageDirectory: 'coverage', - coveragePathIgnorePatterns: ['/node_modules/', 'src/global.d.ts'], - collectCoverageFrom: [ - '**/*.{ts,tsx}', - '!src/shared/proto/**', - '!**/node_modules/**', - '!**/tests/**', - ], - globals: { - 'ts-jest': { - 'tsConfigFile': './tsconfig.test.json', - }, - }, - mapCoverage: true, - moduleDirectories: [ - 'node_modules', - '/src', - ], - moduleFileExtensions: [ - 'js', - 'ts', - 'tsx', - ], - moduleNameMapper: { - '\\.(css)$': 'identity-obj-proxy', - '\\.(vert)$': '/__mocks__/mock.vert', - '\\.(frag)$': '/__mocks__/mock.frag', - }, - roots: [ - '/src', - ], - modulePaths: [ - '/src', - ], - testMatch: [ - '**/tests/**/*.tests.{ts,tsx}', - ], - transform: { - '.(ts|tsx)': '/node_modules/ts-jest/preprocessor.js', - }, -} +module.exports = require('./jestconfig.json') diff --git a/jestconfig.json b/jestconfig.json new file mode 100644 index 00000000..ef8252e0 --- /dev/null +++ b/jestconfig.json @@ -0,0 +1,45 @@ +{ + "coverageDirectory": "coverage", + "coveragePathIgnorePatterns": [ + "/node_modules/", + "src/global.d.ts" + ], + "collectCoverageFrom": [ + "**/*.{ts,tsx}", + "!src/shared/proto/**", + "!**/node_modules/**", + "!**/tests/**" + ], + "globals": { + "ts-jest": { + "tsConfigFile": "./tsconfig.test.json" + } + }, + "mapCoverage": true, + "moduleDirectories": [ + "node_modules", + "/src" + ], + "moduleFileExtensions": [ + "js", + "ts", + "tsx" + ], + "moduleNameMapper": { + "\\.(css)$": "identity-obj-proxy", + "\\.(vert)$": "/__mocks__/mock.vert", + "\\.(frag)$": "/__mocks__/mock.frag" + }, + "roots": [ + "/src" + ], + "modulePaths": [ + "/src" + ], + "testMatch": [ + "**/tests/**/*.tests.{ts,tsx}" + ], + "transform": { + ".(ts|tsx)": "/node_modules/ts-jest/preprocessor.js" + } +} diff --git a/package.json b/package.json index e7d9a98f..2573290b 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ }, "license": "MIT", "devDependencies": { + "@types/buffers": "^0.1.30", "@types/classnames": "^2.2.0", "@types/compression": "^0.0.33", "@types/copy-webpack-plugin": "^4.0.0", @@ -89,6 +90,7 @@ "webpack-hot-middleware": "^2.18.2" }, "dependencies": { + "buffers": "^0.1.1", "classnames": "^2.2.5", "compression": "^1.7.0", "connect-history-api-fallback": "^1.3.0", diff --git a/src/server/nusight_server.ts b/src/server/nusight_server.ts index 753c8920..12d7b475 100644 --- a/src/server/nusight_server.ts +++ b/src/server/nusight_server.ts @@ -1,12 +1,17 @@ +import * as Buffers from 'buffers' import * as fs from 'fs' +import { WriteStream } from 'fs' +import { ReadStream } from 'fs' import { NUClearNetPeer } from 'nuclearnet.js' import { NUClearNetPacket } from 'nuclearnet.js' +import * as stream from 'stream' import { NUClearNetClient } from '../shared/nuclearnet/nuclearnet_client' +import { DirectNUClearNetClient } from './nuclearnet/direct_nuclearnet_client' +import { FakeNUClearNetClient } from './nuclearnet/fake_nuclearnet_client' import { WebSocketServer } from './nuclearnet/web_socket_server' import { WebSocket } from './nuclearnet/web_socket_server' -import { WriteStream } from 'fs' -import { NodeSystemClock } from './time/node_clock' import { Clock } from './time/clock' +import { NodeSystemClock } from './time/node_clock' export class NUsightServer { public constructor(private server: WebSocketServer, private nuclearnetClient: NUClearNetClient) { @@ -93,27 +98,190 @@ class NbsRecorder { const header = Buffer.from([0xE2, 0x98, 0xA2]) // NUClear radiation symbol. // 64bit timestamp in microseconds. - const time = this.clock.performanceNow() * 1e6; - const timeBuffer = new Buffer(8); + const time = this.clock.performanceNow() * 1e6 + const timeBuffer = new Buffer(8) // Convert double into two 32 bit integers. - const MAX_UINT32 = 0xFFFFFFFF; - const highByte = ~~(time / MAX_UINT32); - const lowByte = (time % MAX_UINT32) - highByte; - timeBuffer.writeUInt32LE(lowByte, 0); - timeBuffer.writeUInt32LE(highByte, 4); + const MAX_UINT32 = 0xFFFFFFFF + const highByte = ~~(time / MAX_UINT32) + const lowByte = (time % MAX_UINT32) - highByte + timeBuffer.writeUInt32LE(lowByte, 0) + timeBuffer.writeUInt32LE(highByte, 4) - const remainingByteLength = new Buffer(4); + const remainingByteLength = new Buffer(4) remainingByteLength.writeUInt32LE(timeBuffer.byteLength + packet.hash.byteLength + packet.payload.byteLength, 0) // Write parts to file this.file.write(header) - this.file.write(remainingByteLength); - this.file.write(timeBuffer); - this.file.write(packet.hash); - this.file.write(packet.payload); + this.file.write(remainingByteLength) + this.file.write(timeBuffer) + this.file.write(packet.hash) + this.file.write(packet.payload) } private arePeersEqual(peerA: NUClearNetPeer, peerB: NUClearNetPeer) { return peerA.name === peerB.name && peerA.address === peerB.address && peerA.port === peerB.port } } + +const NUCLEAR_HEADER = 3 + +class NbsPlayback { + private state: PlaybackState + private file: ReadStream + private currentIndex: number + + public constructor(private nuclearnetClient: NUClearNetClient, + private clock: Clock) { + this.state = PlaybackState.Idle + this.currentIndex = 0 + } + + public static of(opts: { fakeNetworking: boolean }) { + const network = opts.fakeNetworking ? FakeNUClearNetClient.of() : DirectNUClearNetClient.of() + return new NbsPlayback(network, NodeSystemClock) + } + + public play(filename: string): () => void { + this.state = PlaybackState.Playing + this.file = fs.createReadStream(filename, { encoding: 'binary' }) + this.playNextPacket() + + return () => { + if (this.state === PlaybackState.Playing) { + this.state = PlaybackState.Idle + } + } + } + + public pause(): () => void { + this.state = PlaybackState.Paused + return () => { + if (this.state === PlaybackState.Paused) { + this.state = PlaybackState.Playing + } + } + } + + private playNextPacket() { + this.file.read(NUCLEAR_HEADER) + } +} + +// type NbsNUClearWritableStreamOpts = { +// } + +const NBS_NUCLEAR_HEADER = Buffer.from([0xE2, 0x98, 0xA2]) // NUClear radiation symbol. + +export class NBsNUClearTransformSteam extends stream.Transform { + private buffers: Buffers + private foundHeader: boolean + private foundPacketSize: boolean + + constructor() { + super({ + objectMode: true, + }) + + this.buffers = new Buffers() + this.foundHeader = false + this.foundPacketSize = false + } + + public _transform(chunk: any, encoding: string, done: (err?: any, data?: any) => void) { + this.buffers.push(chunk) + + let frame + while ((frame = this.getNextFrame(this.buffers)) !== undefined) { + this.push(frame.buffer) + this.buffers.splice(0, frame.offset + frame.buffer.byteLength) + } + + done() + } + + private getNextFrame(buffer: Buffers): { offset: number, buffer: Buffer } | undefined { + const headerIndex = buffer.indexOf(NBS_NUCLEAR_HEADER) + const headerSize = NBS_NUCLEAR_HEADER.byteLength + const packetLengthSize = 4 + const headerAndPacketLengthSize = headerSize + packetLengthSize + if (headerIndex >= 0) { + const chunk = buffer.slice(headerIndex) + if (chunk.length >= headerAndPacketLengthSize) { + const packetSize = chunk.slice(headerSize, headerSize + headerAndPacketLengthSize).readUInt32LE(0) + if (chunk.length >= headerAndPacketLengthSize + packetSize) { + return { + offset: headerIndex, + buffer: chunk.slice(0, headerAndPacketLengthSize + packetSize) + } + } + } + } + return undefined + } +} + +export class NbsNUClearWritableStream extends stream.Writable { + private buffers: Buffers + private buffering: boolean + private payloadLength: number + private headerIndex: number + + public constructor(private nuclearnetClient: NUClearNetClient/*, opts: NbsNUClearWritableStreamOpts*/) { + super() + + this.buffers = new Buffers() + this.buffering = false + this.headerIndex = 0 + } + + public static of(opts: { fakeNetworking: boolean }) { + const network = opts.fakeNetworking ? FakeNUClearNetClient.of() : DirectNUClearNetClient.of() + return new NbsNUClearWritableStream(network) + } + + public _write(chunk: Buffer, encoding: string, cb: Function) { + this.buffers.push(chunk) + + if (!this.buffering) { + const headerIndex = chunk.indexOf(NBS_NUCLEAR_HEADER) + if (headerIndex >= 0) { + console.log(`Found header at ${headerIndex}`) + this.headerIndex = headerIndex + this.buffering = true + } + } else { + console.log(this.buffers.buffers.length) + + const offset = this.headerIndex + if (this.buffers.length >= offset + 7) { + const remainingByteLength = this.buffers.slice(offset + 3, offset + 3 + 4).readUInt32LE(0) + this.payloadLength = remainingByteLength + if (this.buffers.length >= offset + 7 + remainingByteLength) { + const hash = this.buffers.slice(offset + 15, offset + 15 + 8) + const payload = this.buffers.slice(offset + 23, offset + 23 + remainingByteLength - 16) + this.nuclearnetClient.send({ + type: hash, + payload, + }) + this.buffering = false + const end = offset + 7 + remainingByteLength + this.buffers.splice(0, end) + } + } + } + cb() + } + + decode(buffer: Buffer): { type: Buffer, payload: Buffer } { + return { + type: new Buffer(8), + payload: new Buffer(8), + } + } +} + +enum PlaybackState { + Idle = 1, + Paused, + Playing, +} diff --git a/src/server/tests/nbs_nuclear_writeable_stream.tests.ts b/src/server/tests/nbs_nuclear_writeable_stream.tests.ts new file mode 100644 index 00000000..d36c20cc --- /dev/null +++ b/src/server/tests/nbs_nuclear_writeable_stream.tests.ts @@ -0,0 +1,44 @@ +import * as fs from 'fs' +import * as stream from 'stream' +import { FakeNUClearNetClient } from '../nuclearnet/fake_nuclearnet_client' +import { FakeNUClearNetServer } from '../nuclearnet/fake_nuclearnet_server' +import { NbsNUClearWritableStream } from '../nusight_server' +import { NBsNUClearTransformSteam } from '../nusight_server' +import WritableStream = NodeJS.WritableStream + +describe.skip('NbsNUClearWritableStream', () => { + let writeStream: WritableStream + let nuclearnetClient: FakeNUClearNetClient + + beforeEach(() => { + const nuclearnetServer = new FakeNUClearNetServer() + nuclearnetClient = new FakeNUClearNetClient(nuclearnetServer) + writeStream = new NbsNUClearWritableStream(nuclearnetClient) + }) + + it('asdfas', done => { + jest.spyOn(nuclearnetClient, 'send') + const file = fs.createReadStream('/Users/brendan/Lab/NUsight2/recordings/darwin3_WalkAround.nbs') + file.pipe(writeStream).on('finish', () => { + // expect(nuclearnetClient.send).toHaveBeenCalledTimes(149) + done() + }) + }) +}) + +describe('NBsNUClearTransformSteam', () => { + let transform: stream.Transform + + beforeEach(() => { + transform = new NBsNUClearTransformSteam() + }) + + it('Emits 6988 frames', done => { + const file = fs.createReadStream('/Users/brendan/Lab/NUsight2/recordings/darwin3_WalkAround.nbs') + const spy = jest.fn() + file.pipe(transform).on('data', spy).on('finish', () => { + expect(spy).toHaveBeenCalledTimes(6988) + done() + }) + }) +}) diff --git a/src/validator/validate.ts b/src/validator/validate.ts index ca41b51d..08c9cce4 100644 --- a/src/validator/validate.ts +++ b/src/validator/validate.ts @@ -17,7 +17,6 @@ function main() { map.set(packet.hash, (map.get(packet.hash) || 0) + 1) return map }, new Map()) - console.log(types) console.log(`Num packets: ${packets.length}`) console.log('Types:') diff --git a/yarn.lock b/yarn.lock index c22601dc..5c9b216d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -76,6 +76,12 @@ dependencies: "@types/babel-types" "*" +"@types/buffers@^0.1.30": + version "0.1.30" + resolved "https://registry.yarnpkg.com/@types/buffers/-/buffers-0.1.30.tgz#8764e8425ecc52fe131e7b531b5fc100c93bfa10" + dependencies: + "@types/node" "*" + "@types/classnames@^2.2.0": version "2.2.0" resolved "https://registry.yarnpkg.com/@types/classnames/-/classnames-2.2.0.tgz#f2312039e780bdf89d7d4102a26ec11de5ec58aa" @@ -1369,6 +1375,10 @@ buffer@^4.3.0: ieee754 "^1.1.4" isarray "^1.0.0" +buffers@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/buffers/-/buffers-0.1.1.tgz#b24579c3bed4d6d396aeee6d9a8ae7f5482ab7bb" + builtin-modules@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-1.1.1.tgz#270f076c5a72c02f5b65a47df94c5fe3a278892f" From 1ad31695778ec81e8cfbd331b45aad987696ef4e Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Tue, 18 Jul 2017 15:32:56 +1000 Subject: [PATCH 12/24] todo --- src/server/nusight_server.ts | 236 +++++++++++++----- .../nbs_nuclear_writeable_stream.tests.ts | 75 ++++-- 2 files changed, 223 insertions(+), 88 deletions(-) diff --git a/src/server/nusight_server.ts b/src/server/nusight_server.ts index 12d7b475..1942fe50 100644 --- a/src/server/nusight_server.ts +++ b/src/server/nusight_server.ts @@ -12,6 +12,8 @@ import { WebSocketServer } from './nuclearnet/web_socket_server' import { WebSocket } from './nuclearnet/web_socket_server' import { Clock } from './time/clock' import { NodeSystemClock } from './time/node_clock' +import { Stream } from 'stream' +import WritableStream = NodeJS.WritableStream export class NUsightServer { public constructor(private server: WebSocketServer, private nuclearnetClient: NUClearNetClient) { @@ -35,13 +37,18 @@ class NUsightServerClient { this.socket.on('record', this.onRecord) this.socket.on('unrecord', this.onUnrecord) + + this.socket.on('play', this.onPlay) + this.socket.on('pause', this.onPause) + this.socket.on('resume', this.onResume) + this.socket.on('stop', this.onStop) } public static of(socket: WebSocket, nuclearnetClient: NUClearNetClient): NUsightServerClient { return new NUsightServerClient(socket, NodeSystemClock, nuclearnetClient) } - public onRecord = (peer: NUClearNetPeer, requestToken: string) => { + private onRecord = (peer: NUClearNetPeer, requestToken: string) => { const recorder = NbsRecorder.of(peer, this.nuclearnetClient) const filename = `${peer.name.replace(/[^A-Za-z0-9]/g, '_')}_${this.clock.now()}.nbs` console.log('recording', peer, requestToken) @@ -49,7 +56,30 @@ class NUsightServerClient { this.stopRecordingMap.set(requestToken, stopRecording) } - public onUnrecord = (requestToken: string) => { + private onUnrecord = (requestToken: string) => { + const stopRecording = this.stopRecordingMap.get(requestToken) + if (stopRecording) { + console.log('stop recording', requestToken) + stopRecording() + } + } + + private onPlay = (filename: string, requestToken: string) => { + const player = NbsPlayback.of(this.nuclearnetClient) + console.log('playing', filename, requestToken) + const stopPlaying = player.play(`recordings/${filename}`) + this.stopRecordingMap.set(requestToken, stopPlaying) + } + + private onPause = () => { + + } + + private onResume = () => { + + } + + public onStop = (requestToken: string) => { const stopRecording = this.stopRecordingMap.get(requestToken) if (stopRecording) { console.log('stop recording', requestToken) @@ -127,7 +157,8 @@ const NUCLEAR_HEADER = 3 class NbsPlayback { private state: PlaybackState - private file: ReadStream + private inputStream: stream.Transform + private outputStream: WritableStream private currentIndex: number public constructor(private nuclearnetClient: NUClearNetClient, @@ -136,35 +167,40 @@ class NbsPlayback { this.currentIndex = 0 } - public static of(opts: { fakeNetworking: boolean }) { - const network = opts.fakeNetworking ? FakeNUClearNetClient.of() : DirectNUClearNetClient.of() - return new NbsPlayback(network, NodeSystemClock) + public static of(nuclearnetClient: NUClearNetClient) { + return new NbsPlayback(nuclearnetClient, NodeSystemClock) } public play(filename: string): () => void { this.state = PlaybackState.Playing - this.file = fs.createReadStream(filename, { encoding: 'binary' }) - this.playNextPacket() + + this.inputStream = fs.createReadStream(filename, { encoding: 'binary' }) + .pipe(NbsFrameTransformStream.of()) + .pipe(NbsFrameDecoderStream.of()) + .on('finish', () => { + this.state = PlaybackState.Idle + }) + + this.outputStream = this.inputStream.pipe(NbsNUClearPlayback.of(this.nuclearnetClient)) return () => { if (this.state === PlaybackState.Playing) { this.state = PlaybackState.Idle + this.inputStream.end() } } } public pause(): () => void { this.state = PlaybackState.Paused + this.inputStream.pause() return () => { if (this.state === PlaybackState.Paused) { this.state = PlaybackState.Playing + this.inputStream.resume() } } } - - private playNextPacket() { - this.file.read(NUCLEAR_HEADER) - } } // type NbsNUClearWritableStreamOpts = { @@ -172,7 +208,7 @@ class NbsPlayback { const NBS_NUCLEAR_HEADER = Buffer.from([0xE2, 0x98, 0xA2]) // NUClear radiation symbol. -export class NBsNUClearTransformSteam extends stream.Transform { +export class NbsFrameTransformStream extends stream.Transform { private buffers: Buffers private foundHeader: boolean private foundPacketSize: boolean @@ -187,6 +223,10 @@ export class NBsNUClearTransformSteam extends stream.Transform { this.foundPacketSize = false } + public static of(): NbsFrameTransformStream { + return new NbsFrameTransformStream() + } + public _transform(chunk: any, encoding: string, done: (err?: any, data?: any) => void) { this.buffers.push(chunk) @@ -205,13 +245,13 @@ export class NBsNUClearTransformSteam extends stream.Transform { const packetLengthSize = 4 const headerAndPacketLengthSize = headerSize + packetLengthSize if (headerIndex >= 0) { - const chunk = buffer.slice(headerIndex) - if (chunk.length >= headerAndPacketLengthSize) { - const packetSize = chunk.slice(headerSize, headerSize + headerAndPacketLengthSize).readUInt32LE(0) - if (chunk.length >= headerAndPacketLengthSize + packetSize) { + const frame = buffer.slice(headerIndex) + if (frame.length >= headerAndPacketLengthSize) { + const packetSize = frame.slice(headerSize, headerSize + headerAndPacketLengthSize).readUInt32LE(0) + if (frame.length >= headerAndPacketLengthSize + packetSize) { return { offset: headerIndex, - buffer: chunk.slice(0, headerAndPacketLengthSize + packetSize) + buffer: frame.slice(0, headerAndPacketLengthSize + packetSize), } } } @@ -220,66 +260,130 @@ export class NBsNUClearTransformSteam extends stream.Transform { } } -export class NbsNUClearWritableStream extends stream.Writable { - private buffers: Buffers - private buffering: boolean - private payloadLength: number - private headerIndex: number - - public constructor(private nuclearnetClient: NUClearNetClient/*, opts: NbsNUClearWritableStreamOpts*/) { - super() +export class NbsFrameDecoderStream extends stream.Transform { + public constructor() { + super({ + objectMode: true, + }) + } - this.buffers = new Buffers() - this.buffering = false - this.headerIndex = 0 + public static of() { + return new NbsFrameDecoderStream() } - public static of(opts: { fakeNetworking: boolean }) { - const network = opts.fakeNetworking ? FakeNUClearNetClient.of() : DirectNUClearNetClient.of() - return new NbsNUClearWritableStream(network) + public _transform(buffer: Buffer, encoding: string, done: Function) { + this.push({ + header: buffer.slice(0, 3), + size: buffer.slice(3, 7), // TODO: decode + timestamp: buffer.slice(7, 15), // TODO: decode + hash: buffer.slice(15, 23), + payload: buffer.slice(23), + }) + done() } +} - public _write(chunk: Buffer, encoding: string, cb: Function) { - this.buffers.push(chunk) +type NbsFrame = { + header: Buffer + size: number + timestamp: number, + hash: Buffer, + payload: Buffer, +} - if (!this.buffering) { - const headerIndex = chunk.indexOf(NBS_NUCLEAR_HEADER) - if (headerIndex >= 0) { - console.log(`Found header at ${headerIndex}`) - this.headerIndex = headerIndex - this.buffering = true - } - } else { - console.log(this.buffers.buffers.length) - - const offset = this.headerIndex - if (this.buffers.length >= offset + 7) { - const remainingByteLength = this.buffers.slice(offset + 3, offset + 3 + 4).readUInt32LE(0) - this.payloadLength = remainingByteLength - if (this.buffers.length >= offset + 7 + remainingByteLength) { - const hash = this.buffers.slice(offset + 15, offset + 15 + 8) - const payload = this.buffers.slice(offset + 23, offset + 23 + remainingByteLength - 16) - this.nuclearnetClient.send({ - type: hash, - payload, - }) - this.buffering = false - const end = offset + 7 + remainingByteLength - this.buffers.splice(0, end) - } - } - } - cb() +export class NbsNUClearPlayback extends stream.Writable { + private firstFrameTimestamp?: number + private firstLocalTimestamp?: number + + public constructor(private nuclearnetClient: NUClearNetClient, + private clock: Clock) { + super({ + objectMode: true, + }) + } + + public static of(nuclearnetClient: NUClearNetClient) { + return new NbsNUClearPlayback(nuclearnetClient, NodeSystemClock) } - decode(buffer: Buffer): { type: Buffer, payload: Buffer } { - return { - type: new Buffer(8), - payload: new Buffer(8), + public _write(frame: NbsFrame, encoding: string, done: Function) { + if (this.firstFrameTimestamp === undefined || this.firstLocalTimestamp === undefined) { + this.firstFrameTimestamp = frame.timestamp + this.firstLocalTimestamp = this.clock.performanceNow() } + const now = this.clock.performanceNow() + const timeOffset = frame.timestamp - this.firstFrameTimestamp + const timeout = this.firstLocalTimestamp + timeOffset - now + this.clock.setTimeout(() => { + this.nuclearnetClient.send({ + type: frame.hash, + payload: frame.payload, + }) + done() + }, timeout) } } +// export class NbsNUClearWritableStream extends stream.Writable { +// private buffers: Buffers +// private buffering: boolean +// private payloadLength: number +// private headerIndex: number +// +// public constructor(private nuclearnetClient: NUClearNetClient/*, opts: NbsNUClearWritableStreamOpts*/) { +// super() +// +// this.buffers = new Buffers() +// this.buffering = false +// this.headerIndex = 0 +// } +// +// public static of(opts: { fakeNetworking: boolean }) { +// const network = opts.fakeNetworking ? FakeNUClearNetClient.of() : DirectNUClearNetClient.of() +// return new NbsNUClearWritableStream(network) +// } +// +// public _write(chunk: Buffer, encoding: string, cb: Function) { +// this.buffers.push(chunk) +// +// if (!this.buffering) { +// const headerIndex = chunk.indexOf(NBS_NUCLEAR_HEADER) +// if (headerIndex >= 0) { +// console.log(`Found header at ${headerIndex}`) +// this.headerIndex = headerIndex +// this.buffering = true +// } +// } else { +// console.log(this.buffers.buffers.length) +// +// const offset = this.headerIndex +// if (this.buffers.length >= offset + 7) { +// const remainingByteLength = this.buffers.slice(offset + 3, offset + 3 + 4).readUInt32LE(0) +// this.payloadLength = remainingByteLength +// if (this.buffers.length >= offset + 7 + remainingByteLength) { +// const hash = this.buffers.slice(offset + 15, offset + 15 + 8) +// const payload = this.buffers.slice(offset + 23, offset + 23 + remainingByteLength - 16) +// this.nuclearnetClient.send({ +// type: hash, +// payload, +// }) +// this.buffering = false +// const end = offset + 7 + remainingByteLength +// this.buffers.splice(0, end) +// } +// } +// } +// cb() +// } +// +// decode(buffer: Buffer): { type: Buffer, payload: Buffer } { +// return { +// type: new Buffer(8), +// payload: new Buffer(8), +// } +// } +// } + enum PlaybackState { Idle = 1, Paused, diff --git a/src/server/tests/nbs_nuclear_writeable_stream.tests.ts b/src/server/tests/nbs_nuclear_writeable_stream.tests.ts index d36c20cc..363aaed0 100644 --- a/src/server/tests/nbs_nuclear_writeable_stream.tests.ts +++ b/src/server/tests/nbs_nuclear_writeable_stream.tests.ts @@ -1,44 +1,75 @@ import * as fs from 'fs' import * as stream from 'stream' +import { Stream } from 'stream' +import { NUClearNetClient } from '../../shared/nuclearnet/nuclearnet_client' import { FakeNUClearNetClient } from '../nuclearnet/fake_nuclearnet_client' import { FakeNUClearNetServer } from '../nuclearnet/fake_nuclearnet_server' -import { NbsNUClearWritableStream } from '../nusight_server' -import { NBsNUClearTransformSteam } from '../nusight_server' +import { NbsFrameTransformStream } from '../nusight_server' +import { NbsFrameDecoderStream } from '../nusight_server' +import { NbsNUClearPlayback } from '../nusight_server' +import { NodeSystemClock } from '../time/node_clock' import WritableStream = NodeJS.WritableStream -describe.skip('NbsNUClearWritableStream', () => { - let writeStream: WritableStream - let nuclearnetClient: FakeNUClearNetClient +describe('NbsFrameTransformStream', () => { + let transform: stream.Transform beforeEach(() => { - const nuclearnetServer = new FakeNUClearNetServer() - nuclearnetClient = new FakeNUClearNetClient(nuclearnetServer) - writeStream = new NbsNUClearWritableStream(nuclearnetClient) + transform = new NbsFrameTransformStream() }) - it('asdfas', done => { - jest.spyOn(nuclearnetClient, 'send') + it('Emits 6988 frames', done => { const file = fs.createReadStream('/Users/brendan/Lab/NUsight2/recordings/darwin3_WalkAround.nbs') - file.pipe(writeStream).on('finish', () => { - // expect(nuclearnetClient.send).toHaveBeenCalledTimes(149) + const spy = jest.fn() + file.pipe(transform).on('data', spy).on('finish', () => { + expect(spy).toHaveBeenCalledTimes(6988) done() }) }) }) -describe('NBsNUClearTransformSteam', () => { - let transform: stream.Transform +// describe('NbsNUClearWritableStream', () => { +// let transform: stream.Transform +// let writeStream: WritableStream +// let nuclearnetClient: FakeNUClearNetClient +// +// beforeEach(() => { +// const nuclearnetServer = new FakeNUClearNetServer() +// nuclearnetClient = new FakeNUClearNetClient(nuclearnetServer) +// transform = new NbsFrameTransformStream() +// writeStream = new NbsNUClearWritableStream(nuclearnetClient) +// }) +// +// it('asdfas', done => { +// jest.spyOn(nuclearnetClient, 'send') +// const file = fs.createReadStream('/Users/brendan/Lab/NUsight2/recordings/darwin3_WalkAround.nbs') +// file.pipe(transform).pipe(writeStream).on('finish', () => { +// console.log((nuclearnetClient.send as jest.Mock).mock.calls[0][0].type.toString('hex')) +// expect(nuclearnetClient.send).toHaveBeenCalledTimes(6988) +// done() +// }) +// }) +// }) + +describe('NbsNUClearPlayback', () => { + let stream: Stream + let nuclearnetClient: NUClearNetClient beforeEach(() => { - transform = new NBsNUClearTransformSteam() + const nuclearnetServer = new FakeNUClearNetServer() + nuclearnetClient = new FakeNUClearNetClient(nuclearnetServer) + + stream = fs.createReadStream('/Users/brendan/Lab/NUsight2/recordings/darwin3_WalkAround.nbs') + .pipe(new NbsFrameTransformStream()) + .pipe(new NbsFrameDecoderStream) + }) - it('Emits 6988 frames', done => { - const file = fs.createReadStream('/Users/brendan/Lab/NUsight2/recordings/darwin3_WalkAround.nbs') - const spy = jest.fn() - file.pipe(transform).on('data', spy).on('finish', () => { - expect(spy).toHaveBeenCalledTimes(6988) - done() - }) + it.skip('asdf', done => { + jest.spyOn(nuclearnetClient, 'send') + stream + .pipe(new NbsNUClearPlayback(nuclearnetClient, NodeSystemClock)) + .on('finish', () => { + done() + }) }) }) From de5e1837f29248ca1f1345cfbf4414474eddefc5 Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Tue, 18 Jul 2017 17:49:03 +1000 Subject: [PATCH 13/24] . --- src/server/nusight_server.ts | 40 +++--- .../nbs_nuclear_writeable_stream.tests.ts | 38 ++---- src/server/time/fake_node_clock.ts | 124 ++++++++++++++++++ 3 files changed, 153 insertions(+), 49 deletions(-) create mode 100644 src/server/time/fake_node_clock.ts diff --git a/src/server/nusight_server.ts b/src/server/nusight_server.ts index 1942fe50..775686d2 100644 --- a/src/server/nusight_server.ts +++ b/src/server/nusight_server.ts @@ -1,18 +1,15 @@ import * as Buffers from 'buffers' import * as fs from 'fs' import { WriteStream } from 'fs' -import { ReadStream } from 'fs' +import * as Long from 'long' import { NUClearNetPeer } from 'nuclearnet.js' import { NUClearNetPacket } from 'nuclearnet.js' import * as stream from 'stream' import { NUClearNetClient } from '../shared/nuclearnet/nuclearnet_client' -import { DirectNUClearNetClient } from './nuclearnet/direct_nuclearnet_client' -import { FakeNUClearNetClient } from './nuclearnet/fake_nuclearnet_client' import { WebSocketServer } from './nuclearnet/web_socket_server' import { WebSocket } from './nuclearnet/web_socket_server' import { Clock } from './time/clock' import { NodeSystemClock } from './time/node_clock' -import { Stream } from 'stream' import WritableStream = NodeJS.WritableStream export class NUsightServer { @@ -129,13 +126,10 @@ class NbsRecorder { // 64bit timestamp in microseconds. const time = this.clock.performanceNow() * 1e6 + const timeLong = Long.fromNumber(time) const timeBuffer = new Buffer(8) - // Convert double into two 32 bit integers. - const MAX_UINT32 = 0xFFFFFFFF - const highByte = ~~(time / MAX_UINT32) - const lowByte = (time % MAX_UINT32) - highByte - timeBuffer.writeUInt32LE(lowByte, 0) - timeBuffer.writeUInt32LE(highByte, 4) + timeBuffer.writeUInt32LE(timeLong.low, 0) + timeBuffer.writeUInt32LE(timeLong.high, 4) const remainingByteLength = new Buffer(4) remainingByteLength.writeUInt32LE(timeBuffer.byteLength + packet.hash.byteLength + packet.payload.byteLength, 0) @@ -153,22 +147,19 @@ class NbsRecorder { } } -const NUCLEAR_HEADER = 3 - class NbsPlayback { private state: PlaybackState private inputStream: stream.Transform private outputStream: WritableStream private currentIndex: number - public constructor(private nuclearnetClient: NUClearNetClient, - private clock: Clock) { + public constructor(private nuclearnetClient: NUClearNetClient) { this.state = PlaybackState.Idle this.currentIndex = 0 } public static of(nuclearnetClient: NUClearNetClient) { - return new NbsPlayback(nuclearnetClient, NodeSystemClock) + return new NbsPlayback(nuclearnetClient) } public play(filename: string): () => void { @@ -271,16 +262,19 @@ export class NbsFrameDecoderStream extends stream.Transform { return new NbsFrameDecoderStream() } - public _transform(buffer: Buffer, encoding: string, done: Function) { - this.push({ - header: buffer.slice(0, 3), - size: buffer.slice(3, 7), // TODO: decode - timestamp: buffer.slice(7, 15), // TODO: decode - hash: buffer.slice(15, 23), - payload: buffer.slice(23), - }) + public _transform(frame: Buffer, encoding: string, done: (err?: any, data?: any) => void) { + this.push(this.decode(frame)) done() } + + private decode(frame: Buffer) { + const header = frame.slice(0, 3) + const size = frame.readUInt32LE(3) + const timestamp = Long.fromBits(frame.readUInt32LE(7), frame.readUInt32LE(11)).toNumber() + const hash = frame.slice(15, 23) + const payload = frame.slice(23) + return { header, size, timestamp, hash, payload } + } } type NbsFrame = { diff --git a/src/server/tests/nbs_nuclear_writeable_stream.tests.ts b/src/server/tests/nbs_nuclear_writeable_stream.tests.ts index 363aaed0..1cddb982 100644 --- a/src/server/tests/nbs_nuclear_writeable_stream.tests.ts +++ b/src/server/tests/nbs_nuclear_writeable_stream.tests.ts @@ -7,8 +7,10 @@ import { FakeNUClearNetServer } from '../nuclearnet/fake_nuclearnet_server' import { NbsFrameTransformStream } from '../nusight_server' import { NbsFrameDecoderStream } from '../nusight_server' import { NbsNUClearPlayback } from '../nusight_server' -import { NodeSystemClock } from '../time/node_clock' import WritableStream = NodeJS.WritableStream +import { FakeNodeClock } from '../time/fake_node_clock' +import { message } from '../../shared/proto/messages' +import nuclear = message.support.nuclear describe('NbsFrameTransformStream', () => { let transform: stream.Transform @@ -27,29 +29,6 @@ describe('NbsFrameTransformStream', () => { }) }) -// describe('NbsNUClearWritableStream', () => { -// let transform: stream.Transform -// let writeStream: WritableStream -// let nuclearnetClient: FakeNUClearNetClient -// -// beforeEach(() => { -// const nuclearnetServer = new FakeNUClearNetServer() -// nuclearnetClient = new FakeNUClearNetClient(nuclearnetServer) -// transform = new NbsFrameTransformStream() -// writeStream = new NbsNUClearWritableStream(nuclearnetClient) -// }) -// -// it('asdfas', done => { -// jest.spyOn(nuclearnetClient, 'send') -// const file = fs.createReadStream('/Users/brendan/Lab/NUsight2/recordings/darwin3_WalkAround.nbs') -// file.pipe(transform).pipe(writeStream).on('finish', () => { -// console.log((nuclearnetClient.send as jest.Mock).mock.calls[0][0].type.toString('hex')) -// expect(nuclearnetClient.send).toHaveBeenCalledTimes(6988) -// done() -// }) -// }) -// }) - describe('NbsNUClearPlayback', () => { let stream: Stream let nuclearnetClient: NUClearNetClient @@ -64,11 +43,18 @@ describe('NbsNUClearPlayback', () => { }) - it.skip('asdf', done => { + it('asdf', done => { + const fakeClock = FakeNodeClock.of() jest.spyOn(nuclearnetClient, 'send') stream - .pipe(new NbsNUClearPlayback(nuclearnetClient, NodeSystemClock)) + .on('data', () => { + // Ensure that all timers instantly run after each chunk is received. + // Run on the next tick to allow NbsNUClearPlayback to schedule the timers first. + process.nextTick(() => fakeClock.runAllTimers()) + }) + .pipe(new NbsNUClearPlayback(nuclearnetClient, fakeClock)) .on('finish', () => { + expect(nuclearnetClient.send).toHaveBeenCalledTimes(6988) done() }) }) diff --git a/src/server/time/fake_node_clock.ts b/src/server/time/fake_node_clock.ts new file mode 100644 index 00000000..75681165 --- /dev/null +++ b/src/server/time/fake_node_clock.ts @@ -0,0 +1,124 @@ +import { Clock } from './clock' + +type Task = { + id: number, + nextTime: number, + period?: number, + fn: () => void, +} + +export class FakeNodeClock implements Clock { + private nextId: number + private time: number + private tasks: Task[] + + constructor() { + this.nextId = 0 + this.time = 0 + this.tasks = [] + } + + public static of() { + return new FakeNodeClock() + } + + public now(): number { + return this.time + } + + public performanceNow(): number { + return this.time + } + + public setTimeout(fn: () => void, seconds: number): () => void { + const id = this.nextId++ + this.addTask({ id, nextTime: this.now() + seconds, fn }) + return () => this.removeTask(id) + } + + public setInterval(fn: () => void, seconds: number): () => void { + const id = this.nextId++ + this.addTask({ id, nextTime: this.now() + seconds, period: seconds, fn }) + return () => this.removeTask(id) + } + + public setImmediate(fn: () => void): () => void { + const id = this.nextId++ + this.addTask({ id, nextTime: this.now() + 1, fn }) + return () => this.removeTask(id) + } + + public tick(delta: number = 1) { + const newTime = this.now() + delta + + while (this.tasks.length > 0 && this.tasks[0].nextTime <= newTime) { + const task = this.tasks[0] + this.consumeTask(task) + } + + this.time = newTime + } + + public runAllTimers() { + const limit = 1000 + let i = 0 + + while (this.tasks.length > 0) { + if (i > limit) { + throw new Error(`Exceeded clock task limit of ${limit}, possibly caused by a infinite task loop?`) + } + const task = this.tasks[0] + this.consumeTask(task) + i++ + } + } + + private consumeTask(task: Task) { + this.time = task.nextTime + if (task.period != null) { + task.nextTime += task.period + this.sortTasks() + } else { + this.tasks.shift() + } + task.fn() + } + + private sortTasks() { + this.tasks.sort((t1, t2) => { + return t1.nextTime - t2.nextTime + }) + } + + public runOnlyPendingTimers() { + const limit = 1000 + let i = 0 + + while (this.tasks.length > 0 && this.now() <= this.tasks[this.tasks.length - 1].nextTime) { + if (i > limit) { + throw new Error(`Exceeded clock task limit of ${limit}, possibly caused by a infinite task loop?`) + } + const task = this.tasks[0] + this.consumeTask(task) + i++ + } + } + + public runTimersToTime() { + throw new Error('Not implemented') + } + + private addTask(task: Task) { + this.tasks.push(task) + this.sortTasks() + } + + private removeTask(taskId: number) { + for (let i = 0; i < this.tasks.length; i++) { + if (this.tasks[i].id == taskId) { + this.tasks.splice(i, 1) + break + } + } + } +} From df735155a8e31284675d60138be443ae7da826fc Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Tue, 18 Jul 2017 22:27:29 +1000 Subject: [PATCH 14/24] . --- src/server/nbs/nbs_frame_chunker.ts | 56 ++++ src/server/nbs/nbs_frame_codecs.ts | 91 +++++ src/server/nbs/nbs_nuclear_playback.ts | 47 +++ src/server/nbs/nbs_playback_controller.ts | 58 ++++ src/server/nbs/nbs_recorder_controller.ts | 46 +++ .../nbs/tests/nbs_frame_codecs.tests.ts | 34 ++ src/server/nusight_server.ts | 311 +----------------- .../nbs_nuclear_writeable_stream.tests.ts | 18 +- 8 files changed, 345 insertions(+), 316 deletions(-) create mode 100644 src/server/nbs/nbs_frame_chunker.ts create mode 100644 src/server/nbs/nbs_frame_codecs.ts create mode 100644 src/server/nbs/nbs_nuclear_playback.ts create mode 100644 src/server/nbs/nbs_playback_controller.ts create mode 100644 src/server/nbs/nbs_recorder_controller.ts create mode 100644 src/server/nbs/tests/nbs_frame_codecs.tests.ts diff --git a/src/server/nbs/nbs_frame_chunker.ts b/src/server/nbs/nbs_frame_chunker.ts new file mode 100644 index 00000000..ed9f6288 --- /dev/null +++ b/src/server/nbs/nbs_frame_chunker.ts @@ -0,0 +1,56 @@ +import * as Buffers from 'buffers' +import * as stream from 'stream' + +const NBS_NUCLEAR_HEADER = Buffer.from([0xE2, 0x98, 0xA2]) // NUClear radiation symbol. + +export class NbsFrameChunker extends stream.Transform { + private buffers: Buffers + private foundHeader: boolean + private foundPacketSize: boolean + + constructor() { + super({ + objectMode: true, + }) + + this.buffers = new Buffers() + this.foundHeader = false + this.foundPacketSize = false + } + + public static of(): NbsFrameChunker { + return new NbsFrameChunker() + } + + public _transform(chunk: any, encoding: string, done: (err?: any, data?: any) => void) { + this.buffers.push(chunk) + + let frame + while ((frame = this.getNextFrame(this.buffers)) !== undefined) { + this.push(frame.buffer) + this.buffers.splice(0, frame.offset + frame.buffer.byteLength) + } + + done() + } + + private getNextFrame(buffer: Buffers): { offset: number, buffer: Buffer } | undefined { + const headerIndex = buffer.indexOf(NBS_NUCLEAR_HEADER) + const headerSize = NBS_NUCLEAR_HEADER.byteLength + const packetLengthSize = 4 + const headerAndPacketLengthSize = headerSize + packetLengthSize + if (headerIndex >= 0) { + const frame = buffer.slice(headerIndex) + if (frame.length >= headerAndPacketLengthSize) { + const packetSize = frame.slice(headerSize, headerSize + headerAndPacketLengthSize).readUInt32LE(0) + if (frame.length >= headerAndPacketLengthSize + packetSize) { + return { + offset: headerIndex, + buffer: frame.slice(0, headerAndPacketLengthSize + packetSize), + } + } + } + } + return undefined + } +} diff --git a/src/server/nbs/nbs_frame_codecs.ts b/src/server/nbs/nbs_frame_codecs.ts new file mode 100644 index 00000000..13ffce39 --- /dev/null +++ b/src/server/nbs/nbs_frame_codecs.ts @@ -0,0 +1,91 @@ +import * as Long from 'long' +import { NUClearNetPacket } from 'nuclearnet.js' +import * as stream from 'stream' +import { Clock } from '../time/clock' +import { NodeSystemClock } from '../time/node_clock' + +export type NbsFrame = { + header: Buffer + size: number + timestamp: number, + hash: Buffer, + payload: Buffer, +} + +export const NBS_HEADER = Buffer.from([0xE2, 0x98, 0xA2]) // NUClear radiation symbol. + +export class NbsFrameEncoder extends stream.Transform { + public constructor(private clock: Clock) { + super({ + objectMode: true, + }) + } + + public static of() { + return new NbsFrameEncoder(NodeSystemClock) + } + + public _transform(frame: NbsFrame, encoding: string, done: (err?: any, data?: any) => void) { + this.push(encodeFrame(frame)) + done() + } + + public writePacket(packet: NUClearNetPacket) { + this.write(packetToFrame(packet, this.clock.performanceNow())) + } +} + +export class NbsFrameDecoder extends stream.Transform { + public constructor() { + super({ + objectMode: true, + }) + } + + public static of() { + return new NbsFrameDecoder() + } + + public _transform(buffer: Buffer, encoding: string, done: (err?: any, data?: any) => void) { + this.push(decodeFrame(buffer)) + done() + } +} + +// NBS frame format: +// 3 Bytes - NUClear radiation symbol header, useful for synchronisation when attaching to an existing stream. +// 4 Bytes - The remaining packet length i.e. 16 bytes + N payload bytes +// 8 Bytes - 64bit timestamp in microseconds. Note: this is not necessarily a unix timestamp. +// 8 Bytes - 64bit bit hash of the message type. +// N bytes - The binary packet payload. + +export function encodeFrame(frame: NbsFrame): Buffer { + const buffer = new Buffer(7 + frame.size) + NBS_HEADER.copy(buffer) + buffer.writeUInt32LE(frame.size, 3) + const timeLong = Long.fromNumber(frame.timestamp) + buffer.writeUInt32LE(timeLong.low, 7) + buffer.writeUInt32LE(timeLong.high, 11) + frame.hash.copy(buffer, 15) + frame.payload.copy(buffer, 23) + return buffer +} + +export function decodeFrame(buffer: Buffer): NbsFrame { + const header = buffer.slice(0, 3) + const size = buffer.readUInt32LE(3) + const timestamp = Long.fromBits(buffer.readUInt32LE(7), buffer.readUInt32LE(11)).toNumber() + const hash = buffer.slice(15, 23) + const payload = buffer.slice(23) + return { header, size, timestamp, hash, payload } +} + +export function packetToFrame(packet: NUClearNetPacket, timestamp: number): NbsFrame { + return { + header: Buffer.from(NBS_HEADER), + size: packet.payload.byteLength + 16, + timestamp, + hash: packet.hash, + payload: packet.payload, + } +} diff --git a/src/server/nbs/nbs_nuclear_playback.ts b/src/server/nbs/nbs_nuclear_playback.ts new file mode 100644 index 00000000..0bd677b2 --- /dev/null +++ b/src/server/nbs/nbs_nuclear_playback.ts @@ -0,0 +1,47 @@ +import { ReadStream } from 'fs' +import * as stream from 'stream' +import { NUClearNetClient } from '../../shared/nuclearnet/nuclearnet_client' +import { Clock } from '../time/clock' +import { NodeSystemClock } from '../time/node_clock' +import { NbsFrameChunker } from './nbs_frame_chunker' +import { NbsFrame } from './nbs_frame_codecs' +import { NbsFrameDecoder } from './nbs_frame_codecs' + +export class NbsNUClearPlayback extends stream.Writable { + private firstFrameTimestamp?: number + private firstLocalTimestamp?: number + + public constructor(private nuclearnetClient: NUClearNetClient, + private clock: Clock) { + super({ + objectMode: true, + }) + } + + public static of(nuclearnetClient: NUClearNetClient) { + return new NbsNUClearPlayback(nuclearnetClient, NodeSystemClock) + } + + public static fromRawStream(rawStream: ReadStream, nuclearnetClient: NUClearNetClient) { + const playback = NbsNUClearPlayback.of(nuclearnetClient) + rawStream.pipe(new NbsFrameChunker()).pipe(new NbsFrameDecoder()).pipe(playback) + return playback + } + + public _write(frame: NbsFrame, encoding: string, done: Function) { + if (this.firstFrameTimestamp === undefined || this.firstLocalTimestamp === undefined) { + this.firstFrameTimestamp = frame.timestamp + this.firstLocalTimestamp = this.clock.performanceNow() + } + const now = this.clock.performanceNow() + const timeOffset = frame.timestamp - this.firstFrameTimestamp + const timeout = this.firstLocalTimestamp + timeOffset - now + this.clock.setTimeout(() => { + this.nuclearnetClient.send({ + type: frame.hash, + payload: frame.payload, + }) + done() + }, timeout) + } +} diff --git a/src/server/nbs/nbs_playback_controller.ts b/src/server/nbs/nbs_playback_controller.ts new file mode 100644 index 00000000..b2db09c3 --- /dev/null +++ b/src/server/nbs/nbs_playback_controller.ts @@ -0,0 +1,58 @@ +import * as fs from 'fs' +import { ReadStream } from 'fs' +import { NUClearNetClient } from '../../shared/nuclearnet/nuclearnet_client' +import { NbsNUClearPlayback } from './nbs_nuclear_playback' +import WritableStream = NodeJS.WritableStream + +export class NbsPlaybackController { + private state: PlaybackState + private inputStream?: ReadStream + private outputStream: WritableStream + private currentIndex: number + + public constructor(private nuclearnetClient: NUClearNetClient) { + this.state = PlaybackState.Idle + this.currentIndex = 0 + } + + public static of(nuclearnetClient: NUClearNetClient) { + return new NbsPlaybackController(nuclearnetClient) + } + + public play(filename: string): () => void { + this.state = PlaybackState.Playing + + this.inputStream = fs.createReadStream(filename, { encoding: 'binary' }) + this.outputStream = NbsNUClearPlayback.fromRawStream(this.inputStream, this.nuclearnetClient) + .on('finish', () => { + this.state = PlaybackState.Idle + }) + + return () => { + if (this.inputStream) { + this.state = PlaybackState.Idle + this.inputStream.close() + this.inputStream = undefined + } + } + } + + public pause(): () => void { + if (this.inputStream) { + this.state = PlaybackState.Paused + this.inputStream.pause() + } + return () => { + if (this.inputStream && this.state === PlaybackState.Paused) { + this.state = PlaybackState.Playing + this.inputStream.resume() + } + } + } +} + +enum PlaybackState { + Idle = 1, + Paused, + Playing, +} diff --git a/src/server/nbs/nbs_recorder_controller.ts b/src/server/nbs/nbs_recorder_controller.ts new file mode 100644 index 00000000..6fec33af --- /dev/null +++ b/src/server/nbs/nbs_recorder_controller.ts @@ -0,0 +1,46 @@ +import { WriteStream } from 'fs' +import * as fs from 'fs' +import { NUClearNetPacket } from 'nuclearnet.js' +import { NUClearNetPeer } from 'nuclearnet.js' +import { NUClearNetClient } from '../../shared/nuclearnet/nuclearnet_client' +import { NbsFrameEncoder } from './nbs_frame_codecs' + +export class NbsRecorderController { + private frameEncoder: NbsFrameEncoder + private recording: boolean + private file: WriteStream + + public constructor(private peer: NUClearNetPeer, + private nuclearnetClient: NUClearNetClient) { + this.recording = false + } + + public static of(peer: NUClearNetPeer, nuclearnetClient: NUClearNetClient): NbsRecorderController { + return new NbsRecorderController(peer, nuclearnetClient) + } + + public record(filename: string): () => void { + this.frameEncoder = NbsFrameEncoder.of() + this.file = fs.createWriteStream(filename, { defaultEncoding: 'binary' }) + const stopListening = this.nuclearnetClient.onPacket(this.onPacket) + this.frameEncoder.pipe(this.file) + this.recording = true + return () => { + stopListening() + this.recording = false + this.file.end() + } + } + + private onPacket = (packet: NUClearNetPacket) => { + if (!this.arePeersEqual(this.peer, packet.peer)) { + return + } + + this.frameEncoder.writePacket(packet) + } + + private arePeersEqual(peerA: NUClearNetPeer, peerB: NUClearNetPeer) { + return peerA.name === peerB.name && peerA.address === peerB.address && peerA.port === peerB.port + } +} diff --git a/src/server/nbs/tests/nbs_frame_codecs.tests.ts b/src/server/nbs/tests/nbs_frame_codecs.tests.ts new file mode 100644 index 00000000..7a38ce38 --- /dev/null +++ b/src/server/nbs/tests/nbs_frame_codecs.tests.ts @@ -0,0 +1,34 @@ +import { hashType } from '../../nuclearnet/fake_nuclearnet_server' +import { encodeFrame } from '../nbs_frame_codecs' +import { NBS_HEADER } from '../nbs_frame_codecs' +import { decodeFrame } from '../nbs_frame_codecs' + +describe('NbsFrameCodecs', () => { + describe('encoding', () => { + it('encodes frames', () => { + const hash = hashType('message.input.sensors') + const timestamp = 1500379664696000 + const payload = new Buffer(8).fill(0x12) + const buffer = encodeFrame({ + header: NBS_HEADER, + size: 16 + payload.byteLength, + timestamp, + hash, + payload, + }) + expect(buffer.toString('hex')).toEqual('e298a218000000c042f15c9654050010abef8b5398f0d41212121212121212') + }) + }) + + describe('decoding', () => { + it('decodes frames', () => { + const buffer = Buffer.from('e298a218000000c042f15c9654050010abef8b5398f0d41212121212121212', 'hex') + const frame = decodeFrame(buffer) + expect(frame.header.equals(NBS_HEADER)).toBeTruthy() + expect(frame.size).toEqual(24) + expect(frame.timestamp).toEqual(1500379664696000) + expect(frame.hash.equals(hashType('message.input.sensors'))).toBeTruthy() + expect(frame.payload.equals(new Buffer(8).fill(0x12))).toBeTruthy() + }) + }) +}) diff --git a/src/server/nusight_server.ts b/src/server/nusight_server.ts index 775686d2..fa10ca1d 100644 --- a/src/server/nusight_server.ts +++ b/src/server/nusight_server.ts @@ -1,11 +1,7 @@ -import * as Buffers from 'buffers' -import * as fs from 'fs' -import { WriteStream } from 'fs' -import * as Long from 'long' import { NUClearNetPeer } from 'nuclearnet.js' -import { NUClearNetPacket } from 'nuclearnet.js' -import * as stream from 'stream' import { NUClearNetClient } from '../shared/nuclearnet/nuclearnet_client' +import { NbsPlaybackController } from './nbs/nbs_playback_controller' +import { NbsRecorderController } from './nbs/nbs_recorder_controller' import { WebSocketServer } from './nuclearnet/web_socket_server' import { WebSocket } from './nuclearnet/web_socket_server' import { Clock } from './time/clock' @@ -46,7 +42,7 @@ class NUsightServerClient { } private onRecord = (peer: NUClearNetPeer, requestToken: string) => { - const recorder = NbsRecorder.of(peer, this.nuclearnetClient) + const recorder = NbsRecorderController.of(peer, this.nuclearnetClient) const filename = `${peer.name.replace(/[^A-Za-z0-9]/g, '_')}_${this.clock.now()}.nbs` console.log('recording', peer, requestToken) const stopRecording = recorder.record(`recordings/${filename}`) @@ -62,7 +58,7 @@ class NUsightServerClient { } private onPlay = (filename: string, requestToken: string) => { - const player = NbsPlayback.of(this.nuclearnetClient) + const player = NbsPlaybackController.of(this.nuclearnetClient) console.log('playing', filename, requestToken) const stopPlaying = player.play(`recordings/${filename}`) this.stopRecordingMap.set(requestToken, stopPlaying) @@ -84,302 +80,3 @@ class NUsightServerClient { } } } - -class NbsRecorder { - private recording: boolean - private file: WriteStream - - public constructor(private peer: NUClearNetPeer, - private clock: Clock, - private nuclearnetClient: NUClearNetClient) { - this.recording = false - } - - public static of(peer: NUClearNetPeer, nuclearnetClient: NUClearNetClient): NbsRecorder { - return new NbsRecorder(peer, NodeSystemClock, nuclearnetClient) - } - - public record(filename: string): () => void { - this.file = fs.createWriteStream(filename, { defaultEncoding: 'binary' }) - const stopListening = this.nuclearnetClient.onPacket(this.onPacket) - this.recording = true - return () => { - stopListening() - this.recording = false - this.file.end() - } - } - - private onPacket = (packet: NUClearNetPacket) => { - if (!this.arePeersEqual(this.peer, packet.peer)) { - return - } - - // NBS File Format: - // 3 Bytes - NUClear radiation symbol header, useful for synchronisation when attaching to an existing stream. - // 4 Bytes - The remaining packet length i.e. 16 bytes + N payload bytes - // 8 Bytes - 64bit timestamp in microseconds. Note: this is not necessarily a unix timestamp. - // 8 Bytes - 64bit bit hash of the message type. - // N bytes - The binary packet payload. - - const header = Buffer.from([0xE2, 0x98, 0xA2]) // NUClear radiation symbol. - - // 64bit timestamp in microseconds. - const time = this.clock.performanceNow() * 1e6 - const timeLong = Long.fromNumber(time) - const timeBuffer = new Buffer(8) - timeBuffer.writeUInt32LE(timeLong.low, 0) - timeBuffer.writeUInt32LE(timeLong.high, 4) - - const remainingByteLength = new Buffer(4) - remainingByteLength.writeUInt32LE(timeBuffer.byteLength + packet.hash.byteLength + packet.payload.byteLength, 0) - - // Write parts to file - this.file.write(header) - this.file.write(remainingByteLength) - this.file.write(timeBuffer) - this.file.write(packet.hash) - this.file.write(packet.payload) - } - - private arePeersEqual(peerA: NUClearNetPeer, peerB: NUClearNetPeer) { - return peerA.name === peerB.name && peerA.address === peerB.address && peerA.port === peerB.port - } -} - -class NbsPlayback { - private state: PlaybackState - private inputStream: stream.Transform - private outputStream: WritableStream - private currentIndex: number - - public constructor(private nuclearnetClient: NUClearNetClient) { - this.state = PlaybackState.Idle - this.currentIndex = 0 - } - - public static of(nuclearnetClient: NUClearNetClient) { - return new NbsPlayback(nuclearnetClient) - } - - public play(filename: string): () => void { - this.state = PlaybackState.Playing - - this.inputStream = fs.createReadStream(filename, { encoding: 'binary' }) - .pipe(NbsFrameTransformStream.of()) - .pipe(NbsFrameDecoderStream.of()) - .on('finish', () => { - this.state = PlaybackState.Idle - }) - - this.outputStream = this.inputStream.pipe(NbsNUClearPlayback.of(this.nuclearnetClient)) - - return () => { - if (this.state === PlaybackState.Playing) { - this.state = PlaybackState.Idle - this.inputStream.end() - } - } - } - - public pause(): () => void { - this.state = PlaybackState.Paused - this.inputStream.pause() - return () => { - if (this.state === PlaybackState.Paused) { - this.state = PlaybackState.Playing - this.inputStream.resume() - } - } - } -} - -// type NbsNUClearWritableStreamOpts = { -// } - -const NBS_NUCLEAR_HEADER = Buffer.from([0xE2, 0x98, 0xA2]) // NUClear radiation symbol. - -export class NbsFrameTransformStream extends stream.Transform { - private buffers: Buffers - private foundHeader: boolean - private foundPacketSize: boolean - - constructor() { - super({ - objectMode: true, - }) - - this.buffers = new Buffers() - this.foundHeader = false - this.foundPacketSize = false - } - - public static of(): NbsFrameTransformStream { - return new NbsFrameTransformStream() - } - - public _transform(chunk: any, encoding: string, done: (err?: any, data?: any) => void) { - this.buffers.push(chunk) - - let frame - while ((frame = this.getNextFrame(this.buffers)) !== undefined) { - this.push(frame.buffer) - this.buffers.splice(0, frame.offset + frame.buffer.byteLength) - } - - done() - } - - private getNextFrame(buffer: Buffers): { offset: number, buffer: Buffer } | undefined { - const headerIndex = buffer.indexOf(NBS_NUCLEAR_HEADER) - const headerSize = NBS_NUCLEAR_HEADER.byteLength - const packetLengthSize = 4 - const headerAndPacketLengthSize = headerSize + packetLengthSize - if (headerIndex >= 0) { - const frame = buffer.slice(headerIndex) - if (frame.length >= headerAndPacketLengthSize) { - const packetSize = frame.slice(headerSize, headerSize + headerAndPacketLengthSize).readUInt32LE(0) - if (frame.length >= headerAndPacketLengthSize + packetSize) { - return { - offset: headerIndex, - buffer: frame.slice(0, headerAndPacketLengthSize + packetSize), - } - } - } - } - return undefined - } -} - -export class NbsFrameDecoderStream extends stream.Transform { - public constructor() { - super({ - objectMode: true, - }) - } - - public static of() { - return new NbsFrameDecoderStream() - } - - public _transform(frame: Buffer, encoding: string, done: (err?: any, data?: any) => void) { - this.push(this.decode(frame)) - done() - } - - private decode(frame: Buffer) { - const header = frame.slice(0, 3) - const size = frame.readUInt32LE(3) - const timestamp = Long.fromBits(frame.readUInt32LE(7), frame.readUInt32LE(11)).toNumber() - const hash = frame.slice(15, 23) - const payload = frame.slice(23) - return { header, size, timestamp, hash, payload } - } -} - -type NbsFrame = { - header: Buffer - size: number - timestamp: number, - hash: Buffer, - payload: Buffer, -} - -export class NbsNUClearPlayback extends stream.Writable { - private firstFrameTimestamp?: number - private firstLocalTimestamp?: number - - public constructor(private nuclearnetClient: NUClearNetClient, - private clock: Clock) { - super({ - objectMode: true, - }) - } - - public static of(nuclearnetClient: NUClearNetClient) { - return new NbsNUClearPlayback(nuclearnetClient, NodeSystemClock) - } - - public _write(frame: NbsFrame, encoding: string, done: Function) { - if (this.firstFrameTimestamp === undefined || this.firstLocalTimestamp === undefined) { - this.firstFrameTimestamp = frame.timestamp - this.firstLocalTimestamp = this.clock.performanceNow() - } - const now = this.clock.performanceNow() - const timeOffset = frame.timestamp - this.firstFrameTimestamp - const timeout = this.firstLocalTimestamp + timeOffset - now - this.clock.setTimeout(() => { - this.nuclearnetClient.send({ - type: frame.hash, - payload: frame.payload, - }) - done() - }, timeout) - } -} - -// export class NbsNUClearWritableStream extends stream.Writable { -// private buffers: Buffers -// private buffering: boolean -// private payloadLength: number -// private headerIndex: number -// -// public constructor(private nuclearnetClient: NUClearNetClient/*, opts: NbsNUClearWritableStreamOpts*/) { -// super() -// -// this.buffers = new Buffers() -// this.buffering = false -// this.headerIndex = 0 -// } -// -// public static of(opts: { fakeNetworking: boolean }) { -// const network = opts.fakeNetworking ? FakeNUClearNetClient.of() : DirectNUClearNetClient.of() -// return new NbsNUClearWritableStream(network) -// } -// -// public _write(chunk: Buffer, encoding: string, cb: Function) { -// this.buffers.push(chunk) -// -// if (!this.buffering) { -// const headerIndex = chunk.indexOf(NBS_NUCLEAR_HEADER) -// if (headerIndex >= 0) { -// console.log(`Found header at ${headerIndex}`) -// this.headerIndex = headerIndex -// this.buffering = true -// } -// } else { -// console.log(this.buffers.buffers.length) -// -// const offset = this.headerIndex -// if (this.buffers.length >= offset + 7) { -// const remainingByteLength = this.buffers.slice(offset + 3, offset + 3 + 4).readUInt32LE(0) -// this.payloadLength = remainingByteLength -// if (this.buffers.length >= offset + 7 + remainingByteLength) { -// const hash = this.buffers.slice(offset + 15, offset + 15 + 8) -// const payload = this.buffers.slice(offset + 23, offset + 23 + remainingByteLength - 16) -// this.nuclearnetClient.send({ -// type: hash, -// payload, -// }) -// this.buffering = false -// const end = offset + 7 + remainingByteLength -// this.buffers.splice(0, end) -// } -// } -// } -// cb() -// } -// -// decode(buffer: Buffer): { type: Buffer, payload: Buffer } { -// return { -// type: new Buffer(8), -// payload: new Buffer(8), -// } -// } -// } - -enum PlaybackState { - Idle = 1, - Paused, - Playing, -} diff --git a/src/server/tests/nbs_nuclear_writeable_stream.tests.ts b/src/server/tests/nbs_nuclear_writeable_stream.tests.ts index 1cddb982..42e00925 100644 --- a/src/server/tests/nbs_nuclear_writeable_stream.tests.ts +++ b/src/server/tests/nbs_nuclear_writeable_stream.tests.ts @@ -2,21 +2,21 @@ import * as fs from 'fs' import * as stream from 'stream' import { Stream } from 'stream' import { NUClearNetClient } from '../../shared/nuclearnet/nuclearnet_client' +import { message } from '../../shared/proto/messages' +import { NbsFrameChunker } from '../nbs/nbs_frame_chunker' +import { NbsFrameDecoder } from '../nbs/nbs_frame_codecs' +import { NbsNUClearPlayback } from '../nbs/nbs_nuclear_playback' import { FakeNUClearNetClient } from '../nuclearnet/fake_nuclearnet_client' import { FakeNUClearNetServer } from '../nuclearnet/fake_nuclearnet_server' -import { NbsFrameTransformStream } from '../nusight_server' -import { NbsFrameDecoderStream } from '../nusight_server' -import { NbsNUClearPlayback } from '../nusight_server' -import WritableStream = NodeJS.WritableStream import { FakeNodeClock } from '../time/fake_node_clock' -import { message } from '../../shared/proto/messages' +import WritableStream = NodeJS.WritableStream import nuclear = message.support.nuclear -describe('NbsFrameTransformStream', () => { +describe('NbsFrameChunker', () => { let transform: stream.Transform beforeEach(() => { - transform = new NbsFrameTransformStream() + transform = new NbsFrameChunker() }) it('Emits 6988 frames', done => { @@ -38,8 +38,8 @@ describe('NbsNUClearPlayback', () => { nuclearnetClient = new FakeNUClearNetClient(nuclearnetServer) stream = fs.createReadStream('/Users/brendan/Lab/NUsight2/recordings/darwin3_WalkAround.nbs') - .pipe(new NbsFrameTransformStream()) - .pipe(new NbsFrameDecoderStream) + .pipe(new NbsFrameChunker()) + .pipe(new NbsFrameDecoder()) }) From 4a1adfd5c424032de66a2bec808c31f62b8c6165 Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Tue, 18 Jul 2017 23:24:04 +1000 Subject: [PATCH 15/24] . --- src/server/nbs/nbs_frame_chunker.ts | 7 +-- src/server/nbs/nbs_frame_codecs.ts | 59 +++---------------- src/server/nbs/nbs_frame_streams.ts | 46 +++++++++++++++ src/server/nbs/nbs_nuclear_playback.ts | 2 +- .../nbs/tests/nbs_frame_codecs.tests.ts | 7 +-- .../nbs_nuclear_writeable_stream.tests.ts | 2 +- 6 files changed, 59 insertions(+), 64 deletions(-) create mode 100644 src/server/nbs/nbs_frame_streams.ts diff --git a/src/server/nbs/nbs_frame_chunker.ts b/src/server/nbs/nbs_frame_chunker.ts index ed9f6288..50a728a9 100644 --- a/src/server/nbs/nbs_frame_chunker.ts +++ b/src/server/nbs/nbs_frame_chunker.ts @@ -1,7 +1,6 @@ import * as Buffers from 'buffers' import * as stream from 'stream' - -const NBS_NUCLEAR_HEADER = Buffer.from([0xE2, 0x98, 0xA2]) // NUClear radiation symbol. +import { NBS_HEADER } from './nbs_frame_codecs' export class NbsFrameChunker extends stream.Transform { private buffers: Buffers @@ -35,8 +34,8 @@ export class NbsFrameChunker extends stream.Transform { } private getNextFrame(buffer: Buffers): { offset: number, buffer: Buffer } | undefined { - const headerIndex = buffer.indexOf(NBS_NUCLEAR_HEADER) - const headerSize = NBS_NUCLEAR_HEADER.byteLength + const headerIndex = buffer.indexOf(NBS_HEADER) + const headerSize = NBS_HEADER.byteLength const packetLengthSize = 4 const headerAndPacketLengthSize = headerSize + packetLengthSize if (headerIndex >= 0) { diff --git a/src/server/nbs/nbs_frame_codecs.ts b/src/server/nbs/nbs_frame_codecs.ts index 13ffce39..6f2dfe10 100644 --- a/src/server/nbs/nbs_frame_codecs.ts +++ b/src/server/nbs/nbs_frame_codecs.ts @@ -1,57 +1,14 @@ import * as Long from 'long' import { NUClearNetPacket } from 'nuclearnet.js' -import * as stream from 'stream' -import { Clock } from '../time/clock' -import { NodeSystemClock } from '../time/node_clock' + +export const NBS_HEADER = Buffer.from([0xE2, 0x98, 0xA2]) // NUClear radiation symbol. export type NbsFrame = { - header: Buffer - size: number timestamp: number, hash: Buffer, payload: Buffer, } -export const NBS_HEADER = Buffer.from([0xE2, 0x98, 0xA2]) // NUClear radiation symbol. - -export class NbsFrameEncoder extends stream.Transform { - public constructor(private clock: Clock) { - super({ - objectMode: true, - }) - } - - public static of() { - return new NbsFrameEncoder(NodeSystemClock) - } - - public _transform(frame: NbsFrame, encoding: string, done: (err?: any, data?: any) => void) { - this.push(encodeFrame(frame)) - done() - } - - public writePacket(packet: NUClearNetPacket) { - this.write(packetToFrame(packet, this.clock.performanceNow())) - } -} - -export class NbsFrameDecoder extends stream.Transform { - public constructor() { - super({ - objectMode: true, - }) - } - - public static of() { - return new NbsFrameDecoder() - } - - public _transform(buffer: Buffer, encoding: string, done: (err?: any, data?: any) => void) { - this.push(decodeFrame(buffer)) - done() - } -} - // NBS frame format: // 3 Bytes - NUClear radiation symbol header, useful for synchronisation when attaching to an existing stream. // 4 Bytes - The remaining packet length i.e. 16 bytes + N payload bytes @@ -60,9 +17,10 @@ export class NbsFrameDecoder extends stream.Transform { // N bytes - The binary packet payload. export function encodeFrame(frame: NbsFrame): Buffer { - const buffer = new Buffer(7 + frame.size) + const size = 16 + frame.payload.byteLength + const buffer = new Buffer(7 + size) NBS_HEADER.copy(buffer) - buffer.writeUInt32LE(frame.size, 3) + buffer.writeUInt32LE(size, 3) const timeLong = Long.fromNumber(frame.timestamp) buffer.writeUInt32LE(timeLong.low, 7) buffer.writeUInt32LE(timeLong.high, 11) @@ -72,18 +30,15 @@ export function encodeFrame(frame: NbsFrame): Buffer { } export function decodeFrame(buffer: Buffer): NbsFrame { - const header = buffer.slice(0, 3) const size = buffer.readUInt32LE(3) const timestamp = Long.fromBits(buffer.readUInt32LE(7), buffer.readUInt32LE(11)).toNumber() const hash = buffer.slice(15, 23) - const payload = buffer.slice(23) - return { header, size, timestamp, hash, payload } + const payload = buffer.slice(23, 23 + size) + return { timestamp, hash, payload } } export function packetToFrame(packet: NUClearNetPacket, timestamp: number): NbsFrame { return { - header: Buffer.from(NBS_HEADER), - size: packet.payload.byteLength + 16, timestamp, hash: packet.hash, payload: packet.payload, diff --git a/src/server/nbs/nbs_frame_streams.ts b/src/server/nbs/nbs_frame_streams.ts new file mode 100644 index 00000000..de0bd545 --- /dev/null +++ b/src/server/nbs/nbs_frame_streams.ts @@ -0,0 +1,46 @@ +import * as stream from 'stream' +import { Clock } from '../time/clock' +import { NodeSystemClock } from '../time/node_clock' +import { NbsFrame } from './nbs_frame_codecs' +import { encodeFrame } from './nbs_frame_codecs' +import { NUClearNetPacket } from 'nuclearnet.js' +import { packetToFrame } from './nbs_frame_codecs' +import { decodeFrame } from './nbs_frame_codecs' + +export class NbsFrameEncoder extends stream.Transform { + public constructor(private clock: Clock) { + super({ + objectMode: true, + }) + } + + public static of() { + return new NbsFrameEncoder(NodeSystemClock) + } + + public _transform(frame: NbsFrame, encoding: string, done: (err?: any, data?: any) => void) { + this.push(encodeFrame(frame)) + done() + } + + public writePacket(packet: NUClearNetPacket) { + this.write(packetToFrame(packet, this.clock.performanceNow())) + } +} + +export class NbsFrameDecoder extends stream.Transform { + public constructor() { + super({ + objectMode: true, + }) + } + + public static of() { + return new NbsFrameDecoder() + } + + public _transform(buffer: Buffer, encoding: string, done: (err?: any, data?: any) => void) { + this.push(decodeFrame(buffer)) + done() + } +} diff --git a/src/server/nbs/nbs_nuclear_playback.ts b/src/server/nbs/nbs_nuclear_playback.ts index 0bd677b2..dc741f66 100644 --- a/src/server/nbs/nbs_nuclear_playback.ts +++ b/src/server/nbs/nbs_nuclear_playback.ts @@ -5,7 +5,7 @@ import { Clock } from '../time/clock' import { NodeSystemClock } from '../time/node_clock' import { NbsFrameChunker } from './nbs_frame_chunker' import { NbsFrame } from './nbs_frame_codecs' -import { NbsFrameDecoder } from './nbs_frame_codecs' +import { NbsFrameDecoder } from './nbs_frame_streams' export class NbsNUClearPlayback extends stream.Writable { private firstFrameTimestamp?: number diff --git a/src/server/nbs/tests/nbs_frame_codecs.tests.ts b/src/server/nbs/tests/nbs_frame_codecs.tests.ts index 7a38ce38..844bb7d1 100644 --- a/src/server/nbs/tests/nbs_frame_codecs.tests.ts +++ b/src/server/nbs/tests/nbs_frame_codecs.tests.ts @@ -1,6 +1,5 @@ import { hashType } from '../../nuclearnet/fake_nuclearnet_server' import { encodeFrame } from '../nbs_frame_codecs' -import { NBS_HEADER } from '../nbs_frame_codecs' import { decodeFrame } from '../nbs_frame_codecs' describe('NbsFrameCodecs', () => { @@ -10,22 +9,18 @@ describe('NbsFrameCodecs', () => { const timestamp = 1500379664696000 const payload = new Buffer(8).fill(0x12) const buffer = encodeFrame({ - header: NBS_HEADER, - size: 16 + payload.byteLength, timestamp, hash, payload, }) expect(buffer.toString('hex')).toEqual('e298a218000000c042f15c9654050010abef8b5398f0d41212121212121212') }) - }) + }) describe('decoding', () => { it('decodes frames', () => { const buffer = Buffer.from('e298a218000000c042f15c9654050010abef8b5398f0d41212121212121212', 'hex') const frame = decodeFrame(buffer) - expect(frame.header.equals(NBS_HEADER)).toBeTruthy() - expect(frame.size).toEqual(24) expect(frame.timestamp).toEqual(1500379664696000) expect(frame.hash.equals(hashType('message.input.sensors'))).toBeTruthy() expect(frame.payload.equals(new Buffer(8).fill(0x12))).toBeTruthy() diff --git a/src/server/tests/nbs_nuclear_writeable_stream.tests.ts b/src/server/tests/nbs_nuclear_writeable_stream.tests.ts index 42e00925..b1587b07 100644 --- a/src/server/tests/nbs_nuclear_writeable_stream.tests.ts +++ b/src/server/tests/nbs_nuclear_writeable_stream.tests.ts @@ -4,7 +4,7 @@ import { Stream } from 'stream' import { NUClearNetClient } from '../../shared/nuclearnet/nuclearnet_client' import { message } from '../../shared/proto/messages' import { NbsFrameChunker } from '../nbs/nbs_frame_chunker' -import { NbsFrameDecoder } from '../nbs/nbs_frame_codecs' +import { NbsFrameDecoder } from '../nbs/nbs_frame_streams' import { NbsNUClearPlayback } from '../nbs/nbs_nuclear_playback' import { FakeNUClearNetClient } from '../nuclearnet/fake_nuclearnet_client' import { FakeNUClearNetServer } from '../nuclearnet/fake_nuclearnet_server' From 49e020574233f4bac0fd7671fbfa9f25198211b8 Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Tue, 18 Jul 2017 23:53:59 +1000 Subject: [PATCH 16/24] . --- .../nbs/tests/nbs_frame_codecs.tests.ts | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/src/server/nbs/tests/nbs_frame_codecs.tests.ts b/src/server/nbs/tests/nbs_frame_codecs.tests.ts index 844bb7d1..a48ed9e7 100644 --- a/src/server/nbs/tests/nbs_frame_codecs.tests.ts +++ b/src/server/nbs/tests/nbs_frame_codecs.tests.ts @@ -8,22 +8,36 @@ describe('NbsFrameCodecs', () => { const hash = hashType('message.input.sensors') const timestamp = 1500379664696000 const payload = new Buffer(8).fill(0x12) - const buffer = encodeFrame({ - timestamp, - hash, - payload, - }) + const buffer = encodeFrame({ timestamp, hash, payload }) expect(buffer.toString('hex')).toEqual('e298a218000000c042f15c9654050010abef8b5398f0d41212121212121212') }) - }) + }) describe('decoding', () => { it('decodes frames', () => { const buffer = Buffer.from('e298a218000000c042f15c9654050010abef8b5398f0d41212121212121212', 'hex') const frame = decodeFrame(buffer) - expect(frame.timestamp).toEqual(1500379664696000) - expect(frame.hash.equals(hashType('message.input.sensors'))).toBeTruthy() - expect(frame.payload.equals(new Buffer(8).fill(0x12))).toBeTruthy() + expect(frame).toEqual({ + timestamp: 1500379664696000, + hash: hashType('message.input.sensors'), + payload: new Buffer(8).fill(0x12) + }) + }) + }) + + describe('roundtrip', () => { + it('decode than encode should equal original', () => { + const buffer = Buffer.from('e298a218000000c042f15c9654050010abef8b5398f0d41212121212121212', 'hex') + expect(encodeFrame(decodeFrame(buffer))).toEqual(buffer) + }) + + it('encode than decode should equal original', () => { + const frame = { + hash: hashType('message.input.sensors'), + timestamp: 1500379664696000, + payload: new Buffer(8).fill(0x12), + } + expect(decodeFrame(encodeFrame(frame))).toEqual(frame) }) }) }) From 033bbc46f2894efb80d8b6ec42729d88d59ad351 Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Wed, 19 Jul 2017 01:04:21 +1000 Subject: [PATCH 17/24] it works! --- src/server/dev.ts | 15 +++++++++++++++ src/server/nbs/nbs_frame_streams.ts | 2 +- src/server/nbs/nbs_nuclear_playback.ts | 10 ++++++---- src/server/nbs/nbs_recorder_controller.ts | 2 +- src/server/time/node_clock.ts | 6 +++--- 5 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/server/dev.ts b/src/server/dev.ts index 57a3dd07..33086318 100644 --- a/src/server/dev.ts +++ b/src/server/dev.ts @@ -16,6 +16,8 @@ import { FakeNUClearNetClient } from './nuclearnet/fake_nuclearnet_client' import { WebSocketProxyNUClearNetServer } from './nuclearnet/web_socket_proxy_nuclearnet_server' import { WebSocketServer } from './nuclearnet/web_socket_server' import { NUsightServer } from './nusight_server' +import { NbsNUClearPlayback } from './nbs/nbs_nuclear_playback' +import * as fs from 'fs' const compiler = webpack(webpackConfig) @@ -66,3 +68,16 @@ const nuclearnetClient = withSimulators ? FakeNUClearNetClient.of() : DirectNUCl WebSocketProxyNUClearNetServer.of(WebSocketServer.of(sioNetwork.of('/nuclearnet')), nuclearnetClient) NUsightServer.of(WebSocketServer.of(sioNetwork.of('/nusight')), nuclearnetClient) + +async function playback() { + const fake = FakeNUClearNetClient.of() + fake.connect({ name: 'Fake Stream' }) + while (true) { + const file = fs.createReadStream('/Users/brendan/Lab/NUsight2/recordings/darwin3_WalkAround.nbs') + // const file = fs.createReadStream('/Users/brendan/Lab/NUsight2/recordings/darwin3_FollowBall.nbs') + const out = NbsNUClearPlayback.fromRawStream(file, fake) + await new Promise(res => out.on('finish', res)) + } +} + +playback() diff --git a/src/server/nbs/nbs_frame_streams.ts b/src/server/nbs/nbs_frame_streams.ts index de0bd545..caeff7ab 100644 --- a/src/server/nbs/nbs_frame_streams.ts +++ b/src/server/nbs/nbs_frame_streams.ts @@ -24,7 +24,7 @@ export class NbsFrameEncoder extends stream.Transform { } public writePacket(packet: NUClearNetPacket) { - this.write(packetToFrame(packet, this.clock.performanceNow())) + this.write(packetToFrame(packet, this.clock.performanceNow() * 1e6)) } } diff --git a/src/server/nbs/nbs_nuclear_playback.ts b/src/server/nbs/nbs_nuclear_playback.ts index dc741f66..5097ca85 100644 --- a/src/server/nbs/nbs_nuclear_playback.ts +++ b/src/server/nbs/nbs_nuclear_playback.ts @@ -29,13 +29,15 @@ export class NbsNUClearPlayback extends stream.Writable { } public _write(frame: NbsFrame, encoding: string, done: Function) { + const now = this.clock.performanceNow() if (this.firstFrameTimestamp === undefined || this.firstLocalTimestamp === undefined) { this.firstFrameTimestamp = frame.timestamp - this.firstLocalTimestamp = this.clock.performanceNow() + this.firstLocalTimestamp = now } - const now = this.clock.performanceNow() - const timeOffset = frame.timestamp - this.firstFrameTimestamp - const timeout = this.firstLocalTimestamp + timeOffset - now + + const timeOffset = (frame.timestamp - this.firstFrameTimestamp) / 1e6 + const timeout = Math.max(0, this.firstLocalTimestamp + timeOffset - now) + this.clock.setTimeout(() => { this.nuclearnetClient.send({ type: frame.hash, diff --git a/src/server/nbs/nbs_recorder_controller.ts b/src/server/nbs/nbs_recorder_controller.ts index 6fec33af..edf59caa 100644 --- a/src/server/nbs/nbs_recorder_controller.ts +++ b/src/server/nbs/nbs_recorder_controller.ts @@ -3,7 +3,7 @@ import * as fs from 'fs' import { NUClearNetPacket } from 'nuclearnet.js' import { NUClearNetPeer } from 'nuclearnet.js' import { NUClearNetClient } from '../../shared/nuclearnet/nuclearnet_client' -import { NbsFrameEncoder } from './nbs_frame_codecs' +import { NbsFrameEncoder } from './nbs_frame_streams' export class NbsRecorderController { private frameEncoder: NbsFrameEncoder diff --git a/src/server/time/node_clock.ts b/src/server/time/node_clock.ts index 787d2cb8..b484ffb6 100644 --- a/src/server/time/node_clock.ts +++ b/src/server/time/node_clock.ts @@ -3,12 +3,12 @@ import { Clock } from './clock' export type CancelTimer = () => void function setTimeout(cb: (...args: any[]) => void, seconds: number): CancelTimer { - const handle = global.setTimeout(cb, seconds * 1000) + const handle = global.setTimeout(cb, seconds * 1e3) return global.clearTimeout.bind(null, handle) } function setInterval(cb: (...args: any[]) => void, seconds: number): CancelTimer { - const handle = global.setInterval(cb, seconds * 1000) + const handle = global.setInterval(cb, seconds * 1e3) return global.clearInterval.bind(null, handle) } @@ -24,7 +24,7 @@ function performanceNow() { export const NodeSystemClock: Clock = { - now: () => Date.now() / 1000, + now: () => Date.now() / 1e3, performanceNow, setTimeout, setInterval, From f91532c10b576261ca557655543d3defe89e1893 Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Wed, 19 Jul 2017 01:17:05 +1000 Subject: [PATCH 18/24] . --- src/server/nbs/nbs_frame_codecs.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/server/nbs/nbs_frame_codecs.ts b/src/server/nbs/nbs_frame_codecs.ts index 6f2dfe10..6e6b3a49 100644 --- a/src/server/nbs/nbs_frame_codecs.ts +++ b/src/server/nbs/nbs_frame_codecs.ts @@ -4,6 +4,9 @@ import { NUClearNetPacket } from 'nuclearnet.js' export const NBS_HEADER = Buffer.from([0xE2, 0x98, 0xA2]) // NUClear radiation symbol. export type NbsFrame = { + // Omitted redundant header information. + // header: Buffer, + // size: number, timestamp: number, hash: Buffer, payload: Buffer, @@ -19,12 +22,12 @@ export type NbsFrame = { export function encodeFrame(frame: NbsFrame): Buffer { const size = 16 + frame.payload.byteLength const buffer = new Buffer(7 + size) - NBS_HEADER.copy(buffer) + NBS_HEADER.copy(buffer, 0, 0, 3) buffer.writeUInt32LE(size, 3) const timeLong = Long.fromNumber(frame.timestamp) buffer.writeUInt32LE(timeLong.low, 7) buffer.writeUInt32LE(timeLong.high, 11) - frame.hash.copy(buffer, 15) + frame.hash.copy(buffer, 15, 0, 8) frame.payload.copy(buffer, 23) return buffer } From baaec954c076518250753081a8940bf324e0c1ce Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Wed, 19 Jul 2017 10:44:48 +1000 Subject: [PATCH 19/24] . --- src/server/nbs/nbs_frame_codecs.ts | 12 ++++++------ src/server/nbs/nbs_nuclear_playback.ts | 4 ++-- src/server/nbs/tests/nbs_frame_codecs.tests.ts | 2 +- .../tests/nbs_nuclear_writeable_stream.tests.ts | 7 ++----- src/server/time/node_clock.ts | 2 +- 5 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/server/nbs/nbs_frame_codecs.ts b/src/server/nbs/nbs_frame_codecs.ts index 6e6b3a49..809c3291 100644 --- a/src/server/nbs/nbs_frame_codecs.ts +++ b/src/server/nbs/nbs_frame_codecs.ts @@ -7,7 +7,7 @@ export type NbsFrame = { // Omitted redundant header information. // header: Buffer, // size: number, - timestamp: number, + timestampInMicroseconds: number, hash: Buffer, payload: Buffer, } @@ -24,7 +24,7 @@ export function encodeFrame(frame: NbsFrame): Buffer { const buffer = new Buffer(7 + size) NBS_HEADER.copy(buffer, 0, 0, 3) buffer.writeUInt32LE(size, 3) - const timeLong = Long.fromNumber(frame.timestamp) + const timeLong = Long.fromNumber(frame.timestampInMicroseconds) buffer.writeUInt32LE(timeLong.low, 7) buffer.writeUInt32LE(timeLong.high, 11) frame.hash.copy(buffer, 15, 0, 8) @@ -34,15 +34,15 @@ export function encodeFrame(frame: NbsFrame): Buffer { export function decodeFrame(buffer: Buffer): NbsFrame { const size = buffer.readUInt32LE(3) - const timestamp = Long.fromBits(buffer.readUInt32LE(7), buffer.readUInt32LE(11)).toNumber() + const timestampInMicroseconds = Long.fromBits(buffer.readUInt32LE(7), buffer.readUInt32LE(11)).toNumber() const hash = buffer.slice(15, 23) const payload = buffer.slice(23, 23 + size) - return { timestamp, hash, payload } + return { timestampInMicroseconds, hash, payload } } -export function packetToFrame(packet: NUClearNetPacket, timestamp: number): NbsFrame { +export function packetToFrame(packet: NUClearNetPacket, timestampInMicroseconds: number): NbsFrame { return { - timestamp, + timestampInMicroseconds: timestampInMicroseconds, hash: packet.hash, payload: packet.payload, } diff --git a/src/server/nbs/nbs_nuclear_playback.ts b/src/server/nbs/nbs_nuclear_playback.ts index 5097ca85..57357e45 100644 --- a/src/server/nbs/nbs_nuclear_playback.ts +++ b/src/server/nbs/nbs_nuclear_playback.ts @@ -31,11 +31,11 @@ export class NbsNUClearPlayback extends stream.Writable { public _write(frame: NbsFrame, encoding: string, done: Function) { const now = this.clock.performanceNow() if (this.firstFrameTimestamp === undefined || this.firstLocalTimestamp === undefined) { - this.firstFrameTimestamp = frame.timestamp + this.firstFrameTimestamp = frame.timestampInMicroseconds this.firstLocalTimestamp = now } - const timeOffset = (frame.timestamp - this.firstFrameTimestamp) / 1e6 + const timeOffset = (frame.timestampInMicroseconds - this.firstFrameTimestamp) * 1e-6 const timeout = Math.max(0, this.firstLocalTimestamp + timeOffset - now) this.clock.setTimeout(() => { diff --git a/src/server/nbs/tests/nbs_frame_codecs.tests.ts b/src/server/nbs/tests/nbs_frame_codecs.tests.ts index a48ed9e7..ac4770fa 100644 --- a/src/server/nbs/tests/nbs_frame_codecs.tests.ts +++ b/src/server/nbs/tests/nbs_frame_codecs.tests.ts @@ -8,7 +8,7 @@ describe('NbsFrameCodecs', () => { const hash = hashType('message.input.sensors') const timestamp = 1500379664696000 const payload = new Buffer(8).fill(0x12) - const buffer = encodeFrame({ timestamp, hash, payload }) + const buffer = encodeFrame({ timestampInMicroseconds: timestamp, hash, payload }) expect(buffer.toString('hex')).toEqual('e298a218000000c042f15c9654050010abef8b5398f0d41212121212121212') }) }) diff --git a/src/server/tests/nbs_nuclear_writeable_stream.tests.ts b/src/server/tests/nbs_nuclear_writeable_stream.tests.ts index b1587b07..43771e9c 100644 --- a/src/server/tests/nbs_nuclear_writeable_stream.tests.ts +++ b/src/server/tests/nbs_nuclear_writeable_stream.tests.ts @@ -2,15 +2,12 @@ import * as fs from 'fs' import * as stream from 'stream' import { Stream } from 'stream' import { NUClearNetClient } from '../../shared/nuclearnet/nuclearnet_client' -import { message } from '../../shared/proto/messages' import { NbsFrameChunker } from '../nbs/nbs_frame_chunker' import { NbsFrameDecoder } from '../nbs/nbs_frame_streams' import { NbsNUClearPlayback } from '../nbs/nbs_nuclear_playback' import { FakeNUClearNetClient } from '../nuclearnet/fake_nuclearnet_client' import { FakeNUClearNetServer } from '../nuclearnet/fake_nuclearnet_server' import { FakeNodeClock } from '../time/fake_node_clock' -import WritableStream = NodeJS.WritableStream -import nuclear = message.support.nuclear describe('NbsFrameChunker', () => { let transform: stream.Transform @@ -19,7 +16,7 @@ describe('NbsFrameChunker', () => { transform = new NbsFrameChunker() }) - it('Emits 6988 frames', done => { + it('finds 6988 frames within binary stream', done => { const file = fs.createReadStream('/Users/brendan/Lab/NUsight2/recordings/darwin3_WalkAround.nbs') const spy = jest.fn() file.pipe(transform).on('data', spy).on('finish', () => { @@ -43,7 +40,7 @@ describe('NbsNUClearPlayback', () => { }) - it('asdf', done => { + it('sends 6988 messages to NUClearNet', done => { const fakeClock = FakeNodeClock.of() jest.spyOn(nuclearnetClient, 'send') stream diff --git a/src/server/time/node_clock.ts b/src/server/time/node_clock.ts index b484ffb6..cca3cc9e 100644 --- a/src/server/time/node_clock.ts +++ b/src/server/time/node_clock.ts @@ -24,7 +24,7 @@ function performanceNow() { export const NodeSystemClock: Clock = { - now: () => Date.now() / 1e3, + now: () => Date.now() * 1e-3, performanceNow, setTimeout, setInterval, From 905309a476476f18bcc170a40590d15293c3a382 Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Wed, 19 Jul 2017 11:20:58 +1000 Subject: [PATCH 20/24] . --- .../nbs/tests/nbs_frame_codecs.tests.ts | 4 +- src/server/tests/fake_nbs_binary_stream.ts | 19 ++++++++ .../nbs_nuclear_writeable_stream.tests.ts | 45 ++++++++++++++++++- 3 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 src/server/tests/fake_nbs_binary_stream.ts diff --git a/src/server/nbs/tests/nbs_frame_codecs.tests.ts b/src/server/nbs/tests/nbs_frame_codecs.tests.ts index ac4770fa..8c9715ca 100644 --- a/src/server/nbs/tests/nbs_frame_codecs.tests.ts +++ b/src/server/nbs/tests/nbs_frame_codecs.tests.ts @@ -18,7 +18,7 @@ describe('NbsFrameCodecs', () => { const buffer = Buffer.from('e298a218000000c042f15c9654050010abef8b5398f0d41212121212121212', 'hex') const frame = decodeFrame(buffer) expect(frame).toEqual({ - timestamp: 1500379664696000, + timestampInMicroseconds: 1500379664696000, hash: hashType('message.input.sensors'), payload: new Buffer(8).fill(0x12) }) @@ -34,7 +34,7 @@ describe('NbsFrameCodecs', () => { it('encode than decode should equal original', () => { const frame = { hash: hashType('message.input.sensors'), - timestamp: 1500379664696000, + timestampInMicroseconds: 1500379664696000, payload: new Buffer(8).fill(0x12), } expect(decodeFrame(encodeFrame(frame))).toEqual(frame) diff --git a/src/server/tests/fake_nbs_binary_stream.ts b/src/server/tests/fake_nbs_binary_stream.ts new file mode 100644 index 00000000..14891b91 --- /dev/null +++ b/src/server/tests/fake_nbs_binary_stream.ts @@ -0,0 +1,19 @@ +import * as stream from 'stream' + +export class FakeNbsStream extends stream.PassThrough { + public generate(numFrames: number) { + for (let i = 0; i < numFrames; i++) { + const buffer = Buffer.from('e298a218000000c042f15c9654050010abef8b5398f0d41212121212121212', 'hex') + this.write(buffer) + } + this.end() + } + + public generatewithGarbage(numFrames: number) { + for (let i = 0; i < numFrames; i++) { + const buffer = Buffer.from('c96540e298a218000000c042f15c9654050010abef8b5398f0d41212121212121212f350ab21', 'hex') + this.write(buffer) + } + this.end() + } +} diff --git a/src/server/tests/nbs_nuclear_writeable_stream.tests.ts b/src/server/tests/nbs_nuclear_writeable_stream.tests.ts index 43771e9c..5775e980 100644 --- a/src/server/tests/nbs_nuclear_writeable_stream.tests.ts +++ b/src/server/tests/nbs_nuclear_writeable_stream.tests.ts @@ -8,6 +8,7 @@ import { NbsNUClearPlayback } from '../nbs/nbs_nuclear_playback' import { FakeNUClearNetClient } from '../nuclearnet/fake_nuclearnet_client' import { FakeNUClearNetServer } from '../nuclearnet/fake_nuclearnet_server' import { FakeNodeClock } from '../time/fake_node_clock' +import { FakeNbsStream } from './fake_nbs_binary_stream' describe('NbsFrameChunker', () => { let transform: stream.Transform @@ -16,7 +17,7 @@ describe('NbsFrameChunker', () => { transform = new NbsFrameChunker() }) - it('finds 6988 frames within binary stream', done => { + it.skip('finds 6988 frames within binary stream', done => { const file = fs.createReadStream('/Users/brendan/Lab/NUsight2/recordings/darwin3_WalkAround.nbs') const spy = jest.fn() file.pipe(transform).on('data', spy).on('finish', () => { @@ -40,7 +41,7 @@ describe('NbsNUClearPlayback', () => { }) - it('sends 6988 messages to NUClearNet', done => { + it.skip('sends 6988 messages to NUClearNet', done => { const fakeClock = FakeNodeClock.of() jest.spyOn(nuclearnetClient, 'send') stream @@ -55,4 +56,44 @@ describe('NbsNUClearPlayback', () => { done() }) }) + + it('sends all generated messages to NUClearNet', done => { + const fakeClock = FakeNodeClock.of() + jest.spyOn(nuclearnetClient, 'send') + const stream = new FakeNbsStream() + stream + .pipe(new NbsFrameChunker()) + .pipe(new NbsFrameDecoder()) + .on('data', () => { + // Ensure that all timers instantly run after each chunk is received. + // Run on the next tick to allow NbsNUClearPlayback to schedule the timers first. + process.nextTick(() => fakeClock.runAllTimers()) + }) + .pipe(new NbsNUClearPlayback(nuclearnetClient, fakeClock)) + .on('finish', () => { + expect(nuclearnetClient.send).toHaveBeenCalledTimes(1000) + done() + }) + stream.generate(1000) + }) + + it('can handle garbage', done => { + const fakeClock = FakeNodeClock.of() + jest.spyOn(nuclearnetClient, 'send') + const stream = new FakeNbsStream() + stream + .pipe(new NbsFrameChunker()) + .pipe(new NbsFrameDecoder()) + .on('data', () => { + // Ensure that all timers instantly run after each chunk is received. + // Run on the next tick to allow NbsNUClearPlayback to schedule the timers first. + process.nextTick(() => fakeClock.runAllTimers()) + }) + .pipe(new NbsNUClearPlayback(nuclearnetClient, fakeClock)) + .on('finish', () => { + expect(nuclearnetClient.send).toHaveBeenCalledTimes(1000) + done() + }) + stream.generatewithGarbage(1000) + }) }) From e3f4a7fb22f1e3f29af767f02583ef033326d6ad Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Wed, 19 Jul 2017 12:47:38 +1000 Subject: [PATCH 21/24] remove hardcoded FakeNUClearNetClient --- src/server/dev.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/dev.ts b/src/server/dev.ts index 33086318..f683329e 100644 --- a/src/server/dev.ts +++ b/src/server/dev.ts @@ -70,7 +70,7 @@ WebSocketProxyNUClearNetServer.of(WebSocketServer.of(sioNetwork.of('/nuclearnet' NUsightServer.of(WebSocketServer.of(sioNetwork.of('/nusight')), nuclearnetClient) async function playback() { - const fake = FakeNUClearNetClient.of() + const fake = withSimulators ? FakeNUClearNetClient.of() : DirectNUClearNetClient.of() fake.connect({ name: 'Fake Stream' }) while (true) { const file = fs.createReadStream('/Users/brendan/Lab/NUsight2/recordings/darwin3_WalkAround.nbs') From d7b6f912bb46d2651a5eb4d33307a3bc5d1e8820 Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Wed, 19 Jul 2017 12:48:44 +1000 Subject: [PATCH 22/24] Upgrade NUClearNet.js to 1.4.2 --- package.json | 2 +- yarn.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 2573290b..4ca790a3 100644 --- a/package.json +++ b/package.json @@ -98,7 +98,7 @@ "minimist": "^1.2.0", "mobx": "^3.2.0", "mobx-react": "^4.2.2", - "nuclearnet.js": "^1.4.0", + "nuclearnet.js": "^1.4.2", "protobufjs": "^6.7.3", "react": "^15.6.1", "react-dom": "^15.6.1", diff --git a/yarn.lock b/yarn.lock index 5c9b216d..bc28dcec 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4485,9 +4485,9 @@ nth-check@~1.0.1: dependencies: boolbase "~1.0.0" -nuclearnet.js@^1.4.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/nuclearnet.js/-/nuclearnet.js-1.4.0.tgz#f0390e72f3657c2b7066b3b807a4232485548ba5" +nuclearnet.js@^1.4.2: + version "1.4.2" + resolved "https://registry.yarnpkg.com/nuclearnet.js/-/nuclearnet.js-1.4.2.tgz#9a25ea04ac84d866e92bf1851b1754229c6e173c" dependencies: bindings "^1.2.1" nan "^2.0.0" From 27734962cac7556f39019a2dadfcb2bfcd8e5ece Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Wed, 19 Jul 2017 13:07:20 +1000 Subject: [PATCH 23/24] wait until webpack has compiled before starting things --- src/server/dev.ts | 52 ++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/src/server/dev.ts b/src/server/dev.ts index f683329e..741e7a3e 100644 --- a/src/server/dev.ts +++ b/src/server/dev.ts @@ -1,6 +1,7 @@ import * as compression from 'compression' import * as history from 'connect-history-api-fallback' import * as express from 'express' +import * as fs from 'fs' import * as http from 'http' import * as minimist from 'minimist' import * as favicon from 'serve-favicon' @@ -11,13 +12,12 @@ import * as webpackHotMiddleware from 'webpack-hot-middleware' import webpackConfig from '../../webpack.config' import { SensorDataSimulator } from '../simulators/sensor_data_simulator' import { VirtualRobots } from '../simulators/virtual_robots' +import { NbsNUClearPlayback } from './nbs/nbs_nuclear_playback' import { DirectNUClearNetClient } from './nuclearnet/direct_nuclearnet_client' import { FakeNUClearNetClient } from './nuclearnet/fake_nuclearnet_client' import { WebSocketProxyNUClearNetServer } from './nuclearnet/web_socket_proxy_nuclearnet_server' import { WebSocketServer } from './nuclearnet/web_socket_server' import { NUsightServer } from './nusight_server' -import { NbsNUClearPlayback } from './nbs/nbs_nuclear_playback' -import * as fs from 'fs' const compiler = webpack(webpackConfig) @@ -52,32 +52,34 @@ server.listen(port, () => { console.log(`NUsight server started at http://localhost:${port}`) }) -if (withSimulators) { - const virtualRobots = VirtualRobots.of({ - fakeNetworking: true, - numRobots: 3, - simulators: [ - SensorDataSimulator.of(), - ], - }) - virtualRobots.simulateWithFrequency(60) -} +devMiddleware.waitUntilValid(() => { + if (withSimulators) { + const virtualRobots = VirtualRobots.of({ + fakeNetworking: true, + numRobots: 3, + simulators: [ + SensorDataSimulator.of(), + ], + }) + virtualRobots.simulateWithFrequency(60) + } -const nuclearnetClient = withSimulators ? FakeNUClearNetClient.of() : DirectNUClearNetClient.of() + const nuclearnetClient = withSimulators ? FakeNUClearNetClient.of() : DirectNUClearNetClient.of() -WebSocketProxyNUClearNetServer.of(WebSocketServer.of(sioNetwork.of('/nuclearnet')), nuclearnetClient) + WebSocketProxyNUClearNetServer.of(WebSocketServer.of(sioNetwork.of('/nuclearnet')), nuclearnetClient) -NUsightServer.of(WebSocketServer.of(sioNetwork.of('/nusight')), nuclearnetClient) + NUsightServer.of(WebSocketServer.of(sioNetwork.of('/nusight')), nuclearnetClient) -async function playback() { - const fake = withSimulators ? FakeNUClearNetClient.of() : DirectNUClearNetClient.of() - fake.connect({ name: 'Fake Stream' }) - while (true) { - const file = fs.createReadStream('/Users/brendan/Lab/NUsight2/recordings/darwin3_WalkAround.nbs') - // const file = fs.createReadStream('/Users/brendan/Lab/NUsight2/recordings/darwin3_FollowBall.nbs') - const out = NbsNUClearPlayback.fromRawStream(file, fake) - await new Promise(res => out.on('finish', res)) + async function playback() { + const fake = withSimulators ? FakeNUClearNetClient.of() : DirectNUClearNetClient.of() + fake.connect({ name: 'Fake Stream' }) + while (true) { + const file = fs.createReadStream('/Users/brendan/Lab/NUsight2/recordings/darwin3_WalkAround.nbs') + // const file = fs.createReadStream('/Users/brendan/Lab/NUsight2/recordings/darwin3_FollowBall.nbs') + const out = NbsNUClearPlayback.fromRawStream(file, fake) + await new Promise(res => out.on('finish', res)) + } } -} -playback() + playback() +}) From 8df966d9bd4908b97a01c21c8be9128222ec349e Mon Sep 17 00:00:00 2001 From: Brendan Annable Date: Wed, 19 Jul 2017 19:28:29 +1000 Subject: [PATCH 24/24] Linting --- src/client/components/record/controller.ts | 4 +-- src/client/components/record/model.ts | 2 +- src/client/components/record/view.tsx | 12 +++---- src/client/network/nusight_network.ts | 2 +- .../web_socket_proxy_nuclearnet_client.ts | 5 ++- src/server/nbs/nbs_frame_codecs.ts | 2 +- src/server/nusight_server.ts | 3 +- src/server/time/fake_node_clock.ts | 36 +++++++++---------- 8 files changed, 32 insertions(+), 34 deletions(-) diff --git a/src/client/components/record/controller.ts b/src/client/components/record/controller.ts index 61ec6b09..8a4ce1c2 100644 --- a/src/client/components/record/controller.ts +++ b/src/client/components/record/controller.ts @@ -11,14 +11,14 @@ export class RecordController { } @action - onStartRecordingClick(robot: RecordRobotModel) { + public onStartRecordingClick(robot: RecordRobotModel) { const peer = { name: robot.name, address: robot.address, port: robot.port } robot.stopRecording = this.nusightNetwork.record(peer) robot.recording = true } @action - onStopRecordingClick(robot: RecordRobotModel) { + public onStopRecordingClick(robot: RecordRobotModel) { if (robot.stopRecording) { robot.stopRecording() } diff --git a/src/client/components/record/model.ts b/src/client/components/record/model.ts index 43dc0ca9..bc3b6043 100644 --- a/src/client/components/record/model.ts +++ b/src/client/components/record/model.ts @@ -26,7 +26,7 @@ type RecordRobotModelOpts = { } export class RecordRobotModel { - @observable robotModel: RobotModel + @observable private robotModel: RobotModel @observable public recording: boolean public stopRecording?: () => void diff --git a/src/client/components/record/view.tsx b/src/client/components/record/view.tsx index 38138759..57a1d605 100644 --- a/src/client/components/record/view.tsx +++ b/src/client/components/record/view.tsx @@ -15,7 +15,12 @@ type Props = { @observer export class RecordView extends Component { - render() { + public static of(menu: ComponentType<{}>, nusightNetwork: NUsightNetwork, model: RecordModel) { + const controller = RecordController.of(nusightNetwork) + return + } + + public render() { const { menu, controller, model } = this.props const { robots } = model return ( @@ -37,11 +42,6 @@ export class RecordView extends Component { ) } - - public static of(menu: ComponentType<{}>, nusightNetwork: NUsightNetwork, model: RecordModel) { - const controller = RecordController.of(nusightNetwork) - return - } } type RecordMenuBarProps = { diff --git a/src/client/network/nusight_network.ts b/src/client/network/nusight_network.ts index 1b972cac..5073b57b 100644 --- a/src/client/network/nusight_network.ts +++ b/src/client/network/nusight_network.ts @@ -73,7 +73,7 @@ export class NUsightNetwork { } } - getNextRequestToken() { + private getNextRequestToken() { return String(this.nextRequestTokenId++) } } diff --git a/src/client/nuclearnet/web_socket_proxy_nuclearnet_client.ts b/src/client/nuclearnet/web_socket_proxy_nuclearnet_client.ts index 4541715f..2e2a3492 100644 --- a/src/client/nuclearnet/web_socket_proxy_nuclearnet_client.ts +++ b/src/client/nuclearnet/web_socket_proxy_nuclearnet_client.ts @@ -1,11 +1,10 @@ import { NUClearNetOptions } from 'nuclearnet.js' import { NUClearNetSend } from 'nuclearnet.js' +import { NUClearNetPacket } from 'nuclearnet.js' import { NUClearPacketListener } from '../../shared/nuclearnet/nuclearnet_client' import { NUClearEventListener } from '../../shared/nuclearnet/nuclearnet_client' import { NUClearNetClient } from '../../shared/nuclearnet/nuclearnet_client' import { WebSocketClient } from './web_socket_client' -import SocketIOSocket = SocketIOClient.Socket -import { NUClearNetPacket } from 'nuclearnet.js' type PacketListener = (packet: NUClearNetPacket, ack?: () => void) => void @@ -111,7 +110,7 @@ export class WebSocketProxyNUClearNetClient implements NUClearNetClient { } } - onPacket(cb: NUClearPacketListener): () => void { + public onPacket(cb: NUClearPacketListener): () => void { return this.on('nuclear_packet', cb) } diff --git a/src/server/nbs/nbs_frame_codecs.ts b/src/server/nbs/nbs_frame_codecs.ts index 809c3291..5211dfb4 100644 --- a/src/server/nbs/nbs_frame_codecs.ts +++ b/src/server/nbs/nbs_frame_codecs.ts @@ -42,7 +42,7 @@ export function decodeFrame(buffer: Buffer): NbsFrame { export function packetToFrame(packet: NUClearNetPacket, timestampInMicroseconds: number): NbsFrame { return { - timestampInMicroseconds: timestampInMicroseconds, + timestampInMicroseconds, hash: packet.hash, payload: packet.payload, } diff --git a/src/server/nusight_server.ts b/src/server/nusight_server.ts index fa10ca1d..9c44bc74 100644 --- a/src/server/nusight_server.ts +++ b/src/server/nusight_server.ts @@ -6,7 +6,6 @@ import { WebSocketServer } from './nuclearnet/web_socket_server' import { WebSocket } from './nuclearnet/web_socket_server' import { Clock } from './time/clock' import { NodeSystemClock } from './time/node_clock' -import WritableStream = NodeJS.WritableStream export class NUsightServer { public constructor(private server: WebSocketServer, private nuclearnetClient: NUClearNetClient) { @@ -72,7 +71,7 @@ class NUsightServerClient { } - public onStop = (requestToken: string) => { + private onStop = (requestToken: string) => { const stopRecording = this.stopRecordingMap.get(requestToken) if (stopRecording) { console.log('stop recording', requestToken) diff --git a/src/server/time/fake_node_clock.ts b/src/server/time/fake_node_clock.ts index 75681165..40e096e3 100644 --- a/src/server/time/fake_node_clock.ts +++ b/src/server/time/fake_node_clock.ts @@ -4,7 +4,7 @@ type Task = { id: number, nextTime: number, period?: number, - fn: () => void, + fn(): void, } export class FakeNodeClock implements Clock { @@ -73,23 +73,6 @@ export class FakeNodeClock implements Clock { } } - private consumeTask(task: Task) { - this.time = task.nextTime - if (task.period != null) { - task.nextTime += task.period - this.sortTasks() - } else { - this.tasks.shift() - } - task.fn() - } - - private sortTasks() { - this.tasks.sort((t1, t2) => { - return t1.nextTime - t2.nextTime - }) - } - public runOnlyPendingTimers() { const limit = 1000 let i = 0 @@ -113,6 +96,23 @@ export class FakeNodeClock implements Clock { this.sortTasks() } + private sortTasks() { + this.tasks.sort((t1, t2) => { + return t1.nextTime - t2.nextTime + }) + } + + private consumeTask(task: Task) { + this.time = task.nextTime + if (task.period != null) { + task.nextTime += task.period + this.sortTasks() + } else { + this.tasks.shift() + } + task.fn() + } + private removeTask(taskId: number) { for (let i = 0; i < this.tasks.length; i++) { if (this.tasks[i].id == taskId) {