From a9d55df0b6775c18b906fc7b3e3d1cc4df60285e Mon Sep 17 00:00:00 2001 From: drftg <37979265+drftg@users.noreply.github.com> Date: Sat, 14 Apr 2018 17:13:07 +0200 Subject: [PATCH 01/18] Add files via upload Added wake word. A single hotword can wake up MM --- MMM-Assistant.js | 15 ++++++++++++--- node_helper.js | 7 ++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/MMM-Assistant.js b/MMM-Assistant.js index d32bb06..fe1cf0c 100644 --- a/MMM-Assistant.js +++ b/MMM-Assistant.js @@ -444,14 +444,23 @@ Module.register("MMM-Assistant", }, hotwordDetected : function (type) { - if (type == 'ASSISTANT') { + // Start Google Assistant + if (type.hotword == 'ASSISTANT') { this.sendSocketNotification('ACTIVATE_ASSISTANT') this.status = 'ACTIVATE_ASSISTANT' - } else if (type == 'MIRROR') { + // Start command mode + } else if (type.hotword == 'COMMAND') { this.sendSocketNotification('ACTIVATE_COMMAND') this.status = 'ACTIVATE_COMMAND' + // Start snowboy hotword activated shell command + } else if (type.hotword == 'EXECUTE') { + this.sendSocketNotification('EXECUTE', this.config.snowboy.models[type.index-1].parameter) + this.sendSocketNotification('SPEAK', {text: ''}) // kludge, don't know how else to get back to listening + // Send snowboy hotword activated notification to all modules + } else if (type.hotword == 'NOTIFY') { + this.sendSocketNotification('SPEAK', {text: this.config.snowboy.models[type.index-1].parameter}) // kludge, don't know how else to get back to listening + this.sendNotification(this.config.snowboy.models[type.index-1].parameter) } - }, parseCommand: function(msg, cb) { diff --git a/node_helper.js b/node_helper.js index f34a56a..157c66a 100644 --- a/node_helper.js +++ b/node_helper.js @@ -95,6 +95,11 @@ module.exports = NodeHelper.create({ if(this.pause.size == 0) this.activateSpeak(payload.text, payload.option, payload.originalCommand) } break + case 'EXECUTE': + execute(payload, function(callback) { + console.log(callback) + }) + break case 'REBOOT': execute('sudo reboot now', function(callback) { console.log(callback) @@ -202,7 +207,7 @@ module.exports = NodeHelper.create({ detector.on('hotword', (index, hotword, buffer)=>{ record.stop() new Sound(path.resolve(__dirname, 'resources/dong.wav')).play() - this.sendSocketNotification('HOTWORD_DETECTED', hotword) + this.sendSocketNotification('HOTWORD_DETECTED', {hotword:hotword, index:index}) this.sendSocketNotification('MODE', {mode:'HOTWORD_DETECTED'}) if (this.pause.size > 0) this.sendSocketNotification('PAUSED') return From 0d24de752c08590906090b937598ba010eb80713 Mon Sep 17 00:00:00 2001 From: drftg <37979265+drftg@users.noreply.github.com> Date: Sat, 14 Apr 2018 21:45:16 +0200 Subject: [PATCH 02/18] Add files via upload solved for execute and notification --- MMM-Assistant.js | 9 ++++----- node_helper.js | 9 ++++++++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/MMM-Assistant.js b/MMM-Assistant.js index fe1cf0c..9b9a396 100644 --- a/MMM-Assistant.js +++ b/MMM-Assistant.js @@ -413,7 +413,7 @@ Module.register("MMM-Assistant", case 'ERROR': this.status = "ERROR" console.log("[ASSTNT] Error:", payload) - //this.sendSocketNotification("HOTWORD_STANDBY") + this.sendSocketNotification("HOTWORD_STANDBY") break case 'MODE': this.status = payload.mode @@ -455,11 +455,10 @@ Module.register("MMM-Assistant", // Start snowboy hotword activated shell command } else if (type.hotword == 'EXECUTE') { this.sendSocketNotification('EXECUTE', this.config.snowboy.models[type.index-1].parameter) - this.sendSocketNotification('SPEAK', {text: ''}) // kludge, don't know how else to get back to listening // Send snowboy hotword activated notification to all modules - } else if (type.hotword == 'NOTIFY') { - this.sendSocketNotification('SPEAK', {text: this.config.snowboy.models[type.index-1].parameter}) // kludge, don't know how else to get back to listening - this.sendNotification(this.config.snowboy.models[type.index-1].parameter) + } else if (type.hotword == 'NOTIFY') { + this.sendSocketNotification('NOTIFY') + this.sendNotification(this.config.snowboy.models[type.index-1].notification, this.config.snowboy.models[type.index-1].parameter) } }, diff --git a/node_helper.js b/node_helper.js index 157c66a..05dc050 100644 --- a/node_helper.js +++ b/node_helper.js @@ -95,7 +95,13 @@ module.exports = NodeHelper.create({ if(this.pause.size == 0) this.activateSpeak(payload.text, payload.option, payload.originalCommand) } break + case 'NOTIFY': + this.status = 'HOTWORD_DETECTED' + this.sendSocketNotification("HOTWORD_STANDBY") + break case 'EXECUTE': + this.status = 'HOTWORD_DETECTED' + this.sendSocketNotification("HOTWORD_STANDBY") execute(payload, function(callback) { console.log(callback) }) @@ -206,7 +212,8 @@ module.exports = NodeHelper.create({ detector.on('hotword', (index, hotword, buffer)=>{ record.stop() - new Sound(path.resolve(__dirname, 'resources/dong.wav')).play() + var ding = (typeof this.config.snowboy.models[index-1].ding !== 'undefined') ? this.config.snowboy.models[index-1].ding : 'resources/dong.wav' + new Sound(path.resolve(__dirname, ding)).play() this.sendSocketNotification('HOTWORD_DETECTED', {hotword:hotword, index:index}) this.sendSocketNotification('MODE', {mode:'HOTWORD_DETECTED'}) if (this.pause.size > 0) this.sendSocketNotification('PAUSED') From 65d273255857364ebe954e28e7571e19dadfa404 Mon Sep 17 00:00:00 2001 From: drftg <37979265+drftg@users.noreply.github.com> Date: Sun, 15 Apr 2018 20:41:27 +0200 Subject: [PATCH 03/18] Issue #46: GA functionality restored --- node_helper.js | 65 +++++++++++++++----------------------------------- 1 file changed, 19 insertions(+), 46 deletions(-) diff --git a/node_helper.js b/node_helper.js index 05dc050..08274b0 100644 --- a/node_helper.js +++ b/node_helper.js @@ -232,31 +232,24 @@ module.exports = NodeHelper.create({ const assistant = new GoogleAssistant(this.config.assistant.auth) const startConversation = (conversation) => { - //console.log('Say something!'); - let spokenResponseLength = 0; let speakerOpenTime = 0; let speakerTimer; + let openMicAgain = false; - // This is based on: - // ./node_modules/google-assistant/examples/mic-speaker.js - // ./node_modules/google-assistant/examples/speaker-helper.js conversation // send the audio buffer to the speaker .on('audio-data', (data) => { - //record.stop() - const now = new Date().getTime() - if (mode == 'ASSISTANT') { - this.sendSocketNotification('MODE', {mode:'ASSISTANT_SPEAKING'}) - speaker.write(data); - spokenResponseLength += data.length; - const audioTime = spokenResponseLength / (this.config.assistant.conversation.audio.sampleRateOut * 16 / 8) * 1000; - clearTimeout(speakerTimer); - speakerTimer = setTimeout(() => { speaker.end(); }, audioTime - Math.max(0, now - speakerOpenTime)); - } else { - //record.stop() - speaker.end() - } + const now = new Date().getTime(); + speaker.write(data); + + // kill the speaker after enough data has been sent to it and then let it flush out + spokenResponseLength += data.length; + const audioTime = spokenResponseLength / (24000 * 16 / 8) * 1000; + clearTimeout(speakerTimer); + speakerTimer = setTimeout(() => { + speaker.end(); + }, audioTime - Math.max(0, now - speakerOpenTime)); }) // done speaking, close the mic .on('end-of-utterance', () => { record.stop() }) @@ -264,50 +257,27 @@ module.exports = NodeHelper.create({ .on('transcription', (text) => { this.sendSocketNotification('ASSISTANT_TRANSCRIPTION', text) transcription = text - //console.log("[VOX] GA Transcription: ", transcription) // show entire JS object - //--------------------------------------------------------------- - // For account/billing purposes: - // We check if the transcription is complete and update the request - // counter by looking for: "done: true". This should probably be - // moved to MMM-Assistant. - //--------------------------------------------------------------- if (text.done) { gRQC += 1 console.log("[VOX] GA Transcription: ", text.transcription) console.log("[VOX] GA RQC: ", gRQC) } //--------------------------------------------------------------- - //record.stop() - if (mode == 'COMMAND') { - console.log("[ASSTNT] Command understood as: ", transcription) - this.sendSocketNotification('COMMAND', transcription) - } }) // what the assistant answered .on('response', text => console.log('[VOX] GA Response: ', text)) // if we've requested a volume level change, get the percentage of the new level .on('volume-percent', percent => console.log('[VOX] Set Volume [%]: ', percent)) // the device needs to complete an action - //.on('device-action', action => console.log('[VOX] Action:', action)) + .on('device-action', action => console.log('[VOX] Action:', action)) // once the conversation is ended, see if we need to follow up .on('ended', (error, continueConversation) => { - if (this.pause.size > 0) { - record.stop() - speaker.end() - this.sendSocketNotification('PAUSED') - return - } if (error) { console.log('[ASSTNT] Conversation Ended Error:', error) this.sendSocketNotification('ERROR', 'CONVERSATION ENDED') } else if (continueConversation) { - if (mode == 'ASSISTANT') { - assistant.start() - } else { - //@.@ What? There is no stop-conversation in gRpc ????? - } + openMicAgain = true; } else { - record.stop() this.sendSocketNotification('ASSISTANT_FINISHED', mode) } }) @@ -316,7 +286,6 @@ module.exports = NodeHelper.create({ record.stop() speaker.end() // Added by E3V3A: fix attempt for issue #25 --> Need to check for: "Error: Service unavailable" this.sendSocketNotification('ERROR', 'CONVERSATION') - return // added by E3V3A: Do we also need a return? }) // pass the mic audio to the assistant @@ -329,8 +298,12 @@ module.exports = NodeHelper.create({ sampleRate: this.config.assistant.conversation.audio.sampleRateOut, }); speaker - .on('open', () => { speakerOpenTime = new Date().getTime(); }) - .on('close', () => { conversation.end(); }); + .on('open', () => { + clearTimeout(speakerTimer); + spokenResponseLength = 0; + speakerOpenTime = new Date().getTime(); + }) + .on('close', () => { if (openMicAgain) assistant.start(this.config.assistant.conversation); }); }; // Setup the assistant From 3712757ce5bff957ed5c9af306a1c6ee093b2ecc Mon Sep 17 00:00:00 2001 From: drftg <37979265+drftg@users.noreply.github.com> Date: Mon, 16 Apr 2018 10:52:42 +0200 Subject: [PATCH 04/18] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 555d7f2..3d3b0b3 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![MagicMirror2](https://img.shields.io/badge/MagicMirror-2.2.2-lightgray.svg)](https://github.com/MichMich/MagicMirror) [![DocStatus](https://inch-ci.org/github/eouia/MMM-Assistant.svg?branch=master)](https://inch-ci.org/github/eouia/MMM-Assistant) -[![GitHub last commit](https://img.shields.io/github/last-commit/eouia/MMM-Assistant.svg)](https://github.com/eouia/MMM-Assistant) +[![GitHub last commit](https://img.shields.io/github/last-commit/drftg/MMM-Assistant.svg)](https://github.com/eouia/MMM-Assistant) [![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://GitHub.com/eouia/MMM-Assistant/graphs/commit-activity) [![Average time to resolve an issue](http://isitmaintained.com/badge/resolution/eouia/MMM-Assistant.svg)](http://isitmaintained.com/project/eouia/MMM-Assistant "Average time to resolve an issue") [![Dependency Status](https://beta.gemnasium.com/badges/github.com/eouia/MMM-Assistant.svg)](https://beta.gemnasium.com/projects/github.com/eouia/MMM-Assistant) From 426df43a4ae3f8c438977bfd9891463ff6c29fbb Mon Sep 17 00:00:00 2001 From: drftg <37979265+drftg@users.noreply.github.com> Date: Mon, 16 Apr 2018 10:54:03 +0200 Subject: [PATCH 05/18] Update README.md --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3d3b0b3..09b3a0a 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ ## MagicMirror Module: MMM-Assistant [![MagicMirror2](https://img.shields.io/badge/MagicMirror-2.2.2-lightgray.svg)](https://github.com/MichMich/MagicMirror) -[![DocStatus](https://inch-ci.org/github/eouia/MMM-Assistant.svg?branch=master)](https://inch-ci.org/github/eouia/MMM-Assistant) -[![GitHub last commit](https://img.shields.io/github/last-commit/drftg/MMM-Assistant.svg)](https://github.com/eouia/MMM-Assistant) -[![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://GitHub.com/eouia/MMM-Assistant/graphs/commit-activity) -[![Average time to resolve an issue](http://isitmaintained.com/badge/resolution/eouia/MMM-Assistant.svg)](http://isitmaintained.com/project/eouia/MMM-Assistant "Average time to resolve an issue") -[![Dependency Status](https://beta.gemnasium.com/badges/github.com/eouia/MMM-Assistant.svg)](https://beta.gemnasium.com/projects/github.com/eouia/MMM-Assistant) +[![DocStatus](https://inch-ci.org/github/drftg/MMM-Assistant.svg?branch=master)](https://inch-ci.org/github/drftg/MMM-Assistant) +[![GitHub last commit](https://img.shields.io/github/last-commit/drftg/MMM-Assistant.svg)](https://github.com/drftg/MMM-Assistant) +[![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://GitHub.com/drftg/MMM-Assistant/graphs/commit-activity) +[![Average time to resolve an issue](http://isitmaintained.com/badge/resolution/drftg/MMM-Assistant.svg)](http://isitmaintained.com/project/drftg/MMM-Assistant "Average time to resolve an issue") +[![Dependency Status](https://beta.gemnasium.com/badges/github.com/drftg/MMM-Assistant.svg)](https://beta.gemnasium.com/projects/github.com/drftg/MMM-Assistant) A Voice Commander and Google-Assistant for MagicMirror From 484ec6aceba09d67fdc4a7aaac7febee1ad3d6af Mon Sep 17 00:00:00 2001 From: drftg <37979265+drftg@users.noreply.github.com> Date: Mon, 16 Apr 2018 10:56:33 +0200 Subject: [PATCH 06/18] Update README.md --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 09b3a0a..2699fae 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ ## MagicMirror Module: MMM-Assistant [![MagicMirror2](https://img.shields.io/badge/MagicMirror-2.2.2-lightgray.svg)](https://github.com/MichMich/MagicMirror) -[![DocStatus](https://inch-ci.org/github/drftg/MMM-Assistant.svg?branch=master)](https://inch-ci.org/github/drftg/MMM-Assistant) -[![GitHub last commit](https://img.shields.io/github/last-commit/drftg/MMM-Assistant.svg)](https://github.com/drftg/MMM-Assistant) -[![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://GitHub.com/drftg/MMM-Assistant/graphs/commit-activity) -[![Average time to resolve an issue](http://isitmaintained.com/badge/resolution/drftg/MMM-Assistant.svg)](http://isitmaintained.com/project/drftg/MMM-Assistant "Average time to resolve an issue") -[![Dependency Status](https://beta.gemnasium.com/badges/github.com/drftg/MMM-Assistant.svg)](https://beta.gemnasium.com/projects/github.com/drftg/MMM-Assistant) +[![DocStatus](https://inch-ci.org/github/drftg/MMM-Assistant.svg?branch=master)](https://inch-ci.org/eouia/eouia/MMM-Assistant) +[![GitHub last commit](https://img.shields.io/github/last-commit/eouia/MMM-Assistant.svg)](https://github.com/eouia/MMM-Assistant) +[![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://GitHub.com/eouia/MMM-Assistant/graphs/commit-activity) +[![Average time to resolve an issue](http://isitmaintained.com/badge/resolution/eouia/MMM-Assistant.svg)](http://isitmaintained.com/project/eouia/MMM-Assistant "Average time to resolve an issue") +[![Dependency Status](https://beta.gemnasium.com/badges/github.com/eouia/MMM-Assistant.svg)](https://beta.gemnasium.com/projects/github.com/eouia/MMM-Assistant) A Voice Commander and Google-Assistant for MagicMirror From c12bd7569d363967647494e73364e4f3443c3c08 Mon Sep 17 00:00:00 2001 From: drftg Date: Mon, 16 Apr 2018 12:50:45 +0200 Subject: [PATCH 07/18] Updates --- assets/config.txt | 6 ++++++ node_helper.js | 14 ++++++++++++++ package.json | 1 - 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/assets/config.txt b/assets/config.txt index 1935753..5ef4982 100644 --- a/assets/config.txt +++ b/assets/config.txt @@ -27,6 +27,12 @@ file: "resources/snowboy.umdl", // This file define your GA wake word. (See doc notes.) sensitivity: 0.5, hotwords : "ASSISTANT" // Default model: "ASSISTANT". (This is not the wake word!) + }, + { + file: "resources/hello.umdl",// This file define your MM wake word. (See doc notes.) + sensitivity: 0.45, + parameter: 'vcgencmd display_power 1', // monitor on + hotwords : "EXECUTE" } ] }, diff --git a/node_helper.js b/node_helper.js index 08274b0..8d70e45 100644 --- a/node_helper.js +++ b/node_helper.js @@ -232,11 +232,16 @@ module.exports = NodeHelper.create({ const assistant = new GoogleAssistant(this.config.assistant.auth) const startConversation = (conversation) => { + //console.log('Say something!'); + let spokenResponseLength = 0; let speakerOpenTime = 0; let speakerTimer; let openMicAgain = false; + // This is based on: + // ./node_modules/google-assistant/examples/mic-speaker.js + // ./node_modules/google-assistant/examples/speaker-helper.js conversation // send the audio buffer to the speaker .on('audio-data', (data) => { @@ -257,6 +262,13 @@ module.exports = NodeHelper.create({ .on('transcription', (text) => { this.sendSocketNotification('ASSISTANT_TRANSCRIPTION', text) transcription = text + //console.log("[VOX] GA Transcription: ", transcription) // show entire JS object + //--------------------------------------------------------------- + // For account/billing purposes: + // We check if the transcription is complete and update the request + // counter by looking for: "done: true". This should probably be + // moved to MMM-Assistant. + //--------------------------------------------------------------- if (text.done) { gRQC += 1 console.log("[VOX] GA Transcription: ", text.transcription) @@ -278,6 +290,7 @@ module.exports = NodeHelper.create({ } else if (continueConversation) { openMicAgain = true; } else { +// record.stop() this.sendSocketNotification('ASSISTANT_FINISHED', mode) } }) @@ -286,6 +299,7 @@ module.exports = NodeHelper.create({ record.stop() speaker.end() // Added by E3V3A: fix attempt for issue #25 --> Need to check for: "Error: Service unavailable" this.sendSocketNotification('ERROR', 'CONVERSATION') +// return // added by E3V3A: Do we also need a return? }) // pass the mic audio to the assistant diff --git a/package.json b/package.json index fd43e5c..053f1d1 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,6 @@ "url": "https://github.com/eouia/MMM-Assistant/issues" }, "dependencies": { - "electron-rebuild": "^1.7.3", "@google-cloud/speech": "^1.3.0", "google-assistant": "^0.2.2", "node-aplay": "^1.0.3", From 7529e331b3676f89355a809c4d202b0a57a9a3cb Mon Sep 17 00:00:00 2001 From: drftg <37979265+drftg@users.noreply.github.com> Date: Mon, 16 Apr 2018 12:59:59 +0200 Subject: [PATCH 08/18] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2699fae..555d7f2 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ## MagicMirror Module: MMM-Assistant [![MagicMirror2](https://img.shields.io/badge/MagicMirror-2.2.2-lightgray.svg)](https://github.com/MichMich/MagicMirror) -[![DocStatus](https://inch-ci.org/github/drftg/MMM-Assistant.svg?branch=master)](https://inch-ci.org/eouia/eouia/MMM-Assistant) +[![DocStatus](https://inch-ci.org/github/eouia/MMM-Assistant.svg?branch=master)](https://inch-ci.org/github/eouia/MMM-Assistant) [![GitHub last commit](https://img.shields.io/github/last-commit/eouia/MMM-Assistant.svg)](https://github.com/eouia/MMM-Assistant) [![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://GitHub.com/eouia/MMM-Assistant/graphs/commit-activity) [![Average time to resolve an issue](http://isitmaintained.com/badge/resolution/eouia/MMM-Assistant.svg)](http://isitmaintained.com/project/eouia/MMM-Assistant "Average time to resolve an issue") From 9e56cf7cd46a9f6ceebb4be309fa837ed8c8feca Mon Sep 17 00:00:00 2001 From: drftg Date: Tue, 17 Apr 2018 21:19:21 +0200 Subject: [PATCH 09/18] Major reworked version. Every snowboy model in the config now has a list of commands (1 or more) instead of a hotword parameter. Commands can be executed diectly via one hotword but you can also start a voice assistant an speak the commands. Commands have been extended to show or hide a single module by name. Als commands to turn screen on or off have been added. There is now a parameter "modulemap" that containsa mapping from spoken names to actual module names. This is to make it easier for other language STT usage but can also help with mapping "assistant"->"MMM-Assistant". --- MMM-Assistant.js | 130 +++++++++++++++++++++++++++++++++++-------- assets/config.txt | 70 ++++++++++++++++++++--- node_helper.js | 27 ++++++--- translations/en.json | 10 ++++ translations/nl.json | 39 +++++++++++++ 5 files changed, 237 insertions(+), 39 deletions(-) create mode 100644 translations/nl.json diff --git a/MMM-Assistant.js b/MMM-Assistant.js index 9b9a396..9d83c32 100644 --- a/MMM-Assistant.js +++ b/MMM-Assistant.js @@ -88,6 +88,7 @@ Module.register("MMM-Assistant", start: function() { console.log("[ASSTNT] started!") + this.modulemap = new Map(this.config.modulemap) this.commands = [] this.status = "START" this.config = this.configAssignment({}, this.defaults, this.config) @@ -99,6 +100,7 @@ Module.register("MMM-Assistant", getTranslations: function() { return { en: "translations/en.json", + nl: "translations/nl.json", fr: "translations/fr.json", } }, @@ -133,11 +135,21 @@ Module.register("MMM-Assistant", description : this.translate("CMD_HIDE_ALL_MODULES_DESCRIPTION"), callback : 'cmd_asstnt_hideall', }, + { + command: this.translate("CMD_HIDE_MODULE"), + description : this.translate("CMD_HIDE_MODULE_DESCRIPTION"), + callback : 'cmd_asstnt_hide_module', + }, { command: this.translate("CMD_SHOW_ALL_MODULES"), description : this.translate("CMD_SHOW_ALL_MODULES_DESCRIPTION"), callback : 'cmd_asstnt_showall', }, + { + command: this.translate("CMD_SHOW_MODULE"), + description : this.translate("CMD_SHOW_MODULE_DESCRIPTION"), + callback : 'cmd_asstnt_show_module', + }, { command: this.translate("CMD_SAY"), description : this.translate("CMD_SAY_DESCRIPTION"), @@ -153,6 +165,16 @@ Module.register("MMM-Assistant", description : this.translate("CMD_REBOOT_DESCRIPTION"), callback : 'cmd_asstnt_reboot', }, + { + command: this.translate("CMD_WAKE_UP"), + description : this.translate("CMD_WAKE_UP_DESCRIPTION"), + callback : 'cmd_asstnt_wakeup', + }, + { + command: this.translate("CMD_GOTO_SLEEP"), + description : this.translate("CMD_GOTO_SLEEP_DESCRIPTION"), + callback : 'cmd_asstnt_gotosleep', + }, ] commands.forEach((c) => { Register.add(c) @@ -174,6 +196,18 @@ Module.register("MMM-Assistant", this.sendSocketNotification('SHUTDOWN') }, + cmd_asstnt_wakeup : function (command, handler) { + var text = "" + this.sendSocketNotification('EXECUTE', "vcgencmd display_power 1") + if (this.status !== "COMMAND_MODE") this.sendSocketNotification("HOTWORD_STANDBY") + }, + + cmd_asstnt_gotosleep : function (command, handler) { + var text = "" + this.sendSocketNotification('EXECUTE', "vcgencmd display_power 0") + if (this.status !== "COMMAND_MODE") this.sendSocketNotification("HOTWORD_STANDBY") + }, + cmd_asstnt_say : function (command, handler) { handler.response(handler.args.something) }, @@ -182,14 +216,52 @@ Module.register("MMM-Assistant", var text = this.translate("CMD_HIDE_ALL_MODULES_RESULT") var lockString = this.name MM.getModules().enumerate( (m)=> { m.hide(0, {lockString:lockString}) }) - handler.response(text) + if (this.status !== "COMMAND_MODE") { + this.sendSocketNotification("HOTWORD_STANDBY") + } else { + handler.response(text) + } + }, + + cmd_asstnt_hide_module : function (command, handler) { + var text = this.translate("CMD_HIDE_MODULE_RESULT") + var lockString = this.name + var target = handler.args['module'] + MM.getModules().forEach( (m)=> { + if (m.name == target) {m.hide(0, {lockString:lockString})} + else if (m.name == this.modulemap.get(target)) {m.hide(0, {lockString:lockString})} + }) + if (this.status !== "COMMAND_MODE") { + this.sendSocketNotification("HOTWORD_STANDBY") + } else { + handler.response(text) + } }, cmd_asstnt_showall : function (command, handler) { var text = this.translate("CMD_SHOW_ALL_MODULES_RESULT") var lockString = this.name MM.getModules().enumerate( (m)=> { m.show(0, {lockString:lockString}) }) - handler.response(text) + if (this.status !== "COMMAND_MODE") { + this.sendSocketNotification("HOTWORD_STANDBY") + } else { + handler.response(text) + } + }, + + cmd_asstnt_show_module : function (command, handler) { + var text = this.translate("CMD_SHOW_MODULE_RESULT") + var lockString = this.name + var target = handler.args['module'] + MM.getModules().forEach( (m)=> { + if (m.name == target) {m.show(0, {lockString:lockString})} + else if (m.name == this.modulemap.get(target)) {m.show(0, {lockString:lockString})} + }) + if (this.status !== "COMMAND_MODE") { + this.sendSocketNotification("HOTWORD_STANDBY") + } else { + handler.response(text) + } }, cmd_asstnt_list_commands : function (command, handler) { @@ -404,11 +476,19 @@ Module.register("MMM-Assistant", this.sendSocketNotification("HOTWORD_STANDBY") } break + case 'NOTIFY': + this.status = "COMMAND_MODE" + if (payload.notification == "COMMAND") { + this.parseCommand(payload.parameter, this.sendSocketNotification.bind(this)) + } else if (payload.notification == "EXECUTE") { + this.sendSocketNotification("EXECUTE", payload.parameter) + } else { + this.sendNotification(payload.notification, payload.parameter) + } + break case 'COMMAND': - this.status = "COMMAND"; console.log("[ASSTNT] Command:", payload) this.parseCommand(payload, this.sendSocketNotification.bind(this)) - //this.sendSocketNotification("HOTWORD_STANDBY") break; case 'ERROR': this.status = "ERROR" @@ -444,21 +524,25 @@ Module.register("MMM-Assistant", }, hotwordDetected : function (type) { - // Start Google Assistant - if (type.hotword == 'ASSISTANT') { - this.sendSocketNotification('ACTIVATE_ASSISTANT') - this.status = 'ACTIVATE_ASSISTANT' - // Start command mode - } else if (type.hotword == 'COMMAND') { - this.sendSocketNotification('ACTIVATE_COMMAND') - this.status = 'ACTIVATE_COMMAND' - // Start snowboy hotword activated shell command - } else if (type.hotword == 'EXECUTE') { - this.sendSocketNotification('EXECUTE', this.config.snowboy.models[type.index-1].parameter) - // Send snowboy hotword activated notification to all modules - } else if (type.hotword == 'NOTIFY') { - this.sendSocketNotification('NOTIFY') - this.sendNotification(this.config.snowboy.models[type.index-1].notification, this.config.snowboy.models[type.index-1].parameter) + // execute commands + this.config.snowboy.models[type.index-1].commands.forEach( + (command) => { + this.sendSocketNotification('LOG', {title: "[Command] ", message: command}) + if (command.notification == 'ASSISTANT') { + this.sendSocketNotification('ACTIVATE_ASSISTANT') + this.status = 'ACTIVATE_ASSISTANT' + } else if (command.notification == 'MIRROR') { + this.status = 'ACTIVATE_COMMAND' + this.sendSocketNotification('ACTIVATE_COMMAND') + } else { + this.status = "COMMAND_MODE" + this.sendSocketNotification('NOTIFY', command) + } + } + ) + // if last command was not a call for assistant or voice command then activate snowboy + if (this.status == "COMMAND_MODE") { + this.sendSocketNotification("NOTIFY", {notification: "HOTWORD_STANDBY"}) } }, @@ -469,7 +553,7 @@ Module.register("MMM-Assistant", cb("HOTWORD_STANDBY") return } - var msgText = msg + var msgText = msg.toLowerCase() var commandFound = 0 var c for(var i in this.commands) { @@ -534,8 +618,10 @@ Module.register("MMM-Assistant", }, response: function(text, originalCommand, option) { - this.sendSocketNotification('SPEAK', {text:text, option:option, originalCommand:originalCommand} ) - this.status = 'SPEAK' + if (this.status !== "COMMAND_MODE") { + this.sendSocketNotification('SPEAK', {text:text, option:option, originalCommand:originalCommand} ) + this.status = 'SPEAK' + } }, loadCSS: function() { diff --git a/assets/config.txt b/assets/config.txt index 5ef4982..1a5efb7 100644 --- a/assets/config.txt +++ b/assets/config.txt @@ -17,28 +17,80 @@ }, }, snowboy: { + // major change: every model now starts a list of commands so no more "hotword" parameter models: [ { - file: "resources/smart_mirror.umdl",// This file define your MM wake word. (See doc notes.) + file: "resources/hello.pmdl", // This file defines your MM wake word. (See doc note$ sensitivity: 0.5, - hotwords : "MIRROR" // Default model: "MIRROR". (This is not the wake word!) + commands: [ + { + notification: "COMMAND", + parameter: "show all modules" + }, + { + notification: "COMMAND", + parameter: "turn the screen on" + }, + ] }, { - file: "resources/snowboy.umdl", // This file define your GA wake word. (See doc notes.) + file: "resources/u-models/smart_mirror.umdl", // This file defines your MM wake word. (See doc notes.) sensitivity: 0.5, - hotwords : "ASSISTANT" // Default model: "ASSISTANT". (This is not the wake word!) + commands: [ + { + notification: "COMMAND", + parameter: "turn the screen on" + }, + {notification: "MIRROR"} // old hotword + ] }, { - file: "resources/hello.umdl",// This file define your MM wake word. (See doc notes.) - sensitivity: 0.45, - parameter: 'vcgencmd display_power 1', // monitor on - hotwords : "EXECUTE" + file: "resources/u-models/snowboy.umdl", + sensitivity: 0.5, + confirm: "resources/u-hmm-2.wav", + commands: [ // turn the screen on before calling Google Assistant + { + notification: "COMMAND", + parameter: "turn the screen on" + }, + {notification: "ASSISTANT"}, + ] + }, + { + file: "resources/u-models/alexa.umdl", + sensitivity: 0.5, + commands: [ + { + notification: "PLAY", + parameter: "resources/snowboy.wav" + }, + { + notification: "COMMAND", + parameter: "hide all modules" // any spoken command can be sent as text (see translation/??.json for commands) + }, + { + notification: "COMMAND", + parameter: "show module time" // even module name mapping works (see modulemap below) + }, + { + notification: "COMMAND", + parameter: "turn the screen on" + }, + { + notification: "SHOW_ALERT", // any notification can be sent to other modules + parameter: {type: "notification", title: "Important message!", message: "we are awake"} + } + ], } ] }, + modulemap: [ // this table matches spoken module names to their real name + ["time", "clock"], + ["assistant", "MMM-Assistant"] + ], record: { threshold: 0, // Default. No need to change. - verbose: false, // Deafult: true -- for checking recording status. + verbose: false, // Default: true -- for checking recording status. recordProgram: 'rec', // You can use 'arecord', 'sox', but we recommend 'rec' silence: 2.0 // Default. No need to change. }, diff --git a/node_helper.js b/node_helper.js index 8d70e45..73b78be 100644 --- a/node_helper.js +++ b/node_helper.js @@ -96,14 +96,20 @@ module.exports = NodeHelper.create({ } break case 'NOTIFY': - this.status = 'HOTWORD_DETECTED' - this.sendSocketNotification("HOTWORD_STANDBY") + if (payload.notification !== "HOTWORD_STANDBY") { + if (payload.notification == "PLAY") { + new Sound(path.resolve(__dirname, payload.parameter)).play(); + } else { + this.sendSocketNotification("NOTIFY", payload) + } + } else { + this.status = 'HOTWORD_DETECTED' + this.sendSocketNotification("HOTWORD_STANDBY") + } break case 'EXECUTE': - this.status = 'HOTWORD_DETECTED' - this.sendSocketNotification("HOTWORD_STANDBY") execute(payload, function(callback) { - console.log(callback) + console.log("[EXECUTE] ", callback) }) break case 'REBOOT': @@ -119,6 +125,9 @@ module.exports = NodeHelper.create({ case 'TEST': this.test(payload) break + case 'LOG': + this.consoleLog(payload) + break } }, @@ -168,10 +177,12 @@ module.exports = NodeHelper.create({ }) }, + consoleLog: function(payload) { + console.log(payload.title, payload.message) + }, activateHotword: function() { console.log('[ASSTNT] Snowboy Activated') - this.sendSocketNotification('MODE', {mode:'HOTWORD_STARTED'}) new Sound(path.resolve(__dirname, 'resources/ding.wav')).play(); @@ -212,8 +223,8 @@ module.exports = NodeHelper.create({ detector.on('hotword', (index, hotword, buffer)=>{ record.stop() - var ding = (typeof this.config.snowboy.models[index-1].ding !== 'undefined') ? this.config.snowboy.models[index-1].ding : 'resources/dong.wav' - new Sound(path.resolve(__dirname, ding)).play() + var confirm = (typeof this.config.snowboy.models[index-1].confirm !== 'undefined') ? this.config.snowboy.models[index-1].confirm : 'resources/dong.wav' + new Sound(path.resolve(__dirname, confirm)).play() this.sendSocketNotification('HOTWORD_DETECTED', {hotword:hotword, index:index}) this.sendSocketNotification('MODE', {mode:'HOTWORD_DETECTED'}) if (this.pause.size > 0) this.sendSocketNotification('PAUSED') diff --git a/translations/en.json b/translations/en.json index 43cbe25..8017c13 100644 --- a/translations/en.json +++ b/translations/en.json @@ -16,14 +16,24 @@ "CMD_HIDE_ALL_MODULES" : "hide all modules", "CMD_HIDE_ALL_MODULES_DESCRIPTION" : "This command will hide all your available modules.", "CMD_HIDE_ALL_MODULES_RESULT" : "All modules will be hidden.", + "CMD_HIDE_MODULE" : "hide module :module", + "CMD_HIDE_MODULE_DESCRIPTION" : "This command will hide the specified module.", + "CMD_HIDE_MODULE_RESULT" : "The module will be hidden.", "CMD_SHOW_ALL_MODULES" : "show all modules", "CMD_SHOW_ALL_MODULES_DESCRIPTION" : "This command will show all your available modules", "CMD_SHOW_ALL_MODULES_RESULT" : "All modules will be shown.", + "CMD_SHOW_MODULE" : "show module :module", + "CMD_SHOW_MODULE_DESCRIPTION" : "This command will show the specified module", + "CMD_SHOW_MODULE_RESULT" : "The module will be shown.", "CMD_SAY" : "say :something", "CMD_SAY_DESCRIPTION" : "This command will say anything you want.", "CMD_REBOOT" : "reboot", "CMD_REBOOT_DESCRIPTION" : "This command will reboot the magic mirror instantly.", "CMD_SHUTDOWN" : "shut down", "CMD_SHUTDOWN_DESCRIPTION" : "This command will shutdown the magic mirror instantly.", + "CMD_WAKE_UP" : "turn the screen on", + "CMD_WAKE_UP_DESCRIPTION" : "Turn the screen on", + "CMD_GOTO_SLEEP" : "turn the screen off", + "CMD_GOTO_SLEEP_DESCRIPTION" : "Turn the screen off", "TELBOT_CMD_GA_DESCRIPTION" : "Send text to Google Assistant." } diff --git a/translations/nl.json b/translations/nl.json new file mode 100644 index 0000000..be42502 --- /dev/null +++ b/translations/nl.json @@ -0,0 +1,39 @@ +{ + "UNSPEAKABLE" : "Sorry, I cannot pronounce this response.", + "INVALID_FORMAT" : "Sorry, the question and answer based dialog is not supported in command mode. Please ask module developer.", + "INVALID_COMMAND" : "Sorry, I cannot understand your command. Try again.", + "ALIAS" : "This is an alias of {command}", + "CMD_HELP" : "help :command", + "CMD_HELP_COMMAND_PROVIDER" : "This command is provided by {module}", + "CMD_HELP_DESCRIPTION" : "This command could tell you the description of that command.", + "CMD_HELP_COMMAND_EXAMPLE" : "Say command name after 'help' like this. 'help all modules'.", + "CMD_LIST_COMMANDS" : "toon alle commando's", + "CMD_LIST_COMMANDS_DESCRIPTION" : "Dit commando toont alle beschikbare commando's", + "CMD_LIST_COMMANDS_RESULT" : "Dit zijn alle beschikbare commando's", + "CMD_LIST_MODULES" : "toon beschikbare modules", + "CMD_LIST_MODULES_DESCRIPTION" : "Dit commando toont alle beschikbare modules", + "CMD_LIST_MODULES_RESULT" : "Dit zijn alle beschikbare moduels", + "CMD_HIDE_ALL_MODULES" : "verberg alle modules", + "CMD_HIDE_ALL_MODULES_DESCRIPTION" : "Dit commando verbergt alle modules", + "CMD_HIDE_ALL_MODULES_RESULT" : "Alle modules zijn nu verborgen", + "CMD_HIDE_MODULE" : "verberg module :module", + "CMD_HIDE_MODULE_DESCRIPTION" : "Dit commando verbergt de opgegeven module", + "CMD_HIDE_MODULE_RESULT" : "De opgegeven module is nu verborgen", + "CMD_SHOW_ALL_MODULES" : "toon alle modules", + "CMD_SHOW_ALL_MODULES_DESCRIPTION" : "This command will show all your available modules", + "CMD_SHOW_ALL_MODULES_RESULT" : "Alle modules worden zichtbaar", + "CMD_SHOW_MODULE" : "toon module :module", + "CMD_SHOW_MODULE_DESCRIPTION" : "Dit commando maakt de opgegeven module zichtbaar", + "CMD_SHOW_MODULE_RESULT" : "De opgegeven module is nu zichtbaar", + "CMD_SAY" : "zeg :something", + "CMD_SAY_DESCRIPTION" : "Dit commando zegt wat je maar wil", + "CMD_REBOOT" : "herstart", + "CMD_REBOOT_DESCRIPTION" : "Met dit command start de spiegel nieuw op", + "CMD_SHUTDOWN" : "sluit af", + "CMD_SHUTDOWN_DESCRIPTION" : "Dit commando zet de spiegel uit", + "CMD_WAKE_UP" : "wakker worden", + "CMD_WAKE_UP_DESCRIPTION" : "Zet het scherm aan", + "CMD_GOTO_SLEEP" : "ga slapen", + "CMD_GOTO_SLEEP_DESCRIPTION" : "Zet het scherm uit", + "TELBOT_CMD_GA_DESCRIPTION" : "Send text to Google Assistant." +} From 994a7dfc62b81278dc86019a3836ae6f70742ec6 Mon Sep 17 00:00:00 2001 From: drftg <37979265+drftg@users.noreply.github.com> Date: Tue, 17 Apr 2018 21:31:27 +0200 Subject: [PATCH 10/18] hello.pmdl added It is advisable to train this model yourself via snowboy.kitt.ai --- resources/hello.pmdl | Bin 0 -> 8947 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 resources/hello.pmdl diff --git a/resources/hello.pmdl b/resources/hello.pmdl new file mode 100644 index 0000000000000000000000000000000000000000..93b1973a0d89f3740a43bb0d4eb4414abd97db70 GIT binary patch literal 8947 zcmXw-X*gHg`^F6!BD2hu5JED1_ugyT`>RM!B!{9>>L^1|Nu5TS$7BeZDx^XvV<=J~ zgd{12Bc(w!NCTze=l|lrUaS}Qb*&fAwbpu``~CVpOv&q1RTV>FM7LDQdo*!4LNk%M|1>=`Y>M^gIOQlByKv$Bh< zTKR3V^nA$~Piuxl+csGJY)J4-UXJ=(}@i7#6pEQF_H;$T|rSDdtPg#8in0Vq%E5L=xA z^5d#7aPjzwHhbidoS{!3yX*(-xiX8$etQh_JGEg*-dU{l(~-El2?KmNl`8tQ6dcmh zW7nAHv&y_!mcAy*PRAH?307;#$G8ciQhS5xU7^5Ti8IDWx`Wv%Q#m5HAr$R@CxQpn z-T>ZMAR8d#f`*){fEli5`qK}=--mqJ)GI1jHa!jwR133zTK{7!zP$rQca~uZv3??b zE(A0@`3gHOrV!Iri{O>S7`)V4jlB8t5O!^v1^M;oaD#{falZ5v%)OdQwRLO*CG*tT z*01SosN`mLC_;?=vv~>Uumlpt$Z&S|Q&lD{a6PBrAcQTHeb^ooX@a;Ql%uSTPdE60 z)H^05sfUklRih;CI26Bk11GfivH~V!K;@4)89nrvkWgpP zD{Kg~XCcBIEe5>TMWpKcSFHS^1wJe91#K%X;N5#h@co!%W=?4;W$0=RhIF;q@=b;8 z?s2X#k28|u?L*_4_QE&m*ThAiXcgGpOj#Mh@?Q7t2cnPbzN}QAS z8Dww&pGn@asz!~^Ghm#{^0;+)LZcWl~n zan}D^2KU3=h4>hl!IK@LRKTD8V0qwsRIAy^n$*aXLN6Y6iB?5fGYFFyZfy~oFZb9)vDwRuWxU#ie?U8=x2!aWXC`etHqx$0Y*Zb3 zF8oR#;b{Tm(p~Jtr3z%_6bq{_W#H4)7q;~C2hfmDfq~P{N!2P<`o}C&(5-1nly35A zr{0?kXV8h)NA^Lb>jU8U(=#~l-AkPH05WlD=~Qu=B5=sHVX6DN+^T0k*@yAstk$O> zPGZeL0#2L5qi2p$sh3v*%X$eMt9Y5sGFn4?M4M0(s|$dM4A4EZ2(>*8Ma<`L*q}4h z14;ns=?`Q1+peJ?!xXsvNj83B`<->n{tS*w`hmH+l5Ab1AstwG66jb5kfFnNwBO-L zMp*d=R#nJG_NV0Fb^k0pxZ(}|y)TGqJ~%=ZSS$i``wo_OCY0TjWzM-q3bUGVqMR~N zA~`m_$l+`Q1zJks+HfDc`^qC(XR~|Q zUVaA_Q#nK#h3BB$Vq0vPcZn#^ZY0hk`=QuF4WPV38dAScLW#Z+q;1d#165@hi%a#4 zv-$;Y$2C3nZgwhiq@sz?8(|L4y$Js7yayVU0Bd4>gz_CpV|tG!kliolGUkdLpsj+~ zDP4DLzCs_Wv`3KA;drdvTgaq|cT(Y6ZD8`(Dl&4*6=z!f!xq-7vaPX$Wb91|vL6+} zqb6mPm`)Zdk3cxpC=z=$#*rsxBY?NIobmUQf_1e*@VcfR%2I6tz75M5VJe6bJte`< z^g2bZX#}JC^e}S2XCe2!Aqmu2tp{DVmD#DaBfODD4JJlp7D;$MN=sgj2Az)=u)G)7 zP}poE=vre*wl5ICc8?pGcnbkay7W8{ZPp;VL2lUdZ7oTRd`*n>zmxY-InXzA3E}Q` z(9vtFiT2H-B>vY8q`qt?;os5#igTU;r+qTW^GXtkU;79yw^jgCS5ANv9ygc|Iq$f~ zUrrNgsVhk6?0=*(NR6{mivv|REi4XBr>9r2~swIY-q& zk_!^hsDu(*CA64(@>B;{N1TAmzQqwmV>{~Q*fu(_%ZAjXiZg#gJAg|`5^H;Q2~7Ce z2EfHO;`p-!)e1#0iVo_$09P8;NM6Dd*7nF-D2mj&E+O`L!K7Tm405feIK5JqN)#zz z&4y;NbGF7Hf4w(2(>s?j^b=z)wxC{ zN}O#Y04FE5Fl*)V%{qgHpzqHGBy874wDg}C@^QS3DENHEE`t8hajzFDAD`r9_oWfr z{0f4c-XL1m4co1m1uUtDbbEOr^P}Ay`D#6a7R3%A*NdiG+k)xer%d?C%HDWQcQ}ey zt0Ea@D%_ju7%+aZg!$5AfeWYJ^KJ*6q!kU$;r{6z^vdRS;OL(7q-o$b9C14UjUKe1 zyvtLdrO21jl|OG5xpfcl>3M@ueh#{I^EVMczlkgt*hC&G--H}^iL4zg=(Ng4_OAh9 zR!m5UoZixp^+NW7_{ z@K&`agMYT0cTU>EiG)SO=iMFLq&$mrXbUC=b*ik6uOhCHHbyD8+GwTaf$Xk&5%!*P zB>R3$oRt;Y17BYWWG*QC!O)9WQA6)B>}*^DtqrokRl5YHouA1KH!sKCJ0?)i&sL&- zW;a(P;SOXwO~DO+C2T%%mB||OqnnG8$m>LNW_)!F{KhGfe;mb;+O8Bdhs?v*XDT~A zFOm5zJjrv~G68)?ElAkl2_l~?#QmOigv@zU!EVxDi-n)8MQFn)mHkeE?1{}KYi&~4 z{qYmzUF#&=mnaX!bZnvL7Hhn_O91akRYU$EilC#tkWo05&S8naumJZ88JsL6uEU49 zqc<@S9;*cn8J4)?O9Acq=@NZmRy=vrcAL&KP=eQnBk*8XEu3sfN3Qa|c#Ypx1Y|BT ztJ2T#8ryfl^t;=Mf4cxX|G%%SzT8Dp{$ZMpzIO)uzKcf*&pfDEMhW=yDM(&F3u9-; zD6pI6&PI9qd^q}-4?M&s;%|F;kzZH=lpa)KK&}VaxGt3Iv5mmXH>=>+*4K%l+F>s2 zzau~)p%d0WxQ2sRJ9=d28fM^XG|5Y?qBZO!ASiah!Eyso?$#NUS$hi`@6AFdpepcv z5YN-FyaWWEixA!9AhP>?CEF@*O)9!4*-Q6iiS*(cRH!6Kok7!3ZPEk{iORE+M<0_0 z!ERLV5CR7`^}w))PqDY}Y4j4#L%g3?=>ASVEXzK@3PtS2aZD`EE^EbV>Qa37hPiM` zQ3^G$m`!F~ze{Z$)MTdqGb8cC6V#EYWae4pa^(DH7cvU_j`UVrkjd{gsAGIbP>CRVUILvrNF zN5GDRV>l|Mg^Kq-O@HqFj@?t6sMzPqOv&6k@L7%qTJSy<6{yzXE^rJTK^9=1a}KZO zqAJrFd=T5M7bTvgj||PzB$Qp*?-DR>kYXOg6wT{36%@DZ$ zZ7gnO8Yypw?R5D52|PS;joRZc$smnnXpka^QmhrxghU|8MMdnbko#fWl$&-2tCO@S4bo>^<}tWih~P& zt%6R9`PhZ%AnDd?z5UdKD2iyA_{ZvQKR-6%=K0)RPpE(a{Dw5HSFu~_|ia>ycGh;=$E{;869-i(gIwn zIem}=Ok?QV%=p)=*$C-w@6wIraugU!(TCL;4qKmzj%ub03U9bwufpf z3$c2+8dC5(4eFXB=xBo&kX)6*3S1sXZmUDF%RyOO9Hz-v@SOx-UdKTrZYddW>!!S3 z9H1>fI}$^yuT;ME3g+dk6KJUHB8n7z2usK7@Vwt4$a|avVRvrx&bU0Jb9z?bWGhkp zL&B4I)P<7YMip$uf)w;XFco<&)Z=w0szbMoNO-^5gM_7|;Ny2zp^2Y-Xdmtf_s=cD zBOk^fdpQY&szuO+Ut_?qV--6~?-Sy@Lop{Hhl6_nKk8!!@ct(o?ztgCl56f$dTsmZ zu>qDy%kXG?Dva52E+1K6Nkf{-gD{ue#&v2rsK&(>{OHZ(HBZlF0u2Rmy+;tb-tA1l zuhpcq`5)G5|68~#;~cUAlDzP3n$WT%8k)6S!&{aIVxylU(5uJ-di(qhZ+hk7c)KB} z8S4)G4;`cZOJczM?JrqHu?gh#Zz#@ml*6^79AC$37Z4CBg7kmtB(kWTid6HavB3&* zrAU*m+3L;=W!^*sg(2wvy&)(Yav$?oKR`w`mcV}~lb8QZpSiwC8V}4Lh8{iPg#S#8 z43)XE!MAon;8ln!j)?G-ep0ZzO~jC;m8y-TH|Q6ZnjhMnmz`3OSrQ!RL1`UIt{wlA!D2kN99|E5*C)Lr;$L$iqly zx_XBrv-96VlxeUHg)i-dGV4F#alaOn_0$?@oJ`|Ab}whbs8%!-au5bBJ3!tQox>&P zYS|i{Hz4!QY2+>|$h$AG3_cS&3Y`)p&_EQ8b$0Cr^!61{QqdU(EGxpUpcX18E(84! zcF}$7jsu}CDNbPg6Ef)z!-JEuID5>TANdY|#Dl?bX+kls-gAdi{O&~)Gd;5ERVKad zBE?W%;b`URQJC7F29Nr_#W!D-BCS>{P(CYzcV?D8aP(}3Un;GEKPba(*-AJ$&62z= z7XTST7m(4h3G?b}J78yJG5q*w4vaB-jKr18nD>7r0F^uom}66q<{idxNl-O&`=UMl zTR{{|ZS-Q->BymyD+$OukUut_zQUfm-K zd&|GD=@D|cNBcPTJG=mYcwoiXUKkIDif=%rsqZ+<`U#cpJ4L_F6DDCEE9vz)0Zc;c z5jYU^Hv)&!2v8rxlB%g_!523$`0^(21RcicKFvbr|3*RHspI5haVKtYj3f?cKLNKM zJ6v<}DbMZtHF$JX2~8aM0^Oy=iR`{Iu=AHQjJ+#}3jeFYx5~_s{`>d9P;(BG@@FM< zdMdz$CduRE_5_?Bu>|``tl$sNhp=1nEA&#S#B1Rc6%|&`q@>A{z@W>t&$lKf@l6LT zN>xGJuV47UD@jtWl8fv;^MQ@!GOF*10H}{EN5|U^L25@lDfnH9M?_i)F0+Dt-E*+& zurd|4?-`Vkkwk}n^}^k2Hj(k(=Rh?0Z)mdm5sW`{3nv)6B7VREI5sWC#E(&MB2*OV#tJI2ClSh0>HYpq-l(Gp<3yR}&)K_I@oa zJDGxwZbN+3b{W6u`$6bX{1}eSRwS+6O7yJwe?gJ&M=UBAO`BUzGxe&E;E!j)XhJcM zJp4M1+n<-9Oz(W47+yoweHvvnpGP9yNeBl*=97ZBTwGhXgM|5uz{mOW*jiMFnzXb; z>EG26-^2sXYtbWq`|M#+x-&F%dIINnHsBX-JCXknQ^;3b$n5*Wz@%nHZu*uX-sqZ! zeaftGgAm6b3k`uER0p8Hha#z8w1^(83If4Lf8r6LUG&dn5s>Tf7H$j6KtwH{_>Rhw z@yUABey$8Wmb^>(t(;{1PhCKR0{&2Ay(1yHRoGu$ff(tj!f-Wp>?y-g2bv>Lbh{BU z>c0Z#*KHu$X2sCv;x?GP^a+&B=)|9t)Uo}nNVu(%$GA=*Xl*FZ+0;4Ac+L`hCfNsf z>0^E{I{_uOy@LDi&nKM=7SS*2!-2M#Dp_jlOPB7_1;#^nq2qcXtl3daQeM~)y`d8H zO?nJ0$ooKDI4J{ywpF0_l{5FK=1#WSwPCww@}%;a7St=S#P)yhqiWnMP|pg0w)cI8 zo9uRxoyOPTxKjvRJMtCMYo_s;^i>!)6~mh8AO=O)L$l)o+^vlv_>ArY+_X6o%LBq+ zx?B)h*C?QD1tW5+#EhP1LO@i+LSkg&L`VK!4{px84`*E#!NsSFiQk5QN!P8rNPv}s z=h8)Ky{@;6MCuacyx0;J++9V!-p#-_v(}Q%9ZIm_?IrxX<~b#<9fGci15`L)6*Wa~ zAui1=5WD|@pJP2xm2Emnv*+XMdLpPUErV&;G8=9CHN>v>ti|cC7_w3(0Utbz_`%U) zNODyRQZQXX;tjOv;M0kqN=KBuT^T`F>860(`d-*-e-qW*Dkod#9ws{8Q)s`#8hBt# zizZTijJa4$JL$-A z>t+0&C&f-4JWjjdUN9(%$ev3Y%#_j!_|Rl4a@a5%i%D%_y;oHd%eAl2v_J{`Ngtsr z^|iq9Noj0W@ew|Ie40e1v&2&KJ+_{4{{aK1arZMim4a5##)e~O~q*i_uLUz>jDQU(>yZbW0@N1&Bu0&#Hu5B}+% z4Xe*aBcT*Qa@JK1FZgvErFlGH&aYpLKKdT#WcEEKrH*JC>rvfeh|tP#JkCUq(*wPmr8~QEtr242@MN;(``& zwso5VEvaP&k3WBlzbV+$)2|wU^n6#ef8ivu)%RmxeH0>(H=Z3$TOr#BV z@CW7Su7$E?i^%w4MdDS}i1!s#Kmzw+74=HG@Od5>;T=bsvlQT$>IkxOt2`i{8_u{vRu1eCaFKl06sgdA%MFe352@ zr)Sf7!ICgF6kDwlL+cKv!a%LF zsDZa1hnx7LLzMz}PD~r~ZR8HJ5-j8%YkHB^%3Ly-Y))eLPI4d0WzZIn0Q@oJ4jJc4 zDL27y;2HG_kMdin(myU>Pe3aA_d-5$N>gB`HcOF}=bdL}eLYZfZRAY{+A@>gK?s-q z3-HHlczA^xN%&?(Hn){Pc{|<=v&p1N4;=!3WqM+sye_B_o`YTLW1xzQH#8bR=+J`w zsJQ6?)S7BRsd?GVnN}$@mUxj%e}$rQ~C92__5-`3TgZAerpys+EGH+Qqb{<@f4fBmb<0@%hFbHEF{=EXVY>WcV zWxm*GXCcn#8u5$nSZFWUj*68oQ2yJmfaW7xL9h@8)!!j?8~Ia;$1ZnL#U6$ZXU)5)4zzTO#1 zK1jiB5v`PscQV-iE)sdSoP^UJ#YFM@4cu*}fNf1D7~2`VYjR!>i2c$>+m1+s2^SrF zW@jBe5a z&}ZEcB0F9&-j305Snxe39s3cl$`TQJW*EtQ8>|2hKWNl$h%SEuU zwjS|56)@LLJ)ohzINvcj410OZBeIL0U}jGZw<@h2_Rqfwt1F_3SxgjfS;93~Oe5TM zKbJCHgFxP44>W)0e?YQ2oYXE}jpvIg0>DNR~8ki(az*&oWz%!Wf}4AG*our8~pL%6nNr&7iI=&!)f_5;EZe> ze*8~0Gv6r~UYgeA?;DZFqgIb+de>H*))&iVAC*G)^IpSEg(<}0c?7Rv&cDcHkt$X( z9;P<_6$Lz1V^AcT0d4RyU%v=?7ygE#mt5HNg_)d zWp)AC$`RxkFb)(KYqKh`*AQN0g?f)o(oNgmm`&Sk0{okYq52_N@F%kgS$w7Nt$bzj zB(4;mn~XwQQVo=0TO?EKR*zf%R%5P5EJvmL_0iy~OCYRo6TBW(3gXwD2T@fr)pUq3|`EV*aq8S4NlfBV8HZ9^dQ;56#a=WszeLmoEO zK+9hcrT)yP%4SD0M{ZxmTPZW9@wg>2EHg!YE<1o{(=zyD?tkFu=1agLr3xz#3^0Z- zY+ziDK7Z;fjlcQj;K2TMxEQ!{Nk=%;^=KG&WLzXe1yww<&x-gN%i&lj7drP~F4!no ziLAO>f%rp9*6kdJezuyS-ZoXH)vm*=$Y2XlTbBp3*hPSQe-)h`vd2|lVo4NlHR9IE zqRDelsioRcOxxE|oK>vCIQPUM%}W-jI4KzHytD*vT{H}ypG^TLrmo?N%7@IB?Sas+ zLznN|?2TOo-{P$cFXL60ExAKo7HF86gZeTGNpR{N-khF~ Date: Tue, 17 Apr 2018 21:50:43 +0200 Subject: [PATCH 11/18] Version 1.1.0 --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57e295e..6c2a7d9 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,19 @@ All notable changes to this project will be documented in this file. --- +#### [1.1.0] - 2018-04-17 + +- Changed handling of commands in config +- Changed format of models in config.js so an old one will not work (see: assets/config.txt) +- Added commands for screen on and off +- Added commands for showing/hiding a single named module +- Added mapping of spoken module name to actual module name (really helps non-english speakers) +- Added command to play .wav +- Added a single hotword can fire a list of commands and stay in listening mode + for example, a spoken "wake up" can turn on the screen (like PIR) without activating GA or voice command +- Added model specific confirmation sound added in config (parameter "confirm") +- Added translate/nl.json for dutch language + #### [1.0.3] - 2018-03-31 From 2a112623a3d386795ceb3fa96f01459a69d8c059 Mon Sep 17 00:00:00 2001 From: drftg Date: Wed, 18 Apr 2018 22:39:39 +0200 Subject: [PATCH 12/18] Solved weird error handling conversations in GA --- node_helper.js | 32 ++++++++++++++++++++++---------- translations/nl.json | 4 ++-- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/node_helper.js b/node_helper.js index 73b78be..862e415 100644 --- a/node_helper.js +++ b/node_helper.js @@ -173,6 +173,7 @@ module.exports = NodeHelper.create({ } } else { console.log("[ASSTNT] Speak Error", err) + this.sendSocketNotification('MODE', {mode:'SPEAK_ENDED', useAlert:option.useAlert}) } }) }, @@ -237,6 +238,7 @@ module.exports = NodeHelper.create({ activateAssistant: function(mode = 'ASSISTANT') { console.log('[ASSTNT] GA Activated') + var endOfSpeech = false var gRQC = this.googleRequestCounter // Added by E3V3A var transcription = "" this.sendSocketNotification('MODE', {mode:'ASSISTANT_STARTED'}) @@ -256,21 +258,27 @@ module.exports = NodeHelper.create({ conversation // send the audio buffer to the speaker .on('audio-data', (data) => { - const now = new Date().getTime(); - speaker.write(data); + const now = new Date().getTime() + + speaker.write(data) // kill the speaker after enough data has been sent to it and then let it flush out - spokenResponseLength += data.length; - const audioTime = spokenResponseLength / (24000 * 16 / 8) * 1000; - clearTimeout(speakerTimer); + spokenResponseLength += data.length + const audioTime = spokenResponseLength / (24000 * 16 / 8) * 1000 + clearTimeout(speakerTimer) speakerTimer = setTimeout(() => { - speaker.end(); - }, audioTime - Math.max(0, now - speakerOpenTime)); + if (endOfSpeech) { // if spech.end was already called we notify here + this.sendSocketNotification('ASSISTANT_FINISHED', mode) + } + endOfSpeech = true + speaker.end() + }, (audioTime - Math.max(0, now - speakerOpenTime)) + 500) }) // done speaking, close the mic .on('end-of-utterance', () => { record.stop() }) // show each word on console as they are understood, while we say it .on('transcription', (text) => { + endOfSpeech = false this.sendSocketNotification('ASSISTANT_TRANSCRIPTION', text) transcription = text //console.log("[VOX] GA Transcription: ", transcription) // show entire JS object @@ -296,13 +304,17 @@ module.exports = NodeHelper.create({ // once the conversation is ended, see if we need to follow up .on('ended', (error, continueConversation) => { if (error) { + endOfSpeech = true console.log('[ASSTNT] Conversation Ended Error:', error) this.sendSocketNotification('ERROR', 'CONVERSATION ENDED') } else if (continueConversation) { - openMicAgain = true; - } else { -// record.stop() + console.log("[ASSTNT - continue conversation]") + openMicAgain = true + endOfSpeech = false + } else if (endOfSpeech) { this.sendSocketNotification('ASSISTANT_FINISHED', mode) + } else { // weird case where speech.end is called but the spech is still heard + endOfSpeech = true } }) .on('error', (error) => { diff --git a/translations/nl.json b/translations/nl.json index be42502..1f5da47 100644 --- a/translations/nl.json +++ b/translations/nl.json @@ -16,13 +16,13 @@ "CMD_HIDE_ALL_MODULES" : "verberg alle modules", "CMD_HIDE_ALL_MODULES_DESCRIPTION" : "Dit commando verbergt alle modules", "CMD_HIDE_ALL_MODULES_RESULT" : "Alle modules zijn nu verborgen", - "CMD_HIDE_MODULE" : "verberg module :module", + "CMD_HIDE_MODULE" : "verberg :module", "CMD_HIDE_MODULE_DESCRIPTION" : "Dit commando verbergt de opgegeven module", "CMD_HIDE_MODULE_RESULT" : "De opgegeven module is nu verborgen", "CMD_SHOW_ALL_MODULES" : "toon alle modules", "CMD_SHOW_ALL_MODULES_DESCRIPTION" : "This command will show all your available modules", "CMD_SHOW_ALL_MODULES_RESULT" : "Alle modules worden zichtbaar", - "CMD_SHOW_MODULE" : "toon module :module", + "CMD_SHOW_MODULE" : "toon :module", "CMD_SHOW_MODULE_DESCRIPTION" : "Dit commando maakt de opgegeven module zichtbaar", "CMD_SHOW_MODULE_RESULT" : "De opgegeven module is nu zichtbaar", "CMD_SAY" : "zeg :something", From c1d5fa1e8f2b6ece35f699ed4b51a349c22bc9d9 Mon Sep 17 00:00:00 2001 From: drftg Date: Thu, 19 Apr 2018 16:19:04 +0200 Subject: [PATCH 13/18] Made backwards compatible to old comfig.js --- MMM-Assistant.js | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/MMM-Assistant.js b/MMM-Assistant.js index 9d83c32..90c6d44 100644 --- a/MMM-Assistant.js +++ b/MMM-Assistant.js @@ -525,21 +525,31 @@ Module.register("MMM-Assistant", hotwordDetected : function (type) { // execute commands - this.config.snowboy.models[type.index-1].commands.forEach( - (command) => { - this.sendSocketNotification('LOG', {title: "[Command] ", message: command}) - if (command.notification == 'ASSISTANT') { - this.sendSocketNotification('ACTIVATE_ASSISTANT') - this.status = 'ACTIVATE_ASSISTANT' - } else if (command.notification == 'MIRROR') { - this.status = 'ACTIVATE_COMMAND' - this.sendSocketNotification('ACTIVATE_COMMAND') - } else { - this.status = "COMMAND_MODE" - this.sendSocketNotification('NOTIFY', command) + if (typeof this.config.snowboy.models[type.index-1].commands !== 'undefined') { + this.config.snowboy.models[type.index-1].commands.forEach( + (command) => { + this.sendSocketNotification('LOG', {title: "[Command] ", message: command}) + if (command.notification == 'ASSISTANT') { + this.sendSocketNotification('ACTIVATE_ASSISTANT') + this.status = 'ACTIVATE_ASSISTANT' + } else if (command.notification == 'MIRROR') { + this.status = 'ACTIVATE_COMMAND' + this.sendSocketNotification('ACTIVATE_COMMAND') + } else { + this.status = "COMMAND_MODE" + this.sendSocketNotification('NOTIFY', command) + } } + ) + } else if (typeof this.config.snowboy.models[type.index-1].hotwords !== 'undefined') { + if (this.config.snowboy.models[type.index-1].hotwords == 'ASSISTANT') { + this.sendSocketNotification('ACTIVATE_ASSISTANT') + this.status = 'ACTIVATE_ASSISTANT' + } else if (this.config.snowboy.models[type.index-1].hotwords == 'MIRROR') { + this.status = 'ACTIVATE_COMMAND' + this.sendSocketNotification('ACTIVATE_COMMAND') } - ) + } // if last command was not a call for assistant or voice command then activate snowboy if (this.status == "COMMAND_MODE") { this.sendSocketNotification("NOTIFY", {notification: "HOTWORD_STANDBY"}) From 4ac5d7898421fce214215409a4a96125f20545ee Mon Sep 17 00:00:00 2001 From: drftg Date: Thu, 19 Apr 2018 16:39:25 +0200 Subject: [PATCH 14/18] Version 1.1.1 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c2a7d9..a61482d 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,11 @@ All notable changes to this project will be documented in this file. --- +#### [1.1.1] - 2018-04-17 + +- Changed handling of config.js so it is now backwards compatible +- fixed some more GA bugs. It should be stable and robust now and handle conversations correctly + #### [1.1.0] - 2018-04-17 - Changed handling of commands in config From 66ede6bd2478e16a1eaf1b8bb0a907ef609d5865 Mon Sep 17 00:00:00 2001 From: drftg Date: Fri, 20 Apr 2018 10:49:53 +0200 Subject: [PATCH 15/18] Version 1.1.2 --- CHANGELOG.md | 7 ++++++- MMM-Assistant.js | 29 ++++++++++++++++++++++++----- assets/config.txt | 5 +++++ 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a61482d..09e2468 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,12 @@ All notable changes to this project will be documented in this file. --- -#### [1.1.1] - 2018-04-17 +#### [1.1.2] - 2018-04-20 + +- Added timeout for screen +- Added screen section in config to specify timeout and screen on/off commands + +#### [1.1.1] - 2018-04-19 - Changed handling of config.js so it is now backwards compatible - fixed some more GA bugs. It should be stable and robust now and handle conversations correctly diff --git a/MMM-Assistant.js b/MMM-Assistant.js index 90c6d44..2334059 100644 --- a/MMM-Assistant.js +++ b/MMM-Assistant.js @@ -90,6 +90,7 @@ Module.register("MMM-Assistant", console.log("[ASSTNT] started!") this.modulemap = new Map(this.config.modulemap) this.commands = [] +// this.screentimer this.status = "START" this.config = this.configAssignment({}, this.defaults, this.config) this.getCommands( new AssistantCommandRegister(this, this.registerCommand.bind(this)) ) @@ -197,14 +198,32 @@ Module.register("MMM-Assistant", }, cmd_asstnt_wakeup : function (command, handler) { - var text = "" - this.sendSocketNotification('EXECUTE', "vcgencmd display_power 1") + if (typeof this.config.screen.on !== 'undefined') { + this.sendSocketNotification('EXECUTE', this.config.screen.on) + } else { + this.sendSocketNotification('EXECUTE', "vcgencmd display_power 1") + } + if (typeof this.config.screen.timeoff !== 'undefined') { + if (typeof this.config.screen.timeoff !== 0) { + clearTimeout(this.screenTimer) + this.screenTimer = setTimeout(() => { + if (typeof this.config.screen.off !== 'undefined') { + this.sendSocketNotification('EXECUTE', this.config.screen.off) + } else { + this.sendSocketNotification('EXECUTE', "vcgencmd display_power 0") + } + }, this.config.screen.timeoff * 1000) + } + } if (this.status !== "COMMAND_MODE") this.sendSocketNotification("HOTWORD_STANDBY") }, - cmd_asstnt_gotosleep : function (command, handler) { - var text = "" - this.sendSocketNotification('EXECUTE', "vcgencmd display_power 0") + cmd_asstnt_gotosleep : function (command = "", handler = "") { + if (typeof this.config.screen.off !== 'undefined') { + this.sendSocketNotification('EXECUTE', this.config.screen.off) + } else { + this.sendSocketNotification('EXECUTE', "vcgencmd display_power 0") + } if (this.status !== "COMMAND_MODE") this.sendSocketNotification("HOTWORD_STANDBY") }, diff --git a/assets/config.txt b/assets/config.txt index 1a5efb7..ea55eea 100644 --- a/assets/config.txt +++ b/assets/config.txt @@ -84,6 +84,11 @@ } ] }, + screen: { + on: "vcgencmd display_power 1", + off: "vcgencmd display_power 0", + timeoff: 120 // seconds before screen is turned off + }, modulemap: [ // this table matches spoken module names to their real name ["time", "clock"], ["assistant", "MMM-Assistant"] From 5683ab0ed151e006853a34bed7eda0ef5b6bdcf9 Mon Sep 17 00:00:00 2001 From: drftg Date: Fri, 20 Apr 2018 20:49:51 +0200 Subject: [PATCH 16/18] Version 1.1.3 --- CHANGELOG.md | 7 ++++++ MMM-Assistant.js | 5 ++++ node_helper.js | 58 ++++++++++++++++++++++++++++++++++++++------ package.json | 5 ++-- translations/nl.json | 16 ++++++------ 5 files changed, 73 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09e2468..8d657b2 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,13 @@ All notable changes to this project will be documented in this file. --- +#### [1.1.3] - 2018-04-20 + +- Added Google TTS. Many more languages available than pico2tts + Command conversation can now be in languages other than en-GB,de-DE,es-ES,fr-FR,it-IT like nl-NL +- Added config switch to choose between pico and Google TTS +- Fixed one more bug in GA. Saying "nevermind" just stopped the conversation without returning to listening mode + #### [1.1.2] - 2018-04-20 - Added timeout for screen diff --git a/MMM-Assistant.js b/MMM-Assistant.js index 2334059..81dd200 100644 --- a/MMM-Assistant.js +++ b/MMM-Assistant.js @@ -228,6 +228,7 @@ Module.register("MMM-Assistant", }, cmd_asstnt_say : function (command, handler) { + this.sendSocketNotification('LOG', {title: "SAY", message: handler.args.something}) handler.response(handler.args.something) }, @@ -514,6 +515,10 @@ Module.register("MMM-Assistant", console.log("[ASSTNT] Error:", payload) this.sendSocketNotification("HOTWORD_STANDBY") break + case 'LOG': + // straight back to node helper + this.sendSocketNotification(notification, payload) + break case 'MODE': this.status = payload.mode if (payload.mode == 'SPEAK_ENDED') { diff --git a/node_helper.js b/node_helper.js index 862e415..58f48f6 100644 --- a/node_helper.js +++ b/node_helper.js @@ -19,6 +19,7 @@ const Sound = require('node-aplay') // Deprecated const path = require('path') const fs = require('fs') const record = require('node-record-lpcm16') +const textToSpeech = require('@google-cloud/text-to-speech') const Detector = require('snowboy').Detector const Models = require('snowboy').Models const Speaker = require('speaker') @@ -140,8 +141,13 @@ module.exports = NodeHelper.create({ option.language = (typeof commandOption.language !== 'undefined') ? commandOption.language : this.config.speak.language option.useAlert = (typeof commandOption.useAlert !== 'undefined') ? commandOption.useAlert : this.config.speak.useAlert option.originalCommand = (originalCommand) ? originalCommand : "" + option.auth = this.config.stt.auth[0] + option.useGoogle = this.config.speak.useGoogle + // Use the small footprint Text-to-Speech (TTS): pico2wave - var commandTmpl = 'pico2wave -l "{{lang}}" -w {{file}} "{{text}}" && aplay {{file}}' + var commandTmpl = 'pico2wave -l "{{lang}}" -w {{file}} "{{text}}" && aplay {{file}}' + var commandGetSpeech = 'pico2wave -l "{{lang}}" -w {{file}} "{{text}}"' + var commandSpeak = 'aplay {{file}}' function getTmpFile() { var random = Math.random().toString(36).slice(2), @@ -155,12 +161,42 @@ module.exports = NodeHelper.create({ text = text.replace(/\"/g, "'") text = text.trim() - var file = getTmpFile(), + var file = getTmpFile() + var command = "" + + if (option.useGoogle) { + let client = new textToSpeech.TextToSpeechClient(option.auth) + + const request = { + input: {text: text}, + voice: {languageCode: lang, ssmlGender: 'NEUTRAL'}, + audioConfig: {audioEncoding: 'LINEAR16'}, + } + + client.synthesizeSpeech(request, (err, response) => { + if (err) { + console.error('ERROR:', err); + return; + } + fs.writeFile(file, response.audioContent, 'binary', err => { + if (err) { + console.error('ERROR:', err); + return; + } + command = commandSpeak.replace(/\{\{file\}\}/g, file) + exec(command, function(err) { + cb && cb(err) + fs.unlink(file, ()=>{}) + }) + }) + }) + } else { command = commandTmpl.replace('{{lang}}', lang).replace('{{text}}', text).replace(/\{\{file\}\}/g, file) exec(command, function(err) { cb && cb(err) fs.unlink(file, ()=>{}) - }) + }) + } } this.sendSocketNotification('MODE', {mode:'SPEAK_STARTED', useAlert:option.useAlert, originalCommand:option.originalCommand, text:text}) @@ -179,7 +215,9 @@ module.exports = NodeHelper.create({ }, consoleLog: function(payload) { - console.log(payload.title, payload.message) + // helper for logging via notification + // USAGE: sendSocketNotification("LOG", {title: "", message: ""}) + if (this.config.debug) console.log(payload.title, payload.message) }, activateHotword: function() { @@ -236,6 +274,7 @@ module.exports = NodeHelper.create({ }, activateAssistant: function(mode = 'ASSISTANT') { +// this.sendSocketNotification("LOG", {title: "[ASSTNT]", message: "GA Activated"}) console.log('[ASSTNT] GA Activated') var endOfSpeech = false @@ -279,9 +318,7 @@ module.exports = NodeHelper.create({ // show each word on console as they are understood, while we say it .on('transcription', (text) => { endOfSpeech = false - this.sendSocketNotification('ASSISTANT_TRANSCRIPTION', text) - transcription = text - //console.log("[VOX] GA Transcription: ", transcription) // show entire JS object +// console.log("[VOX] GA Transcription: ", text) // show entire JS object //--------------------------------------------------------------- // For account/billing purposes: // We check if the transcription is complete and update the request @@ -296,7 +333,12 @@ module.exports = NodeHelper.create({ //--------------------------------------------------------------- }) // what the assistant answered - .on('response', text => console.log('[VOX] GA Response: ', text)) + .on('response', text => { + // another special case. Uttering "nevermind" produces no response + if (text == "") { + this.sendSocketNotification('ASSISTANT_FINISHED', mode) + } + }) // if we've requested a volume level change, get the percentage of the new level .on('volume-percent', percent => console.log('[VOX] Set Volume [%]: ', percent)) // the device needs to complete an action diff --git a/package.json b/package.json index 053f1d1..e71bf96 100644 --- a/package.json +++ b/package.json @@ -24,12 +24,13 @@ "url": "https://github.com/eouia/MMM-Assistant/issues" }, "dependencies": { - "@google-cloud/speech": "^1.3.0", + "@google-cloud/speech": "^1.4.0", + "@google-cloud/text-to-speech": "^0.1.0", "google-assistant": "^0.2.2", "node-aplay": "^1.0.3", "node-record-lpcm16": "^0.3.0", "snowboy": "^1.2.0", - "speaker": "^0.4.0" + "speaker": "^0.4.1" }, "devDependencies": { "electron-rebuild": "^1.7.3", diff --git a/translations/nl.json b/translations/nl.json index 1f5da47..4e325e3 100644 --- a/translations/nl.json +++ b/translations/nl.json @@ -1,13 +1,13 @@ { - "UNSPEAKABLE" : "Sorry, I cannot pronounce this response.", - "INVALID_FORMAT" : "Sorry, the question and answer based dialog is not supported in command mode. Please ask module developer.", - "INVALID_COMMAND" : "Sorry, I cannot understand your command. Try again.", - "ALIAS" : "This is an alias of {command}", + "UNSPEAKABLE" : "Sorry, dit kan ik niet uitspreken", + "INVALID_FORMAT" : "Sorry, de vraag antwoord dialoog wordt niet ondersteund in commando modus.", + "INVALID_COMMAND" : "Sorry, ik begrijp niet wat je zegt. Probeer het nog eens", + "ALIAS" : "Dit is een andere benaming voor {command}", "CMD_HELP" : "help :command", - "CMD_HELP_COMMAND_PROVIDER" : "This command is provided by {module}", - "CMD_HELP_DESCRIPTION" : "This command could tell you the description of that command.", - "CMD_HELP_COMMAND_EXAMPLE" : "Say command name after 'help' like this. 'help all modules'.", - "CMD_LIST_COMMANDS" : "toon alle commando's", + "CMD_HELP_COMMAND_PROVIDER" : "Deze opdracht wordt u aangeboden door {module}", + "CMD_HELP_DESCRIPTION" : "Dit commando kan een omschrijving geven van een opgegeven commando", + "CMD_HELP_COMMAND_EXAMPLE" : "Zeg help gevolgd door een commando. Zoals: 'help toon beschikbare modules'.", + "CMD_LIST_COMMANDS" : "toon beschikbare commando's", "CMD_LIST_COMMANDS_DESCRIPTION" : "Dit commando toont alle beschikbare commando's", "CMD_LIST_COMMANDS_RESULT" : "Dit zijn alle beschikbare commando's", "CMD_LIST_MODULES" : "toon beschikbare modules", From a6bc50e89f8706c14387683d68271917c60f8c0a Mon Sep 17 00:00:00 2001 From: drftg Date: Fri, 20 Apr 2018 20:52:05 +0200 Subject: [PATCH 17/18] Version 1.1.3 --- assets/config.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/assets/config.txt b/assets/config.txt index ea55eea..b53d93f 100644 --- a/assets/config.txt +++ b/assets/config.txt @@ -112,6 +112,7 @@ }, }, speak: { + useGoogle: true, // [true] Enable Google TTS useAlert: true, // [true] Enable this to show the understood text of your speech language: 'en-US', // [en-US] To set the default GA speech reply language. }, From 5b4f90e2139bcec3a7de4e93999248c00314c2c0 Mon Sep 17 00:00:00 2001 From: drftg Date: Wed, 25 Apr 2018 21:14:59 +0200 Subject: [PATCH 18/18] Version 1.1.4 --- CHANGELOG.md | 7 +++++++ MMM-Assistant.js | 50 +++++++++++++++++++++++++++++++++++++------- assets/config.txt | 5 +++-- node_helper.js | 5 ++--- translations/en.json | 5 +++++ translations/nl.json | 5 +++++ 6 files changed, 65 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d657b2..73b28f8 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,13 @@ All notable changes to this project will be documented in this file. --- +#### [1.1.4] - 2018-04-25 + +- Added command command to stay awake. This cancels the screen timeout +- Added modulemap can now be a set of modules +- Added command to hide all modules except the selected one. This can be a set from the modulemap so + effectively this can be used to show named pages + #### [1.1.3] - 2018-04-20 - Added Google TTS. Many more languages available than pico2tts diff --git a/MMM-Assistant.js b/MMM-Assistant.js index 81dd200..10d9c1e 100644 --- a/MMM-Assistant.js +++ b/MMM-Assistant.js @@ -131,6 +131,11 @@ Module.register("MMM-Assistant", description: this.translate("CMD_LIST_MODULES_DESCRIPTION"), callback : 'cmd_asstnt_list_modules' }, + { + command: this.translate("CMD_HIDE_ALL_MODULES_EXCEPT"), + description : this.translate("CMD_HIDE_ALL_MODULES_EXCEPT_DESCRIPTION"), + callback : 'cmd_asstnt_hideall_except', + }, { command: this.translate("CMD_HIDE_ALL_MODULES"), description : this.translate("CMD_HIDE_ALL_MODULES_DESCRIPTION"), @@ -171,6 +176,11 @@ Module.register("MMM-Assistant", description : this.translate("CMD_WAKE_UP_DESCRIPTION"), callback : 'cmd_asstnt_wakeup', }, + { + command: this.translate("CMD_STAY_AWAKE"), + description : this.translate("CMD_STAY_AWAKE_DESCRIPTION"), + callback : 'cmd_asstnt_stay_awake', + }, { command: this.translate("CMD_GOTO_SLEEP"), description : this.translate("CMD_GOTO_SLEEP_DESCRIPTION"), @@ -218,7 +228,12 @@ Module.register("MMM-Assistant", if (this.status !== "COMMAND_MODE") this.sendSocketNotification("HOTWORD_STANDBY") }, - cmd_asstnt_gotosleep : function (command = "", handler = "") { + cmd_asstnt_stay_awake : function (command, handler) { + clearTimeout(this.screenTimer) + if (this.status !== "COMMAND_MODE") this.sendSocketNotification("HOTWORD_STANDBY") + }, + + cmd_asstnt_gotosleep : function (command, handler) { if (typeof this.config.screen.off !== 'undefined') { this.sendSocketNotification('EXECUTE', this.config.screen.off) } else { @@ -247,9 +262,10 @@ Module.register("MMM-Assistant", var text = this.translate("CMD_HIDE_MODULE_RESULT") var lockString = this.name var target = handler.args['module'] - MM.getModules().forEach( (m)=> { - if (m.name == target) {m.hide(0, {lockString:lockString})} - else if (m.name == this.modulemap.get(target)) {m.hide(0, {lockString:lockString})} + var moduleSet = this.modulemap.get(target) + if (typeof moduleSet == 'undefined') moduleSet = target + MM.getModules().withClass(moduleSet).forEach( (m)=> { + m.hide(0, {lockString:lockString}) }) if (this.status !== "COMMAND_MODE") { this.sendSocketNotification("HOTWORD_STANDBY") @@ -273,9 +289,29 @@ Module.register("MMM-Assistant", var text = this.translate("CMD_SHOW_MODULE_RESULT") var lockString = this.name var target = handler.args['module'] - MM.getModules().forEach( (m)=> { - if (m.name == target) {m.show(0, {lockString:lockString})} - else if (m.name == this.modulemap.get(target)) {m.show(0, {lockString:lockString})} + var moduleSet = this.modulemap.get(target) + if (typeof moduleSet == 'undefined') moduleSet = target + MM.getModules().withClass(moduleSet).forEach( (m)=> { + m.show(0, {lockString:lockString}) + }) + if (this.status !== "COMMAND_MODE") { + this.sendSocketNotification("HOTWORD_STANDBY") + } else { + handler.response(text) + } + }, + + cmd_asstnt_hideall_except : function (command, handler) { + var text = this.translate("CMD_HIDE_ALL_MODULES_EXCEPT_RESULT") + var lockString = this.name + var target = handler.args['module'] + var moduleSet = this.modulemap.get(target) + if (typeof moduleSet == 'undefined') moduleSet = target + MM.getModules().exceptWithClass(moduleSet).forEach( (m)=> { + if (m.name != this.name) {m.hide(1000, {lockString:lockString})} + }) + MM.getModules().withClass(moduleSet).forEach( (m)=> { + m.show(1000, {lockString:lockString}) }) if (this.status !== "COMMAND_MODE") { this.sendSocketNotification("HOTWORD_STANDBY") diff --git a/assets/config.txt b/assets/config.txt index b53d93f..c1d2b71 100644 --- a/assets/config.txt +++ b/assets/config.txt @@ -25,7 +25,7 @@ commands: [ { notification: "COMMAND", - parameter: "show all modules" + parameter: "show overview" }, { notification: "COMMAND", @@ -91,7 +91,8 @@ }, modulemap: [ // this table matches spoken module names to their real name ["time", "clock"], - ["assistant", "MMM-Assistant"] + ["assistant", "MMM-Assistant"], + ["overview", ["clock", "currentweather"]] ], record: { threshold: 0, // Default. No need to change. diff --git a/node_helper.js b/node_helper.js index 58f48f6..0b9c061 100644 --- a/node_helper.js +++ b/node_helper.js @@ -133,7 +133,7 @@ module.exports = NodeHelper.create({ }, test: function(test) { - this.sendSocketNotification('COMMAND', test) + console.log("TESTING") }, activateSpeak: function(text, commandOption={}, originalCommand = "") { @@ -217,7 +217,7 @@ module.exports = NodeHelper.create({ consoleLog: function(payload) { // helper for logging via notification // USAGE: sendSocketNotification("LOG", {title: "", message: ""}) - if (this.config.debug) console.log(payload.title, payload.message) + console.log(payload.title, payload.message) }, activateHotword: function() { @@ -274,7 +274,6 @@ module.exports = NodeHelper.create({ }, activateAssistant: function(mode = 'ASSISTANT') { -// this.sendSocketNotification("LOG", {title: "[ASSTNT]", message: "GA Activated"}) console.log('[ASSTNT] GA Activated') var endOfSpeech = false diff --git a/translations/en.json b/translations/en.json index 8017c13..30ce487 100644 --- a/translations/en.json +++ b/translations/en.json @@ -16,6 +16,9 @@ "CMD_HIDE_ALL_MODULES" : "hide all modules", "CMD_HIDE_ALL_MODULES_DESCRIPTION" : "This command will hide all your available modules.", "CMD_HIDE_ALL_MODULES_RESULT" : "All modules will be hidden.", + "CMD_HIDE_ALL_MODULES_EXCEPT" : "show :module", + "CMD_HIDE_ALL_MODULES_EXCEPT_DESCRIPTION" : "This command will hide all modules except the selected module(s)", + "CMD_HIDE_ALL_MODULES_EXCEPT_RESULT" : "All modules will be hidden except the selected ones", "CMD_HIDE_MODULE" : "hide module :module", "CMD_HIDE_MODULE_DESCRIPTION" : "This command will hide the specified module.", "CMD_HIDE_MODULE_RESULT" : "The module will be hidden.", @@ -33,6 +36,8 @@ "CMD_SHUTDOWN_DESCRIPTION" : "This command will shutdown the magic mirror instantly.", "CMD_WAKE_UP" : "turn the screen on", "CMD_WAKE_UP_DESCRIPTION" : "Turn the screen on", + "CMD_STAY_AWAKE" : "stay awake", + "CMD_STAY_AWAKE_DESCRIPTION" : "Do not turn the screen off", "CMD_GOTO_SLEEP" : "turn the screen off", "CMD_GOTO_SLEEP_DESCRIPTION" : "Turn the screen off", "TELBOT_CMD_GA_DESCRIPTION" : "Send text to Google Assistant." diff --git a/translations/nl.json b/translations/nl.json index 4e325e3..6bd91da 100644 --- a/translations/nl.json +++ b/translations/nl.json @@ -16,6 +16,9 @@ "CMD_HIDE_ALL_MODULES" : "verberg alle modules", "CMD_HIDE_ALL_MODULES_DESCRIPTION" : "Dit commando verbergt alle modules", "CMD_HIDE_ALL_MODULES_RESULT" : "Alle modules zijn nu verborgen", + "CMD_HIDE_ALL_MODULES_EXCEPT" : "thema :module", + "CMD_HIDE_ALL_MODULES_EXCEPT_DESCRIPTION" : "Dit commando verbergt alle modules behalve de opgegeven module(s)", + "CMD_HIDE_ALL_MODULES_EXCEPT_RESULT" : "Alle modules zijn nu verborgen behalve de opgegeven module(s)", "CMD_HIDE_MODULE" : "verberg :module", "CMD_HIDE_MODULE_DESCRIPTION" : "Dit commando verbergt de opgegeven module", "CMD_HIDE_MODULE_RESULT" : "De opgegeven module is nu verborgen", @@ -31,6 +34,8 @@ "CMD_REBOOT_DESCRIPTION" : "Met dit command start de spiegel nieuw op", "CMD_SHUTDOWN" : "sluit af", "CMD_SHUTDOWN_DESCRIPTION" : "Dit commando zet de spiegel uit", + "CMD_STAY_AWAKE" : "blijf wakker", + "CMD_STAY_AWAKE_DESCRIPTION" : "Laat het scherm aanstaan", "CMD_WAKE_UP" : "wakker worden", "CMD_WAKE_UP_DESCRIPTION" : "Zet het scherm aan", "CMD_GOTO_SLEEP" : "ga slapen",