Add SHOW_ALERT functionality

[sdmydbr9]

Hi, I know it already does show the output on the screen but I am trying to implement it to show a notification or alert using the default alert module by sending a SHOW_ALERT notification, I believe this can be achieved by sending a SHOW_ALERT notification from the nodehelper or the mainjs, I tried few different ways but I can’t seem to get it to work so I thought it’s best to reach out to you.
My objective is: To show Alert or notification of the response from the module, I tried updating the code in the following ways

class PostProcess {
  constructor (callbacks) {
    this._callbacks = callbacks
  }

  sendNotification(noti, payload) {
    this._callbacks.sendNotification(noti, payload)
  }

  shellExec(scr) {
    this._callbacks.shellExec(scr)
  }

  getModule(mName = null) {
    var modules = this.getModules()
    if (mName == null) mName = "MMM-OpenAI"
    for (var i = 0; i < modules.length; i++) {
      if (modules[i] && modules[i].name == mName) return modules[i]
    }
  }

  getModules() {
    return this._callbacks.getModules()
  }
}

Module.register('MMM-OpenAI', {
  defaultNotification: {
    "OPENAI_REQUEST": (payload, sender) => {
      const defaultObj = {
        method: 'TEXT',
        callback: null,
        request: {},
        prompt: null,
        stealth: false,
      }
      let requested = {}
      if (typeof payload === 'string') {
        requested = {...defaultObj, ...{ prompt: payload }}
      } else {
        requested = {...defaultObj, ...payload}
      }
      if (requested.prompt && typeof requested.prompt === 'string') requested.request.prompt = requested.prompt
      return requested
    },
  },

  defaults: {
    stealth: false,
    defaultBehaviour: true,
    imageAreaHeight: '400px',
    resultLife: 10 * 60 * 1000,
    method: 'TEXT',  // 'IMAGE'
    defaultRequest: {
      text: {
        model: 'text-davinci-003',
        prompt: '',
        n: 1, // Warning; consumeing tokens
        max_tokens: 512, // Warning; consuming tokens
/* // You should know what you are doing.
        temperature: 1, // 0~2
        top_p: 0,
        suffix: null,
        stream: false, // Not prepared for this result, RESERVED. (Don't use it)
        logprobs: null,
        echo: false,
        stop: "\n",
        presence_penalty: 0,
        best_of: 1, //Warning; consuming tokens
*/
      },
      image: {
        prompt: '',
        n: 1,
        size: '512x512', // '256x256', '512x512'
        response_format:'url', // 'b64_json'
      }
    },

    notificationHook: {
      'AI_EXAMPLE': {
        method: 'TEXT',
        request: {
          prompt: 'Hello, there. Introduce yourself',
        }
      }
    },

    postProcessing: (processor, payload) => {
      console.log('[OPENAI] Nothing on postprocessing. (default)')
    },

    telegramCommandText: 'txtai',
    telegramCommandImage: 'imgai'
  },

  getStyles: function () {
    return ['MMM-OpenAI.css']
  },

  start: function() {
    this.requestPool = new Map()
    this.notificationSet = { ...this.config.notificationHook, ...this.defaultNotification }
    this.resultTimer = null

    this.postProcessor = new PostProcess({
      sendNotification: (noti, payload) => {
        this.sendNotification(noti, payload)
      },
      shellExec: (scr) => {
        if (!scr) return false
        this.sendSocketNotification("SHELL_EXEC", scr)
      },
      getModules: () => {
        return MM.getModules()
      },
    })
  },

  getDom: function () {
    let dom = document.createElement('div')
    dom.classList.add('bodice', 'OpenAI')
    if (!this.lastResponse) return dom
    if (this.lastResponse.stealth) return dom
    let {error, response, request, options, responseTime} = this.lastResponse
    if (error) return dom

    let req = document.createElement('div')
    req.classList.add('prompt')
    req.innerHTML = request?.prompt

    let res = document.createElement('div')
    res.classList.add('response')

    if (options.method === 'TEXT') {
      res.innerHTML = response.choices[0].text.replace(/^\n\n/, '').replaceAll('\n', '<br/>')
      res.classList.add('response_text')
    } else if (options.method === 'IMAGE') {
      let url = (response.data[0]?.url) ? response.data[0]?.url : `data:image/png;base64,${response.data[0].b64_json}`
      res.style.backgroundImage = `url("${url}")`
      res.style.setProperty('--imageAreaHeight', this.config.imageAreaHeight)
      res.classList.add('response_image')
    }
    dom.appendChild(req)
    dom.appendChild(res)

    return dom
  },

  getCommands: function(commander) {
    commander.add({
      command: this.config.telegramCommandText,
      description: 'Ask to OpenAI with text',
      callback: 'command_text',
    })
    commander.add({
      command: this.config.telegramCommandImage,
      description: 'Request image to OpenAI',
      callback: 'command_image'
    })
  },

  command_text: function(command, handler) {
    if (handler.args) {
      let request = { prompt: handler.args }
      this.request({
        method: 'TEXT', 
        requestable: request, 
        handler
      })
    } else {
      handler.reply('There is no prompt to ask to OpenAI.')
    }
  },

  command_image: function(command, handler) {
    if (handler.args) {
      let request = { prompt: handler.args }
      this.request({
        method: 'IMAGE',
        requestable: request,
        handler
      })
    } else {
      handler.reply('There is no prompt to ask to OpenAI.')
    }
  },

  request: function({method, requestable, handler = null, stealth = this.config.stealth}) {
    let t = (method === 'TEXT') ? 'text' : ((method === 'IMAGE') ? 'image' : false)
    if (!t) return false
    let id = Date.now()
    if (handler) this.requestPool.set(id, handler)
    this.sendSocketNotification('REQUEST', {
      request: { ...this.config.defaultRequest[t], ...requestable },
      options: {id, method, stealth}
    })
    return true
  },
socketNotificationReceived: function(notification, payload) {
  if (notification === "SHOW_ALERT") {
    console.log ("received SHOW_ALERT notification with payload:", payload);
    this.sendNotification(notification, payload);
  }
},

socketNotificationReceived: function(notification, payload) {
  if (notification === "RESPONSE") {
    Log.info("Received RESPONSE: " + JSON.stringify(payload));
    console.log("payload.response.choices:", payload.response.choices);
    if (payload.options && payload.options.showAlert && payload.response?.data?.choices?.length) {
      let message = payload.response.choices[0].text.trim();
      let timer = payload.options.alertTimer || 5000; // default 5 seconds
      console.log("Sending  SHOW_ALERT notification from mainjs with message:", message);
      this.sendNotification("SHOW_ALERT", { title: "OpenAI Response", message: message, timer: timer });
    }
  }
},


  socketNotificationReceived: function (notification, payload) { 
   if (notification === 'RESPONSE') {
      if (payload.options?.id && this.requestPool.has(payload.options.id)) {
        let handler = this.requestPool.get(payload.options.id)
        if (payload.error) {
          payload.response = null
          console.log('[OPENAI] Error:', payload.error)
        }
        if (typeof handler === 'function') {
          handler(payload)
        } else if (typeof handler?.reply === 'function') {
          if (payload.error) {
            handler.reply('TEXT', 'Something wrong, see the log.')
          } else if (payload.options.method === 'IMAGE') {
            handler.reply('PHOTO_URL', payload.response.data[0].url, { caption: payload.request.prompt })
          } else {
            handler.reply('TEXT', payload.response.choices[0].text)
          }
        } else {
          console.log('[OPENAI] Invalid Handler)', payload.response)
        }
        this.requestPool.delete(payload.options.id)
      } else {
        console.log('[OPENAI] Response without handler (Usually timedout or missing callback)', payload.response)
      }
      if (typeof this.config.postProcessing === 'function') {
        this.config.postProcessing(this.postProcessor, payload)
      }

      this.lastResponse = payload
      if (!payload.stealth) this.updateDom(500)
      if (this.config.resultLife > 0) {
        clearTimeout(this.resultTimer)
        this.resultTimer = setTimeout(() => {
          this.lastResponse = null
          this.updateDom(500)
        }, this.config.resultLife)
      }
    } // RESPONSE 
  },

  notificationReceived: function (notification, payload, sender) {
    for(const [key, factory] of Object.entries(this.notificationSet)) {
      if (notification === key) {
        let requestable = (typeof factory === 'function') ? factory(payload, sender) : ((typeof factory === 'object') ? factory : {})
        if (!(requestable?.request && requestable.request?.prompt)) {
          console.log('[OPENAI] Invalid payload', payload)
          if (typeof requestable?.callback === 'function') requestable.callback(false)
          return
        }
        let r = this.request({
          method: requestable?.method ?? null,
          requestable: requestable.request, 
          handler: requestable?.callback ?? null,
          stealth: requestable?.stealth || this.config.stealth
        })
        if (!r) {
          if (typeof payload.callback === 'function') payload.callback(false)
          this.sendNotification('OPENAI_REQUEST_FAILED_BY_NOTIFICATION', { payload, sender: sender.identifier })
        }
        return
      }
    }
  }



})

And for the nodehelper I have added some console log and I could get it to print it on the console but the mainjs is not receiving the notification for some reason. Here is my nodehelper

 require('dotenv').config()
const NodeHelper = require('node_helper')
const { Configuration, OpenAIApi } = require("openai")
const API_KEY = process.env.OPENAI_API_KEY
const ORGANIZATION = process.env.OPENAI_ORGANIZATION

const configuration = new Configuration({
  organization: ORGANIZATION,
  apiKey: API_KEY,
})

const openai = new OpenAIApi(configuration)

module.exports = NodeHelper.create({
  start: function () {
  },

  socketNotificationReceived: function (notification, payload) {
    if (notification === 'REQUEST') {
      this.job(payload).catch((error) => {
        console.log(error)
      })
    }
  },

  job: async function (payload) {
    let e = null
    let {request, options} = payload
    let response = null
    console.log('----------------------')
    console.log(request)
    console.log(options)
    try {
      response = (options?.method === 'IMAGE')
        ? await openai.createImage(request)
        : await openai.createCompletion(request)
    } catch (error) {
      console.log('error')
      console.log(error)
      e = error
    } finally {
      console.log(response?.data ?? e)
      if (!options.stealth) {
        this.sendSocketNotification('RESPONSE', {
          error: e,
          response: response?.data ?? e,
          request: request,
          options: options,
          responseTimestamp: Date.now(),
          stealth: options.stealth
        })
      }
      if (options.stealth === true && response) {
        let payload = {
          message: response.data.choices[0].text.trim(),
        }
        console.log('sending SHOW_ALERT notification')
        this.sendSocketNotification('SHOW_ALERT', payload)
        console.log(`[OPENAI] stealth mode, this is node helepr sending notification: ${payload.message}`)
      }
      if (options.stealth === true && e) {
        let payload = {
          message: "Error occurred while processing your request",
        }
        console.log('sending SHOW_ALERT notification for error')
        this.sendSocketNotification('SHOW_ALERT', payload)
        console.log(`[OPENAI] stealth mode, sending error notification`)
      }
    }
  }
})

I know it’s a little clumsy, not really sure what iam doing wrong here. Please help me out

[eouia]

So what you want is;
When the response from OpenAI arrives, the default alert module shows the result. And this should be done without other modules.(Only with MMM-OpenAI and alert module) right?

You don't need to modify the source. You can use postProcessing in the module config. (and with the stealth option)

postProcessing: (helper, responseObj) => {
  helper.sendNotification('SHOW_ALERT', {
   title: 'Response from OpenAI',
   message: responseObj.response.choices[0].text
  })
}

[eouia]

I'll show working demo in a few minutes. :)

[eouia]

By the way, How did you send the request? with TelegramBot? or some other module? (On request time, stealth:true option might be needed.)

[sdmydbr9]

I sent it using the telegrambot.

[eouia]

So you don't want the default MMM-OpenAI display right? Instead of module view, you want to use alert right?
(anyway, Almost done. wait a sec. :) )

[sdmydbr9]

Yes thats correct, just show the alert instead of the pop up. 😊

[eouia]

[eouia]

// config.js
{
	module: "MMM-OpenAI",
	position: 'top_right',
	config: {
		stealth: true,  // <- This is needed to hide default module view.
		postProcessing: function (handler, responseObj) {
			handler.sendNotification('SHOW_ALERT', { 
				title: responseObj.request.prompt,
				message: responseObj.response.choices[0].text
			})
		}
	}
},

[eouia]

To handle image togehter, you need to add some more logic, but I think you can catch the concept.

[eouia]

Here is the more advanced version;

{
	module: "MMM-OpenAI",
	position: 'top_right',
	config: {
		stealth: true,  // <- This is needed to hide default module view.
		postProcessing: function (handler, responseObj) {
			if (responseObj.error) return // When the error happens, just do nothing.
			let method = responseObj.options.method
			let alertPayload = {
				title: responseObj.request.prompt,
				imageUrl: (method === 'IMAGE') ? responseObj.response.data[0].url : null,
				message: (method === 'TEXT') ? responseObj.response.choices[0].text : null,
				timer: 5 * 60 * 1000
			}
			handler.sendNotification('SHOW_ALERT', alertPayload)
		}
	}
},

[sdmydbr9]

Thank you so much, I will try them. You're a life saver☺️

[sdmydbr9]

Hi again, I was using the new chatai using telegram and the first few conversations are okay but after 2 or 3 conversations the chatai throws an error in the telegram. Here are the logs of the mirror, I am not sure whats causing it.

ended: true,
0|MagicMirror  |       _ending: true,
0|MagicMirror  |       _redirectCount: 0,
0|MagicMirror  |       _redirects: [],
0|MagicMirror  |       _requestBodyLength: 684,
0|MagicMirror  |       _requestBodyBuffers: [],
0|MagicMirror  |       _onNativeResponse: [Function (anonymous)],
0|MagicMirror  |       _currentRequest: [Circular *1],
0|MagicMirror  |       _currentUrl: 'https://api.openai.com/v1/chat/completions',
0|MagicMirror  |       [Symbol(kCapture)]: false
0|MagicMirror  |     },
0|MagicMirror  |     [Symbol(kCapture)]: false,
0|MagicMirror  |     [Symbol(kNeedDrain)]: false,
0|MagicMirror  |     [Symbol(corked)]: 0,
0|MagicMirror  |     [Symbol(kOutHeaders)]: [Object: null prototype] {
0|MagicMirror  |       accept: [Array],
0|MagicMirror  |       'content-type': [Array],
0|MagicMirror  |       'user-agent': [Array],
0|MagicMirror  |       authorization: [Array],
0|MagicMirror  |       'content-length': [Array],
0|MagicMirror  |       host: [Array]
0|MagicMirror  |     }
0|MagicMirror  |   },
0|MagicMirror  |   response: {
0|MagicMirror  |     status: 400,
0|MagicMirror  |     statusText: 'Bad Request',
0|MagicMirror  |     headers: {
0|MagicMirror  |       date: 'Sat, 04 Mar 2023 09:37:45 GMT',
0|MagicMirror  |       'content-type': 'application/json',
0|MagicMirror  |       'content-length': '329',
0|MagicMirror  |       connection: 'close',
0|MagicMirror  |       'access-control-allow-origin': '*',
0|MagicMirror  |       'openai-organization': 'user-5qcgsq4ynyd5liocxywa2zfq',
0|MagicMirror  |       'openai-processing-ms': '85',
0|MagicMirror  |       'openai-version': '2020-10-01',
0|MagicMirror  |       'strict-transport-security': 'max-age=15724800; includeSubDomains',
0|MagicMirror  |       'x-request-id': '0b6bf5d9ddca8e65110457c0cca0d9a3'
0|MagicMirror  |     },
0|MagicMirror  |     config: {
0|MagicMirror  |       transitional: [Object],
0|MagicMirror  |       adapter: [Function: httpAdapter],
0|MagicMirror  |       transformRequest: [Array],
0|MagicMirror  |       transformResponse: [Array],
0|MagicMirror  |       timeout: 0,
0|MagicMirror  |       xsrfCookieName: 'XSRF-TOKEN',
0|MagicMirror  |       xsrfHeaderName: 'X-XSRF-TOKEN',
0|MagicMirror  |       maxContentLength: -1,
0|MagicMirror  |       maxBodyLength: -1,
0|MagicMirror  |       validateStatus: [Function: validateStatus],
0|MagicMirror  |       headers: [Object],
0|MagicMirror  |       method: 'post',
0|MagicMirror  |       data: `{"model":"gpt-3.5-turbo","messages":[{"role":"system","content":"I want you to act like Jarvis from Iron man series . I want you to respond and answer like Jarvis using the tone, manner and vocabulary Jarvis would use. Do not write any explanations. Only answer like Jarvis . You must know all of the knowledge of Jarvis . My first sentence is Hi Jarvis."},{"role":"user","content":"hi"},{"role":"assistant","content":"Greetings. How may I be of assistance?"},{"role":"user","content":"good morning"},{"role":"assistant","content":"Good morning, sir/madam. It's a pleasure to interact with you. How may I assist you today?"},{"role":"user","content":"how are you"}],"max_tokens":4000}`,
0|MagicMirror  |       url: 'https://api.openai.com/v1/chat/completions'
0|MagicMirror  |     },
0|MagicMirror  |     request: <ref *1> ClientRequest {
0|MagicMirror  |       _events: [Object: null prototype],
0|MagicMirror  |       _eventsCount: 7,
0|MagicMirror  |       _maxListeners: undefined,
0|MagicMirror  |       outputData: [],
0|MagicMirror  |       outputSize: 0,
0|MagicMirror  |       writable: true,
0|MagicMirror  |       destroyed: false,
0|MagicMirror  |       _last: true,
0|MagicMirror  |       chunkedEncoding: false,
0|MagicMirror  |       shouldKeepAlive: false,
0|MagicMirror  |       maxRequestsOnConnectionReached: false,
0|MagicMirror  |       _defaultKeepAlive: true,
0|MagicMirror  |       useChunkedEncodingByDefault: true,
0|MagicMirror  |       sendDate: false,
0|MagicMirror  |       _removedConnection: false,
0|MagicMirror  |       _removedContLen: false,
0|MagicMirror  |       _removedTE: false,
0|MagicMirror  |       _contentLength: null,
0|MagicMirror  |       _hasBody: true,
0|MagicMirror  |       _trailer: '',
0|MagicMirror  |       finished: true,
0|MagicMirror  |       _headerSent: true,
0|MagicMirror  |       _closed: false,
0|MagicMirror  |       socket: [TLSSocket],
0|MagicMirror  |       _header: 'POST /v1/chat/completions HTTP/1.1\r\n' +
0|MagicMirror  |         'Accept: application/json, text/plain, */*\r\n' +
0|MagicMirror  |         'Content-Type: application/json\r\n' +
0|MagicMirror  |         'User-Agent: OpenAI/NodeJS/3.2.1\r\n' +
0|MagicMirror  |         'Authorization: Bearer sk-apikey\r\n' +
0|MagicMirror  |         'Content-Length: 684\r\n' +
0|MagicMirror  |         'Host: api.openai.com\r\n' +
0|MagicMirror  |         'Connection: close\r\n' +
0|MagicMirror  |         '\r\n',
0|MagicMirror  |       _keepAliveTimeout: 0,
0|MagicMirror  |       _onPendingData: [Function: nop],
0|MagicMirror  |       agent: [Agent],
0|MagicMirror  |       socketPath: undefined,
0|MagicMirror  |       method: 'POST',
0|MagicMirror  |       maxHeaderSize: undefined,
0|MagicMirror  |       insecureHTTPParser: undefined,
0|MagicMirror  |       path: '/v1/chat/completions',
0|MagicMirror  |       _ended: true,
0|MagicMirror  |       res: [IncomingMessage],
0|MagicMirror  |       aborted: false,
0|MagicMirror  |       timeoutCb: null,
0|MagicMirror  |       upgradeOrConnect: false,
0|MagicMirror  |       parser: null,
0|MagicMirror  |       maxHeadersCount: null,
0|MagicMirror  |       reusedSocket: false,
0|MagicMirror  |       host: 'api.openai.com',
0|MagicMirror  |       protocol: 'https:',
0|MagicMirror  |       _redirectable: [Writable],
0|MagicMirror  |       [Symbol(kCapture)]: false,
0|MagicMirror  |       [Symbol(kNeedDrain)]: false,
0|MagicMirror  |       [Symbol(corked)]: 0,
0|MagicMirror  |       [Symbol(kOutHeaders)]: [Object: null prototype]
0|MagicMirror  |     },
0|MagicMirror  |     data: { error: [Object] }
0|MagicMirror  |   },
0|MagicMirror  |   isAxiosError: true,
0|MagicMirror  |   toJSON: [Function: toJSON

[eouia]

400 error usually happens on throwing wrong request data

[eouia]

What conversation did you make? Show me whole conversation when you met the error

[sdmydbr9]

[sdmydbr9]

I was using telegram bot chatai, any idea or workaround for this?

[eouia]

Hmmm. I will check, but this kind of error is hard to reproduce. I'll make a debug/test version to gather symptom and reason easy. Wait a few days.
BTW, has your first conversation always got a success? and Second consistently failed? Or totally random?

[sdmydbr9]

The first and the second conversation is always successful. And if I reset and try again it's responding again. And after 2 conversations it stops responding. Here is a screenshot after resetting

[eouia]

I found what was wrong from your log.
Your trial produce the error with this reason;

error: {
    message: "This model's maximum context length is 4096 tokens. However, you requested 4130 tokens (130 in the messages, 4000 in the completion). Please reduce the length of the messages or completion.",
    type: 'invalid_request_error',
    param: 'messages',
    code: 'context_length_exceeded'
  }
}

Too long token (conversation)

[eouia]

Anyway, MMM-Clap looks working

Screen-2023-03-08-181154.1.mp4

[eouia]

I tried the whisper api, and I switched to google STT, whisper seems to throw error sometimes, returning different text or wrong transcription sometimes another language.

Ah, so? I'm also considering Picovoice's Porcupine and Leopard for VAD and STT. Anyway I'm researching.

[sdmydbr9]

Okay I will wait for your hotword module.I think Google stt will be a better approach, but maybe you have better ways to use the whisper api, try it.

[eouia]

demo.mov

the new hotword detector... almost done.

[sdmydbr9]

demo.mov

the new hotword detector... almost done.

Wow, cool.. waiting 😁