From 9a9165484682105a152af6405c8f90869d1db09a Mon Sep 17 00:00:00 2001
From: Elliot Corvinova <blueskymonster@users.noreply.github.com>
Date: Wed, 27 Mar 2024 19:45:42 -0700
Subject: [PATCH 1/2] Update repo to modern ooba OpenAI API streaming standards

---
 README.md            |  12 ++--
 browser_example.html |   2 +-
 node_example.mjs     |  14 ++++-
 oobapi-ws.mjs        | 132 ++++++++++++++++++++++++++++---------------
 package-lock.json    |   4 +-
 5 files changed, 106 insertions(+), 58 deletions(-)

diff --git a/README.md b/README.md
index 6039e80..cb49afa 100644
--- a/README.md
+++ b/README.md
@@ -1,11 +1,11 @@
 # oobapi-ws
-[oobabooga/text-generation-webui](https://github.com/oobabooga/text-generation-webui) websocket api wrapper
+[oobabooga/text-generation-webui](https://github.com/oobabooga/text-generation-webui) ~websocket~ streaming api wrapper
 
-  This is a simple isomorphic wrapper for the oobabooga websocket api (must be enabled when [launching ooba](https://github.com/oobabooga/text-generation-webui#api) with `--api`)
+  This is a simple isomorphic wrapper for the oobabooga ~websocket~ streaming api (must be enabled when [launching ooba](https://github.com/oobabooga/text-generation-webui#api) with `--api`)
   
 ## Usage
 
-  As this wrapper is meant to function the same in the browser as well as in node it must be passed a `WebSocket` object when instantiated. This can be either the browser context `WebSocket` or one from the [`ws` module](https://github.com/websockets/ws). See `node_example.mjs` and `browser_example.html` for more details.
+  ~~As this wrapper is meant to function the same in the browser as well as in node it must be passed a `WebSocket` object when instantiated. This can be either the browser context `WebSocket` or one from the [`ws` module](https://github.com/websockets/ws). See `node_example.mjs` and `browser_example.html` for more details.~~ No longer uses websockets.
 
   Ｗhen instantiating the api you can specify a host and a port, as well as default generation options．
   ```js
@@ -13,13 +13,13 @@
   const generation_options = {max_new_tokens: 1000}
   const host = 'remote.host.net'
   const port = 1337
-  const api = await ooba(WebSocket)({host,port}, generation_options)
+  const api = ooba({host,port}, generation_options)
   ```
   or if you're running ooba locally and you want to use the default options you can omit both
   ```js
-  const api = await ooba(WebSocket)()
+  const api = ooba()
   ```
-  this promise will return once a websocket connection to `localhost:5005` is established
+  this will return an api object that is ready to make streaming requests.
 
   in order to generate text simply call `api.generate()` with a prompt
   ```js
diff --git a/browser_example.html b/browser_example.html
index a6cef0c..1398977 100644
--- a/browser_example.html
+++ b/browser_example.html
@@ -13,7 +13,7 @@
     <script type="module">
       import ooba from './oobapi-ws.mjs'
 
-      const api = await ooba(WebSocket)()
+      const api = ooba()
       const prompt = "In order to make homemade bread, follow these steps:\n1)"
       document.body.innerText += prompt
 
diff --git a/node_example.mjs b/node_example.mjs
index 94ca3b0..514d021 100644
--- a/node_example.mjs
+++ b/node_example.mjs
@@ -1,11 +1,19 @@
-import WebSocket from 'ws'
 import ooba from './oobapi-ws.mjs'
 
-const api = await ooba(WebSocket)()
+const api = ooba()
 
-const prompt = "In order to make homemade bread, follow these steps:\n1)"
+const prompt = "[INST]In order to make homemade bread, follow these steps:[/INST]\n)"
 
+let response_complete = false
 process.stdout.write(prompt)
 api.ontoken = token => process.stdout.write(token)
+api.onend = () => response_complete = true
+api.onerror = () => response_complete = true
 
 api.generate(prompt)
+
+const delay = ms => new Promise(res => setTimeout(res, ms))
+
+while (!response_complete) {
+  await delay(100)
+}
diff --git a/oobapi-ws.mjs b/oobapi-ws.mjs
index 2dc5e3d..afe038a 100644
--- a/oobapi-ws.mjs
+++ b/oobapi-ws.mjs
@@ -1,69 +1,109 @@
+import http from 'http'
 //default options from https://github.com/oobabooga/text-generation-webui/blob/main/api-example-stream.py
 const default_options = {
-  prompt: "In order to make homemade bread, follow these steps:\n1)",
-  max_new_tokens: 250,
-  do_sample: true,
-  temperature: 1.3,
-  top_p: 0.1,
+  model: "unsloth_mistral-7b-instruct-v0.2-bnb-4bit",
+  prompt: "[INST]In five sentences or less, explain what buffer solutions are and how they are relevant to biology.[/INST]",
+  best_of: 1,
+  echo: false,
+  frequency_penalty: 0,
+  logit_bias: {},
+  logprobs: 0,
+  max_tokens: 250,
+  n: 1,
+  presence_penalty: 0,
+  stream: true,
+  suffix: "",
+  temperature: 1,
+  top_p: 1,
+  min_p: 0,
+  dynamic_temperature: false,
+  dynatemp_low: 1,
+  dynatemp_high: 1,
+  dynatemp_exponent: 1,
+  smoothing_factor: 0,
+  smoothing_curve: 1,
+  stop: [],
+  top_k: 0,
+  repetition_penalty: 1,
+  repetition_penalty_range: 1024,
   typical_p: 1,
-  epsilon_cutoff: 0,  // In units of 1e-4
-  eta_cutoff: 0,  // In units of 1e-4
-  repetition_penalty: 1.18,
-  top_k: 40,
-  min_length: 0,
-  no_repeat_ngram_size: 0,
-  num_beams: 1,
+  tfs: 1,
+  top_a: 0,
+  epsilon_cutoff: 0,
+  eta_cutoff: 0,
+  guidance_scale: 1,
+  negative_prompt: "",
   penalty_alpha: 0,
-  length_penalty: 1,
-  early_stopping: false,
   mirostat_mode: 0,
   mirostat_tau: 5,
   mirostat_eta: 0.1,
+  temperature_last: false,
+  do_sample: true,
   seed: -1,
-  add_bos_token: true,
-  truncation_length: 2048,
+  encoder_repetition_penalty: 1,
+  no_repeat_ngram_size: 0,
+  min_length: 0,
+  num_beams: 1,
+  length_penalty: 1,
+  early_stopping: false,
+  truncation_length: 0,
+  max_tokens_second: 0,
+  prompt_lookup_num_tokens: 0,
+  custom_token_bans: "",
+  sampler_priority: [],
   ban_eos_token: false,
+  add_bos_token: true,
   skip_special_tokens: true,
-  stopping_strings: []
+  grammar_string: ""
 }
 
-const api = WebSocket => ({host='localhost',port=5005}={}, parameters=default_options) => new Promise(res => {
-  const URI = `ws://${host}:${port}/api/v1/stream`
-  const socket = new WebSocket(URI)
+const api = ({host='127.0.0.1', port=5000}={}, parameters=default_options) => {
+  const URI = `http://${host}:${port}/v1/completions`
 
-  //browser compatibility shim
-  if (!socket.on) socket.on = socket.addEventListener
 
   let api = {parameters}
   api.generate = prompt => {
-    //check for string prompt and wrap in an object
+    // check for string prompt and wrap in an object
     if (typeof prompt === 'string') prompt = {prompt}
-    socket.send(JSON.stringify(Object.assign({},api.parameters,prompt)))
-  }
-
-  socket.on('message', (data) => {
-    //browser compatibility shim
-    if (data.data) data = data.data
-    const parsed = JSON.parse(data)
-    switch (parsed.event) {
-      case 'text_stream':
-        api.ontoken?.(parsed.text)
-        api.debugstream?.(parsed)
-        break;
-      case 'stream_end':
-        api.onend?.(parsed)
+    const postData = JSON.stringify(Object.assign({},api.parameters, prompt))
+    const http_options = {
+      hostname: host,
+      port,
+      path: '/v1/completions',
+      method: 'POST',
+      headers: {
+        'Accept': 'application/json',
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(postData)
+      }
     }
-  })
 
-  socket.on('error', (error) => {
-    console.error(`WebSocket error: `,error)
-  })
+    const req = http.request(http_options, (res) => {
+      res.setEncoding('utf8')
+
+      res.on('data', (chunk) => {
+        // Remove the 'data:' prefix from the chunk
+        const parsed = JSON.parse(chunk.slice(5))
+        // TODO: handle multiples choices when n != 1
+        const response = parsed.choices[0]
+        if (response.finish_reason == null) {
+          api.ontoken?.(response.text)
+          api.debugstream?.(parsed)
+        } else {
+          api.onend?.(parsed)
+        }
+      })
+    })
 
-  socket.on('close', () => {
-    console.error(`WebSocket closed`)
-  })
+    req.on('error', (error) => {
+      api.onerror?.({error})
+      console.error("Request error:", error)
+    })
 
-  socket.on('open', () => res(api))
-})
+    req.write(postData)
+    req.end()
+  }
+  return api
+}
 
 export default api
diff --git a/package-lock.json b/package-lock.json
index 2f01a86..e1f91ab 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "oobapi-ws",
-  "version": "0.0.1",
+  "version": "1.0.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "oobapi-ws",
-      "version": "0.0.1",
+      "version": "1.0.1",
       "license": "MIT",
       "devDependencies": {
         "ws": "^8.13.0"

From cbc1c13c36eb4a8f9f9a6ea59aa88adadebe897a Mon Sep 17 00:00:00 2001
From: Elliot Corvinova <blueskymonster@users.noreply.github.com>
Date: Thu, 18 Apr 2024 09:45:21 -0700
Subject: [PATCH 2/2] Prepare repo for move to Hallucinate Games repo

---
 LICENSE                            |  2 +-
 README.md                          | 10 ++++-----
 browser_example.html               |  2 +-
 node_example.mjs                   |  2 +-
 oobapi-ws.mjs => oobapi-stream.mjs |  4 ++--
 package-lock.json                  | 32 ++++-------------------------
 package.json                       | 33 +++++++++++++++++++-----------
 7 files changed, 34 insertions(+), 51 deletions(-)
 rename oobapi-ws.mjs => oobapi-stream.mjs (95%)

diff --git a/LICENSE b/LICENSE
index 60c7e25..fbffc49 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 thot-experiment
+Copyright (c) 2024 Hallucinate, LLC
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/README.md b/README.md
index cb49afa..b43942c 100644
--- a/README.md
+++ b/README.md
@@ -1,15 +1,13 @@
-# oobapi-ws
-[oobabooga/text-generation-webui](https://github.com/oobabooga/text-generation-webui) ~websocket~ streaming api wrapper
+# oobapi-stream
+[oobabooga/text-generation-webui](https://github.com/oobabooga/text-generation-webui) streaming api wrapper
 
-  This is a simple isomorphic wrapper for the oobabooga ~websocket~ streaming api (must be enabled when [launching ooba](https://github.com/oobabooga/text-generation-webui#api) with `--api`)
+  This is a simple isomorphic wrapper for the oobabooga api that handles the necessary HTTP shenanigans to stream token predictions to you as they come instead of getting a result all at once. To use it, API mode must be enabled when [launching ooba](https://github.com/oobabooga/text-generation-webui#api) by using `--api`
   
 ## Usage
 
-  ~~As this wrapper is meant to function the same in the browser as well as in node it must be passed a `WebSocket` object when instantiated. This can be either the browser context `WebSocket` or one from the [`ws` module](https://github.com/websockets/ws). See `node_example.mjs` and `browser_example.html` for more details.~~ No longer uses websockets.
-
   Ｗhen instantiating the api you can specify a host and a port, as well as default generation options．
   ```js
-  import ooba from 'oobapi-ws'
+  import ooba from 'oobapi-stream'
   const generation_options = {max_new_tokens: 1000}
   const host = 'remote.host.net'
   const port = 1337
diff --git a/browser_example.html b/browser_example.html
index 1398977..1be77cd 100644
--- a/browser_example.html
+++ b/browser_example.html
@@ -11,7 +11,7 @@
   </head>
   <body>
     <script type="module">
-      import ooba from './oobapi-ws.mjs'
+      import ooba from './oobapi-stream.mjs'
 
       const api = ooba()
       const prompt = "In order to make homemade bread, follow these steps:\n1)"
diff --git a/node_example.mjs b/node_example.mjs
index 514d021..aa68482 100644
--- a/node_example.mjs
+++ b/node_example.mjs
@@ -1,4 +1,4 @@
-import ooba from './oobapi-ws.mjs'
+import ooba from './oobapi-stream.mjs'
 
 const api = ooba()
 
diff --git a/oobapi-ws.mjs b/oobapi-stream.mjs
similarity index 95%
rename from oobapi-ws.mjs
rename to oobapi-stream.mjs
index afe038a..9a32af7 100644
--- a/oobapi-ws.mjs
+++ b/oobapi-stream.mjs
@@ -1,5 +1,5 @@
 import http from 'http'
-//default options from https://github.com/oobabooga/text-generation-webui/blob/main/api-example-stream.py
+// default options (model, prompt, and stream excepted) from OpenAI API documentation
 const default_options = {
   model: "unsloth_mistral-7b-instruct-v0.2-bnb-4bit",
   prompt: "[INST]In five sentences or less, explain what buffer solutions are and how they are relevant to biology.[/INST]",
@@ -11,7 +11,7 @@ const default_options = {
   max_tokens: 250,
   n: 1,
   presence_penalty: 0,
-  stream: true,
+  stream: true, // Do not overwrite this option.
   suffix: "",
   temperature: 1,
   top_p: 1,
diff --git a/package-lock.json b/package-lock.json
index e1f91ab..5494856 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,37 +1,13 @@
 {
-  "name": "oobapi-ws",
-  "version": "1.0.1",
+  "name": "oobapi-stream",
+  "version": "1.0.2",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
-      "name": "oobapi-ws",
+      "name": "oobapi-stream",
       "version": "1.0.1",
-      "license": "MIT",
-      "devDependencies": {
-        "ws": "^8.13.0"
-      }
-    },
-    "node_modules/ws": {
-      "version": "8.13.0",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz",
-      "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==",
-      "dev": true,
-      "engines": {
-        "node": ">=10.0.0"
-      },
-      "peerDependencies": {
-        "bufferutil": "^4.0.1",
-        "utf-8-validate": ">=5.0.2"
-      },
-      "peerDependenciesMeta": {
-        "bufferutil": {
-          "optional": true
-        },
-        "utf-8-validate": {
-          "optional": true
-        }
-      }
+      "license": "MIT"
     }
   }
 }
diff --git a/package.json b/package.json
index 8c8036c..ed56c8b 100644
--- a/package.json
+++ b/package.json
@@ -1,14 +1,14 @@
 {
-  "name": "oobapi-ws",
-  "version": "1.0.1",
-  "description": "oobabooga/text-generation-webui websocket api wrapper",
-  "main": "oobapi-ws.mjs",
+  "name": "oobapi-stream",
+  "version": "1.0.3",
+  "description": "oobabooga/text-generation-webui streaming api wrapper",
+  "main": "oobapi-stream.mjs",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/thot-experiment/oobapi-ws.git"
+    "url": "git+https://github.com/hallucinate-games/oobapi-stream.git"
   },
   "keywords": [
     "oobabooga",
@@ -21,15 +21,24 @@
     "alpaca",
     "large",
     "language",
-    "model"
+    "model",
+    "stream"
+  ],
+  "contributors": [
+    {
+      "name": "Matt Vana",
+      "email": "thot@hallucinate.games",
+      "url": "https://github.com/thot-experiment"
+    },
+    {
+      "name": "Elliot Corvinova",
+      "email": "elliot@hallucinate.games",
+      "url": "https://github.com/blueskymonster"
+    }
   ],
-  "author": "thot-experiment",
   "license": "MIT",
-  "devDependencies": {
-    "ws": "^8.13.0"
-  },
   "bugs": {
-    "url": "https://github.com/thot-experiment/oobapi-ws/issues"
+    "url": "https://github.com/hallucinate-games/oobapi-stream/issues"
   },
-  "homepage": "https://github.com/thot-experiment/oobapi-ws#readme"
+  "homepage": "https://github.com/hallucinate-games/oobapi-streams#readme"
 }