wip: database

Author: Bas950

packages/util/package.json

@@ -32,6 +32,11 @@
     "access": "public"
   },
   "dependencies": {
-    "defu": "^6.1.4"
+    "@fast-csv/parse": "^5.0.0",
+    "ky": "^1.7.2",
+    "yauzl": "^3.1.3"
+  },
+  "devDependencies": {
+    "@types/yauzl": "^2.10.3"
   }
 }

packages/util/src/browser.ts

@@ -4,6 +4,11 @@ import { numberToCountryCode } from './functions/numberToCountryCode.js'
 import { numberToDir } from './functions/numberToDir.js'
 import { parseIp } from './functions/parseIp.js'
 
+/**
+ * Sets up an IP lookup function based on the specified data type.
+ * @template T - The type of data to return ('country' or 'geocode')
+ * @returns A function that takes an IP address and returns location data
+ */
 export function setup<T extends 'country' | 'geocode'>(): (ipInput: string) => Promise<T extends 'country' ? { country: string } | null : { latitude: number, longitude: number, country: string } | null> {
   const CDN_URL = __CDN_URL__
   const MAIN_RECORD_SIZE = __DATA_TYPE__ === 'country' ? 2 : 8
@@ -16,12 +21,20 @@ export function setup<T extends 'country' | 'geocode'>(): (ipInput: string) => P
     6: CDN_URL,
   }
 
+  /**
+   * Loads the index for the specified IP version.
+   * @param ipVersion - The IP version (4 or 6)
+   * @returns A promise that resolves to the loaded index
+   */
   async function loadIndex(ipVersion: 4 | 6) {
-    //* Determine the base URL for downloading the index
     const baseUrl = getBaseUrl()
     return downloadIndex(baseUrl, ipVersion)
   }
 
+  /**
+   * Determines the base URL for downloading the index.
+   * @returns The base URL as a string
+   */
   function getBaseUrl(): string {
     //* If we are not in the DOM we just use the CDN_URL to download the index
     if (typeof document === 'undefined' || !document.currentScript) {
@@ -37,6 +50,12 @@ export function setup<T extends 'country' | 'geocode'>(): (ipInput: string) => P
     return document.currentScript.src.split('/').slice(0, -1).join('/')
   }
 
+  /**
+   * Downloads the index file for the specified IP version.
+   * @param baseUrl - The base URL for downloading
+   * @param version - The IP version (4 or 6)
+   * @returns A promise that resolves to the downloaded index
+   */
   async function downloadIndex(baseUrl: string, version: 4 | 6) {
     const result = await fetchArrayBuffer(
       new URL(`indexes/${version}.idx`, baseUrl),
@@ -50,6 +69,11 @@ export function setup<T extends 'country' | 'geocode'>(): (ipInput: string) => P
     return (INDEXES[version] = new BigUint64Array(buffer))
   }
 
+  /**
+   * Performs an IP lookup and returns location data.
+   * @param ipInput - The IP address to look up
+   * @returns A promise that resolves to location data or null if not found
+   */
   return async function IpLookup(ipInput: string) {
     const { version, ip } = parseIp(ipInput)
 
@@ -100,6 +124,15 @@ export function setup<T extends 'country' | 'geocode'>(): (ipInput: string) => P
     return null
   } as (ipInput: string) => Promise<T extends 'country' ? { country: string } | null : { latitude: number, longitude: number, country: string } | null>
 
+  /**
+   * Retrieves the end IP for a given record.
+   * @param dataBuffer - The buffer containing IP data
+   * @param ipVersion - The IP version (4 or 6)
+   * @param recordCount - The total number of records
+   * @param recordIndex - The index of the current record
+   * @param ipSize - The size of an IP address in bytes
+   * @returns The end IP as a number
+   */
   function getEndIp(dataBuffer: ArrayBuffer, ipVersion: 4 | 6, recordCount: number, recordIndex: number, ipSize: number) {
     const endIpBuffer = dataBuffer.slice(
       (recordCount + recordIndex) * ipSize,
@@ -110,6 +143,15 @@ export function setup<T extends 'country' | 'geocode'>(): (ipInput: string) => P
       : new BigUint64Array(endIpBuffer)[0]!
   }
 
+  /**
+   * Parses a record and returns location data.
+   * @param dataBuffer - The buffer containing record data
+   * @param recordCount - The total number of records
+   * @param recordIndex - The index of the current record
+   * @param ipSize - The size of an IP address in bytes
+   * @param MAIN_RECORD_SIZE - The size of the main record data
+   * @returns Location data object
+   */
   function parseRecord(dataBuffer: ArrayBuffer, recordCount: number, recordIndex: number, ipSize: number, MAIN_RECORD_SIZE: number) {
     const recordBuffer = dataBuffer.slice(
       recordCount * ipSize * 2 + recordIndex * MAIN_RECORD_SIZE,

packages/util/src/db.ts

@@ -1,20 +1,39 @@
+import type { IpLocationApiInputSettings, IpLocationApiSettings } from './functions/getSettings.js'
+/* eslint-disable jsdoc/check-param-names */
 import { existsSync } from 'node:fs'
 import { mkdir } from 'node:fs/promises'
-import { getSettings, type IpLocationApiSettings } from './functions/getSettings.js'
+import { createDatabase } from './functions/database/createDatabase.js'
+import { downloadAndExtractDatabase } from './functions/database/downloadAndExtractDatabase.js'
+import { getSettings } from './functions/getSettings.js'
 
-export async function update(settings?: Partial<IpLocationApiSettings>) {
-  const { dataDir, tmpDataDir } = getSettings(settings)
+/**
+ * Updates the GeoIP database based on the provided settings.
+ * @param inputSettings - Partial settings to override the default ones.
+ * @returns An array of extracted file names or false if no update was needed.
+ */
+export async function update(inputSettings?: Partial<IpLocationApiInputSettings>): Promise<false | string[]> {
+  const settings = getSettings(inputSettings)
+  await ensureDirectoriesExist(settings)
+  const { files, dataType } = await downloadAndExtractDatabase(settings)
+  if (!files)
+    return false
 
-  if (!existsSync(dataDir)) {
-    // TODO add debug log
-    await mkdir(dataDir, { recursive: true })
-  }
+  // TODO: Add debug log
+  await createDatabase(files, settings, dataType)
+  // TODO: Add debug log
 
-  if (!existsSync(tmpDataDir)) {
-    // TODO add debug log
-    await mkdir(tmpDataDir, { recursive: true })
-  }
+  return files
+}
 
-  // TODO add debug log
-  // TODO download db and save
+/**
+ * Ensures that the necessary directories exist.
+ * @param param0 - Object containing fieldDir and tmpDataDir paths.
+ */
+async function ensureDirectoriesExist({ fieldDir, tmpDataDir }: IpLocationApiSettings): Promise<void> {
+  for (const dir of [fieldDir, tmpDataDir]) {
+    if (!existsSync(dir)) {
+      // TODO: Add debug log to indicate that the directory is being created
+      await mkdir(dir, { recursive: true })
+    }
+  }
 }

packages/util/src/functions/binarySearch.ts

@@ -1,8 +1,17 @@
+/**
+ * Performs a binary search on a sorted array of BigUint64 or Uint32 values.
+ *
+ * @param array - The sorted array to search in (BigUint64Array or Uint32Array).
+ * @param target - The value to search for (number or bigint).
+ * @returns The index of the target if found, or the index where it would be inserted if not found.
+ *          Returns null if the target is smaller than all elements in the array.
+ */
 export function binarySearch(array: BigUint64Array | Uint32Array, target: number | bigint): number | null {
   let low = 0
   let high = array.length - 1
 
   while (low <= high) {
+    //* Bitwise right shift by 1 is equivalent to Math.floor((low + high) / 2)
     const mid = (low + high) >> 1
     const midValue = array[mid]!
 
@@ -13,9 +22,10 @@ export function binarySearch(array: BigUint64Array | Uint32Array, target: number
       low = mid + 1
     }
     else {
-      return mid
+      return mid //* Target found
     }
   }
 
+  //* If target not found, return the insertion point or null if smaller than all elements
   return high >= 0 ? high : null
 }

packages/util/src/functions/database/createBlockDatabase.ts

@@ -0,0 +1,15 @@
+import type { IpLocationApiSettings } from '../getSettings.js'
+import type { LocationData } from './createDatabase.js'
+
+/**
+ * Creates a block database from the provided file and location data.
+ * @param file - The file path of the block database CSV
+ * @param locationData - Array of location data records
+ * @param locationIdList - List of location IDs to process
+ * @param settings - IP location API settings
+ */
+export async function createBlockDatabase(file: string, locationData: Record<number, LocationData | string>[], locationIdList: number[], settings: IpLocationApiSettings): Promise<void> {
+  const version = file.endsWith('v4.csv') ? 4 : 6
+  //* TODO: Implement the block database creation logic
+  console.log(`Creating block database for IP${version}...`)
+}