wip: database

Author: Bas950

packages/util/package.json

@@ -33,6 +33,7 @@
   },
   "dependencies": {
     "@fast-csv/parse": "^5.0.0",
+    "ip-address": "^9.0.5",
     "ky": "^1.7.2",
     "yauzl": "^3.1.3"
   },

packages/util/src/constants.ts

@@ -0,0 +1,37 @@
+import type { IpLocationApiSettings, LocalDatabase } from './functions/getSettings.js'
+
+export const MAXMIND_URL = 'https://download.maxmind.com/app/geoip_download'
+export const DATABASE_SUFFIX_SHA = '.zip.sha256'
+export const DATABASE_SUFFIX_ZIP = '.zip'
+export const MAIN_FIELDS = ['latitude', 'longitude', 'area', 'postcode'] as const
+export const LOCATION_FIELDS = ['country', 'region1', 'region1_name', 'region2', 'region2_name', 'metro', 'timezone', 'city', 'eu'] as const
+export const v4: LocalDatabase = {
+  version: 4,
+  recordSize: 0,
+  fileLineMax: 0,
+  folderLineMax: 0,
+}
+export const v6: LocalDatabase = {
+  version: 6,
+  recordSize: 0,
+  fileLineMax: 0,
+  folderLineMax: 0,
+}
+export const DEFAULT_SETTINGS: IpLocationApiSettings = {
+  licenseKey: 'redist',
+  series: 'GeoLite2',
+  dataDir: '../data',
+  tmpDataDir: '../tmp',
+  fields: ['country'],
+  language: 'en',
+  smallMemory: false,
+  smallMemoryFileSize: 4096,
+  //* Generated settings
+  fieldDir: '',
+  dataType: 'Country',
+  locationFile: false,
+  mainRecordSize: 2,
+  locationRecordSize: 0,
+  v4,
+  v6,
+}

packages/util/src/db.ts

@@ -1,7 +1,9 @@
 import type { IpLocationApiInputSettings, IpLocationApiSettings } from './functions/getSettings.js'
 /* eslint-disable jsdoc/check-param-names */
 import { existsSync } from 'node:fs'
-import { mkdir } from 'node:fs/promises'
+import { cp, mkdir, readdir, rename, rm, writeFile } from 'node:fs/promises'
+import path from 'node:path'
+import { DATABASE_SUFFIX_SHA } from './constants.js'
 import { createDatabase } from './functions/database/createDatabase.js'
 import { downloadAndExtractDatabase } from './functions/database/downloadAndExtractDatabase.js'
 import { getSettings } from './functions/getSettings.js'
@@ -11,18 +13,34 @@ import { getSettings } from './functions/getSettings.js'
  * @param inputSettings - Partial settings to override the default ones.
  * @returns An array of extracted file names or false if no update was needed.
  */
-export async function update(inputSettings?: Partial<IpLocationApiInputSettings>): Promise<false | string[]> {
+export async function update(inputSettings?: Partial<IpLocationApiInputSettings>): Promise<void> {
   const settings = getSettings(inputSettings)
   await ensureDirectoriesExist(settings)
-  const { files, dataType } = await downloadAndExtractDatabase(settings)
+  const { files, sha256 } = await downloadAndExtractDatabase(settings)
   if (!files)
-    return false
+    return
 
   // TODO: Add debug log
-  await createDatabase(files, settings, dataType)
+  await createDatabase(files, settings)
   // TODO: Add debug log
 
-  return files
+  //* Save the sha256 hash of the database
+  await writeFile(path.join(settings.fieldDir, `${settings.series}-${settings.dataType}-CSV${DATABASE_SUFFIX_SHA}`), sha256)
+
+  //* Rename the temporary files to the correct name
+  const tmpFiles = (await readdir(settings.fieldDir)).filter(file => file.endsWith('.tmp'))
+  for (const file of tmpFiles) {
+    await rename(path.join(settings.fieldDir, file), path.join(settings.fieldDir, file.replace('.tmp', '')))
+  }
+
+  if (settings.smallMemory) {
+    //* Copy the temporary files to the correct name
+    await cp(path.join(settings.fieldDir, 'v4-tmp'), path.join(settings.fieldDir, 'v4'), { recursive: true, force: true })
+    await cp(path.join(settings.fieldDir, 'v6-tmp'), path.join(settings.fieldDir, 'v6'), { recursive: true, force: true })
+    //* Remove the temporary files
+    await rm(path.join(settings.fieldDir, 'v4-tmp'), { recursive: true, force: true, maxRetries: 3 })
+    await rm(path.join(settings.fieldDir, 'v6-tmp'), { recursive: true, force: true, maxRetries: 3 })
+  }
 }
 
 /**