feat: add back logs

Bas950 · Bas950 · commit ac2dca2ae5ca · 2024-11-12T14:46:14.000+01:00
diff --git a/packages/country-extra/vite.config.ts b/packages/country-extra/vite.config.ts
@@ -52,6 +52,7 @@ export default defineConfig({
           dataDir: resolve('../../data/country'),
           tmpDataDir: resolve('../../tmp/country'),
           fields: ['country'],
+          silent: true,
         }
         await update(settings)
         await createBrowserIndex('country', settings, resolve('./indexes'))
diff --git a/packages/country/vite.config.ts b/packages/country/vite.config.ts
@@ -52,6 +52,7 @@ export default defineConfig({
           dataDir: resolve('../../data/country'),
           tmpDataDir: resolve('../../tmp/country'),
           fields: ['country'],
+          silent: true,
         }
         await update(settings)
         await createBrowserIndex('country', settings, resolve('./indexes'))
diff --git a/packages/geocode-extra/vite.config.ts b/packages/geocode-extra/vite.config.ts
@@ -52,6 +52,7 @@ export default defineConfig({
           dataDir: resolve('../../data/geocode'),
           tmpDataDir: resolve('../../tmp/geocode'),
           fields: ['latitude', 'longitude'],
+          silent: true,
         }
         await update(settings)
         await createBrowserIndex('geocode', settings, resolve('./indexes'))
diff --git a/packages/geocode/vite.config.ts b/packages/geocode/vite.config.ts
@@ -52,6 +52,7 @@ export default defineConfig({
           dataDir: resolve('../../data/geocode'),
           tmpDataDir: resolve('../../tmp/geocode'),
           fields: ['latitude', 'longitude'],
+          silent: true,
         }
         await update(settings)
         await createBrowserIndex('geocode', settings, resolve('./indexes'))
diff --git a/packages/ip-location-api/src/functions/lookup.ts b/packages/ip-location-api/src/functions/lookup.ts
@@ -1,7 +1,7 @@
 import { Buffer } from 'node:buffer'
 import { open } from 'node:fs/promises'
 import { join } from 'node:path'
-import { binarySearch, getSmallMemoryFile, type IpLocationApiSettings, type LocalDatabase, number37ToString, parseIp, SAVED_SETTINGS } from '@iplookup/util'
+import { binarySearch, getSmallMemoryFile, type IpLocationApiSettings, type LocalDatabase, log, number37ToString, parseIp, SAVED_SETTINGS } from '@iplookup/util'
 import { LOADED_DATA } from './reload.js'
 
 /**
@@ -193,7 +193,7 @@ async function lineToFile(line: number, db: LocalDatabase, settings: IpLocationA
   const buffer = Buffer.alloc(db.recordSize)
   await fd.read(buffer, 0, db.recordSize, offset)
   fd.close().catch(() => {
-    // TODO console.warn
+    log('warn', 'Failed to close file descriptor')
   })
   return buffer
 }
@@ -375,8 +375,7 @@ async function setCountryInfo(geodata: GeoData, settings: IpLocationApiSettings)
       /* c8 ignore next 5 */ //* We don't check the try-catch as it's an optional peer dependency
     }
     catch (error) {
-      // TODO add correct debug message
-      console.error('Error importing countries-list', error)
+      log('warn', 'Error importing countries-list', error)
     }
   }
   return geodata
diff --git a/packages/ip-location-api/src/index.all.smallMemory.test.ts b/packages/ip-location-api/src/index.all.smallMemory.test.ts
@@ -14,8 +14,9 @@ describe('lookup (all, smallMemory)', () => {
       fields: 'all',
       smallMemory: true,
       addCountryInfo: true,
+      // silent: true,
     })
-  }, 15 * 60_000)
+  }, 25 * 60_000)
 
   afterAll(async () => {
     clear()
diff --git a/packages/ip-location-api/src/index.city.test.ts b/packages/ip-location-api/src/index.city.test.ts
@@ -12,8 +12,9 @@ describe('lookup (city)', () => {
       dataDir: resolve(__dirname, '../data', id),
       tmpDataDir: resolve(__dirname, '../tmp', id),
       fields: ['city', 'country'],
+      silent: true,
     })
-  }, 15 * 60_000)
+  }, 25 * 60_000)
 
   afterAll(async () => {
     clear()
diff --git a/packages/ip-location-api/src/index.country.smallMemory.test.ts b/packages/ip-location-api/src/index.country.smallMemory.test.ts
@@ -13,6 +13,7 @@ describe('lookup (country, smallMemory)', () => {
       tmpDataDir: resolve(__dirname, '../tmp', id),
       fields: ['country'],
       smallMemory: true,
+      silent: true,
     })
   }, 5 * 60_000)
 
diff --git a/packages/ip-location-api/src/index.country.test.ts b/packages/ip-location-api/src/index.country.test.ts
@@ -12,6 +12,7 @@ describe('lookup (country)', () => {
       dataDir: resolve(__dirname, '../data', id),
       tmpDataDir: resolve(__dirname, '../tmp', id),
       fields: ['country'],
+      silent: true,
     })
   }, 5 * 60_000)
 
diff --git a/packages/util/src/browser.ts b/packages/util/src/browser.ts
@@ -1,5 +1,6 @@
 import { binarySearch } from './functions/binarySearch.js'
 import { fetchArrayBuffer } from './functions/fetchArrayBuffer.js'
+import { log } from './functions/log.js'
 import { numberToCountryCode } from './functions/numberToCountryCode.js'
 import { numberToDir } from './functions/numberToDir.js'
 import { parseIp } from './functions/parseIp.js'
@@ -32,13 +33,15 @@ export function setup<T extends 'country' | 'geocode'>(): (ipInput: string) => P
     //* Get the index for the IP version
     const index = INDEXES[version] ?? (await loadIndex(version))
     if (!index) {
-      // TODO add debug log
+      log('warn', 'No index found')
       return null
     }
 
     //* If the IP is less than the first index, return null
-    if (!(ip >= index[0]!))
+    if (!(ip >= index[0]!)) {
+      log('warn', `IP ${ipInput} is out of range`)
       return null
+    }
 
     //* Binary search to find the correct line in the index
     const lineIndex = binarySearch(index, ip)
@@ -51,7 +54,7 @@ export function setup<T extends 'country' | 'geocode'>(): (ipInput: string) => P
       new URL(`${DATA_URL[version]}/indexes/${version}/${numberToDir(lineIndex)}`),
     )
     if (!dataResponse) {
-      // TODO Add debug log
+      log('warn', 'Index file not found, is it corrupted?')
       return null
     }
 
@@ -115,8 +118,10 @@ export function setup<T extends 'country' | 'geocode'>(): (ipInput: string) => P
     const result = await fetchArrayBuffer(
       new URL(`${baseUrl}/indexes/${version}.idx`),
     )
-    if (!result)
-      return // TODO add debug log
+    if (!result) {
+      log('warn', 'Index file not found, is it corrupted?')
+      return null
+    }
 
     const { versionHeader, buffer } = result
     if (versionHeader)
diff --git a/packages/util/src/constants.ts b/packages/util/src/constants.ts
@@ -35,4 +35,10 @@ export const DEFAULT_SETTINGS: IpLocationApiSettings = {
   v4,
   v6,
   addCountryInfo: false,
+  silent: false,
+}
+// eslint-disable-next-line import/no-mutable-exports
+export let SAVED_SETTINGS: IpLocationApiSettings = DEFAULT_SETTINGS
+export function setSavedSettings(settings: IpLocationApiSettings) {
+  return SAVED_SETTINGS = settings
 }
diff --git a/packages/util/src/db.ts b/packages/util/src/db.ts
@@ -7,6 +7,7 @@ import { createDatabase } from './functions/database/createDatabase.js'
 import { downloadAndExtractDatabase } from './functions/database/downloadAndExtractDatabase.js'
 import { ensureDirectoriesExist } from './functions/database/ensureDirectoriesExist.js'
 import { getSettings } from './functions/getSettings.js'
+import { log } from './functions/log.js'
 
 /**
  * Updates the GeoIP database based on the provided settings.
@@ -15,13 +16,14 @@ import { getSettings } from './functions/getSettings.js'
 export async function update(inputSettings?: Partial<IpLocationApiInputSettings>): Promise<void> {
   const settings = getSettings(inputSettings)
   await ensureDirectoriesExist(settings)
+  log('info', 'Downloading database...')
   const { files, sha256 } = await downloadAndExtractDatabase(settings)
   if (!files || files.length === 0)
     return
 
-  // TODO: Add debug log
+  log('info', 'Creating database...')
   await createDatabase(files, settings)
-  // TODO: Add debug log
+  log('info', 'Database created')
 
   //* Save the sha256 hash of the database
   await writeFile(path.join(settings.fieldDir, `${settings.series}-${settings.dataType}-CSV${DATABASE_SUFFIX_SHA}`), sha256)
@@ -40,4 +42,6 @@ export async function update(inputSettings?: Partial<IpLocationApiInputSettings>
     await rm(path.join(settings.fieldDir, 'v4-tmp'), { recursive: true, force: true, maxRetries: 3 })
     await rm(path.join(settings.fieldDir, 'v6-tmp'), { recursive: true, force: true, maxRetries: 3 })
   }
+
+  log('info', 'Database successfully updated')
 }
diff --git a/packages/util/src/functions/database/createBlockDatabase.ts b/packages/util/src/functions/database/createBlockDatabase.ts
@@ -10,6 +10,7 @@ import { Address4, Address6 } from 'ip-address'
 import { aton4 } from '../aton4.js'
 import { aton6 } from '../aton6.js'
 import { getPostcodeDatabase } from '../getPostcodeDatabase.js'
+import { log } from '../log.js'
 import { makeDatabase } from '../makeDatabase.js'
 import { createSmallMemoryFile } from './createSmallMemoryFile.js'
 
@@ -41,6 +42,19 @@ export async function createBlockDatabase(
   settings: IpLocationApiSettings,
 ): Promise<void> {
   const version = file.endsWith('v4.csv') ? 4 : 6
+  log('info', `Creating block database for IPv${version}...`)
+
+  //* First count total lines
+  let totalLines = 0
+  await new Promise<void>((resolve) => {
+    createReadStream(path.join(settings.tmpDataDir, file))
+      .pipe(parse({ headers: true }))
+      .on('data', () => { totalLines++ })
+      .on('end', resolve)
+  })
+
+  log('info', `Found ${totalLines.toLocaleString()} total records to process`)
+
   const readStream = createReadStream(path.join(settings.tmpDataDir, file))
   const writeStreamDat1 = createWriteStream(path.join(settings.fieldDir, `${version}-1.dat.tmp`), { highWaterMark: 1024 * 1024 })
 
@@ -60,16 +74,14 @@ export async function createBlockDatabase(
   }
 
   return new Promise<void>((resolve, reject) => {
-    //* Count the number of checks to ensure all streams are written before resolving
     let checkCount = 0
     function check() {
-      if (++checkCount === 3)
+      if (++checkCount === 3) {
+        log('info', `Completed processing IPv${version} block database`)
         resolve()
+      }
     }
 
-    /**
-     * Represents the data from the previous row for comparison and optimization.
-     */
     let previousData: {
       countryCode?: string
       end: number | bigint
@@ -84,10 +96,21 @@ export async function createBlockDatabase(
       counter?: number
     } | undefined
     let lineCount = 0
+    let processedCount = 0
+    const logInterval = 100000 // Log every 100k records
 
     readStream.pipe(parse({ headers: true }))
-      .on('error', reject)
+      .on('error', (error) => {
+        log('error', `Error processing IPv${version} block database: ${error.message}`)
+        reject(error)
+      })
       .on('data', (row: BlockDatabaseRow) => {
+        processedCount++
+        if (processedCount % logInterval === 0) {
+          const percentage = ((processedCount / totalLines) * 100).toFixed(2)
+          log('info', `Processed ${processedCount.toLocaleString()} IPv${version} records (${percentage}%)...`)
+        }
+
         const addr = version === 4 ? new Address4(row.network!) : new Address6(row.network!)
         const start = version === 4 ? aton4(addr.startAddress().correctForm()) : aton6(addr.startAddress().correctForm())
         const end = version === 4 ? aton4(addr.endAddress().correctForm()) : aton6(addr.endAddress().correctForm())
@@ -196,12 +219,16 @@ export async function createBlockDatabase(
             }
           }
           else {
-            if (!locationId)
-              return // TODO add debug log
+            if (!locationId) {
+              log('warn', 'No location ID found')
+              return
+            }
 
             const dataMap = locationDataMap[locationId]
-            if (!dataMap)
-              return // TODO add debug log
+            if (!dataMap) {
+              log('warn', `Invalid location ID ${locationId}`)
+              return
+            }
 
             //* Assign a counter if it doesn't exist
             if (!dataMap.counter) {
@@ -294,6 +321,8 @@ export async function createBlockDatabase(
         }
       })
       .on('end', () => {
+        log('info', `Finished processing ${processedCount.toLocaleString()} IPv${version} records (100%)`)
+
         if (settings.smallMemory && previousData?.buffer2 && previousData?.buffer3) {
           writeStreamSmallMemory = createSmallMemoryFile(writeStreamSmallMemory!, version, lineCount++, previousData.buffer2, previousData.buffer3, settings)
           writeStreamSmallMemory?.end(check)
diff --git a/packages/util/src/functions/database/downloadAndExtractDatabase.ts b/packages/util/src/functions/database/downloadAndExtractDatabase.ts
@@ -7,6 +7,7 @@ import { Writable } from 'node:stream'
 import ky from 'ky'
 import { open } from 'yauzl'
 import { DATABASE_SUFFIX_SHA, DATABASE_SUFFIX_ZIP, MAXMIND_URL } from '../../constants.js'
+import { log } from '../log.js'
 
 /**
  * Downloads and extracts the database if an update is needed.
@@ -26,7 +27,9 @@ export async function downloadAndExtractDatabase(settings: IpLocationApiSettings
   }
 
   const zipPath = await downloadDatabase(settings, edition)
+  log('info', `Decompressing ${edition}...`)
   const files = await extractDatabase(zipPath, settings.tmpDataDir, src)
+  log('info', `Decompressed ${edition}, extracted ${files.length} files`)
   return { files, sha256: remoteHash }
 }
 
@@ -153,7 +156,7 @@ async function extractDatabase(zipPath: string, outputDir: string, filesToExtrac
         //* Check if the entry matches any files to extract
         const matchedFile = filesToExtract.find(file => entry.fileName.endsWith(file))
         if (matchedFile) {
-          // TODO: Add debug log
+          log('info', `Extracting ${matchedFile}...`)
           //* Open the read stream for the entry
           zipfile.openReadStream(entry, (err, readStream) => {
             if (err)
diff --git a/packages/util/src/functions/database/ensureDirectoriesExist.ts b/packages/util/src/functions/database/ensureDirectoriesExist.ts
@@ -1,6 +1,7 @@
 import type { IpLocationApiSettings } from '../getSettings.js'
 import { existsSync } from 'node:fs'
 import { mkdir } from 'node:fs/promises'
+import { log } from '../log.js'
 
 /**
  * Ensures that the necessary directories exist.
@@ -11,7 +12,7 @@ import { mkdir } from 'node:fs/promises'
 export async function ensureDirectoriesExist({ fieldDir, tmpDataDir }: IpLocationApiSettings): Promise<void> {
   for (const dir of [fieldDir, tmpDataDir]) {
     if (!existsSync(dir)) {
-      // TODO: Add debug log to indicate that the directory is being created
+      log('info', `Creating directory ${dir}`)
       await mkdir(dir, { recursive: true })
     }
   }
diff --git a/packages/util/src/functions/getPostcodeDatabase.ts b/packages/util/src/functions/getPostcodeDatabase.ts
@@ -1,3 +1,5 @@
+import { log } from './log.js'
+
 const isPostNumReg = /^\d+$/
 const isPostNumReg2 = /^(\d+)[-\s](\d+)$/
 const isPostStrReg = /^[A-Z\d]+$/
@@ -53,7 +55,7 @@ export function getPostcodeDatabase(postcode: string): [number, number] {
 
   const stringMatch2 = isPostStrReg2.exec(postcode)
   if (!stringMatch2) {
-    // TODO: add invalid postcode debug log
+    log('warn', `Invalid postcode ${postcode}`)
     return [0, 0] //* Invalid postcode
   }
 
diff --git a/packages/util/src/functions/getSettings.ts b/packages/util/src/functions/getSettings.ts
diff --git a/packages/util/src/functions/log.test.ts b/packages/util/src/functions/log.test.ts
diff --git a/packages/util/src/functions/log.ts b/packages/util/src/functions/log.ts
diff --git a/packages/util/src/index.ts b/packages/util/src/index.ts

Original file line number	Diff line number	Diff line change
`@@ -52,6 +52,7 @@ export default defineConfig({`
`52`	`52`	`dataDir: resolve('../../data/country'),`
`53`	`53`	`tmpDataDir: resolve('../../tmp/country'),`
`54`	`54`	`fields: ['country'],`
	`55`	`+ silent: true,`
`55`	`56`	`}`
`56`	`57`	`await update(settings)`
`57`	`58`	`await createBrowserIndex('country', settings, resolve('./indexes'))`