✨ geoip to redis (NGC-1726)

Author: paulsouche

package.json

@@ -54,6 +54,7 @@
     "multer": "^1.4.5-lts.1",
     "nanoid": "3.3.4",
     "nodemon": "^3.0.3",
+    "papaparse": "^5.5.2",
     "publicodes": "1.1.1",
     "redis-mock": "^0.56.3",
     "request-ip": "^3.3.0",
@@ -83,6 +84,7 @@
     "@types/morgan": "^1.9.9",
     "@types/multer": "^1.4.12",
     "@types/node": "^20.11.7",
+    "@types/papaparse": "^5.3.15",
     "@types/redis-mock": "^0.17.3",
     "@types/request-ip": "^0.0.41",
     "@types/supertest": "^6.0.2",

prisma/index.ts

@@ -8,6 +8,7 @@ const main = async () => {
     await import('./scripts/grant-roles'),
     await import('./scripts/add-integrations-api-scopes'),
     await import('./scripts/add-integrations-email-whitelist'),
+    await import('./scripts/geolocation'),
   ]
 
   try {

prisma/scripts/geolocation.ts

@@ -0,0 +1,57 @@
+import axios from 'axios'
+import { createGunzip } from 'node:zlib'
+import Papa from 'papaparse'
+import { redis } from '../../src/adapters/redis/client'
+import { KEYS } from '../../src/adapters/redis/constant'
+import { converIpToNumber } from '../../src/features/geolocation/geolocation.service'
+
+export const exec = async () => {
+  try {
+    const geoipUrl = process.env.GEOIP_URL
+
+    if (!geoipUrl) {
+      throw new Error('GEOIP_URL is required')
+    }
+
+    const today = new Date()
+
+    const datasourceStream = (
+      await axios.get(
+        geoipUrl.replace(
+          '{{YYYY-MM}}',
+          `${today.getFullYear()}-${(today.getMonth() + 1).toString().padStart(2, '0')}`
+        ),
+        {
+          responseType: 'stream',
+        }
+      )
+    ).data.pipe(createGunzip())
+
+    const parseStream = Papa.parse(Papa.NODE_STREAM_INPUT, {
+      header: false,
+    })
+
+    datasourceStream.pipe(parseStream)
+
+    const sortedArray = []
+
+    for await (const chunk of parseStream) {
+      const [ipStart, _, countryCode] = chunk
+
+      // Ignore IPv6 addresses
+      if (!ipStart.includes('.')) {
+        continue
+      }
+
+      const ipStartNumber = converIpToNumber(ipStart)
+      sortedArray.push({ ipStartNum: ipStartNumber, countryCode })
+    }
+
+    await redis.set(KEYS.geolocation, JSON.stringify(sortedArray))
+    await redis.persist(KEYS.geolocation)
+    console.log(`Stored ${sortedArray.length} IP to redis`)
+  } catch (err) {
+    console.error('Geolocation error', err)
+    throw err
+  }
+}

src/adapters/redis/constant.ts

@@ -1,3 +1,7 @@
 export const CHANNELS = {
   apiEvents: 'api-events',
 }
+
+export const KEYS = {
+  geolocation: 'geolocation',
+}

src/features/geolocation/geolocation.service.ts

@@ -0,0 +1,8 @@
+export const converIpToNumber = (ip: string) => {
+  return ip
+    .split('.')
+    .reduce(
+      (accumulator, octet) => (accumulator << 8) + Number.parseInt(octet, 10),
+      0
+    )
+}

yarn.lock

@@ -4008,6 +4008,13 @@
   dependencies:
     undici-types "~5.26.4"
 
+"@types/papaparse@^5.3.15":
+  version "5.3.15"
+  resolved "https://registry.yarnpkg.com/@types/papaparse/-/papaparse-5.3.15.tgz#7cafa16757a1d121422deefbb10b6310b224ecc4"
+  integrity sha512-JHe6vF6x/8Z85nCX4yFdDslN11d+1pr12E526X8WAfhadOeaOTx5AuIkvDKIBopfvlzpzkdMx4YyvSKCM9oqtw==
+  dependencies:
+    "@types/node" "*"
+
 "@types/pg-pool@2.0.4":
   version "2.0.4"
   resolved "https://registry.yarnpkg.com/@types/pg-pool/-/pg-pool-2.0.4.tgz#b5c60f678094ff3acf3442628a7f708928fcf263"
@@ -8888,6 +8895,11 @@ package-json-from-dist@^1.0.0:
   resolved "https://registry.yarnpkg.com/package-json-from-dist/-/package-json-from-dist-1.0.0.tgz#e501cd3094b278495eb4258d4c9f6d5ac3019f00"
   integrity sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw==
 
+papaparse@^5.5.2:
+  version "5.5.2"
+  resolved "https://registry.yarnpkg.com/papaparse/-/papaparse-5.5.2.tgz#fb67cc5a03ba8930cb435dc4641a25d6804bd4d7"
+  integrity sha512-PZXg8UuAc4PcVwLosEEDYjPyfWnTEhOrUfdv+3Bx+NuAb+5NhDmXzg5fHWmdCh1mP5p7JAZfFr3IMQfcntNAdA==
+
 param-case@^3.0.4:
   version "3.0.4"
   resolved "https://registry.yarnpkg.com/param-case/-/param-case-3.0.4.tgz#7d17fe4aa12bde34d4a77d91acfb6219caad01c5"