From 1d5aaed5deebd297b13b6cc5a61af3999aee6e07 Mon Sep 17 00:00:00 2001
From: matias martinez <matias@xlabs.xyz>
Date: Thu, 4 Apr 2024 14:02:11 -0300
Subject: [PATCH 1/8] turn high cardinality tags into fields

---
 scripts/influxdb.ts | 50 ++++-----------------------------------------
 1 file changed, 4 insertions(+), 46 deletions(-)

diff --git a/scripts/influxdb.ts b/scripts/influxdb.ts
index c139ada..d93d0ba 100644
--- a/scripts/influxdb.ts
+++ b/scripts/influxdb.ts
@@ -15,7 +15,6 @@ import {
 import * as anchor from '@coral-xyz/anchor'
 import { envOrErr } from './claim_sdk'
 import { BN } from '@coral-xyz/anchor'
-import { inspect } from 'util'
 import { InfluxDB, Point, QueryApi } from '@influxdata/influxdb-client'
 
 const ENDPOINT = envOrErr('ENDPOINT')
@@ -27,7 +26,6 @@ const INFLUX_ORG = envOrErr('INFLUX_ORG')
 const INFLUX_BUCKET = envOrErr('INFLUX_BUCKET')
 const TIME_WINDOW_SECS = Number.parseInt(envOrErr('TIME_WINDOW_SECS'), 10)
 const CHUNK_SIZE = Number.parseInt(envOrErr('CHUNK_SIZE'), 10)
-const LOW_BALANCE_THRESHOLD = envOrErr('LOW_BALANCE_THRESHOLD')
 // based off airdrop allocation commit 16d0c19f3951427f04cc015d38805f356fcb88b1
 const MAX_AMOUNT_PER_ECOSYSTEM = new Map<string, BN>([
   ['discord', new BN('87000000000')],
@@ -118,15 +116,6 @@ async function main() {
     console.log('No double claims detected')
   }
 
-  const lowBalanceEventPoint = createLowBalanceEventPoint(formattedTxnEvents)
-
-  if (lowBalanceEventPoint) {
-    console.log('Detected low balance event')
-    writeApi.writePoint(lowBalanceEventPoint)
-  } else {
-    console.log('No low balance detected')
-  }
-
   const txnEventPoints = createTxnEventPoints(formattedTxnEvents)
   txnEventPoints.forEach((txnEventPoint) => {
     writeApi.writePoint(txnEventPoint)
@@ -199,11 +188,11 @@ function createTxnEventPoints(formattedTxnEvents: FormattedTxnEventInfo[]) {
     }
 
     const point = new Point('txn_event')
-      .tag('claimant', claimant!)
       .tag('ecosystem', ecosystem)
-      .tag('address', address)
       .tag('network', CLUSTER)
       .tag('eventCategory', eventCategory)
+      .stringField('claimant', claimant)
+      .stringField('address', address)
       .stringField('signature', signature)
       .intField('amount', amountValue)
       .stringField('eventDetails', JSON.stringify(formattedEvent))
@@ -246,9 +235,8 @@ function createDoubleClaimPoint(formattedTxnEvents: FormattedTxnEventInfo[]) {
 
       const point = new Point('double_claim_event')
         .tag('ecosystem', ecosystem)
-        .tag('address', address)
         .tag('network', CLUSTER)
-        .tag('service', 'token-dispenser-event-subscriber')
+        .stringField('address', address)
         .stringField('details', JSON.stringify(txnEventInfos))
         .timestamp(new Date(blockTime * 1000))
       doubleClaimPoints.push(point)
@@ -262,44 +250,14 @@ function createDoubleClaimPoint(formattedTxnEvents: FormattedTxnEventInfo[]) {
 function createFailedTxnEventPoints(failedTxns: TxnInfo[]) {
   return failedTxns.map((errorLog) => {
     const point = new Point('failed_txn_event')
-      .tag('signature', errorLog.signature)
       .tag('network', CLUSTER)
-      .tag('service', 'token-dispenser-event-subscriber')
+      .stringField('signature', errorLog.signature)
       .stringField('errorDetails', JSON.stringify(errorLog))
       .timestamp(new Date(errorLog.blockTime * 1000))
     return point
   })
 }
 
-function createLowBalanceEventPoint(
-  formattedTxnEvents: FormattedTxnEventInfo[]
-) {
-  if (formattedTxnEvents.length === 0) {
-    return undefined
-  }
-
-  const mostRecentEvent = formattedTxnEvents.sort((a, b) => b.slot - a.slot)[0]
-
-  if (
-    mostRecentEvent.remainingBalance &&
-    new BN(mostRecentEvent.remainingBalance).lt(new BN(LOW_BALANCE_THRESHOLD))
-  ) {
-    const point = new Point('low_balance_event')
-      .tag('signature', mostRecentEvent.signature)
-      .tag('network', CLUSTER)
-      .tag('service', 'token-dispenser-event-subscriber')
-      .intField(
-        'remainingBalance',
-        parseInt(mostRecentEvent.remainingBalance, 10)
-      )
-      .stringField('eventDetails', JSON.stringify(mostRecentEvent))
-      .timestamp(new Date(mostRecentEvent.blockTime * 1000).toISOString())
-    return point
-  }
-
-  return undefined
-}
-
 ;(async () => {
   try {
     await main()

From ba74562a51929aec35f2100c85e5ec9be59e8447 Mon Sep 17 00:00:00 2001
From: matias martinez <matias@xlabs.xyz>
Date: Thu, 4 Apr 2024 17:41:16 -0300
Subject: [PATCH 2/8] minor changes

---
 scripts/influxdb.ts | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/scripts/influxdb.ts b/scripts/influxdb.ts
index d93d0ba..4729e46 100644
--- a/scripts/influxdb.ts
+++ b/scripts/influxdb.ts
@@ -11,6 +11,7 @@ import {
   TokenDispenserEventSubscriber,
   FormattedTxnEventInfo,
   TxnInfo,
+  TxnEventInfo,
 } from './claim_sdk/eventSubscriber'
 import * as anchor from '@coral-xyz/anchor'
 import { envOrErr } from './claim_sdk'
@@ -26,6 +27,11 @@ const INFLUX_ORG = envOrErr('INFLUX_ORG')
 const INFLUX_BUCKET = envOrErr('INFLUX_BUCKET')
 const TIME_WINDOW_SECS = Number.parseInt(envOrErr('TIME_WINDOW_SECS'), 10)
 const CHUNK_SIZE = Number.parseInt(envOrErr('CHUNK_SIZE'), 10)
+const LAST_TXN_MEASUREMENT =
+  process.env.LAST_TXN_MEASUREMENT ?? 'latest_txn_seen'
+const TXN_MEASUREMENT = process.env.TXN_MEASUREMENT ?? 'txn_event'
+const FAILED_TXN_MEASUREMENT =
+  process.env.FAILED_TXN_MEASUREMENT ?? 'failed_txn_event'
 // based off airdrop allocation commit 16d0c19f3951427f04cc015d38805f356fcb88b1
 const MAX_AMOUNT_PER_ECOSYSTEM = new Map<string, BN>([
   ['discord', new BN('87000000000')],
@@ -137,7 +143,7 @@ async function main() {
   )
 
   console.log('Last signature processed:', latestSignature)
-  const latestTxPoint = new Point('latest_txn_seen')
+  const latestTxPoint = new Point(LAST_TXN_MEASUREMENT)
     .tag('network', CLUSTER)
     .stringField('signature', latestSignature)
 
@@ -161,8 +167,8 @@ async function getLatestTxSignature(
 ): Promise<string | undefined> {
   const query = `from(bucket: "${bucket}")
     |> range(start: -1d)
-    |> filter(fn: (r) => r._measurement == "latest_txn_seen")
-    |> filter(fn: (r) => r.network == "${network}")
+    |> filter(fn: (r) => r._measurement == "${LAST_TXN_MEASUREMENT}")
+    |> filter(fn: (r) => r.network == "${network}" and r._field == "signature")
     |> sort(columns: ["_time"], desc: true)
     |> first()
     |> limit(n:1)`
@@ -182,15 +188,16 @@ function createTxnEventPoints(formattedTxnEvents: FormattedTxnEventInfo[]) {
     const { ecosystem, address, amount } = formattedEvent.claimInfo!
     let eventCategory = 'normal'
     let amountValue = parseInt(amount, 10)
-
-    if (MAX_AMOUNT_PER_ECOSYSTEM.get(ecosystem)!.lt(new BN(amount))) {
+    const maxAmount = MAX_AMOUNT_PER_ECOSYSTEM.get(ecosystem)
+    if (amount && maxAmount && maxAmount.lt(new BN(amount))) {
       eventCategory = 'max_transfer_exceeded'
     }
 
-    const point = new Point('txn_event')
+    const point = new Point(TXN_MEASUREMENT)
       .tag('ecosystem', ecosystem)
       .tag('network', CLUSTER)
       .tag('eventCategory', eventCategory)
+      .tag('address', address)
       .stringField('claimant', claimant)
       .stringField('address', address)
       .stringField('signature', signature)
@@ -249,7 +256,7 @@ function createDoubleClaimPoint(formattedTxnEvents: FormattedTxnEventInfo[]) {
 
 function createFailedTxnEventPoints(failedTxns: TxnInfo[]) {
   return failedTxns.map((errorLog) => {
-    const point = new Point('failed_txn_event')
+    const point = new Point(FAILED_TXN_MEASUREMENT)
       .tag('network', CLUSTER)
       .stringField('signature', errorLog.signature)
       .stringField('errorDetails', JSON.stringify(errorLog))

From 66f010d3898381b0e3816e417d12855cfae0de90 Mon Sep 17 00:00:00 2001
From: matias martinez <matias@xlabs.xyz>
Date: Thu, 4 Apr 2024 19:20:17 -0300
Subject: [PATCH 3/8] fixer script

---
 scripts/.env.sample       |  10 +++-
 scripts/influxdb.fixer.ts | 100 ++++++++++++++++++++++++++++++++++++++
 scripts/influxdb.ts       |   2 -
 scripts/package.json      |   1 +
 4 files changed, 110 insertions(+), 3 deletions(-)
 create mode 100644 scripts/influxdb.fixer.ts

diff --git a/scripts/.env.sample b/scripts/.env.sample
index ad9470a..5de138a 100644
--- a/scripts/.env.sample
+++ b/scripts/.env.sample
@@ -8,8 +8,16 @@ export LOW_BALANCE_THRESHOLD=10000
 export INFLUX_URL=""
 export INFLUX_TOKEN=""
 export INFLUX_ORG=""
-export INFLU_BUCKET=""
+export INFLUX_BUCKET=""
 
 # Datadog Event Subscriber Configs
 export TIME_WINDOW_SECS=600
 export CHUNK_SIZE=50
+
+# Tag converter
+export MEASUREMENT="txn_event"
+export NEW_MEASUREMENT_VERSION="v1"
+export START_HOURS_AGO="72"
+export END_HOURS_AGO="0"
+export HOUR_WINDOW="1"
+export TAGS_TO_CONVERT="address"
diff --git a/scripts/influxdb.fixer.ts b/scripts/influxdb.fixer.ts
new file mode 100644
index 0000000..602d907
--- /dev/null
+++ b/scripts/influxdb.fixer.ts
@@ -0,0 +1,100 @@
+/**
+ *
+ * To Run:
+ * copy .env.sample .env
+ * Then update the parameters in .env accordingly
+ * source .env
+ * ts-node ./scripts/influxdb.fixer.ts
+ */
+import { envOrErr } from './claim_sdk'
+import { InfluxDB, QueryApi } from '@influxdata/influxdb-client'
+
+const INFLUX_URL = envOrErr('INFLUX_URL')
+const INFLUX_TOKEN = envOrErr('INFLUX_TOKEN')
+const INFLUX_ORG = envOrErr('INFLUX_ORG')
+const INFLUX_BUCKET = envOrErr('INFLUX_BUCKET')
+
+const MEASUREMENT = process.env.MEASUREMENT ?? 'txn_event'
+const NEW_MEASUREMENT_VERSION = process.env.NEW_MEASUREMENT_VERSION ?? 'v1'
+
+const START_HOURS_AGO = process.env.START_HOURS_AGO ?? '72' // 3 days
+const END_HOURS_AGO = process.env.END_HOURS_AGO ?? '0' // now
+const HOUR_WINDOW = process.env.HOUR_WINDOW ?? '1'
+
+const TAGS_TO_CONVERT = process.env.TAGS_TO_CONVERT ?? 'address'
+
+async function main() {
+  console.log(
+    `Connecting to influx <${INFLUX_URL}/${INFLUX_ORG}/${INFLUX_BUCKET}>`
+  )
+
+  console.log(
+    `Migrating ${MEASUREMENT} to ${MEASUREMENT}_${NEW_MEASUREMENT_VERSION}`
+  )
+
+  const influxDB = new InfluxDB({ url: INFLUX_URL, token: INFLUX_TOKEN })
+  const readApi = influxDB.getQueryApi(INFLUX_ORG)
+  const tagsToConvert = TAGS_TO_CONVERT.split(',')
+
+  let startHoursAgo = Number.parseInt(START_HOURS_AGO)
+  const endHoursAgo = Number.parseInt(END_HOURS_AGO)
+  let currentEndHoursago = ''
+
+  try {
+    while (startHoursAgo > endHoursAgo) {
+      const currentStartHoursAgo = `-${startHoursAgo}h`
+      currentEndHoursago = `-${startHoursAgo - Number.parseInt(HOUR_WINDOW)}h`
+
+      await mapAndWrite(
+        INFLUX_BUCKET,
+        MEASUREMENT,
+        NEW_MEASUREMENT_VERSION,
+        tagsToConvert,
+        currentStartHoursAgo,
+        currentEndHoursago,
+        readApi
+      )
+      startHoursAgo -= Number.parseInt(HOUR_WINDOW)
+    }
+  } catch (error) {
+    console.error(`Error: ${error}`)
+  }
+
+  console.log(
+    `Done -> StartHoursAgo: ${startHoursAgo} - EndHoursAgo: ${currentEndHoursago}`
+  )
+}
+
+async function mapAndWrite(
+  bucket: string,
+  measurement: string,
+  newMeasurementVersion: string,
+  tagsToConvert: string[],
+  start: string,
+  end: string,
+  readApi: QueryApi
+): Promise<void> {
+  const stream = `from(bucket: "${bucket}")
+  |> range(start: ${start}, stop: ${end})
+  |> filter(fn: (r) => r._measurement == "${measurement}")`
+
+  const mapper = `${stream}
+  |> map(fn: (r) => ({ r with _measurement: "${measurement}_${newMeasurementVersion}", ${tagsToConvert
+    .map((tag) => `${tag}: r.${tag}`)
+    .join(', ')} }))
+  |> drop(columns: [${tagsToConvert.map((tag) => `"${tag}"`).join(',')}])
+  |> to(bucket: "${bucket}")
+  `
+  console.log(` Executing the following query: ${mapper}`)
+  const response = await readApi.queryRaw(mapper)
+  console.log(response)
+}
+
+;(async () => {
+  try {
+    await main()
+  } catch (e) {
+    console.error(`error from influxdb.ts: ${e}`)
+    process.exit(1)
+  }
+})()
diff --git a/scripts/influxdb.ts b/scripts/influxdb.ts
index 4729e46..4c7ba30 100644
--- a/scripts/influxdb.ts
+++ b/scripts/influxdb.ts
@@ -11,7 +11,6 @@ import {
   TokenDispenserEventSubscriber,
   FormattedTxnEventInfo,
   TxnInfo,
-  TxnEventInfo,
 } from './claim_sdk/eventSubscriber'
 import * as anchor from '@coral-xyz/anchor'
 import { envOrErr } from './claim_sdk'
@@ -197,7 +196,6 @@ function createTxnEventPoints(formattedTxnEvents: FormattedTxnEventInfo[]) {
       .tag('ecosystem', ecosystem)
       .tag('network', CLUSTER)
       .tag('eventCategory', eventCategory)
-      .tag('address', address)
       .stringField('claimant', claimant)
       .stringField('address', address)
       .stringField('signature', signature)
diff --git a/scripts/package.json b/scripts/package.json
index 5fb3d0b..0df2a89 100644
--- a/scripts/package.json
+++ b/scripts/package.json
@@ -6,6 +6,7 @@
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
     "airdrop:metrics": "ts-node ./influxdb.ts",
+    "airdrop:metrics:fix-tags": "ts-node ./influxdb.fixer.ts",
     "prettier": "prettier . --write",
     "docker:login": "aws ecr get-login-password --region us-east-2 --profile $AWS_PROFILE_PRODUCTION | docker login --username AWS --password-stdin $AWS_ACCOUNT_PRODUCTION.dkr.ecr.us-east-2.amazonaws.com",
     "docker:build": "docker build --platform=linux/amd64 --tag $(npm run --silent docker:prod-tag) . --file ./Dockerfile",

From 4bc7c276de6c8f3dc8af2e4e76f3dade3aa04482 Mon Sep 17 00:00:00 2001
From: matias martinez <matias@xlabs.xyz>
Date: Fri, 5 Apr 2024 17:42:09 -0300
Subject: [PATCH 4/8] fix runner

---
 scripts/package-lock.json | 170 +++++++++++++++++++++++++++++++++++++-
 scripts/package.json      |   5 +-
 2 files changed, 170 insertions(+), 5 deletions(-)

diff --git a/scripts/package-lock.json b/scripts/package-lock.json
index c4a178f..037c41b 100644
--- a/scripts/package-lock.json
+++ b/scripts/package-lock.json
@@ -1,17 +1,18 @@
 {
   "name": "airdrop-metrics-job",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "airdrop-metrics-job",
-      "version": "0.1.1",
+      "version": "0.1.2",
       "license": "ISC",
       "dependencies": {
         "@coral-xyz/anchor": "^0.29.0",
         "@influxdata/influxdb-client": "^1.33.2",
-        "hi-base32": "^0.5.1"
+        "hi-base32": "^0.5.1",
+        "ts-node": "^10.9.2"
       },
       "devDependencies": {
         "prettier": "^2.8.8"
@@ -67,11 +68,44 @@
         "@solana/web3.js": "^1.68.0"
       }
     },
+    "node_modules/@cspotcode/source-map-support": {
+      "version": "0.8.1",
+      "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz",
+      "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==",
+      "dependencies": {
+        "@jridgewell/trace-mapping": "0.3.9"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/@influxdata/influxdb-client": {
       "version": "1.33.2",
       "resolved": "https://registry.npmjs.org/@influxdata/influxdb-client/-/influxdb-client-1.33.2.tgz",
       "integrity": "sha512-RT5SxH+grHAazo/YK3UTuWK/frPWRM0N7vkrCUyqVprDgQzlLP+bSK4ak2Jv3QVF/pazTnsxWjvtKZdwskV5Xw=="
     },
+    "node_modules/@jridgewell/resolve-uri": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz",
+      "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==",
+      "engines": {
+        "node": ">=6.0.0"
+      }
+    },
+    "node_modules/@jridgewell/sourcemap-codec": {
+      "version": "1.4.15",
+      "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz",
+      "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg=="
+    },
+    "node_modules/@jridgewell/trace-mapping": {
+      "version": "0.3.9",
+      "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz",
+      "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==",
+      "dependencies": {
+        "@jridgewell/resolve-uri": "^3.0.3",
+        "@jridgewell/sourcemap-codec": "^1.4.10"
+      }
+    },
     "node_modules/@noble/curves": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.4.0.tgz",
@@ -132,6 +166,26 @@
       "resolved": "https://registry.npmjs.org/superstruct/-/superstruct-0.14.2.tgz",
       "integrity": "sha512-nPewA6m9mR3d6k7WkZ8N8zpTWfenFH3q9pA2PkuiZxINr9DKB2+40wEQf0ixn8VaGuJ78AB6iWOtStI+/4FKZQ=="
     },
+    "node_modules/@tsconfig/node10": {
+      "version": "1.0.11",
+      "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz",
+      "integrity": "sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw=="
+    },
+    "node_modules/@tsconfig/node12": {
+      "version": "1.0.11",
+      "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz",
+      "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag=="
+    },
+    "node_modules/@tsconfig/node14": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz",
+      "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow=="
+    },
+    "node_modules/@tsconfig/node16": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz",
+      "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA=="
+    },
     "node_modules/@types/connect": {
       "version": "3.4.38",
       "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz",
@@ -153,6 +207,25 @@
         "@types/node": "*"
       }
     },
+    "node_modules/acorn": {
+      "version": "8.11.3",
+      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz",
+      "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==",
+      "bin": {
+        "acorn": "bin/acorn"
+      },
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    },
+    "node_modules/acorn-walk": {
+      "version": "8.3.2",
+      "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.2.tgz",
+      "integrity": "sha512-cjkyv4OtNCIeqhHrfS81QWXoCBPExR/J62oyEqepVw8WaQeSqpW2uhuLPh1m9eWhDuOo/jUXVTlifvesOWp/4A==",
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    },
     "node_modules/agentkeepalive": {
       "version": "4.5.0",
       "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz",
@@ -164,6 +237,11 @@
         "node": ">= 8.0.0"
       }
     },
+    "node_modules/arg": {
+      "version": "4.1.3",
+      "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz",
+      "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA=="
+    },
     "node_modules/base-x": {
       "version": "3.0.9",
       "resolved": "https://registry.npmjs.org/base-x/-/base-x-3.0.9.tgz",
@@ -294,6 +372,11 @@
       "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
       "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ=="
     },
+    "node_modules/create-require": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz",
+      "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ=="
+    },
     "node_modules/cross-fetch": {
       "version": "3.1.8",
       "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz",
@@ -324,6 +407,14 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/diff": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
+      "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
+      "engines": {
+        "node": ">=0.3.1"
+      }
+    },
     "node_modules/dot-case": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/dot-case/-/dot-case-3.0.4.tgz",
@@ -470,6 +561,11 @@
         "tslib": "^2.0.3"
       }
     },
+    "node_modules/make-error": {
+      "version": "1.3.6",
+      "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz",
+      "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw=="
+    },
     "node_modules/ms": {
       "version": "2.1.3",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
@@ -631,11 +727,66 @@
       "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
       "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
     },
+    "node_modules/ts-node": {
+      "version": "10.9.2",
+      "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz",
+      "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==",
+      "dependencies": {
+        "@cspotcode/source-map-support": "^0.8.0",
+        "@tsconfig/node10": "^1.0.7",
+        "@tsconfig/node12": "^1.0.7",
+        "@tsconfig/node14": "^1.0.0",
+        "@tsconfig/node16": "^1.0.2",
+        "acorn": "^8.4.1",
+        "acorn-walk": "^8.1.1",
+        "arg": "^4.1.0",
+        "create-require": "^1.1.0",
+        "diff": "^4.0.1",
+        "make-error": "^1.1.1",
+        "v8-compile-cache-lib": "^3.0.1",
+        "yn": "3.1.1"
+      },
+      "bin": {
+        "ts-node": "dist/bin.js",
+        "ts-node-cwd": "dist/bin-cwd.js",
+        "ts-node-esm": "dist/bin-esm.js",
+        "ts-node-script": "dist/bin-script.js",
+        "ts-node-transpile-only": "dist/bin-transpile.js",
+        "ts-script": "dist/bin-script-deprecated.js"
+      },
+      "peerDependencies": {
+        "@swc/core": ">=1.2.50",
+        "@swc/wasm": ">=1.2.50",
+        "@types/node": "*",
+        "typescript": ">=2.7"
+      },
+      "peerDependenciesMeta": {
+        "@swc/core": {
+          "optional": true
+        },
+        "@swc/wasm": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/tslib": {
       "version": "2.6.2",
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
       "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
     },
+    "node_modules/typescript": {
+      "version": "5.4.4",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.4.tgz",
+      "integrity": "sha512-dGE2Vv8cpVvw28v8HCPqyb08EzbBURxDpuhJvTrusShUfGnhHBafDsLdS1EhhxyL6BJQE+2cT3dDPAv+MQ6oLw==",
+      "peer": true,
+      "bin": {
+        "tsc": "bin/tsc",
+        "tsserver": "bin/tsserver"
+      },
+      "engines": {
+        "node": ">=14.17"
+      }
+    },
     "node_modules/utf-8-validate": {
       "version": "5.0.10",
       "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.10.tgz",
@@ -657,6 +808,11 @@
         "uuid": "dist/bin/uuid"
       }
     },
+    "node_modules/v8-compile-cache-lib": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",
+      "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg=="
+    },
     "node_modules/webidl-conversions": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
@@ -690,6 +846,14 @@
           "optional": true
         }
       }
+    },
+    "node_modules/yn": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz",
+      "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==",
+      "engines": {
+        "node": ">=6"
+      }
     }
   }
 }
diff --git a/scripts/package.json b/scripts/package.json
index 0df2a89..040435a 100644
--- a/scripts/package.json
+++ b/scripts/package.json
@@ -1,6 +1,6 @@
 {
   "name": "airdrop-metrics-job",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -21,7 +21,8 @@
   "dependencies": {
     "@coral-xyz/anchor": "^0.29.0",
     "@influxdata/influxdb-client": "^1.33.2",
-    "hi-base32": "^0.5.1"
+    "hi-base32": "^0.5.1",
+    "ts-node": "^10.9.2"
   },
   "devDependencies": {
     "prettier": "^2.8.8"

From 6987d01e0220ea6e2981e80a3686343b56b62ba9 Mon Sep 17 00:00:00 2001
From: matias martinez <matias@xlabs.xyz>
Date: Fri, 5 Apr 2024 19:29:19 -0300
Subject: [PATCH 5/8] use to

---
 scripts/.env.sample       | 10 ++++++++++
 scripts/influxdb.fixer.ts | 34 ++++++++++++++++++++++------------
 2 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/scripts/.env.sample b/scripts/.env.sample
index 5de138a..dbd2a0f 100644
--- a/scripts/.env.sample
+++ b/scripts/.env.sample
@@ -21,3 +21,13 @@ export START_HOURS_AGO="72"
 export END_HOURS_AGO="0"
 export HOUR_WINDOW="1"
 export TAGS_TO_CONVERT="address"
+
+export NEW_MEASUREMENT_VERSION=v1
+
+#export MEASUREMENT="failed_txn_event"
+#export TAGS_TO_KEEP="network"
+#export FIELDS="signature,errorDetails"
+export MEASUREMENT="txn_event"
+export TAGS_TO_KEEP="ecosystem,network,eventCategory"
+export FIELDS="signature,amount,eventDetails,address,claimant"
+
diff --git a/scripts/influxdb.fixer.ts b/scripts/influxdb.fixer.ts
index 602d907..b3bc9ab 100644
--- a/scripts/influxdb.fixer.ts
+++ b/scripts/influxdb.fixer.ts
@@ -16,13 +16,15 @@ const INFLUX_BUCKET = envOrErr('INFLUX_BUCKET')
 
 const MEASUREMENT = process.env.MEASUREMENT ?? 'txn_event'
 const NEW_MEASUREMENT_VERSION = process.env.NEW_MEASUREMENT_VERSION ?? 'v1'
+const TAGS_TO_KEEP =
+  process.env.TAGS_TO_KEEP ?? 'ecosystem,network,eventCategory'
+const FIELDS =
+  process.env.FIELDS ?? 'signature,amount,eventDetails,address,claimant'
 
 const START_HOURS_AGO = process.env.START_HOURS_AGO ?? '72' // 3 days
 const END_HOURS_AGO = process.env.END_HOURS_AGO ?? '0' // now
 const HOUR_WINDOW = process.env.HOUR_WINDOW ?? '1'
 
-const TAGS_TO_CONVERT = process.env.TAGS_TO_CONVERT ?? 'address'
-
 async function main() {
   console.log(
     `Connecting to influx <${INFLUX_URL}/${INFLUX_ORG}/${INFLUX_BUCKET}>`
@@ -34,7 +36,8 @@ async function main() {
 
   const influxDB = new InfluxDB({ url: INFLUX_URL, token: INFLUX_TOKEN })
   const readApi = influxDB.getQueryApi(INFLUX_ORG)
-  const tagsToConvert = TAGS_TO_CONVERT.split(',')
+  const tagsToKeep = TAGS_TO_KEEP.split(',')
+  const fields = FIELDS.split(',')
 
   let startHoursAgo = Number.parseInt(START_HOURS_AGO)
   const endHoursAgo = Number.parseInt(END_HOURS_AGO)
@@ -49,7 +52,8 @@ async function main() {
         INFLUX_BUCKET,
         MEASUREMENT,
         NEW_MEASUREMENT_VERSION,
-        tagsToConvert,
+        tagsToKeep,
+        fields,
         currentStartHoursAgo,
         currentEndHoursago,
         readApi
@@ -69,22 +73,28 @@ async function mapAndWrite(
   bucket: string,
   measurement: string,
   newMeasurementVersion: string,
-  tagsToConvert: string[],
+  tagsToKeep: string[],
+  fields: string[],
   start: string,
   end: string,
   readApi: QueryApi
 ): Promise<void> {
   const stream = `from(bucket: "${bucket}")
   |> range(start: ${start}, stop: ${end})
-  |> filter(fn: (r) => r._measurement == "${measurement}")`
+  |> filter(fn: (r) => r._measurement == "${measurement}")
+  |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value")`
 
   const mapper = `${stream}
-  |> map(fn: (r) => ({ r with _measurement: "${measurement}_${newMeasurementVersion}", ${tagsToConvert
-    .map((tag) => `${tag}: r.${tag}`)
-    .join(', ')} }))
-  |> drop(columns: [${tagsToConvert.map((tag) => `"${tag}"`).join(',')}])
-  |> to(bucket: "${bucket}")
-  `
+  |> set(key: "_measurement", value: "${measurement}_${newMeasurementVersion}")
+  |> to(
+      bucket: "${bucket}",
+      timeColumn: "_time",
+      tagColumns: [${tagsToKeep.map((tag) => `"${tag}"`).join(',')}],
+      fieldFn: (r) => ({ ${fields
+        .map((field) => `${field}: r.${field}`)
+        .join(',')} })
+    )`
+
   console.log(` Executing the following query: ${mapper}`)
   const response = await readApi.queryRaw(mapper)
   console.log(response)

From a4a50d13397b5722c541c031846a25608252aa5b Mon Sep 17 00:00:00 2001
From: matias martinez <matias@xlabs.xyz>
Date: Tue, 9 Apr 2024 14:35:29 -0300
Subject: [PATCH 6/8] use date time window

---
 scripts/influxdb.fixer.ts | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/scripts/influxdb.fixer.ts b/scripts/influxdb.fixer.ts
index b3bc9ab..96449d2 100644
--- a/scripts/influxdb.fixer.ts
+++ b/scripts/influxdb.fixer.ts
@@ -21,9 +21,9 @@ const TAGS_TO_KEEP =
 const FIELDS =
   process.env.FIELDS ?? 'signature,amount,eventDetails,address,claimant'
 
-const START_HOURS_AGO = process.env.START_HOURS_AGO ?? '72' // 3 days
-const END_HOURS_AGO = process.env.END_HOURS_AGO ?? '0' // now
-const HOUR_WINDOW = process.env.HOUR_WINDOW ?? '1'
+const START = process.env.START ?? '2024-04-09T00:00:00.000Z'
+const END = process.env.END ?? '2024-04-09T23:59:59.000Z'
+const HOUR_WINDOW = parseInt(process.env.HOUR_WINDOW ?? '1')
 
 async function main() {
   console.log(
@@ -39,14 +39,13 @@ async function main() {
   const tagsToKeep = TAGS_TO_KEEP.split(',')
   const fields = FIELDS.split(',')
 
-  let startHoursAgo = Number.parseInt(START_HOURS_AGO)
-  const endHoursAgo = Number.parseInt(END_HOURS_AGO)
-  let currentEndHoursago = ''
+  let startTime = new Date(START)
+  const endTime = new Date(END)
+  let currentEndDateTime = new Date(startTime)
 
   try {
-    while (startHoursAgo > endHoursAgo) {
-      const currentStartHoursAgo = `-${startHoursAgo}h`
-      currentEndHoursago = `-${startHoursAgo - Number.parseInt(HOUR_WINDOW)}h`
+    while (startTime < endTime) {
+      currentEndDateTime.setHours(startTime.getHours() + HOUR_WINDOW)
 
       await mapAndWrite(
         INFLUX_BUCKET,
@@ -54,18 +53,19 @@ async function main() {
         NEW_MEASUREMENT_VERSION,
         tagsToKeep,
         fields,
-        currentStartHoursAgo,
-        currentEndHoursago,
+        startTime,
+        currentEndDateTime,
         readApi
       )
-      startHoursAgo -= Number.parseInt(HOUR_WINDOW)
+      startTime.setHours(startTime.getHours() + HOUR_WINDOW)
+      await new Promise((resolve) => setTimeout(resolve, 1500))
     }
   } catch (error) {
     console.error(`Error: ${error}`)
   }
 
   console.log(
-    `Done -> StartHoursAgo: ${startHoursAgo} - EndHoursAgo: ${currentEndHoursago}`
+    `Done -> StartHoursAgo: -${startTime} - EndHoursAgo: ${currentEndDateTime}`
   )
 }
 
@@ -75,12 +75,12 @@ async function mapAndWrite(
   newMeasurementVersion: string,
   tagsToKeep: string[],
   fields: string[],
-  start: string,
-  end: string,
+  start: Date,
+  end: Date,
   readApi: QueryApi
 ): Promise<void> {
   const stream = `from(bucket: "${bucket}")
-  |> range(start: ${start}, stop: ${end})
+  |> range(start: ${start.toISOString()}, stop: ${end.toISOString()})
   |> filter(fn: (r) => r._measurement == "${measurement}")
   |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value")`
 
@@ -96,8 +96,8 @@ async function mapAndWrite(
     )`
 
   console.log(` Executing the following query: ${mapper}`)
-  const response = await readApi.queryRaw(mapper)
-  console.log(response)
+  await readApi.queryRaw(mapper)
+  console.log(`Done ${start} - ${end}`)
 }
 
 ;(async () => {

From 18e5699df98e1990420648de7a030d97ee8dd611 Mon Sep 17 00:00:00 2001
From: matias martinez <matias@xlabs.xyz>
Date: Wed, 10 Apr 2024 17:18:55 -0300
Subject: [PATCH 7/8] dedupe

---
 scripts/influxdb.ts | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/scripts/influxdb.ts b/scripts/influxdb.ts
index 4c7ba30..c9aa37d 100644
--- a/scripts/influxdb.ts
+++ b/scripts/influxdb.ts
@@ -182,7 +182,18 @@ async function getLatestTxSignature(
 }
 
 function createTxnEventPoints(formattedTxnEvents: FormattedTxnEventInfo[]) {
+  const timestamps: Record<string, boolean> = {}
+
   return formattedTxnEvents.map((formattedEvent) => {
+    let timestamp = formattedEvent.blockTime * 1000
+    if (timestamps[timestamp]) {
+      // see https://docs.influxdata.com/influxdb/v2/write-data/best-practices/duplicate-points/
+      while (timestamps[timestamp]) {
+        timestamp = timestamp + 1
+      }
+    }
+    timestamps[timestamp] = true
+
     const { signature, claimant } = formattedEvent
     const { ecosystem, address, amount } = formattedEvent.claimInfo!
     let eventCategory = 'normal'
@@ -201,7 +212,7 @@ function createTxnEventPoints(formattedTxnEvents: FormattedTxnEventInfo[]) {
       .stringField('signature', signature)
       .intField('amount', amountValue)
       .stringField('eventDetails', JSON.stringify(formattedEvent))
-      .timestamp(new Date(formattedEvent.blockTime * 1000))
+      .timestamp(new Date(timestamp))
 
     return point
   })
@@ -253,12 +264,22 @@ function createDoubleClaimPoint(formattedTxnEvents: FormattedTxnEventInfo[]) {
 }
 
 function createFailedTxnEventPoints(failedTxns: TxnInfo[]) {
+  const timestamps: Record<string, boolean> = {}
+
   return failedTxns.map((errorLog) => {
+    let timestamp = errorLog.blockTime * 1000
+    if (timestamps[timestamp]) {
+      // see https://docs.influxdata.com/influxdb/v2/write-data/best-practices/duplicate-points/
+      while (timestamps[timestamp]) {
+        timestamp = timestamp + 1
+      }
+    }
+    timestamps[timestamp] = true
     const point = new Point(FAILED_TXN_MEASUREMENT)
       .tag('network', CLUSTER)
       .stringField('signature', errorLog.signature)
       .stringField('errorDetails', JSON.stringify(errorLog))
-      .timestamp(new Date(errorLog.blockTime * 1000))
+      .timestamp(new Date(timestamp))
     return point
   })
 }

From 0fc687d7037da9f6e1d63ba5a559104317dad8d5 Mon Sep 17 00:00:00 2001
From: matias martinez <matias@xlabs.xyz>
Date: Wed, 10 Apr 2024 18:05:53 -0300
Subject: [PATCH 8/8] bump worker

---
 scripts/.env.sample                  | 1 -
 scripts/claim_sdk/eventSubscriber.ts | 2 +-
 scripts/package.json                 | 2 +-
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/scripts/.env.sample b/scripts/.env.sample
index dbd2a0f..4e21287 100644
--- a/scripts/.env.sample
+++ b/scripts/.env.sample
@@ -30,4 +30,3 @@ export NEW_MEASUREMENT_VERSION=v1
 export MEASUREMENT="txn_event"
 export TAGS_TO_KEEP="ecosystem,network,eventCategory"
 export FIELDS="signature,amount,eventDetails,address,claimant"
-
diff --git a/scripts/claim_sdk/eventSubscriber.ts b/scripts/claim_sdk/eventSubscriber.ts
index e6ce27d..41060e6 100644
--- a/scripts/claim_sdk/eventSubscriber.ts
+++ b/scripts/claim_sdk/eventSubscriber.ts
@@ -275,7 +275,7 @@ function formatClaimInfo(
       address: claimInfo.identity.injective.address,
       amount: claimInfo.amount.toString(),
     }
-  } else if (claimInfo.identity.algorand?.pubkey) {
+  } else if (claimInfo.identity.algorand) {
     return {
       ecosystem: 'algorand',
       address: base32encode(claimInfo.identity.algorand.pubkey),
diff --git a/scripts/package.json b/scripts/package.json
index 040435a..b079091 100644
--- a/scripts/package.json
+++ b/scripts/package.json
@@ -1,6 +1,6 @@
 {
   "name": "airdrop-metrics-job",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "",
   "main": "index.js",
   "scripts": {