From faae0e2b385520414433def08e88bbe8a78b4a11 Mon Sep 17 00:00:00 2001
From: anonymoussprocket <anonymoussprocket@gmail.com>
Date: Sat, 20 Mar 2021 03:18:02 -0400
Subject: [PATCH] - add throttling - add and disable profiling

---
 conseilUtil.js | 20 ++++++++++++--
 package.json   | 74 ++++++++++++++++++++++++++------------------------
 2 files changed, 56 insertions(+), 38 deletions(-)

diff --git a/conseilUtil.js b/conseilUtil.js
index b022581..874e22b 100644
--- a/conseilUtil.js
+++ b/conseilUtil.js
@@ -2,6 +2,8 @@ const conseiljs = require('conseiljs')
 const fetch = require('node-fetch')
 const log = require('loglevel')
 const BigNumber = require('bignumber.js')
+const pThrottle = require('p-throttle')
+// const { performance } = require('perf_hooks'); // NOTE: disabled for prod
 
 const logger = log.getLogger('conseiljs')
 logger.setLevel('error', false)
@@ -11,6 +13,8 @@ const conseilServer = 'https://conseil-prod.cryptonomic-infra.tech'
 const conseilApiKey = 'aa73fa8a-8626-4f43-a605-ff63130f37b1' // signup at nautilus.cloud
 const tezosNode = ''
 
+const throttleConseil = pThrottle({ limit: 15, interval: 1200 })
+
 const mainnet = require('./config').networkConfig
 
 /**
@@ -275,12 +279,13 @@ const getArtisticUniverse = async (max_time) => {
 
     const objectQueries = queryChunks.map(c => makeObjectQuery(c))
 
+    // const a = performance.now()
     let universe = []
     await Promise.all(
         objectQueries.map(async (q) =>  {
-            const r = []
+            let r = []
             try {
-                r = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', q)
+                r = await throttleConseilQuery('big_map_contents', q)
                     .then(result => result.map(row => {
                         const objectId = row['value'].toString().replace(/^Pair ([0-9]{1,}) .*/, '$1')
                         const objectUrl = row['value'].toString().replace(/.* 0x([0-9a-z]{1,}) \}$/, '$1')
@@ -288,10 +293,15 @@ const getArtisticUniverse = async (max_time) => {
 
                     universe.push({ objectId, ipfsHash, minter: artistMap[objectId], swaps: swapMap[objectId] !== undefined ? swapMap[objectId] : []})
                     })) // NOTE: it's a work in progress, this will drop failed requests and return a smaller set than expected
+            } catch (error) {
+                console.log('failed at query', q, 'with error', error)
             } finally {
                 return r
             }}))
 
+    // const b = performance.now()
+    // console.log(`time ${b - a}`)
+
     return universe
 }
 
@@ -353,6 +363,12 @@ const chunkArray = (arr, len) => { // TODO: move to util.js
     return chunks;
 }
 
+const throttleConseilQuery = throttleConseil(async (table, query) => {
+    const result = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', table, query)
+
+    return Promise.resolve(result)
+})
+
 module.exports = {
     getCollectionForAddress,
     gethDaoBalanceForAddress,
diff --git a/package.json b/package.json
index 04d9450..301e8ef 100644
--- a/package.json
+++ b/package.json
@@ -1,38 +1,40 @@
 {
-    "name": "hicetnunc-apiv2",
-    "version": "1.0.0",
-    "description": "",
-    "main": "index.js",
-    "scripts": {
-      "start": "node index.js"
-    },
-    "repository": {
-      "type": "git",
-      "url": "git+https://github.com/hicetnunc2000/hicetnunc.git"
-    },
-    "author": "@hicetnunc2000",
-    "license": "MIT",
-    "bugs": {
-      "url": "https://github.com/hicetnunc2000/hicetnunc/issues"
-    },
-    "homepage": "https://github.com/hicetnunc2000/hicetnunc#readme",
-    "dependencies": {
-      "axios": "^0.21.1",
-      "bignumber.js": "9.0.1",
-      "cloud-local-storage": "0.0.11",
-      "conseiljs": "5.0.7-2",
-      "cors": "^2.8.5",
-      "dotenv": "^8.2.0",
-      "express": "^4.17.1",
-      "fetch": "^1.1.0",
-      "lodash": "^4.17.21",
-      "loglevel": "1.7.1",
-      "node-fetch": "2.6.1",
-      "serverless-dotenv-plugin": "^3.8.1",
-      "serverless-http": "^2.7.0"
-    },
-    "engines": {
-      "node": "12.20.1",
-      "npm": "6.14.10"
-    }
+  "name": "hicetnunc-apiv2",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "start": "node index.js"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/hicetnunc2000/hicetnunc-api.git"
+  },
+  "author": "@hicetnunc2000",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/hicetnunc2000/hicetnunc-api/issues"
+  },
+  "homepage": "https://github.com/hicetnunc2000/hicetnunc-api#readme",
+  "dependencies": {
+    "axios": "^0.21.1",
+    "bignumber.js": "9.0.1",
+    "cloud-local-storage": "0.0.11",
+    "conseiljs": "5.0.7-2",
+    "cors": "^2.8.5",
+    "dotenv": "^8.2.0",
+    "express": "^4.17.1",
+    "fetch": "^1.1.0",
+    "lodash": "^4.17.21",
+    "loglevel": "1.7.1",
+    "node-fetch": "2.6.1",
+    "perf_hooks": "0.0.1",
+    "p-throttle": "4.1.1",
+    "serverless-dotenv-plugin": "^3.8.1",
+    "serverless-http": "^2.7.0"
+  },
+  "engines": {
+    "node": "12.20.1",
+    "npm": "6.14.10"
+  }
 }