redis configuration with bottleneck and shutdown method

Configure for shutdown of session after execution
Addtl Testing Logic
2024-03-17 23:25:38 -04:00 · 2024-03-17 23:25:02 -04:00 · 2024-03-17 23:23:37 -04:00 · 2024-03-17 23:23:18 -04:00
5 changed files with 173 additions and 6 deletions
--- a/package-lock.json
+++ b/package-lock.json
@ -12,6 +12,7 @@
        "dotenv": "^16.4.5",
        "form-data": "^4.0.0",
        "fs": "^0.0.1-security",
+        "ioredis": "^5.3.2",
        "path": "^0.12.7",
        "qs": "^6.11.2",
        "sqlite": "^5.1.1",
@ -29,6 +30,11 @@
      "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==",
      "optional": true
    },
+    "node_modules/@ioredis/commands": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
+      "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg=="
+    },
    "node_modules/@npmcli/fs": {
      "version": "1.1.1",
      "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-1.1.1.tgz",
@ -360,6 +366,14 @@
        "wrap-ansi": "^7.0.0"
      }
    },
+    "node_modules/cluster-key-slot": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
+      "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
    "node_modules/color-convert": {
      "version": "2.0.1",
      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
@ -439,7 +453,6 @@
      "version": "4.3.4",
      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
-      "optional": true,
      "dependencies": {
        "ms": "2.1.2"
      },
@ -504,6 +517,14 @@
      "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==",
      "optional": true
    },
+    "node_modules/denque": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
+      "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
+      "engines": {
+        "node": ">=0.10"
+      }
+    },
    "node_modules/detect-libc": {
      "version": "2.0.2",
      "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz",
@ -924,6 +945,29 @@
      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
      "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew=="
    },
+    "node_modules/ioredis": {
+      "version": "5.3.2",
+      "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz",
+      "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==",
+      "dependencies": {
+        "@ioredis/commands": "^1.1.1",
+        "cluster-key-slot": "^1.1.0",
+        "debug": "^4.3.4",
+        "denque": "^2.1.0",
+        "lodash.defaults": "^4.2.0",
+        "lodash.isarguments": "^3.1.0",
+        "redis-errors": "^1.2.0",
+        "redis-parser": "^3.0.0",
+        "standard-as-callback": "^2.1.0"
+      },
+      "engines": {
+        "node": ">=12.22.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/ioredis"
+      }
+    },
    "node_modules/ip-address": {
      "version": "9.0.5",
      "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz",
@ -981,6 +1025,16 @@
      "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==",
      "optional": true
    },
+    "node_modules/lodash.defaults": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
+      "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ=="
+    },
+    "node_modules/lodash.isarguments": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
+      "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg=="
+    },
    "node_modules/lru-cache": {
      "version": "6.0.0",
      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
@ -1176,8 +1230,7 @@
    "node_modules/ms": {
      "version": "2.1.2",
      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
-      "optional": true
+      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
    },
    "node_modules/napi-build-utils": {
      "version": "1.0.2",
@ -1461,6 +1514,25 @@
        "node": ">= 6"
      }
    },
+    "node_modules/redis-errors": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
+      "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==",
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/redis-parser": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
+      "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==",
+      "dependencies": {
+        "redis-errors": "^1.0.0"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
    "node_modules/require-directory": {
      "version": "2.1.1",
      "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
@ -1705,6 +1777,11 @@
        "node": ">= 8"
      }
    },
+    "node_modules/standard-as-callback": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz",
+      "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="
+    },
    "node_modules/string_decoder": {
      "version": "1.3.0",
      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
@ -2027,6 +2104,11 @@
      "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==",
      "optional": true
    },
+    "@ioredis/commands": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
+      "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg=="
+    },
    "@npmcli/fs": {
      "version": "1.1.1",
      "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-1.1.1.tgz",
@ -2280,6 +2362,11 @@
        "wrap-ansi": "^7.0.0"
      }
    },
+    "cluster-key-slot": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
+      "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="
+    },
    "color-convert": {
      "version": "2.0.1",
      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
@ -2346,7 +2433,6 @@
      "version": "4.3.4",
      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
-      "optional": true,
      "requires": {
        "ms": "2.1.2"
      }
@ -2385,6 +2471,11 @@
      "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==",
      "optional": true
    },
+    "denque": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
+      "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="
+    },
    "detect-libc": {
      "version": "2.0.2",
      "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz",
@ -2690,6 +2781,22 @@
      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
      "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew=="
    },
+    "ioredis": {
+      "version": "5.3.2",
+      "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz",
+      "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==",
+      "requires": {
+        "@ioredis/commands": "^1.1.1",
+        "cluster-key-slot": "^1.1.0",
+        "debug": "^4.3.4",
+        "denque": "^2.1.0",
+        "lodash.defaults": "^4.2.0",
+        "lodash.isarguments": "^3.1.0",
+        "redis-errors": "^1.2.0",
+        "redis-parser": "^3.0.0",
+        "standard-as-callback": "^2.1.0"
+      }
+    },
    "ip-address": {
      "version": "9.0.5",
      "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz",
@ -2735,6 +2842,16 @@
      "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==",
      "optional": true
    },
+    "lodash.defaults": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
+      "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ=="
+    },
+    "lodash.isarguments": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
+      "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg=="
+    },
    "lru-cache": {
      "version": "6.0.0",
      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
@ -2877,8 +2994,7 @@
    "ms": {
      "version": "2.1.2",
      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
-      "optional": true
+      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
    },
    "napi-build-utils": {
      "version": "1.0.2",
@ -3104,6 +3220,19 @@
        "util-deprecate": "^1.0.1"
      }
    },
+    "redis-errors": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
+      "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w=="
+    },
+    "redis-parser": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
+      "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==",
+      "requires": {
+        "redis-errors": "^1.0.0"
+      }
+    },
    "require-directory": {
      "version": "2.1.1",
      "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
@ -3254,6 +3383,11 @@
        "minipass": "^3.1.1"
      }
    },
+    "standard-as-callback": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz",
+      "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="
+    },
    "string_decoder": {
      "version": "1.3.0",
      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
--- a/package.json
+++ b/package.json
@ -7,6 +7,7 @@
    "dotenv": "^16.4.5",
    "form-data": "^4.0.0",
    "fs": "^0.0.1-security",
+    "ioredis": "^5.3.2",
    "path": "^0.12.7",
    "qs": "^6.11.2",
    "sqlite": "^5.1.1",
--- a/src/index.ts
+++ b/src/index.ts
@ -41,6 +41,8 @@ async function startApplication() {
    // Initialize and start your workflow
    const workflowOrchestrator = new WorkflowOrchestrator();
    await workflowOrchestrator.executeWorkflow();
+
+    await rDramaSessionManager.shutdown()
 }

 startApplication()
--- a/src/rdrama/session/SessionManager.ts
+++ b/src/rdrama/session/SessionManager.ts
@ -1,6 +1,10 @@
 import axios, { AxiosInstance, AxiosError, AxiosResponse, AxiosRequestConfig } from 'axios';
 import axiosRetry from 'axios-retry';
 import Bottleneck from 'bottleneck';
+import dotenv from 'dotenv';
+
+// Load environment variables from .env file
+dotenv.config();

 class SessionManager {
    private static instance: SessionManager;
@ -8,8 +12,19 @@ class SessionManager {
    private limiter: Bottleneck;

    private constructor() {
+
+
        // Initialize the Bottleneck limiter
        this.limiter = new Bottleneck({
+            id: "rDramaAPI-limiter",
+            datastore: "ioredis",
+            clearDatastore: false,
+            clientOptions: {
+                host: process.env.REDIS_HOST,
+                port: Number(process.env.REDIS_PORT),
+                password: process.env.REDIS_PASSWORD || undefined, // Use undefined if no password is set
+                enableOfflineQueue: true
+            },
            maxConcurrent: 1, // Maximum number of concurrent requests
            minTime: 1000 // Minimum time between dispatches of requests in milliseconds
        });
@ -38,6 +53,10 @@ class SessionManager {
        return SessionManager.instance;
    }

+    public async shutdown(): Promise<void> {
+        await this.limiter.disconnect();
+    }
+
    private retryDelayStrategy(retryCount: number, error: AxiosError): number {
        const retryAfter = error.response?.headers['retry-after'];
        if (retryAfter) {
@ -52,6 +71,7 @@ class SessionManager {
        return status === 429 || status >= 500;
    }

+
    private wrapAxiosInstance(instance: AxiosInstance): void {
        // Wrap the get method
        const originalGet = instance.get;
@ -81,6 +101,8 @@ class SessionManager {
    public setAuthorizationToken(token: string): void {
        this.axiosInstance.defaults.headers.common['Authorization'] = `${token}`;
    }
+
+
 }

 export default SessionManager;
--- a/tests/api/redisAPITest.ts
+++ b/tests/api/redisAPITest.ts
@ -1,10 +1,18 @@
 import { CommentFetcher } from "../../src/rdrama/services/CommentFetcher";
+import SessionManager from "../../src/rdrama/session/SessionManager"

 main();

 async function main(): Promise<void> {
+    console.log('1')
+    const axiosSession = SessionManager.getInstance()
+    console.log('2')
    let maxPages = 5
    for (let page = 1; page <= maxPages; page++) {
        const newComments = await CommentFetcher.fetchComments(page)
    }
+    console.log('3')
+    await axiosSession.shutdown()
+    console.log('4')
+
 }
Author	SHA1	Message	Date
j	39b9d3d9e2	redis configuration with bottleneck and shutdown method	2024-03-17 23:25:38 -04:00
j	76e9014173	Configure for shutdown of session after execution	2024-03-17 23:25:02 -04:00
j	98830cb0ff	Addtl Testing Logic	2024-03-17 23:23:37 -04:00
j	99d39524bd	added Ioredis	2024-03-17 23:23:18 -04:00