From 78a63b91a6d03aa7203c54357c5ab466026ec50b Mon Sep 17 00:00:00 2001 From: j Date: Sun, 17 Mar 2024 14:06:41 -0400 Subject: [PATCH 1/8] Supporting Testing Configuration --- package.json | 5 +++-- tsconfig.tests.json | 10 ++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 tsconfig.tests.json diff --git a/package.json b/package.json index 44e0d2d..579c4d0 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ }, "scripts": { "build": "tsc && copyfiles -u 3 \"src/db/migrations/*.sql\" dist/db/migrations && copyfiles -u 3 \"src/db/seed/*.sql\" dist/db/seed && copyfiles -u 2 \"src/messages/*.txt\" dist/messages", - "start": "node dist/index.js" + "start": "node dist/index.js", + "test": "tsc -p tsconfig.tests.json" } -} +} \ No newline at end of file diff --git a/tsconfig.tests.json b/tsconfig.tests.json new file mode 100644 index 0000000..8dd932a --- /dev/null +++ b/tsconfig.tests.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist-tests", + "rootDir": "./" + }, + "include": [ + "tests/**/*" + ] +} \ No newline at end of file From bf7a7a96804a7db9c28009251e434791d43bd35f Mon Sep 17 00:00:00 2001 From: j Date: Sun, 17 Mar 2024 14:11:00 -0400 Subject: [PATCH 2/8] Inital commit of test file --- tests/api/redisAPITest.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 tests/api/redisAPITest.ts diff --git a/tests/api/redisAPITest.ts b/tests/api/redisAPITest.ts new file mode 100644 index 0000000..c80798c --- /dev/null +++ b/tests/api/redisAPITest.ts @@ -0,0 +1,10 @@ +import { CommentFetcher } from "../../src/rdrama/services/CommentFetcher"; + +main(); + +async function main(): Promise { + let maxPages = 5 + for (let page = 1; page <= maxPages; page++) { + const newComments = await CommentFetcher.fetchComments(page) + } +} \ No newline at end of file From 53930d5bfb0cb3d0def27df42d357bd2cda89738 Mon Sep 17 00:00:00 2001 From: j Date: Sun, 17 Mar 2024 14:37:52 -0400 Subject: [PATCH 3/8] Bottleneck Install --- package-lock.json | 23 +++++++++++++++++------ package.json | 3 ++- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 291a059..e61336b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "axios": "^1.6.7", "axios-request-throttle": "^1.0.0", "axios-retry": "^4.0.0", + "bottleneck": "^2.19.5", "dotenv": "^16.4.5", "form-data": "^4.0.0", "fs": "^0.0.1-security", @@ -246,6 +247,11 @@ "readable-stream": "^3.4.0" } }, + "node_modules/bottleneck": { + "version": "2.19.5", + "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", + "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==" + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -597,9 +603,9 @@ "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" }, "node_modules/follow-redirects": { - "version": "1.15.5", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.5.tgz", - "integrity": "sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==", + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", "funding": [ { "type": "individual", @@ -2190,6 +2196,11 @@ "readable-stream": "^3.4.0" } }, + "bottleneck": { + "version": "2.19.5", + "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", + "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==" + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -2449,9 +2460,9 @@ "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" }, "follow-redirects": { - "version": "1.15.5", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.5.tgz", - "integrity": "sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==" + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==" }, "form-data": { "version": "4.0.0", diff --git a/package.json b/package.json index 579c4d0..c4dff4f 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "axios": "^1.6.7", "axios-request-throttle": "^1.0.0", "axios-retry": "^4.0.0", + "bottleneck": "^2.19.5", "dotenv": "^16.4.5", "form-data": "^4.0.0", "fs": "^0.0.1-security", @@ -21,4 +22,4 @@ "start": "node dist/index.js", "test": "tsc -p tsconfig.tests.json" } -} \ No newline at end of file +} From d672797bbdf57541a58c66a85041473212bc61b4 Mon Sep 17 00:00:00 2001 From: j Date: Sun, 17 Mar 2024 14:38:30 -0400 Subject: [PATCH 4/8] AxiosThrottle Removed, Bottleneck Configured --- src/rdrama/session/SessionManager.ts | 42 +++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/src/rdrama/session/SessionManager.ts b/src/rdrama/session/SessionManager.ts index 56a314c..dd4801f 100644 --- a/src/rdrama/session/SessionManager.ts +++ b/src/rdrama/session/SessionManager.ts @@ -1,13 +1,18 @@ -import axios, {AxiosInstance, AxiosError} from 'axios'; +import axios, { AxiosInstance, AxiosError, AxiosResponse, AxiosRequestConfig } from 'axios'; import axiosRetry from 'axios-retry'; -import axiosThrottle from 'axios-request-throttle'; +import Bottleneck from 'bottleneck'; class SessionManager { private static instance: SessionManager; public readonly axiosInstance: AxiosInstance; + private limiter: Bottleneck; private constructor() { - axiosThrottle.use(axios, { requestsPerSecond: 1 }); // Throttle setup + // Initialize the Bottleneck limiter + this.limiter = new Bottleneck({ + maxConcurrent: 1, // Maximum number of concurrent requests + minTime: 1000 // Minimum time between dispatches of requests in milliseconds + }); this.axiosInstance = axios.create({ baseURL: 'https://rdrama.net/', @@ -16,6 +21,9 @@ class SessionManager { }, }); + // Wrap axios requests with the limiter + this.wrapAxiosInstance(this.axiosInstance); + axiosRetry(this.axiosInstance, { retries: 3, retryDelay: this.retryDelayStrategy, @@ -41,7 +49,33 @@ class SessionManager { private retryCondition(error: AxiosError): boolean { const status = error.response?.status ?? 0; - return status === 429 || status >= 400; + return status === 429 || status >= 500; + } + + private wrapAxiosInstance(instance: AxiosInstance): void { + // Wrap the get method + const originalGet = instance.get; + instance.get = >(url: string, config?: AxiosRequestConfig): Promise => { + return this.limiter.schedule(() => originalGet.apply(instance, [url, config])) as Promise; + }; + + // Wrap the post method + const originalPost = instance.post; + instance.post = >(url: string, data?: any, config?: AxiosRequestConfig): Promise => { + return this.limiter.schedule(() => originalPost.apply(instance, [url, data, config])) as Promise; + }; + + // Wrap the put method + const originalPut = instance.put; + instance.put = >(url: string, data?: any, config?: AxiosRequestConfig): Promise => { + return this.limiter.schedule(() => originalPut.apply(instance, [url, data, config])) as Promise; + }; + + // Wrap the delete method + const originalDelete = instance.delete; + instance.delete = >(url: string, config?: AxiosRequestConfig): Promise => { + return this.limiter.schedule(() => originalDelete.apply(instance, [url, config])) as Promise; + }; } public setAuthorizationToken(token: string): void { From 99d39524bd0003ac11ecd0fbc20dae4e20f48c93 Mon Sep 17 00:00:00 2001 From: j Date: Sun, 17 Mar 2024 23:23:18 -0400 Subject: [PATCH 5/8] added Ioredis --- package-lock.json | 146 ++++++++++++++++++++++++++++++++++++++++++++-- package.json | 1 + 2 files changed, 141 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index e61336b..84e66dc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "dotenv": "^16.4.5", "form-data": "^4.0.0", "fs": "^0.0.1-security", + "ioredis": "^5.3.2", "path": "^0.12.7", "qs": "^6.11.2", "sqlite": "^5.1.1", @@ -29,6 +30,11 @@ "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==", "optional": true }, + "node_modules/@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, "node_modules/@npmcli/fs": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-1.1.1.tgz", @@ -360,6 +366,14 @@ "wrap-ansi": "^7.0.0" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -439,7 +453,6 @@ "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "optional": true, "dependencies": { "ms": "2.1.2" }, @@ -504,6 +517,14 @@ "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", "optional": true }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/detect-libc": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz", @@ -924,6 +945,29 @@ "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" }, + "node_modules/ioredis": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/ip-address": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", @@ -981,6 +1025,16 @@ "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==", "optional": true }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" + }, "node_modules/lru-cache": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", @@ -1176,8 +1230,7 @@ "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "optional": true + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node_modules/napi-build-utils": { "version": "1.0.2", @@ -1461,6 +1514,25 @@ "node": ">= 6" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -1705,6 +1777,11 @@ "node": ">= 8" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -2027,6 +2104,11 @@ "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==", "optional": true }, + "@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, "@npmcli/fs": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-1.1.1.tgz", @@ -2280,6 +2362,11 @@ "wrap-ansi": "^7.0.0" } }, + "cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==" + }, "color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -2346,7 +2433,6 @@ "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "optional": true, "requires": { "ms": "2.1.2" } @@ -2385,6 +2471,11 @@ "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", "optional": true }, + "denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" + }, "detect-libc": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz", @@ -2690,6 +2781,22 @@ "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" }, + "ioredis": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", + "requires": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + } + }, "ip-address": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", @@ -2735,6 +2842,16 @@ "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==", "optional": true }, + "lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + }, + "lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" + }, "lru-cache": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", @@ -2877,8 +2994,7 @@ "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "optional": true + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "napi-build-utils": { "version": "1.0.2", @@ -3104,6 +3220,19 @@ "util-deprecate": "^1.0.1" } }, + "redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==" + }, + "redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "requires": { + "redis-errors": "^1.0.0" + } + }, "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -3254,6 +3383,11 @@ "minipass": "^3.1.1" } }, + "standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", diff --git a/package.json b/package.json index c4dff4f..86944bb 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "dotenv": "^16.4.5", "form-data": "^4.0.0", "fs": "^0.0.1-security", + "ioredis": "^5.3.2", "path": "^0.12.7", "qs": "^6.11.2", "sqlite": "^5.1.1", From 98830cb0ffe42fd9d47e27b2ab7cba6f28d39a4a Mon Sep 17 00:00:00 2001 From: j Date: Sun, 17 Mar 2024 23:23:37 -0400 Subject: [PATCH 6/8] Addtl Testing Logic --- tests/api/redisAPITest.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/api/redisAPITest.ts b/tests/api/redisAPITest.ts index c80798c..ad2d55f 100644 --- a/tests/api/redisAPITest.ts +++ b/tests/api/redisAPITest.ts @@ -1,10 +1,18 @@ import { CommentFetcher } from "../../src/rdrama/services/CommentFetcher"; +import SessionManager from "../../src/rdrama/session/SessionManager" main(); async function main(): Promise { + console.log('1') + const axiosSession = SessionManager.getInstance() + console.log('2') let maxPages = 5 for (let page = 1; page <= maxPages; page++) { const newComments = await CommentFetcher.fetchComments(page) } + console.log('3') + await axiosSession.shutdown() + console.log('4') + } \ No newline at end of file From 76e90141738de34110e360caf43baad9bdccd61b Mon Sep 17 00:00:00 2001 From: j Date: Sun, 17 Mar 2024 23:25:02 -0400 Subject: [PATCH 7/8] Configure for shutdown of session after execution --- src/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/index.ts b/src/index.ts index 08f99bc..11f8ed8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -41,6 +41,8 @@ async function startApplication() { // Initialize and start your workflow const workflowOrchestrator = new WorkflowOrchestrator(); await workflowOrchestrator.executeWorkflow(); + + await rDramaSessionManager.shutdown() } startApplication() From 39b9d3d9e28c2b7a8923acfd7b48d37cfd7da06f Mon Sep 17 00:00:00 2001 From: j Date: Sun, 17 Mar 2024 23:25:38 -0400 Subject: [PATCH 8/8] redis configuration with bottleneck and shutdown method --- src/rdrama/session/SessionManager.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/rdrama/session/SessionManager.ts b/src/rdrama/session/SessionManager.ts index dd4801f..8986b1b 100644 --- a/src/rdrama/session/SessionManager.ts +++ b/src/rdrama/session/SessionManager.ts @@ -1,6 +1,10 @@ import axios, { AxiosInstance, AxiosError, AxiosResponse, AxiosRequestConfig } from 'axios'; import axiosRetry from 'axios-retry'; import Bottleneck from 'bottleneck'; +import dotenv from 'dotenv'; + +// Load environment variables from .env file +dotenv.config(); class SessionManager { private static instance: SessionManager; @@ -8,8 +12,19 @@ class SessionManager { private limiter: Bottleneck; private constructor() { + + // Initialize the Bottleneck limiter this.limiter = new Bottleneck({ + id: "rDramaAPI-limiter", + datastore: "ioredis", + clearDatastore: false, + clientOptions: { + host: process.env.REDIS_HOST, + port: Number(process.env.REDIS_PORT), + password: process.env.REDIS_PASSWORD || undefined, // Use undefined if no password is set + enableOfflineQueue: true + }, maxConcurrent: 1, // Maximum number of concurrent requests minTime: 1000 // Minimum time between dispatches of requests in milliseconds }); @@ -38,6 +53,10 @@ class SessionManager { return SessionManager.instance; } + public async shutdown(): Promise { + await this.limiter.disconnect(); + } + private retryDelayStrategy(retryCount: number, error: AxiosError): number { const retryAfter = error.response?.headers['retry-after']; if (retryAfter) { @@ -52,6 +71,7 @@ class SessionManager { return status === 429 || status >= 500; } + private wrapAxiosInstance(instance: AxiosInstance): void { // Wrap the get method const originalGet = instance.get; @@ -81,6 +101,8 @@ class SessionManager { public setAuthorizationToken(token: string): void { this.axiosInstance.defaults.headers.common['Authorization'] = `${token}`; } + + } export default SessionManager;