Removed unneeded files
parent
6ec7b5ef60
commit
7ef10dbb0f
|
|
@ -1,24 +0,0 @@
|
||||||
CREATE TABLE IF NOT EXISTS comments (
|
|
||||||
db_id INTEGER PRIMARY KEY,
|
|
||||||
id INTEGER UNIQUE NOT NULL,
|
|
||||||
author_id INTEGER NOT NULL,
|
|
||||||
author_name TEXT NOT NULL,
|
|
||||||
body TEXT NOT NULL,
|
|
||||||
body_html TEXT NOT NULL,
|
|
||||||
created_utc INTEGER NOT NULL,
|
|
||||||
deleted_utc INTEGER DEFAULT 0,
|
|
||||||
distinguished BOOLEAN DEFAULT FALSE,
|
|
||||||
downvotes INTEGER DEFAULT 0,
|
|
||||||
edited_utc INTEGER DEFAULT 0,
|
|
||||||
is_banned BOOLEAN DEFAULT FALSE,
|
|
||||||
is_bot BOOLEAN DEFAULT FALSE,
|
|
||||||
is_nsfw BOOLEAN DEFAULT FALSE,
|
|
||||||
level INTEGER DEFAULT 0,
|
|
||||||
permalink TEXT NOT NULL,
|
|
||||||
pinned TEXT,
|
|
||||||
post_id INTEGER,
|
|
||||||
replies TEXT, -- Storing as JSON; consider relational integrity and querying needs.
|
|
||||||
reports TEXT, -- Storing as JSON; same considerations as 'replies'.
|
|
||||||
score INTEGER DEFAULT 0,
|
|
||||||
upvotes INTEGER DEFAULT 0
|
|
||||||
);
|
|
||||||
|
|
@ -1,28 +0,0 @@
|
||||||
CREATE TABLE IF NOT EXISTS user_mentions (
|
|
||||||
-- Unique identifier for each record. Auto-incremented to ensure uniqueness.
|
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
||||||
|
|
||||||
-- The unique identifier of the comment from the r/Drama platform. Not unique in this table
|
|
||||||
-- because a single comment can mention multiple users.
|
|
||||||
rdrama_comment_id TEXT NOT NULL,
|
|
||||||
|
|
||||||
-- The mentioned Reddit username in a standardized format (e.g., u/username). Lowercased
|
|
||||||
-- to ensure consistency and prevent duplicate entries due to case differences.
|
|
||||||
username TEXT NOT NULL,
|
|
||||||
|
|
||||||
-- The content of the message sent to the mentioned user, if any. Allows tracking
|
|
||||||
-- of what communication has been made, useful for audit purposes or resending messages.
|
|
||||||
message TEXT,
|
|
||||||
|
|
||||||
-- Timestamp when the mention was processed and, if applicable, when a message was sent.
|
|
||||||
-- Defaults to the current timestamp at the time of record creation.
|
|
||||||
sent_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
|
|
||||||
-- Enforces uniqueness for each comment-username pair to prevent processing and notifying
|
|
||||||
-- the same user mention in the same comment more than once.
|
|
||||||
CONSTRAINT unique_comment_user UNIQUE (rdrama_comment_id, username)
|
|
||||||
);
|
|
||||||
|
|
||||||
-- Consider adding indexes based on query patterns for improved performance, such as:
|
|
||||||
-- CREATE INDEX idx_username ON user_mentions(username);
|
|
||||||
-- CREATE INDEX idx_rdrama_comment_id ON user_mentions(rdrama_comment_id);
|
|
||||||
|
|
@ -1,4 +0,0 @@
|
||||||
CREATE TABLE IF NOT EXISTS maintenance_log (
|
|
||||||
task_name TEXT PRIMARY KEY,
|
|
||||||
last_run TIMESTAMP NOT NULL
|
|
||||||
);
|
|
||||||
|
|
@ -1,9 +0,0 @@
|
||||||
CREATE TABLE IF NOT EXISTS oauth_tokens (
|
|
||||||
id INTEGER PRIMARY KEY,
|
|
||||||
token_identifier TEXT NOT NULL UNIQUE, -- Static identifier for the OAuth token
|
|
||||||
access_token TEXT NOT NULL,
|
|
||||||
token_type TEXT NOT NULL,
|
|
||||||
expires_in INTEGER NOT NULL,
|
|
||||||
expiry_timestamp INTEGER NOT NULL,
|
|
||||||
scope TEXT NOT NULL
|
|
||||||
);
|
|
||||||
|
|
@ -1,69 +0,0 @@
|
||||||
import { DatabaseService } from './Database';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Manages and executes database maintenance tasks such as purging old comments.
|
|
||||||
* This service is responsible for periodically running maintenance tasks based on specified intervals.
|
|
||||||
*/
|
|
||||||
export class DatabaseMaintenanceService {
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A list of maintenance tasks to be executed, each with a name, action, and interval.
|
|
||||||
*/
|
|
||||||
private maintenanceTasks = [
|
|
||||||
{
|
|
||||||
name: 'PurgeOldComments',
|
|
||||||
action: this.purgeOldComments.bind(this),
|
|
||||||
interval: 24 * 60 * 60 * 1000, // 24 hours in milliseconds
|
|
||||||
},
|
|
||||||
// Add more tasks here as needed
|
|
||||||
];
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Executes all maintenance tasks that are due based on their defined intervals.
|
|
||||||
*/
|
|
||||||
public async runMaintenanceTasks() {
|
|
||||||
for (const task of this.maintenanceTasks) {
|
|
||||||
const shouldRun = await this.shouldRunTask(task.name, task.interval);
|
|
||||||
if (shouldRun) {
|
|
||||||
await task.action();
|
|
||||||
await this.updateLastRunTimestamp(task.name);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Determines whether a specific maintenance task should run based on its last execution time and defined interval.
|
|
||||||
*
|
|
||||||
* @param {string} taskName - The name of the task to check.
|
|
||||||
* @param {number} interval - The interval in milliseconds to determine if the task should run.
|
|
||||||
* @returns {Promise<boolean>} True if the task should run, otherwise false.
|
|
||||||
*/
|
|
||||||
private async shouldRunTask(taskName: string, interval: number): Promise<boolean> {
|
|
||||||
// Use the DatabaseService to check the last run timestamp from the maintenance_log table
|
|
||||||
const lastRun = await DatabaseService.getLastRunTimestamp(taskName);
|
|
||||||
if (!lastRun) return true; // Task has never run
|
|
||||||
|
|
||||||
const now = Date.now();
|
|
||||||
return (now - lastRun.getTime()) > interval;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Purges old comments from the database.
|
|
||||||
*/
|
|
||||||
private async purgeOldComments() {
|
|
||||||
console.log("Purging old comments...");
|
|
||||||
// Use the DatabaseService for the SQL operation
|
|
||||||
await DatabaseService.purgeOldComments();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Updates the last run timestamp for a specific maintenance task.
|
|
||||||
*
|
|
||||||
* @param {string} taskName - The name of the task for which to update the last run timestamp.
|
|
||||||
*/
|
|
||||||
private async updateLastRunTimestamp(taskName: string) {
|
|
||||||
// Use the DatabaseService to update the last run timestamp in the maintenance_log table
|
|
||||||
await DatabaseService.updateLastRunTimestamp(taskName);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
import SessionManager from '../session/SessionManager';
|
|
||||||
import { Comment } from '../models/Comment';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* CommentFetcher is responsible for fetching comments from the r/Drama API.
|
|
||||||
* It utilizes a SessionManager instance to manage API requests with appropriate
|
|
||||||
* headers and configurations, including rate limiting and retries.
|
|
||||||
*/
|
|
||||||
export class CommentFetcher {
|
|
||||||
/**
|
|
||||||
* Fetches comments from the r/Drama API across multiple pages.
|
|
||||||
* Each page's fetch operation is timed for performance analysis.
|
|
||||||
*
|
|
||||||
* @returns {Promise<Comment[]>} A promise that resolves to an array of comments fetched from the API.
|
|
||||||
* @throws {Error} Throws an error if there is a failure in fetching comments from the API.
|
|
||||||
*/
|
|
||||||
static async fetchComments(page: number): Promise<Comment[]> {
|
|
||||||
console.time(`Fetching page: ${page}`);
|
|
||||||
try {
|
|
||||||
const axiosInstance = SessionManager.getInstance().axiosInstance;
|
|
||||||
const response = await axiosInstance.get(`/comments?page=${page}`);
|
|
||||||
console.timeEnd(`Fetching page: ${page}`);
|
|
||||||
return response.data.data;
|
|
||||||
} catch (error) {
|
|
||||||
console.error(`Failed to fetch comments for page ${page}:`, error);
|
|
||||||
console.timeEnd(`Fetching page: ${page}`);
|
|
||||||
throw error; //Rethrow
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,46 +0,0 @@
|
||||||
import { Comment } from '../models/Comment';
|
|
||||||
import { DatabaseService } from '../../db/services/Database';
|
|
||||||
import { CommentFetcher } from './CommentFetcher';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* CommentProcessor handles the retrieval and processing of comments from the r/Drama API.
|
|
||||||
* It manages API requests through CommentFetcher, including rate limiting and retries, and coordinates
|
|
||||||
* with DatabaseService for checking the existence and persisting new comments.
|
|
||||||
*/
|
|
||||||
export class CommentProcessor {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Fetches comments from the r/Drama API across multiple pages, up to the specified maximum.
|
|
||||||
* Iterates through pages starting from the first page until the maximum page limit is reached
|
|
||||||
* or there are no more comments to fetch. Each page's fetch operation is timed for performance analysis.
|
|
||||||
*
|
|
||||||
* @returns {Promise<Comment[]>} A promise that resolves to an array of comments fetched from the API.
|
|
||||||
* @throws {Error} Throws an error if there is a failure in fetching comments from the API.
|
|
||||||
*/
|
|
||||||
static async processComments(maxPages: number = 10): Promise<Comment[]> {
|
|
||||||
let comments: Comment[] = [];
|
|
||||||
let stopFetching = false;
|
|
||||||
|
|
||||||
for (let page = 1; page <= maxPages && !stopFetching; page++) {
|
|
||||||
const newComments = await CommentFetcher.fetchComments(page)
|
|
||||||
|
|
||||||
// Check each new comment against the database and existing comments in this batch
|
|
||||||
for (const comment of newComments) {
|
|
||||||
// Check if the comment was already processed in this batch
|
|
||||||
if (comments.some(c => c.id === comment.id)) continue;
|
|
||||||
|
|
||||||
const exists = await DatabaseService.commentExists(comment.id.toString());
|
|
||||||
if (exists) {
|
|
||||||
stopFetching = true;
|
|
||||||
break; // Stop processing this batch of comments
|
|
||||||
}
|
|
||||||
await DatabaseService.insertComment(comment)
|
|
||||||
comments.push(comment);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (newComments.length === 0) break; // No more comments to fetch
|
|
||||||
|
|
||||||
}
|
|
||||||
return comments;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue