From 4937f625d2eb241821682f08fc8ec35a6bcb4dd1 Mon Sep 17 00:00:00 2001 From: float-trip Date: Tue, 11 Jul 2023 15:37:58 +0000 Subject: [PATCH] 'distributor' -> 'batcher' + slow down new ID checks --- batcher.py | 7 ++++++- fetcher.py | 6 +++++- id_service.proto | 1 - 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/batcher.py b/batcher.py index f13f548..3e4fa2c 100644 --- a/batcher.py +++ b/batcher.py @@ -27,6 +27,11 @@ with open(os.path.join(DIR, "batcher_config.yaml")) as f: class IDBatcher: + """ + Periodically gets the newest comment/post ID from reddit, compares it to the + last fetched one, and adds all intermediate values into a queue. + """ + def __init__(self): self.lock = threading.Lock() self.ids_queue = deque() @@ -43,7 +48,7 @@ class IDBatcher: self.fetch_new_ids() - schedule.every(5).seconds.do(self.fetch_new_ids) + schedule.every(30).seconds.do(self.fetch_new_ids) thread = threading.Thread(target=self.start_scheduler) thread.start() diff --git a/fetcher.py b/fetcher.py index 533d828..f6345e0 100644 --- a/fetcher.py +++ b/fetcher.py @@ -126,11 +126,15 @@ class RedditClient: class FetcherClient: + """ + Retrieves lists of IDs from the batcher, fetches them from reddit, and submits the JSON back. + """ + def __init__(self, client_id, reddit_client): self.client_id = client_id self.reddit_client = reddit_client self.channel = grpc.insecure_channel( - config["distributor_uri"], + config["batcher_uri"], options=[ ("grpc.max_send_message_length", 100 * 10**6), ("grpc.max_receive_message_length", 1 * 10**6), diff --git a/id_service.proto b/id_service.proto index 92845b7..d966d9b 100644 --- a/id_service.proto +++ b/id_service.proto @@ -10,7 +10,6 @@ service IDService { rpc SubmitBatch (SubmitRequest) returns (SubmitResponse) {} } -// The BatchRequest message contains the client id. message BatchRequest { string client_id = 1; }