From 83f890ec79167b047b73a93aacbb070111453196 Mon Sep 17 00:00:00 2001 From: bendiller Date: Tue, 1 Oct 2019 15:40:53 -0600 Subject: [PATCH 1/5] Add checks for valid response and retries to fetch_posts() --- bot/cogs/reddit.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/bot/cogs/reddit.py b/bot/cogs/reddit.py index 63a57c5c66..c626ad48c5 100644 --- a/bot/cogs/reddit.py +++ b/bot/cogs/reddit.py @@ -21,6 +21,7 @@ class Reddit(Cog): HEADERS = {"User-Agent": "Discord Bot: PythonDiscord (https://pythondiscord.com/)"} URL = "https://www.reddit.com" + MAX_FETCH_RETRIES = 3 def __init__(self, bot: Bot): self.bot = bot @@ -42,16 +43,19 @@ async def fetch_posts(self, route: str, *, amount: int = 25, params: dict = None if params is None: params = {} - response = await self.bot.http_session.get( - url=f"{self.URL}/{route}.json", - headers=self.HEADERS, - params=params - ) - - content = await response.json() - posts = content["data"]["children"] + for _ in range(self.MAX_FETCH_RETRIES): + response = await self.bot.http_session.get( + url=f"{self.URL}/{route}.json", + headers=self.HEADERS, + params=params + ) + if response.status == 200 and response.content_type == 'application/json': + # Got appropriate response - process and return. + content = await response.json() + posts = content["data"]["children"] + return posts[:amount] - return posts[:amount] + return list() # Failed to get appropriate response within allowed number of retries. async def send_top_posts( self, channel: TextChannel, subreddit: Subreddit, content: str = None, time: str = "all" From 2bc08cc669e886fe749590e75d4030a5dbcb3f71 Mon Sep 17 00:00:00 2001 From: bendiller Date: Wed, 2 Oct 2019 18:56:04 -0600 Subject: [PATCH 2/5] Add logging for invalid response (after all retries are exhausted) --- bot/cogs/reddit.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bot/cogs/reddit.py b/bot/cogs/reddit.py index c626ad48c5..5a5f43da94 100644 --- a/bot/cogs/reddit.py +++ b/bot/cogs/reddit.py @@ -43,9 +43,10 @@ async def fetch_posts(self, route: str, *, amount: int = 25, params: dict = None if params is None: params = {} + url = f"{self.URL}/{route}.json" for _ in range(self.MAX_FETCH_RETRIES): response = await self.bot.http_session.get( - url=f"{self.URL}/{route}.json", + url=url, headers=self.HEADERS, params=params ) @@ -55,6 +56,7 @@ async def fetch_posts(self, route: str, *, amount: int = 25, params: dict = None posts = content["data"]["children"] return posts[:amount] + log.debug(f"Invalid response from: {url} - status code {response.status}, mimetype {response.content_type}") return list() # Failed to get appropriate response within allowed number of retries. async def send_top_posts( From 0b59585bfd4a117dc1f3e6c680b20e37026a097e Mon Sep 17 00:00:00 2001 From: bendiller Date: Wed, 2 Oct 2019 19:13:13 -0600 Subject: [PATCH 3/5] Add sleep(3) between retries, with bot indicating typing during sleep --- bot/cogs/reddit.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/bot/cogs/reddit.py b/bot/cogs/reddit.py index 5a5f43da94..f072da3542 100644 --- a/bot/cogs/reddit.py +++ b/bot/cogs/reddit.py @@ -34,7 +34,9 @@ def __init__(self, bot: Bot): self.new_posts_task = None self.top_weekly_posts_task = None - async def fetch_posts(self, route: str, *, amount: int = 25, params: dict = None) -> List[dict]: + async def fetch_posts( + self, channel: TextChannel, route: str, *, amount: int = 25, params: dict = None + ) -> List[dict]: """A helper method to fetch a certain amount of Reddit posts at a given route.""" # Reddit's JSON responses only provide 25 posts at most. if not 25 >= amount > 0: @@ -55,6 +57,8 @@ async def fetch_posts(self, route: str, *, amount: int = 25, params: dict = None content = await response.json() posts = content["data"]["children"] return posts[:amount] + async with channel.typing(): + await asyncio.sleep(3) log.debug(f"Invalid response from: {url} - status code {response.status}, mimetype {response.content_type}") return list() # Failed to get appropriate response within allowed number of retries. @@ -69,6 +73,7 @@ async def send_top_posts( # Get the posts posts = await self.fetch_posts( + channel=channel, route=f"{subreddit}/top", amount=5, params={ @@ -116,7 +121,7 @@ async def send_top_posts( embed=embed ) - async def poll_new_posts(self) -> None: + async def poll_new_posts(self, channel: TextChannel) -> None: """Periodically search for new subreddit posts.""" while True: await asyncio.sleep(RedditConfig.request_delay) @@ -137,7 +142,7 @@ async def poll_new_posts(self) -> None: self.prev_lengths[subreddit] = content_length # Now we can actually fetch the new data - posts = await self.fetch_posts(f"{subreddit}/new") + posts = await self.fetch_posts(channel, f"{subreddit}/new") new_posts = [] # Only show new posts if we've checked before. @@ -266,7 +271,7 @@ async def on_ready(self) -> None: if self.reddit_channel is not None: if self.new_posts_task is None: - self.new_posts_task = self.bot.loop.create_task(self.poll_new_posts()) + self.new_posts_task = self.bot.loop.create_task(self.poll_new_posts(self.reddit_channel)) if self.top_weekly_posts_task is None: self.top_weekly_posts_task = self.bot.loop.create_task(self.poll_top_weekly_posts()) else: From 241f74c7a51adc0e73e49a9e91ee9a3ae501a8ea Mon Sep 17 00:00:00 2001 From: bendiller Date: Thu, 3 Oct 2019 15:49:55 -0600 Subject: [PATCH 4/5] Move asyncio.sleep() to avoid disturbing function signatures. --- bot/cogs/reddit.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/bot/cogs/reddit.py b/bot/cogs/reddit.py index f072da3542..08a725900f 100644 --- a/bot/cogs/reddit.py +++ b/bot/cogs/reddit.py @@ -34,9 +34,7 @@ def __init__(self, bot: Bot): self.new_posts_task = None self.top_weekly_posts_task = None - async def fetch_posts( - self, channel: TextChannel, route: str, *, amount: int = 25, params: dict = None - ) -> List[dict]: + async def fetch_posts(self, route: str, *, amount: int = 25, params: dict = None) -> List[dict]: """A helper method to fetch a certain amount of Reddit posts at a given route.""" # Reddit's JSON responses only provide 25 posts at most. if not 25 >= amount > 0: @@ -57,8 +55,7 @@ async def fetch_posts( content = await response.json() posts = content["data"]["children"] return posts[:amount] - async with channel.typing(): - await asyncio.sleep(3) + await asyncio.sleep(3) log.debug(f"Invalid response from: {url} - status code {response.status}, mimetype {response.content_type}") return list() # Failed to get appropriate response within allowed number of retries. @@ -72,14 +69,14 @@ async def send_top_posts( embed.description = "" # Get the posts - posts = await self.fetch_posts( - channel=channel, - route=f"{subreddit}/top", - amount=5, - params={ - "t": time - } - ) + async with channel.typing(): + posts = await self.fetch_posts( + route=f"{subreddit}/top", + amount=5, + params={ + "t": time + } + ) if not posts: embed.title = random.choice(ERROR_REPLIES) @@ -121,7 +118,7 @@ async def send_top_posts( embed=embed ) - async def poll_new_posts(self, channel: TextChannel) -> None: + async def poll_new_posts(self) -> None: """Periodically search for new subreddit posts.""" while True: await asyncio.sleep(RedditConfig.request_delay) @@ -142,7 +139,7 @@ async def poll_new_posts(self, channel: TextChannel) -> None: self.prev_lengths[subreddit] = content_length # Now we can actually fetch the new data - posts = await self.fetch_posts(channel, f"{subreddit}/new") + posts = await self.fetch_posts(f"{subreddit}/new") new_posts = [] # Only show new posts if we've checked before. @@ -271,7 +268,7 @@ async def on_ready(self) -> None: if self.reddit_channel is not None: if self.new_posts_task is None: - self.new_posts_task = self.bot.loop.create_task(self.poll_new_posts(self.reddit_channel)) + self.new_posts_task = self.bot.loop.create_task(self.poll_new_posts()) if self.top_weekly_posts_task is None: self.top_weekly_posts_task = self.bot.loop.create_task(self.poll_top_weekly_posts()) else: From df4906c9bd54dfd12201e3684cdfafb74693c06f Mon Sep 17 00:00:00 2001 From: Ben Diller Date: Thu, 3 Oct 2019 15:58:47 -0600 Subject: [PATCH 5/5] Improve readability Co-Authored-By: Mark --- bot/cogs/reddit.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/cogs/reddit.py b/bot/cogs/reddit.py index 08a725900f..6880aab854 100644 --- a/bot/cogs/reddit.py +++ b/bot/cogs/reddit.py @@ -55,6 +55,7 @@ async def fetch_posts(self, route: str, *, amount: int = 25, params: dict = None content = await response.json() posts = content["data"]["children"] return posts[:amount] + await asyncio.sleep(3) log.debug(f"Invalid response from: {url} - status code {response.status}, mimetype {response.content_type}")