From 629db4c25d456f9ba9a5a9370b11620eac585735 Mon Sep 17 00:00:00 2001 From: mitsuha_s Date: Sat, 11 Jun 2022 16:46:19 +0000 Subject: [PATCH 01/20] fix methods in rss.py --- rss.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/rss.py b/rss.py index 8389972..a6b243e 100644 --- a/rss.py +++ b/rss.py @@ -2,10 +2,11 @@ import feedparser class FeedItem(): - def __init__(self, url: str, title: str, description: str) -> None: + def __init__(self, url: str, title: str, description: str, date: str) -> None: self.url = url self.title = title self.description = description + self.date = date class Feed(): def __init__(self, url: str, items: list[FeedItem]) -> None: @@ -18,14 +19,12 @@ class RssReader(): items = self.__get_items(f.entries) return Feed(url, items) - def __convert_to_feed_item(self, item: dict) -> FeedItem: - if 'title' in item: - title = item['title'] - if 'link' in item: - url = item['link'] - if 'summary' in item: - description = item['summary'] - return FeedItem(url, title, description) + def __convert_to_feed_item(self, item: feedparser.util.FeedParserDict) -> FeedItem: + title = item.get('title', '') + url = item.get('link', '') + description = item.get('summary', '') + date = item.get('published', '') + return FeedItem(url, title, description, date) def __get_items(self, items: list) -> list: list_items = [] -- 2.43.5 From 94d77a42f45895d925d9259bbc96186471547226 Mon Sep 17 00:00:00 2001 From: mitsuha_s Date: Sat, 11 Jun 2022 22:51:39 +0000 Subject: [PATCH 02/20] implement update manager --- .gitignore | 1 + database.py | 10 ++++++++-- rss.py | 6 +++--- update_manager.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 5 deletions(-) create mode 100644 update_manager.py diff --git a/.gitignore b/.gitignore index 4bb55d6..040e37d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ # Python /.venv +/__pycache__ # Database /*.db \ No newline at end of file diff --git a/database.py b/database.py index 6c74564..234d88e 100644 --- a/database.py +++ b/database.py @@ -1,6 +1,7 @@ import sqlite3 from exceptions import DisplayableException +from rss import FeedItem class Database: @@ -97,6 +98,11 @@ class Database: return 0 return int(row[0]) + def find_feed_subscribers(self, feed_id: int) -> list: + """Return feed subscribers""" + self.cur.execute('SELECT telegram_id FROM users WHERE id IN (SELECT user_id FROM subscriptions WHERE feed_id = ?)', [feed_id]) + return self.cur.fetchall() + def find_feeds(self) -> list: """Get a list of feeds.""" self.cur.execute('SELECT * FROM feeds') @@ -113,10 +119,10 @@ class Database: self.cur.execute('SELECT * FROM feeds_last_items WHERE feed_id = ?', [feed_id]) return self.cur.fetchall() - def update_feed_items(self, feed_id: int, new_items: list) -> None: + def update_feed_items(self, feed_id: int, new_items: list[FeedItem]) -> None: """Replace last feed items with a list items that receive.""" for i in range(len(new_items)): - new_items[i] = (feed_id,) + new_items[i] + new_items[i] = [feed_id] + list(new_items[i].__dict__.values()) self.cur.execute('DELETE FROM feeds_last_items WHERE feed_id = ?', [feed_id]) self.cur.executemany( diff --git a/rss.py b/rss.py index a6b243e..cb76d7c 100644 --- a/rss.py +++ b/rss.py @@ -1,19 +1,19 @@ import feedparser -class FeedItem(): +class FeedItem: def __init__(self, url: str, title: str, description: str, date: str) -> None: self.url = url self.title = title self.description = description self.date = date -class Feed(): +class Feed: def __init__(self, url: str, items: list[FeedItem]) -> None: self.url = url self.items = items -class RssReader(): +class RssReader: def get_feed(self, url: str) -> Feed: f = feedparser.parse(url) items = self.__get_items(f.entries) diff --git a/update_manager.py b/update_manager.py new file mode 100644 index 0000000..3cae06a --- /dev/null +++ b/update_manager.py @@ -0,0 +1,44 @@ +from rss import RssReader, FeedItem +from database import Database +from telegram import Notifier + + +class UpdateManager: + def __init__(self, database: Database, notifier: Notifier, rss_reader: RssReader) -> None: + self.database = database + self.notifier = notifier + self.rss_reader = rss_reader + + def update(self): + feeds = self.database.find_feeds() + + for feed_id, feed_url in feeds: + new_items = self.rss_reader.get_feed(feed_url).items + old_items = self.database.find_feed_items(feed_id) + + diff = self.__calculate_difference(new_items, old_items) + + if not diff: + continue + + chat_ids = self.database.find_feed_subscribers(feed_id) + chat_ids = list(map(lambda x: x[0], chat_ids)) + self.notifier.send_updates(chat_ids, diff) + self.database.update_feed_items(feed_id, new_items) + + def __calculate_difference(self, new_items: list[FeedItem], old_items: list[tuple]) -> list[FeedItem]: + if not old_items: + return new_items + + diff = [] + urls = list(map(lambda x: x[1], old_items)) + + for item in new_items: + if item.url not in urls: + diff.append(item) + + return diff + + + + -- 2.43.5 From 18f680a9e97f6c20d1e4a1d2c6d8733d790224ae Mon Sep 17 00:00:00 2001 From: mitsuha_s Date: Sun, 12 Jun 2022 13:02:18 +0000 Subject: [PATCH 03/20] correct return values in some methods from database.py --- database.py | 16 ++++++++++++---- update_manager.py | 16 +++++----------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/database.py b/database.py index 234d88e..c08c6b6 100644 --- a/database.py +++ b/database.py @@ -98,27 +98,35 @@ class Database: return 0 return int(row[0]) - def find_feed_subscribers(self, feed_id: int) -> list: + def find_feed_subscribers(self, feed_id: int) -> list[int]: """Return feed subscribers""" self.cur.execute('SELECT telegram_id FROM users WHERE id IN (SELECT user_id FROM subscriptions WHERE feed_id = ?)', [feed_id]) - return self.cur.fetchall() + subscribers = self.cur.fetchall() + return list(map(lambda x: x[0], subscribers)) def find_feeds(self) -> list: """Get a list of feeds.""" self.cur.execute('SELECT * FROM feeds') return self.cur.fetchall() - def find_user_feeds(self, user_id: int) -> list: + def find_user_feeds(self, user_id: int) -> list[tuple]: """Return a list of feeds the user is subscribed to.""" self.cur.execute('SELECT * FROM feeds WHERE id IN (SELECT feed_id FROM subscriptions WHERE user_id = ?)', [user_id]) return self.cur.fetchall() - def find_feed_items(self, feed_id: int) -> list: + def find_feed_items(self, feed_id: int) -> list[tuple]: """Get last feed items.""" self.cur.execute('SELECT * FROM feeds_last_items WHERE feed_id = ?', [feed_id]) return self.cur.fetchall() + def find_feed_items_url(self, feed_id: int) -> list[str]: + """Return urls last feed items""" + items = self.find_feed_items(feed_id) + if not items: + return items + return list(map(lambda x: x[1], items)) + def update_feed_items(self, feed_id: int, new_items: list[FeedItem]) -> None: """Replace last feed items with a list items that receive.""" for i in range(len(new_items)): diff --git a/update_manager.py b/update_manager.py index 3cae06a..0589290 100644 --- a/update_manager.py +++ b/update_manager.py @@ -14,31 +14,25 @@ class UpdateManager: for feed_id, feed_url in feeds: new_items = self.rss_reader.get_feed(feed_url).items - old_items = self.database.find_feed_items(feed_id) + old_items_url = self.database.find_feed_items_url(feed_id) - diff = self.__calculate_difference(new_items, old_items) + diff = self.__calculate_difference(new_items, old_items_url) if not diff: continue chat_ids = self.database.find_feed_subscribers(feed_id) - chat_ids = list(map(lambda x: x[0], chat_ids)) self.notifier.send_updates(chat_ids, diff) self.database.update_feed_items(feed_id, new_items) - def __calculate_difference(self, new_items: list[FeedItem], old_items: list[tuple]) -> list[FeedItem]: - if not old_items: + def __calculate_difference(self, new_items: list[FeedItem], old_items_url: list[str]) -> list[FeedItem]: + if not old_items_url: return new_items diff = [] - urls = list(map(lambda x: x[1], old_items)) for item in new_items: - if item.url not in urls: + if item.url not in old_items_url: diff.append(item) return diff - - - - -- 2.43.5 From b2c49a583a6b013baf3791ce4230a107c4ee9b22 Mon Sep 17 00:00:00 2001 From: mitsuha_s Date: Sun, 12 Jun 2022 13:18:55 +0000 Subject: [PATCH 04/20] refactor rss.py --- rss.py | 38 ++++++++++++++------------------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/rss.py b/rss.py index cb76d7c..c889d98 100644 --- a/rss.py +++ b/rss.py @@ -2,33 +2,23 @@ import feedparser class FeedItem: - def __init__(self, url: str, title: str, description: str, date: str) -> None: - self.url = url - self.title = title - self.description = description - self.date = date + def __init__(self, item: feedparser.util.FeedParserDict) -> None: + self.url = item.get('link', '') + self.title = item.get('title', '') + self.description = item.get('summary', '') + self.date = item.get('published', '') class Feed: - def __init__(self, url: str, items: list[FeedItem]) -> None: + def __init__(self, url: str, feed: feedparser.util.FeedParserDict) -> None: self.url = url - self.items = items + self.items = self.__get_items(feed.entries) + + def __get_items(self, items: list[feedparser.util.FeedParserDict]) -> list[FeedItem]: + list_items = [] + for item in items: + list_items.append(FeedItem(item)) + return list_items class RssReader: def get_feed(self, url: str) -> Feed: - f = feedparser.parse(url) - items = self.__get_items(f.entries) - return Feed(url, items) - - def __convert_to_feed_item(self, item: feedparser.util.FeedParserDict) -> FeedItem: - title = item.get('title', '') - url = item.get('link', '') - description = item.get('summary', '') - date = item.get('published', '') - return FeedItem(url, title, description, date) - - def __get_items(self, items: list) -> list: - list_items = [] - for item in items: - list_items.append(self.__convert_to_feed_item(item)) - return list_items - + return Feed(url, feedparser.parse(url)) -- 2.43.5 From 94c1447093df4e179216934c2ed74a9f8c92b7fc Mon Sep 17 00:00:00 2001 From: mitsuha_s Date: Sun, 12 Jun 2022 17:13:58 +0000 Subject: [PATCH 05/20] refactor a bit in rss.py --- database.py | 2 +- rss.py | 18 +++++++----------- update_manager.py | 10 +++++----- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/database.py b/database.py index c08c6b6..66b45e8 100644 --- a/database.py +++ b/database.py @@ -120,7 +120,7 @@ class Database: self.cur.execute('SELECT * FROM feeds_last_items WHERE feed_id = ?', [feed_id]) return self.cur.fetchall() - def find_feed_items_url(self, feed_id: int) -> list[str]: + def find_feed_items_urls(self, feed_id: int) -> list[str]: """Return urls last feed items""" items = self.find_feed_items(feed_id) if not items: diff --git a/rss.py b/rss.py index c889d98..f2ca995 100644 --- a/rss.py +++ b/rss.py @@ -1,24 +1,20 @@ -import feedparser +from feedparser import FeedParserDict, parse class FeedItem: - def __init__(self, item: feedparser.util.FeedParserDict) -> None: + def __init__(self, item: FeedParserDict) -> None: self.url = item.get('link', '') self.title = item.get('title', '') self.description = item.get('summary', '') self.date = item.get('published', '') class Feed: - def __init__(self, url: str, feed: feedparser.util.FeedParserDict) -> None: + def __init__(self, url: str, feed: FeedParserDict) -> None: self.url = url - self.items = self.__get_items(feed.entries) - - def __get_items(self, items: list[feedparser.util.FeedParserDict]) -> list[FeedItem]: - list_items = [] - for item in items: - list_items.append(FeedItem(item)) - return list_items + self.items = [] + for item in feed: + self.items.append(FeedItem(item)) class RssReader: def get_feed(self, url: str) -> Feed: - return Feed(url, feedparser.parse(url)) + return Feed(url, parse(url)) diff --git a/update_manager.py b/update_manager.py index 0589290..81ac61e 100644 --- a/update_manager.py +++ b/update_manager.py @@ -14,9 +14,9 @@ class UpdateManager: for feed_id, feed_url in feeds: new_items = self.rss_reader.get_feed(feed_url).items - old_items_url = self.database.find_feed_items_url(feed_id) + old_items_urls = self.database.find_feed_items_urls(feed_id) - diff = self.__calculate_difference(new_items, old_items_url) + diff = self.__calculate_difference(new_items, old_items_urls) if not diff: continue @@ -25,14 +25,14 @@ class UpdateManager: self.notifier.send_updates(chat_ids, diff) self.database.update_feed_items(feed_id, new_items) - def __calculate_difference(self, new_items: list[FeedItem], old_items_url: list[str]) -> list[FeedItem]: - if not old_items_url: + def __calculate_difference(self, new_items: list[FeedItem], old_items_urls: list[str]) -> list[FeedItem]: + if not old_items_urls: return new_items diff = [] for item in new_items: - if item.url not in old_items_url: + if item.url not in old_items_urls: diff.append(item) return diff -- 2.43.5 From f02b8d85c67f050c533c9081b8fbbda2852138bc Mon Sep 17 00:00:00 2001 From: mitsuha_s Date: Sun, 12 Jun 2022 20:08:11 +0000 Subject: [PATCH 06/20] add a row factory to the database module --- database.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/database.py b/database.py index 66b45e8..040ee7b 100644 --- a/database.py +++ b/database.py @@ -11,6 +11,7 @@ class Database: """Create a database file if not exists.""" # TODO: think about removing check_same_thread=False self.conn = sqlite3.connect(path, check_same_thread=False) + self.conn.row_factory = sqlite3.Row self.cur = self.conn.cursor() self.__init_schema() @@ -26,7 +27,7 @@ class Database: row = self.cur.fetchone() if row is None: return None - return row[0] + return row['id'] def add_feed(self, url: str) -> int: """Add a feed to the database and return its id.""" @@ -40,7 +41,7 @@ class Database: row = self.cur.fetchone() if row is None: return None - return row[0] + return row['id'] def subscribe_user_by_url(self, user_id: int, url: str) -> None: """Subscribe user to the feed creating it if does not exist yet.""" @@ -92,30 +93,28 @@ class Database: def get_feed_subscribers_count(self, feed_id: int) -> int: """Count feed subscribers.""" - self.cur.execute('SELECT COUNT(user_id) FROM subscriptions WHERE feed_id = ?', [feed_id]) + self.cur.execute('SELECT COUNT(user_id) AS amount_subscribers FROM subscriptions WHERE feed_id = ?', [feed_id]) row = self.cur.fetchone() - if row is None: - return 0 - return int(row[0]) + return row['amount_subscribers'] def find_feed_subscribers(self, feed_id: int) -> list[int]: """Return feed subscribers""" self.cur.execute('SELECT telegram_id FROM users WHERE id IN (SELECT user_id FROM subscriptions WHERE feed_id = ?)', [feed_id]) subscribers = self.cur.fetchall() - return list(map(lambda x: x[0], subscribers)) + return list(map(lambda x: x['telegram_id'], subscribers)) - def find_feeds(self) -> list: + def find_feeds(self) -> list[sqlite3.Row]: """Get a list of feeds.""" self.cur.execute('SELECT * FROM feeds') return self.cur.fetchall() - def find_user_feeds(self, user_id: int) -> list[tuple]: + def find_user_feeds(self, user_id: int) -> list[sqlite3.Row]: """Return a list of feeds the user is subscribed to.""" self.cur.execute('SELECT * FROM feeds WHERE id IN (SELECT feed_id FROM subscriptions WHERE user_id = ?)', [user_id]) return self.cur.fetchall() - def find_feed_items(self, feed_id: int) -> list[tuple]: + def find_feed_items(self, feed_id: int) -> list[sqlite3.Row]: """Get last feed items.""" self.cur.execute('SELECT * FROM feeds_last_items WHERE feed_id = ?', [feed_id]) return self.cur.fetchall() @@ -125,7 +124,7 @@ class Database: items = self.find_feed_items(feed_id) if not items: return items - return list(map(lambda x: x[1], items)) + return list(map(lambda x: x['url'], items)) def update_feed_items(self, feed_id: int, new_items: list[FeedItem]) -> None: """Replace last feed items with a list items that receive.""" -- 2.43.5 From e9a311ac67e60055101aa979de3ed227565e15d3 Mon Sep 17 00:00:00 2001 From: mitsuha_s Date: Sun, 12 Jun 2022 20:48:05 +0000 Subject: [PATCH 07/20] add feed title to Feed class --- rss.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rss.py b/rss.py index f2ca995..1f6422b 100644 --- a/rss.py +++ b/rss.py @@ -12,7 +12,8 @@ class Feed: def __init__(self, url: str, feed: FeedParserDict) -> None: self.url = url self.items = [] - for item in feed: + self.title = feed.feed.get('title', '') + for item in feed.entries: self.items.append(FeedItem(item)) class RssReader: -- 2.43.5 From eae90190e1c902759f425f87b7415cfab403a5f0 Mon Sep 17 00:00:00 2001 From: mitsuha_s Date: Sun, 12 Jun 2022 22:13:30 +0000 Subject: [PATCH 08/20] add sending the feed title to the user --- telegram.py | 10 +++++++--- update_manager.py | 11 ++++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/telegram.py b/telegram.py index bf39403..fb87c38 100644 --- a/telegram.py +++ b/telegram.py @@ -58,8 +58,8 @@ class CommandProcessor: feeds = self.database.find_user_feeds(data['user_id']) feed_list = '' - for feed in feeds: - feed_list += '* ' + str(feed[0]) + ': ' + feed[1] + '\n' + for count, feed in enumerate(feeds, start=1): + feed_list += '* ' + str(count) + ': ' + feed['url'] + '\n' self.bot.reply_to(message, 'Your feeds:\n' + feed_list) @@ -97,9 +97,13 @@ class Notifier: def __init__(self, token: str): self.bot: TeleBot = TeleBot(token) - def send_updates(self, chat_ids: list[int], updates: list[FeedItem]): + def send_updates(self, chat_ids: list[int], updates: list[FeedItem], feed_title: str): """Send notification about new items to the user""" for chat_id in chat_ids: + self.bot.send_message( + chat_id=chat_id, + text=f'Updates from the {feed_title} feed:' + ) for update in updates: self.__send_update(chat_id, update) self.sent_counter += 1 diff --git a/update_manager.py b/update_manager.py index 81ac61e..a928438 100644 --- a/update_manager.py +++ b/update_manager.py @@ -5,15 +5,16 @@ from telegram import Notifier class UpdateManager: def __init__(self, database: Database, notifier: Notifier, rss_reader: RssReader) -> None: - self.database = database - self.notifier = notifier - self.rss_reader = rss_reader + self.database: Database = database + self.notifier: Notifier = notifier + self.rss_reader: RssReader = rss_reader def update(self): feeds = self.database.find_feeds() for feed_id, feed_url in feeds: - new_items = self.rss_reader.get_feed(feed_url).items + feed = self.rss_reader.get_feed(feed_url) + new_items = feed.items old_items_urls = self.database.find_feed_items_urls(feed_id) diff = self.__calculate_difference(new_items, old_items_urls) @@ -22,7 +23,7 @@ class UpdateManager: continue chat_ids = self.database.find_feed_subscribers(feed_id) - self.notifier.send_updates(chat_ids, diff) + self.notifier.send_updates(chat_ids, diff, feed.title) self.database.update_feed_items(feed_id, new_items) def __calculate_difference(self, new_items: list[FeedItem], old_items_urls: list[str]) -> list[FeedItem]: -- 2.43.5 From 4e6cb3f28f2416204416d9514a237cbb02587ca5 Mon Sep 17 00:00:00 2001 From: mitsuha_s Date: Sun, 12 Jun 2022 22:34:02 +0000 Subject: [PATCH 09/20] add update.py --- update.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 update.py diff --git a/update.py b/update.py new file mode 100644 index 0000000..0cc00d6 --- /dev/null +++ b/update.py @@ -0,0 +1,15 @@ +import os +from rss import RssReader +from update_manager import UpdateManager +from database import Database +from telegram import Notifier + +token = os.getenv('TELEGRAM_TOKEN') +db_path = os.getenv('DATABASE_PATH') + +db = Database(db_path) +notifier = Notifier(token) +rss_reader = RssReader() + +updater = UpdateManager(db, notifier, rss_reader) +updater.update() \ No newline at end of file -- 2.43.5 From 82641cc655dea5da8fa1efa111ad88bc71508aba Mon Sep 17 00:00:00 2001 From: mitsuha_s Date: Sun, 12 Jun 2022 22:42:41 +0000 Subject: [PATCH 10/20] delete the date field from the database schema --- database.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/database.py b/database.py index 040ee7b..9604624 100644 --- a/database.py +++ b/database.py @@ -129,11 +129,11 @@ class Database: def update_feed_items(self, feed_id: int, new_items: list[FeedItem]) -> None: """Replace last feed items with a list items that receive.""" for i in range(len(new_items)): - new_items[i] = [feed_id] + list(new_items[i].__dict__.values()) + new_items[i] = [feed_id] + list(new_items[i].__dict__.values())[:-1] self.cur.execute('DELETE FROM feeds_last_items WHERE feed_id = ?', [feed_id]) self.cur.executemany( - 'INSERT INTO feeds_last_items (feed_id, url, title, description, date) VALUES (?, ?, ?, ?, ?)', new_items) + 'INSERT INTO feeds_last_items (feed_id, url, title, description) VALUES (?, ?, ?, ?)', new_items) self.conn.commit() def __init_schema(self): @@ -157,7 +157,6 @@ class Database: ' url TEXT NOT NULL UNIQUE,' ' title TEXT,' ' description TEXT,' - ' date NUMERIC,' ' FOREIGN KEY(feed_id) REFERENCES feeds(id)' ')' ) -- 2.43.5 From 42de1efed1a111226322c5f7557d39c7f6970898 Mon Sep 17 00:00:00 2001 From: mitsuha_s Date: Sun, 12 Jun 2022 22:52:16 +0000 Subject: [PATCH 11/20] add a date display in update sent to the user --- rss.py | 5 ++++- telegram.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/rss.py b/rss.py index 1f6422b..0587255 100644 --- a/rss.py +++ b/rss.py @@ -6,7 +6,10 @@ class FeedItem: self.url = item.get('link', '') self.title = item.get('title', '') self.description = item.get('summary', '') - self.date = item.get('published', '') + if 'published' in item: + self.date = item.published_parsed() + else: + self.date = '' class Feed: def __init__(self, url: str, feed: FeedParserDict) -> None: diff --git a/telegram.py b/telegram.py index fb87c38..e147b24 100644 --- a/telegram.py +++ b/telegram.py @@ -123,7 +123,8 @@ class Notifier: def __format_message(item: FeedItem) -> str: return ( f"{item.title}\n\n" - f"{item.description}" + f"{item.description}\n" + f"{item.date}" ) -- 2.43.5 From 4f25c2dd3fdf8d7eec42faca03b4c14d868fbd9b Mon Sep 17 00:00:00 2001 From: mitsuha_s Date: Mon, 4 Jul 2022 18:38:23 +0000 Subject: [PATCH 12/20] minor fixes --- rss.py | 4 +++- telegram.py | 12 +++++++----- update.py | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/rss.py b/rss.py index 0587255..514bb8d 100644 --- a/rss.py +++ b/rss.py @@ -9,7 +9,8 @@ class FeedItem: if 'published' in item: self.date = item.published_parsed() else: - self.date = '' + self.date = None + class Feed: def __init__(self, url: str, feed: FeedParserDict) -> None: @@ -19,6 +20,7 @@ class Feed: for item in feed.entries: self.items.append(FeedItem(item)) + class RssReader: def get_feed(self, url: str) -> Feed: return Feed(url, parse(url)) diff --git a/telegram.py b/telegram.py index e147b24..e929497 100644 --- a/telegram.py +++ b/telegram.py @@ -58,8 +58,8 @@ class CommandProcessor: feeds = self.database.find_user_feeds(data['user_id']) feed_list = '' - for count, feed in enumerate(feeds, start=1): - feed_list += '* ' + str(count) + ': ' + feed['url'] + '\n' + for index, feed in enumerate(feeds, start=1): + feed_list += '* ' + str(index) + ': ' + f"{feed['title']} str: return ( f"{item.title}\n\n" - f"{item.description}\n" - f"{item.date}" + f"{item.date}\n" + f"{item.description}" ) diff --git a/update.py b/update.py index 0cc00d6..c2c3624 100644 --- a/update.py +++ b/update.py @@ -12,4 +12,4 @@ notifier = Notifier(token) rss_reader = RssReader() updater = UpdateManager(db, notifier, rss_reader) -updater.update() \ No newline at end of file +updater.update() -- 2.43.5 From d25ca99da1f6ffa72684c51b6f42e20f3a76022f Mon Sep 17 00:00:00 2001 From: mitsuha_s Date: Fri, 8 Jul 2022 20:16:33 +0000 Subject: [PATCH 13/20] fix html code --- telegram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telegram.py b/telegram.py index e929497..4293146 100644 --- a/telegram.py +++ b/telegram.py @@ -59,7 +59,7 @@ class CommandProcessor: feed_list = '' for index, feed in enumerate(feeds, start=1): - feed_list += '* ' + str(index) + ': ' + f"{feed['title']}{feed['title']} Date: Fri, 8 Jul 2022 20:57:41 +0000 Subject: [PATCH 14/20] add docstrings --- database.py | 3 ++- telegram.py | 4 ++-- update_manager.py | 6 +++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/database.py b/database.py index 9604624..77a9038 100644 --- a/database.py +++ b/database.py @@ -60,6 +60,7 @@ class Database: self.conn.commit() def unsubscribe_user_by_url(self, user_id: int, url: str) -> None: + """Subscribe a user to the feed by url.""" feed_id = self.find_feed_by_url(url) if feed_id is None: raise DisplayableException('Feed does not exist') @@ -128,7 +129,7 @@ class Database: def update_feed_items(self, feed_id: int, new_items: list[FeedItem]) -> None: """Replace last feed items with a list items that receive.""" - for i in range(len(new_items)): + for i, _ in enumerate(new_items): new_items[i] = [feed_id] + list(new_items[i].__dict__.values())[:-1] self.cur.execute('DELETE FROM feeds_last_items WHERE feed_id = ?', [feed_id]) diff --git a/telegram.py b/telegram.py index 4293146..98bfd4f 100644 --- a/telegram.py +++ b/telegram.py @@ -59,7 +59,7 @@ class CommandProcessor: feed_list = '' for index, feed in enumerate(feeds, start=1): - feed_list += '* ' + str(index) + ': ' + f'''{feed['title']}{feed['title']}''' + '\n' self.bot.reply_to(message, 'Your feeds:\n' + feed_list) @@ -105,7 +105,7 @@ class Notifier: text=f'Updates from the {feed_title} feed:' ) self.sent_counter += 1 - + for update in updates: self.__send_update(chat_id, update) self.sent_counter += 1 diff --git a/update_manager.py b/update_manager.py index a928438..c8898f2 100644 --- a/update_manager.py +++ b/update_manager.py @@ -4,12 +4,15 @@ from telegram import Notifier class UpdateManager: + """Implement the feed update.""" + def __init__(self, database: Database, notifier: Notifier, rss_reader: RssReader) -> None: self.database: Database = database self.notifier: Notifier = notifier self.rss_reader: RssReader = rss_reader def update(self): + """Send new feed items to the user.""" feeds = self.database.find_feeds() for feed_id, feed_url in feeds: @@ -27,9 +30,10 @@ class UpdateManager: self.database.update_feed_items(feed_id, new_items) def __calculate_difference(self, new_items: list[FeedItem], old_items_urls: list[str]) -> list[FeedItem]: + """Calculate new feed items.""" if not old_items_urls: return new_items - + diff = [] for item in new_items: -- 2.43.5 From 25af6c21760ae7447d65fdb39842afeb8e30da89 Mon Sep 17 00:00:00 2001 From: mitsuha_s Date: Fri, 8 Jul 2022 20:59:52 +0000 Subject: [PATCH 15/20] little changes in pylama.ini --- pylama.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylama.ini b/pylama.ini index a17cf17..22b0398 100644 --- a/pylama.ini +++ b/pylama.ini @@ -3,7 +3,7 @@ format = pylint skip = .venv/* linters = pyflakes,pylint,pycodestyle -ignore = F0401,C0114,R0903 +ignore = F0401,C0114,R0903,C0115,C0116,W0511 [pylama:pylint] max_line_length = 120 -- 2.43.5 From 53fa8caf1e4902382a923b56f089de38e876a7c6 Mon Sep 17 00:00:00 2001 From: mitsuha_s Date: Fri, 8 Jul 2022 21:04:37 +0000 Subject: [PATCH 16/20] little code style changes --- database.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/database.py b/database.py index 77a9038..6a77669 100644 --- a/database.py +++ b/database.py @@ -100,7 +100,8 @@ class Database: def find_feed_subscribers(self, feed_id: int) -> list[int]: """Return feed subscribers""" - self.cur.execute('SELECT telegram_id FROM users WHERE id IN (SELECT user_id FROM subscriptions WHERE feed_id = ?)', [feed_id]) + self.cur.execute('SELECT telegram_id FROM users WHERE id IN (SELECT user_id FROM subscriptions WHERE feed_id = ?)', + [feed_id]) subscribers = self.cur.fetchall() return list(map(lambda x: x['telegram_id'], subscribers)) -- 2.43.5 From 5bb83c2a09e892c976e1ef8f7d061593a39778e2 Mon Sep 17 00:00:00 2001 From: mitsuha_s Date: Fri, 8 Jul 2022 21:09:10 +0000 Subject: [PATCH 17/20] change max_line_length in pylama.ini --- database.py | 2 +- pylama.ini | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/database.py b/database.py index 6a77669..72136f4 100644 --- a/database.py +++ b/database.py @@ -100,7 +100,7 @@ class Database: def find_feed_subscribers(self, feed_id: int) -> list[int]: """Return feed subscribers""" - self.cur.execute('SELECT telegram_id FROM users WHERE id IN (SELECT user_id FROM subscriptions WHERE feed_id = ?)', + self.cur.execute('SELECT telegram_id FROM users WHERE id IN (SELECT user_id FROM subscriptions WHERE feed_id = ?)', [feed_id]) subscribers = self.cur.fetchall() return list(map(lambda x: x['telegram_id'], subscribers)) diff --git a/pylama.ini b/pylama.ini index 22b0398..225dd7b 100644 --- a/pylama.ini +++ b/pylama.ini @@ -6,9 +6,9 @@ linters = pyflakes,pylint,pycodestyle ignore = F0401,C0114,R0903,C0115,C0116,W0511 [pylama:pylint] -max_line_length = 120 +max_line_length = 130 score = yes [pylama:pycodestyle] # Maximum length of each line -max_line_length = 120 +max_line_length = 130 -- 2.43.5 From 8a4bec01c164abc7f2aba34bc6b5af50dccb6b13 Mon Sep 17 00:00:00 2001 From: mitsuha_s Date: Fri, 8 Jul 2022 21:49:13 +0000 Subject: [PATCH 18/20] refactor send_updates method in Notifier class and add __register_request_and_wait method --- telegram.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/telegram.py b/telegram.py index 98bfd4f..e8ddc0a 100644 --- a/telegram.py +++ b/telegram.py @@ -90,7 +90,7 @@ class CommandProcessor: class Notifier: """Sends notifications to users about new RSS feed items.""" - BATCH_LIMIT: int = 29 + BATCH_LIMIT: int = 30 sent_counter: int = 0 @@ -99,7 +99,11 @@ class Notifier: def send_updates(self, chat_ids: list[int], updates: list[FeedItem], feed_title: str): """Send notification about new items to the user""" + if not updates: + return + for chat_id in chat_ids: + self.__register_request_and_wait() self.bot.send_message( chat_id=chat_id, text=f'Updates from the {feed_title} feed:' @@ -107,12 +111,9 @@ class Notifier: self.sent_counter += 1 for update in updates: + self.__register_request_and_wait() self.__send_update(chat_id, update) self.sent_counter += 1 - if self.sent_counter >= self.BATCH_LIMIT: - # TODO: probably implement better later - time.sleep(1) - self.sent_counter = 0 def __send_update(self, chat_id: int, update: FeedItem): self.bot.send_message( @@ -121,6 +122,12 @@ class Notifier: parse_mode='HTML' ) + def __register_request_and_wait(self): + if self.sent_counter >= self.BATCH_LIMIT: + # TODO: probably implement better later + time.sleep(1) + self.sent_counter = 0 + @staticmethod def __format_message(item: FeedItem) -> str: return ( -- 2.43.5 From c4809a20ea6c7232529680310bb85a3f3040fd7f Mon Sep 17 00:00:00 2001 From: mitsuha_s Date: Fri, 8 Jul 2022 22:09:11 +0000 Subject: [PATCH 19/20] rename __register_request_and_wait method to __count_request_and_wait and add counter increment to --- telegram.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/telegram.py b/telegram.py index e8ddc0a..cddcb58 100644 --- a/telegram.py +++ b/telegram.py @@ -108,12 +108,11 @@ class Notifier: chat_id=chat_id, text=f'Updates from the {feed_title} feed:' ) - self.sent_counter += 1 + for update in updates: self.__register_request_and_wait() self.__send_update(chat_id, update) - self.sent_counter += 1 def __send_update(self, chat_id: int, update: FeedItem): self.bot.send_message( @@ -122,11 +121,12 @@ class Notifier: parse_mode='HTML' ) - def __register_request_and_wait(self): + def __count_request_and_wait(self): if self.sent_counter >= self.BATCH_LIMIT: # TODO: probably implement better later time.sleep(1) self.sent_counter = 0 + self.sent_counter += 1 @staticmethod def __format_message(item: FeedItem) -> str: -- 2.43.5 From 1749bd5f5e47bf6a625cf4889bf00514d01f5ca4 Mon Sep 17 00:00:00 2001 From: mitsuha_s Date: Fri, 8 Jul 2022 22:15:46 +0000 Subject: [PATCH 20/20] fix method calls --- telegram.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/telegram.py b/telegram.py index cddcb58..74ef36a 100644 --- a/telegram.py +++ b/telegram.py @@ -103,15 +103,14 @@ class Notifier: return for chat_id in chat_ids: - self.__register_request_and_wait() + self.__count_request_and_wait() self.bot.send_message( chat_id=chat_id, text=f'Updates from the {feed_title} feed:' ) - for update in updates: - self.__register_request_and_wait() + self.__count_request_and_wait() self.__send_update(chat_id, update) def __send_update(self, chat_id: int, update: FeedItem): -- 2.43.5