Update manager (#23)

Reviewed-on: #23
Reviewed-by: Alexey Skobkin <skobkin-ru@ya.ru>

.gitignore

@@ -3,6 +3,7 @@
 
 # Python
 /.venv
+/__pycache__
 
 # Database
 /*.db

database.py

@@ -1,6 +1,7 @@
 import sqlite3
 
 from exceptions import DisplayableException
+from rss import FeedItem
 
 
 class Database:
@@ -10,6 +11,7 @@ class Database:
         """Create a database file if not exists."""
         # TODO: think about removing check_same_thread=False
         self.conn = sqlite3.connect(path, check_same_thread=False)
+        self.conn.row_factory = sqlite3.Row
         self.cur = self.conn.cursor()
         self.__init_schema()
 
@@ -25,7 +27,7 @@ class Database:
         row = self.cur.fetchone()
         if row is None:
             return None
-        return row[0]
+        return row['id']
 
     def add_feed(self, url: str) -> int:
         """Add a feed to the database and return its id."""
@@ -39,7 +41,7 @@ class Database:
         row = self.cur.fetchone()
         if row is None:
             return None
-        return row[0]
+        return row['id']
 
     def subscribe_user_by_url(self, user_id: int, url: str) -> None:
         """Subscribe user to the feed creating it if does not exist yet."""
@@ -58,6 +60,7 @@ class Database:
         self.conn.commit()
 
     def unsubscribe_user_by_url(self, user_id: int, url: str) -> None:
+        """Subscribe a user to the feed by url."""
         feed_id = self.find_feed_by_url(url)
         if feed_id is None:
             raise DisplayableException('Feed does not exist')
@@ -91,36 +94,48 @@ class Database:
 
     def get_feed_subscribers_count(self, feed_id: int) -> int:
         """Count feed subscribers."""
-        self.cur.execute('SELECT COUNT(user_id) FROM subscriptions WHERE feed_id = ?', [feed_id])
+        self.cur.execute('SELECT COUNT(user_id) AS amount_subscribers FROM subscriptions WHERE feed_id = ?', [feed_id])
         row = self.cur.fetchone()
-        if row is None:
-            return 0
-        return int(row[0])
+        return row['amount_subscribers']
 
-    def find_feeds(self) -> list:
+    def find_feed_subscribers(self, feed_id: int) -> list[int]:
+        """Return feed subscribers"""
+        self.cur.execute('SELECT telegram_id FROM users WHERE id IN (SELECT user_id FROM subscriptions WHERE feed_id = ?)',
+                         [feed_id])
+        subscribers = self.cur.fetchall()
+        return list(map(lambda x: x['telegram_id'], subscribers))
+
+    def find_feeds(self) -> list[sqlite3.Row]:
         """Get a list of feeds."""
         self.cur.execute('SELECT * FROM feeds')
         return self.cur.fetchall()
 
-    def find_user_feeds(self, user_id: int) -> list:
+    def find_user_feeds(self, user_id: int) -> list[sqlite3.Row]:
         """Return a list of feeds the user is subscribed to."""
         self.cur.execute('SELECT * FROM feeds WHERE id IN (SELECT feed_id FROM subscriptions WHERE user_id = ?)',
                          [user_id])
         return self.cur.fetchall()
 
-    def find_feed_items(self, feed_id: int) -> list:
+    def find_feed_items(self, feed_id: int) -> list[sqlite3.Row]:
         """Get last feed items."""
         self.cur.execute('SELECT * FROM feeds_last_items WHERE feed_id = ?', [feed_id])
         return self.cur.fetchall()
 
-    def update_feed_items(self, feed_id: int, new_items: list) -> None:
+    def find_feed_items_urls(self, feed_id: int) -> list[str]:
+        """Return urls last feed items"""
+        items = self.find_feed_items(feed_id)
+        if not items:
+            return items
+        return list(map(lambda x: x['url'], items))
+
+    def update_feed_items(self, feed_id: int, new_items: list[FeedItem]) -> None:
         """Replace last feed items with a list items that receive."""
-        for i in range(len(new_items)):
-            new_items[i] = (feed_id,) + new_items[i]
+        for i, _ in enumerate(new_items):
+            new_items[i] = [feed_id] + list(new_items[i].__dict__.values())[:-1]
 
         self.cur.execute('DELETE FROM feeds_last_items WHERE feed_id = ?', [feed_id])
         self.cur.executemany(
-            'INSERT INTO feeds_last_items (feed_id, url, title, description, date) VALUES (?, ?, ?, ?, ?)', new_items)
+            'INSERT INTO feeds_last_items (feed_id, url, title, description) VALUES (?, ?, ?, ?)', new_items)
         self.conn.commit()
 
     def __init_schema(self):
@@ -144,7 +159,6 @@ class Database:
             '   url TEXT NOT NULL UNIQUE,'
             '   title TEXT,'
             '   description TEXT,'
-            '   date NUMERIC,'
             '   FOREIGN KEY(feed_id) REFERENCES feeds(id)'
             ')'
         )

pylama.ini

@@ -3,12 +3,12 @@
 format = pylint
 skip = .venv/*
 linters = pyflakes,pylint,pycodestyle
-ignore = F0401,C0114,R0903
+ignore = F0401,C0114,R0903,C0115,C0116,W0511
 
 [pylama:pylint]
-max_line_length = 120
+max_line_length = 130
 score = yes
 
 [pylama:pycodestyle]
 # Maximum length of each line
-max_line_length = 120
+max_line_length = 130

rss.py

@@ -1,35 +1,26 @@
-import feedparser
+from feedparser import FeedParserDict, parse
 
 
-class FeedItem():
-    def __init__(self, url: str, title: str, description: str) -> None:
+class FeedItem:
+    def __init__(self, item: FeedParserDict) -> None:
+        self.url = item.get('link', '')
+        self.title = item.get('title', '')
+        self.description = item.get('summary', '')
+        if 'published' in item:
+            self.date = item.published_parsed()
+        else:
+            self.date = None
+
+
+class Feed:
+    def __init__(self, url: str, feed: FeedParserDict) -> None:
         self.url = url
-        self.title = title
-        self.description = description
+        self.items = []
+        self.title = feed.feed.get('title', '')
+        for item in feed.entries:
+            self.items.append(FeedItem(item))
 
-class Feed():
-    def __init__(self, url: str, items: list[FeedItem]) -> None:
-        self.url = url
-        self.items = items
 
-class RssReader():
+class RssReader:
     def get_feed(self, url: str) -> Feed:
-        f = feedparser.parse(url)
-        items = self.__get_items(f.entries)
-        return Feed(url, items)
-        
-    def __convert_to_feed_item(self, item: dict) -> FeedItem:
-        if 'title' in item:
-            title = item['title']
-        if 'link' in item:
-            url = item['link']
-        if 'summary' in item:
-            description = item['summary']
-        return FeedItem(url, title, description)
-
-    def __get_items(self, items: list) -> list:
-        list_items = []
-        for item in items:
-            list_items.append(self.__convert_to_feed_item(item))
-        return list_items
-
+        return Feed(url, parse(url))

telegram.py

@@ -58,8 +58,8 @@ class CommandProcessor:
         feeds = self.database.find_user_feeds(data['user_id'])
 
         feed_list = ''
-        for feed in feeds:
-            feed_list += '* ' + str(feed[0]) + ': ' + feed[1] + '\n'
+        for index, feed in enumerate(feeds, start=1):
+            feed_list += '* ' + str(index) + ': ' + f'''<a href="{feed['url']}">{feed['title']}</a>''' + '\n'
 
         self.bot.reply_to(message, 'Your feeds:\n' + feed_list)
 
@@ -97,16 +97,21 @@ class Notifier:
     def __init__(self, token: str):
         self.bot: TeleBot = TeleBot(token)
 
-    def send_updates(self, chat_ids: list[int], updates: list[FeedItem]):
+    def send_updates(self, chat_ids: list[int], updates: list[FeedItem], feed_title: str):
         """Send notification about new items to the user"""
+        if not updates:
+            return
+
         for chat_id in chat_ids:
+            self.__count_request_and_wait()
+            self.bot.send_message(
+                chat_id=chat_id,
+                text=f'Updates from the {feed_title} feed:'
+            )
+
             for update in updates:
+                self.__count_request_and_wait()
                 self.__send_update(chat_id, update)
-                self.sent_counter += 1
-                if self.sent_counter >= self.BATCH_LIMIT:
-                    # TODO: probably implement better later
-                    time.sleep(1)
-                    self.sent_counter = 0
 
     def __send_update(self, chat_id: int, update: FeedItem):
         self.bot.send_message(
@@ -115,10 +120,18 @@ class Notifier:
             parse_mode='HTML'
         )
 
+    def __count_request_and_wait(self):
+        if self.sent_counter >= self.BATCH_LIMIT:
+            # TODO: probably implement better later
+            time.sleep(1)
+            self.sent_counter = 0
+        self.sent_counter += 1
+
     @staticmethod
     def __format_message(item: FeedItem) -> str:
         return (
             f"<strong><a href=\"{item.url}\">{item.title}</a></strong>\n\n"
+            f"{item.date}\n"
             f"{item.description}"
         )
 

update.py

@@ -0,0 +1,15 @@
+import os
+from rss import RssReader
+from update_manager import UpdateManager
+from database import Database
+from telegram import Notifier
+
+token = os.getenv('TELEGRAM_TOKEN')
+db_path = os.getenv('DATABASE_PATH')
+
+db = Database(db_path)
+notifier = Notifier(token)
+rss_reader = RssReader()
+
+updater = UpdateManager(db, notifier, rss_reader)
+updater.update()

update_manager.py

@@ -0,0 +1,43 @@
+from rss import RssReader, FeedItem
+from database import Database
+from telegram import Notifier
+
+
+class UpdateManager:
+    """Implement the feed update."""
+
+    def __init__(self, database: Database, notifier: Notifier, rss_reader: RssReader) -> None:
+        self.database: Database = database
+        self.notifier: Notifier = notifier
+        self.rss_reader: RssReader = rss_reader
+
+    def update(self):
+        """Send new feed items to the user."""
+        feeds = self.database.find_feeds()
+
+        for feed_id, feed_url in feeds:
+            feed = self.rss_reader.get_feed(feed_url)
+            new_items = feed.items
+            old_items_urls = self.database.find_feed_items_urls(feed_id)
+
+            diff = self.__calculate_difference(new_items, old_items_urls)
+
+            if not diff:
+                continue
+
+            chat_ids = self.database.find_feed_subscribers(feed_id)
+            self.notifier.send_updates(chat_ids, diff, feed.title)
+            self.database.update_feed_items(feed_id, new_items)
+
+    def __calculate_difference(self, new_items: list[FeedItem], old_items_urls: list[str]) -> list[FeedItem]:
+        """Calculate new feed items."""
+        if not old_items_urls:
+            return new_items
+
+        diff = []
+
+        for item in new_items:
+            if item.url not in old_items_urls:
+                diff.append(item)
+
+        return diff