- Implement creating and filling articles and links tables.

- Implement initial loading of the content.
- Implement browsing categories and feeds using the DB.

API.py

@@ -18,9 +18,6 @@ class Fetcher:
         result = dict([(item["id"], (item["count"], item["newestItemTimestampUsec"], Utils.timestampToDate(
             item["newestItemTimestampUsec"]))) for item in response.json()["unreadcounts"]])
         self.unreadCounts = result
-        self.categories = sorted([{"id": item, "name": item[13:], "count": self.unreadCounts[item][0], "date": self.unreadCounts[item][2]}
-                                 for item in self.unreadCounts.keys() if item[0:13] == "user/-/label/"],
-                                 key=lambda item: item["date"], reverse=True)
 
     def getSubscriptions(self):
         response = httpx.get(self.URL+"/reader/api/0/subscription/list?output=json", headers=self.headers)
@@ -36,7 +33,8 @@ class Fetcher:
     def articlesFromCategory(self, category, number):
         if category not in self.articles:
             response = httpx.get(self.URL+"/reader/api/0/stream/contents?n="+number+"&s="+category, headers=self.headers)
-            self.articles[category] = response.json()["items"]
+            return response.json()["items"]
+            # self.articles[category] = response.json()["items"]
 
     async def articlesFromCategoryAsync(self, category, number):
         if category not in self.articles:

App.py

@@ -6,8 +6,8 @@ import subprocess
 import os
 from concurrent.futures import ThreadPoolExecutor
 from API import Fetcher
-from DB import DB
+from Cache import Cache
-from Render import Article
+from Render import Article, Render
 import Utils
 warnings.filterwarnings("ignore")
 
@@ -16,8 +16,8 @@ class LeftPane(urwid.ListBox):
     def __init__(self, categories):
         super().__init__(self)
         items = [urwid.AttrMap(urwid.Columns([
-            (16, urwid.Text(category["date"])), urwid.Text(category["name"]), (5, urwid.Text(str(category["count"])))]),
+            (16, urwid.Text(category[4])), urwid.Text(category[1]), (5, urwid.Text(str(category[2])))]),
-                               (category["id"], category["name"], category["count"]), "reveal focus") for category in categories]
+                               (category[0], category[1], category[2]), "reveal focus") for category in categories]
         walker = urwid.SimpleListWalker(items)
         self.body = walker
         self.categoryPosition = 0
@@ -25,7 +25,22 @@ class LeftPane(urwid.ListBox):
         self.currentCategory = ""
         focus_widget, idx = self.get_focus()
         if self.isCategoryView:
+            try:
                 self.currentCategory = focus_widget.attr_map[None][0]
+            except BaseException:
+                self.currentCategory = None
+
+    def fill(self, items, is_category_view):
+        def getAttrs(item):
+            if is_category_view is True:
+                return (item[0], item[1], item[2])
+            else:
+                return (item[0], item[1])
+        items = [urwid.AttrMap(urwid.Columns([
+            (16, urwid.Text(item[4])), urwid.Text(item[1]), (5, urwid.Text(str(item[2])))]),
+                               getAttrs(item), "reveal focus") for item in items]
+        walker = urwid.SimpleListWalker(items)
+        self.body = walker
 
     def setArticlesPaneTitle(self, text):
         tui.rightBox.set_title(text)
@@ -40,19 +55,14 @@ class LeftPane(urwid.ListBox):
 
     async def setCategoryArticles(self, attrMap):
         itemId = attrMap[0]
-        number = attrMap[2]
         name = attrMap[1]
-        await tui.fetcher.articlesFromCategoryAsync(itemId, str(number))
+        tui.articles = tui.cache.getArticlesFromCategory(itemId)
-        focus_widget, idx = self.get_focus()
-        currentCategory = focus_widget.attr_map[None][0]
-        if itemId == currentCategory:
-            tui.articles = tui.fetcher.articles[currentCategory]
         tui.articleView.fill(tui.articles)
         self.setArticlesPaneTitle(name)
 
     def setFeedArticles(self, attrMap):
         itemId = attrMap[0]
-        tui.articles = tui.fetcher.articlesFromFeed(itemId, self.currentCategory)
+        tui.articles = tui.cache.getArticlesFromFeed(itemId)
         if tui.articles is not None:
             tui.articleView.fill(tui.articles)
             self.setArticlesPaneTitle(attrMap[1])
@@ -84,19 +94,14 @@ class LeftPane(urwid.ListBox):
                 self.setFeedArticles(focus_widget.attr_map[None])
             return
         elif key in ("l", "right"):
-            if self.isCategoryView and tui.fetcher.checkCategory(self.currentCategory):
+            if self.isCategoryView:
                 self.isCategoryView = False
                 focus_widget, idx = self.get_focus()
                 self.categoryPosition = idx
                 categoryId = focus_widget.attr_map[None][0]
                 categoryName = focus_widget.attr_map[None][1]
-                feeds = tui.fetcher.feedsFromCategory(categoryId)
+                feeds = tui.cache.getFeeds(categoryId)
-                feedItems = [urwid.AttrMap(urwid.Columns([(16, urwid.Text(tui.fetcher.unreadCounts[feed["id"]][2])),
+                self.fill(feeds, False)
-                                                          urwid.Text(feed["title"]),
-                                                          (5, urwid.Text(str(tui.fetcher.unreadCounts[feed["id"]][0])))]),
-                                           (feed["id"], feed["title"]), "reveal focus") for feed in feeds]
-                walker = urwid.SimpleListWalker(feedItems)
-                self.body = walker
                 focus_widget, idx = self.get_focus()
                 self.setFeedArticles(focus_widget.attr_map[None])
                 tui.leftBox.set_title(categoryName)
@@ -104,24 +109,11 @@ class LeftPane(urwid.ListBox):
         elif key in ("h", "left"):
             if not self.isCategoryView:
                 self.isCategoryView = True
-                items = [
+                self.fill(tui.categories, True)
-                    urwid.AttrMap(
-                        urwid.Columns(
-                            [(16, urwid.Text(category["date"])),
-                             urwid.Text(category["name"]),
-                             (5, urwid.Text(str(category["count"])))]),
-                        (category["id"],
-                         category["name"],
-                         category["count"]),
-                        "reveal focus") for category in tui.fetcher.categories]
-                walker = urwid.SimpleFocusListWalker(items)
-                self.body = walker
                 tui.leftBox.set_title("Categories")
                 self.set_focus(self.categoryPosition)
                 focus_widget, idx = self.get_focus()
-                tui.articles = self.getArticlesFromCategory(focus_widget.attr_map[None][0])
+                asyncio.create_task(self.setCategoryArticles(focus_widget.attr_map[None]))
-                tui.articleView.fill(tui.articles)
-                self.setArticlesPaneTitle(self.currentCategory[13:])
             return
 
         return super().keypress(size, key)
@@ -142,9 +134,9 @@ class RightPane(urwid.ListBox):
             urwid.AttrMap(
                 urwid.Columns(
                     [(2, urwid.Text("")),
-                     (16, urwid.Text(Utils.timestampToDate(article["timestampUsec"]))),
+                     (16, urwid.Text(article[3])),
-                     urwid.Text(article["title"])]),
+                     urwid.Text(article[1])]),
-                article["id"],
+                article[0],
                 "reveal focus") for article in articles]
         walker = urwid.SimpleListWalker(items)
         self.body = walker
@@ -282,14 +274,14 @@ class TUI(urwid.Frame):
 
         self.overlay = None
         self.fetcher = Fetcher(URL, token)
-        self.DB = DB(self.fetcher)
+        self.cache = Cache(self.fetcher)
-        self.DB.refresh()
+        self.categories = self.cache.getCategories()
 
         self.leftPaneItems = {}
         self.activePane = False
         self.executor = ThreadPoolExecutor(max_workers=1)
         self.loop = None
-        self.feedView = LeftPane(self.fetcher.categories)
+        self.feedView = LeftPane([])
         self.articleView = RightPane([])
         self.leftBox = urwid.LineBox(self.feedView, title="Categories")
         self.rightBox = urwid.LineBox(self.articleView, title="Articles")
@@ -301,14 +293,20 @@ class TUI(urwid.Frame):
 
         super().__init__(self.body)
 
-    def run(self):
+    def initialize_panes(self):
+        try:
+            self.feedView.fill(self.cache.getCategories(), True)
             focus_widget, idx = self.feedView.get_focus()
             item = focus_widget.attr_map[None][0]
             name = focus_widget.attr_map[None][1]
-        self.fetcher.articlesFromCategory(item, str(focus_widget.attr_map[None][2]))
+            self.articles = self.cache.getArticlesFromCategory(item)
-        self.articles = tui.fetcher.articles[item]
             self.articleView.fill(self.articles)
             self.feedView.setArticlesPaneTitle(name)
+        except BaseException:
+            pass
+
+    def run(self):
+        self.initialize_panes()
         self.loop.run()
         self.executor.shutdown(wait=False)
 
@@ -336,11 +334,12 @@ class TUI(urwid.Frame):
                 urwid.LineBox(olb),
                 self.body, align="center", width=("relative", 50), valign="middle", height=3)
             self.body = overlay
-            articles = tui.fetcher.fetch()
+            articles = tui.cache.refresh()
             for text in articles:
                 olb.body = urwid.SimpleListWalker([urwid.Text(text)])
                 tui.loop.entering_idle()
             self.body = overlay.bottom_w
+            self.initialize_panes()
 
 
 tui = TUI.create()

Cache.py

@@ -0,0 +1,115 @@
+import sqlite3
+from sqlite3 import Error
+import os
+from Render import Article
+
+
+class Cache:
+    def __init__(self, api):
+        self.conn = None
+        self.api = api
+        create_categories = """create table if not exists categories (
+                                   id text primary key,
+                                   name text not null,
+                                   unread_count integer,
+                                   timestamp integer,
+                                   date text
+                               )
+                            """
+        create_feeds = """create table if not exists feeds (
+                              id text primary key,
+                              name text not null,
+                              unread_count integer,
+                              timestamp integer,
+                              date text,
+                              category_id text,
+                              foreign key (category_id) references categories (id)
+                          )
+                       """
+        create_articles = """create table if not exists articles (
+                              id text primary key,
+                              title text not null,
+                              timestamp integer,
+                              date text,
+                              content text,
+                              url text,
+                              is_read boolean,
+                              origin text,
+                              category_id text,
+                              foreign key (category_id) references categories (id),
+                              foreign key (origin) references feeds (id)
+                          )
+                       """
+        create_links = """create table if not exists links (
+                              id text,
+                              url text not null,
+                              foreign key (id) references articles (id)
+                          )
+                       """
+        try:
+            self.conn = sqlite3.connect(os.path.expanduser("~") + '/.config/inomnibus/cache.db')
+            self.conn.cursor().execute(create_categories)
+            self.conn.cursor().execute(create_feeds)
+            self.conn.cursor().execute(create_articles)
+            self.conn.cursor().execute(create_links)
+
+        except Error as e:
+            print(e)
+
+    def getArticlesFromFeed(self, feed_id):
+        cur = self.conn.cursor()
+        cur.execute("""select * from articles where origin = ?""", (feed_id,))
+        return cur.fetchall()
+
+    def getArticlesFromCategory(self, category_id):
+        cur = self.conn.cursor()
+        cur.execute("""select * from articles where category_id = ?""", (category_id,))
+        return cur.fetchall()
+
+    def getCategories(self):
+        cur = self.conn.cursor()
+        cur.execute("""select * from categories where unread_count != 0 order by timestamp desc""")
+        return cur.fetchall()
+
+    def getFeeds(self, category_id):
+        cur = self.conn.cursor()
+        cur.execute("""select * from feeds where category_id = ? and unread_count != 0 order by timestamp desc""", (category_id,))
+        return cur.fetchall()
+
+    def refresh(self):
+        self.api.refresh()
+        for item in self.api.unreadCounts.keys():
+            if item[0:13] != "user/-/label/":
+                continue
+            data = self.api.unreadCounts[item]
+            cur = self.conn.cursor()
+            cur.execute("""insert into categories(id,name,unread_count,timestamp,date) values(:id,:name,:count,:ts,:d)
+                        on conflict(id) do update set unread_count = :count, timestamp = :ts, date = :d;
+                        """,
+                        {"id": item, "name": item[13:], "count": data[0], "ts": data[1], "d": data[2]})
+        self.conn.commit()
+        for item in self.api.feeds:
+            data = self.api.unreadCounts[item["id"]]
+            cur.execute("""insert into feeds(id,name,unread_count,timestamp,date,category_id) values(:id,:name,:count,:ts,:d,:c_id)
+                        on conflict(id) do update set unread_count = :count, timestamp = :ts, date = :d,category_id = :c_id;
+                        """,
+                        {"id": item["id"], "name": item["title"], "count": data[0], "ts": data[1], "d": data[2],
+                            "c_id": item["categories"][0]["id"]})
+        self.conn.commit()
+        cur.execute("""select id, name, unread_count from categories""")
+        for row in cur.fetchall():
+            yield "Fetching category " + row[1]
+            articles = self.api.articlesFromCategory(row[0], str(row[2]))
+            for article in articles:
+                articleObj = Article(article)
+                cur.execute("""insert into articles(id,title,timestamp,date,content,url,is_read,origin,category_id)
+                            values(:id,:t,:ts,:d,:c,:u,:r,:o,:c_id) on conflict(id) do update
+                            set id = :id, title = :t, timestamp = :ts, date = :d, content = :c, url = :u, is_read = :r, category_id = :c_id,
+                            origin = :o;
+                            """,
+                            {"id": articleObj.id, "t": articleObj.title, "ts": articleObj.timestamp, "d": articleObj.date,
+                             "c": articleObj.text, "u": articleObj.url, "r": articleObj.is_read, "o": articleObj.origin, "c_id": row[0]})
+                for link in articleObj.links:
+                    cur.execute("""insert into links(id,url) values(?,?)""", (articleObj.id, link))
+
+            self.conn.commit()

DB.py

@@ -1,55 +0,0 @@
-import sqlite3
-from sqlite3 import Error
-import os
-
-
-class DB:
-    def __init__(self, api):
-        self.conn = None
-        self.api = api
-        create_categories = """create table if not exists categories (
-                                   id text primary key,
-                                   name text not null,
-                                   unread_count integer,
-                                   timestamp integer,
-                                   date text
-                               )
-                            """
-        create_feeds = """create table if not exists feeds (
-                              id text primary key,
-                              name text not null,
-                              unread_count integer,
-                              timestamp integer,
-                              date text,
-                              category_id text,
-                              foreign key (category_id) references categories (id)
-                          )
-                       """
-        try:
-            self.conn = sqlite3.connect(os.path.expanduser("~") + '/.config/inomnibus/cache.db')
-            self.conn.cursor().execute(create_categories)
-            self.conn.cursor().execute(create_feeds)
-
-        except Error as e:
-            print(e)
-
-    def refresh(self):
-        self.api.refresh()
-        for item in self.api.unreadCounts.keys():
-            if item[0:13] != "user/-/label/":
-                continue
-            data = self.api.unreadCounts[item]
-            cur = self.conn.cursor()
-            cur.execute("""insert into categories(id,name,unread_count,timestamp,date) values(:id,:name,:count,:ts,:d)
-                        on conflict(id) do update set unread_count = :count, timestamp = :ts, date = :d;
-                        """,
-                        {"id": item, "name": item[13:], "count": data[0], "ts": data[1], "d": data[2]})
-        self.conn.commit()
-        for item in self.api.feeds:
-            data = self.api.unreadCounts[item["id"]]
-            cur.execute("""insert into feeds(id,name,unread_count,timestamp,date,category_id) values(:id,:name,:count,:ts,:d,:c_id)
-                        on conflict(id) do update set unread_count = :count, timestamp = :ts, date = :d,category_id = :c_id;
-                        """,
-                        {"id": item["id"], "name": item["title"], "count": data[0], "ts": data[1], "d": data[2],
-                            "c_id": item["categories"][0]["id"]})
-        self.conn.commit()

Render.py

@@ -5,26 +5,10 @@ from bs4 import BeautifulSoup
 from RedditCommentsParser import RedditComments
 
 
-class Article:
+class Render:
-    def __init__(self, articleObj):
+    def __init__(self, id, content, url, links):
-        content = articleObj["summary"]["content"]
+        if Utils.checkReddit(self.url):
-        soup = BeautifulSoup(content)
+            comments_link = Utils.checkRedditComments(links)
-        links = soup.find_all(href=True)
-        media = soup.find_all(src=True)
-        links_set = set()
-        for link in links:
-            links_set.add(link['href'])
-        for m in media:
-            links_set.add(m['src'])
-        self.links = list(links_set)
-        self.text = get_text(content)
-        self.title = articleObj["title"]
-        self.date = Utils.timestampToDate(articleObj["timestampUsec"])
-        self.url = articleObj["canonical"][0]["href"]
-        if Utils.checkStreamingVideo(self.url):
-            self.links.append(self.url)
-        elif Utils.checkReddit(self.url):
-            comments_link = Utils.checkRedditComments(self.links)
             if comments_link:
                 commentsObj = RedditComments(comments_link)
                 commentsObj.getComments()
@@ -68,3 +52,27 @@ class Article:
         else:
             self.currentPageNumber -= 1
         return self.chunks[self.currentPageNumber - 1]
+
+
+class Article:
+    def __init__(self, articleJSON):
+        content = articleJSON["summary"]["content"]
+        soup = BeautifulSoup(content)
+        links = soup.find_all(href=True)
+        media = soup.find_all(src=True)
+        links_set = set()
+        for link in links:
+            links_set.add(link['href'])
+        for m in media:
+            links_set.add(m['src'])
+        self.links = list(links_set)
+        self.text = get_text(content)
+        self.title = articleJSON["title"]
+        self.timestamp = articleJSON["timestampUsec"]
+        self.date = Utils.timestampToDate(self.timestamp)
+        self.url = articleJSON["canonical"][0]["href"]
+        self.is_read = "user/-/state/com.google/read" in articleJSON["categories"]
+        self.id = articleJSON["id"]
+        self.origin = articleJSON["origin"]["streamId"]
+        if Utils.checkStreamingVideo(self.url):
+            self.links.append(self.url)