Implement downloading pictures to local cache and opening them from there.

- Implement display read articles as blue.

API.py

@@ -7,18 +7,17 @@ class Fetcher:
         self.URL = URL
         self.token = token
         self.headers = {"Authorization": "GoogleLogin auth="+token}
+        self.articles = {}
+
+    def refresh(self):
         self.getUnreadCounts()
         self.getSubscriptions()
-        self.articles = {}
 
     def getUnreadCounts(self):
         response = httpx.get(self.URL+"/reader/api/0/unread-count?output=json", headers=self.headers)
         result = dict([(item["id"], (item["count"], item["newestItemTimestampUsec"], Utils.timestampToDate(
             item["newestItemTimestampUsec"]))) for item in response.json()["unreadcounts"]])
         self.unreadCounts = result
-        self.categories = sorted([{"id": item, "name": item[13:], "count": self.unreadCounts[item][0], "date": self.unreadCounts[item][2]}
-                                 for item in self.unreadCounts.keys() if item[0:13] == "user/-/label/"],
-                                 key=lambda item: item["date"], reverse=True)
 
     def getSubscriptions(self):
         response = httpx.get(self.URL+"/reader/api/0/subscription/list?output=json", headers=self.headers)
@@ -27,28 +26,42 @@ class Fetcher:
     def checkCategory(self, category):
         return category in self.articles.keys()
 
-    def feedsFromCategory(self, category):
-        return sorted([item for item in self.feeds if item["categories"][0]["id"] == category and self.unreadCounts[item["id"]][0] != 0],
-                      key=lambda item: self.unreadCounts[item["id"]][2], reverse=True)
-
-    def articlesFromCategory(self, category, number):
-        if category not in self.articles:
-            response = httpx.get(self.URL+"/reader/api/0/stream/contents?n="+number+"&s="+category, headers=self.headers)
-            self.articles[category] = response.json()["items"]
+    def articlesFromCategory(self, category):
+        response = httpx.get(self.URL+"/reader/api/0/stream/contents?n=1000"+"&s="+category, headers=self.headers)
+        return response.json()["items"]
 
-    async def articlesFromCategoryAsync(self, category, number):
-        if category not in self.articles:
-            response = await httpx.AsyncClient().get(self.URL+"/reader/api/0/stream/contents?n="+number +
-                                                     "&s="+category, headers=self.headers)
-            self.articles[category] = response.json()["items"]
+    def getFavorites(self):
+        response = httpx.get(self.URL+"/reader/api/0/stream/contents?s=user/-/state/com.google/starred&n=100", headers=self.headers)
+        json = response.json()
+        return (json["updated"], Utils.timestampToDate(json["updated"]*1000000), json["items"])
 
-    def articlesFromFeed(self, feed, category):
+    def markStreamAsRead(self, streamId, ts):
         try:
-            return [article for article in self.articles[category] if article["origin"]["streamId"] == feed]
+            response = httpx.post(self.URL+"/reader/api/0/mark-all-as-read", data={"s": streamId, "ts": ts}, headers=self.headers)
+            if response.status_code == 200:
+                return True
+            else:
+                return False
         except BaseException:
-            return None
+            return False
 
-    def fetch(self):
-        for category in self.categories:
-            yield "Fetching category " + category["name"]
-            self.articlesFromCategory(category["id"], str(self.unreadCounts[category["id"]][0]))
+    def toggleArticleStatus(self, articleId, is_read):
+        return self.toggleArticleTag(articleId, is_read, "user/-/state/com.google/read")
+
+    def toggleArticleStarred(self, articleId, is_favorite):
+        return self.toggleArticleTag(articleId, is_favorite, "user/-/state/com.google/starred")
+
+    def toggleArticleTag(self, articleId, is_set, tag):
+        if is_set == 0:
+            tag_op = 'r'
+        else:
+            tag_op = 'a'
+        try:
+            response = httpx.post(self.URL+"/reader/api/0/edit-tag", data={"i": articleId, tag_op: tag},
+                                  headers=self.headers)
+            if response.status_code == 200:
+                return True
+            else:
+                return False
+        except BaseException:
+            return False

App.py

@@ -1,12 +1,13 @@
 import urwid
 import yaml
-import asyncio
 import warnings
 import subprocess
 import os
 from concurrent.futures import ThreadPoolExecutor
 from API import Fetcher
-from Render import Article
+from Cache import Cache
+from Render import Render
+from FileCache import FileCache
 import Utils
 warnings.filterwarnings("ignore")
 
@@ -15,115 +16,172 @@ class LeftPane(urwid.ListBox):
     def __init__(self, categories):
         super().__init__(self)
         items = [urwid.AttrMap(urwid.Columns([
-            (16, urwid.Text(category["date"])), urwid.Text(category["name"]), (5, urwid.Text(str(category["count"])))]),
-                               (category["id"], category["name"], category["count"]), "reveal focus") for category in categories]
+            (16, urwid.Text(category[4])), urwid.Text(category[1]), (5, urwid.Text(str(category[2])))]),
+                               (category[0], category[1], category[2]), "reveal focus") for category in categories]
         walker = urwid.SimpleListWalker(items)
         self.body = walker
         self.categoryPosition = 0
         self.isCategoryView = True
         self.currentCategory = ""
+
+    def processAttrMap(self, attrMap):
+        res = attrMap[None]
+        if res == "favorite":
+            return attrMap["attrs"]
+        else:
+            return res
+
+    def fill(self, items, is_category_view):
+        def getAttrs(item):
+            if is_category_view is True:
+                return (item[0], item[1], item[2])
+            else:
+                return (item[0], item[1])
+        if is_category_view is True:
+            items = [
+                urwid.AttrMap(
+                    urwid.Columns(
+                        [(16, urwid.Text(items[0][4])),
+                         urwid.Text(items[0][1]),
+                         (5, urwid.Text(str(items[0][2])))]),
+                    {None: "favorite", "attrs": getAttrs(items[0])},
+                    "reveal focus"),
+                *
+                [urwid.AttrMap(
+                     urwid.Columns([(16, urwid.Text(item[4])),
+                                    urwid.Text(item[1]),
+                                    (5, urwid.Text(str(item[2])))]),
+                     getAttrs(item),
+                     "reveal focus") for item in items[1:]]]
+        else:
+            items = [urwid.AttrMap(urwid.Columns([
+                (16, urwid.Text(item[4])), urwid.Text(item[1]), (5, urwid.Text(str(item[2])))]),
+                                getAttrs(item), "reveal focus") for item in items]
+        walker = urwid.SimpleListWalker(items)
+        self.body = walker
         focus_widget, idx = self.get_focus()
         if self.isCategoryView:
-            self.currentCategory = focus_widget.attr_map[None][0]
+            try:
+                self.currentCategory = focus_widget.attr_map[None][0]
+            except BaseException:
+                self.currentCategory = None
+
+    def findById(self, id):
+        idx = 0
+        for idx, item in zip(range(len(self.body)), self.body):
+            if item.attr_map[None][0] == id:
+                break
+        return idx
 
     def setArticlesPaneTitle(self, text):
         tui.rightBox.set_title(text)
 
-    def getArticlesFromCategory(self, category, number=0):
-        tui.fetcher.articlesFromCategory(category, str(number))
-        articles = tui.fetcher.articles[category]
-        return articles
-
-    def getArticlesFromFeed(self, feed):
-        return tui.fetcher.articlesFromFeed(feed, self.currentCategory)
-
-    async def setCategoryArticles(self, attrMap):
+    def setCategoryArticles(self, attrMap):
+        Utils.writeLog(attrMap)
         itemId = attrMap[0]
-        number = attrMap[2]
         name = attrMap[1]
-        await tui.fetcher.articlesFromCategoryAsync(itemId, str(number))
-        focus_widget, idx = self.get_focus()
-        currentCategory = focus_widget.attr_map[None][0]
-        if itemId == currentCategory:
-            tui.articles = tui.fetcher.articles[currentCategory]
-            tui.articleView.fill(tui.articles)
-            self.setArticlesPaneTitle(name)
+        tui.articles = tui.cache.getArticlesFromCategory(itemId, tui.show_read)
+        tui.articleView.fill(tui.articles, True)
+        self.setArticlesPaneTitle(name)
 
     def setFeedArticles(self, attrMap):
         itemId = attrMap[0]
-        tui.articles = tui.fetcher.articlesFromFeed(itemId, self.currentCategory)
+        tui.articles = tui.cache.getArticlesFromFeed(itemId, tui.show_read)
         if tui.articles is not None:
-            tui.articleView.fill(tui.articles)
+            tui.articleView.fill(tui.articles, False)
             self.setArticlesPaneTitle(attrMap[1])
 
-    def keypress(self, size, key):
+    def keypress(self, size, key):  # noqa
         if key in ("j", "down"):
             item_size = len(self.body)
-            focus_widget, idx = self.get_focus()
-            if idx < item_size - 1:
-                idx = idx + 1
-                self.set_focus(idx)
-            focus_widget, idx = self.get_focus()
-            if self.isCategoryView:
-                self.currentCategory = focus_widget.attr_map[None][0]
-                asyncio.create_task(self.setCategoryArticles(focus_widget.attr_map[None]))
-            else:
-                self.setFeedArticles(focus_widget.attr_map[None])
+            try:
+                focus_widget, idx = self.get_focus()
+                if idx < item_size - 1:
+                    idx = idx + 1
+                    self.set_focus(idx)
+                focus_widget, idx = self.get_focus()
+                if self.isCategoryView:
+                    self.currentCategory = self.processAttrMap(focus_widget.attr_map)[0]
+                    self.setCategoryArticles(self.processAttrMap(focus_widget.attr_map))
+                else:
+                    self.setFeedArticles(focus_widget.attr_map[None])
+            except BaseException:
+                pass
             return
         elif key in ("k", "up"):
-            focus_widget, idx = self.get_focus()
-            if idx > 0:
-                idx = idx - 1
-                self.set_focus(idx)
-            focus_widget, idx = self.get_focus()
-            if self.isCategoryView:
-                self.currentCategory = focus_widget.attr_map[None][0]
-                asyncio.create_task(self.setCategoryArticles(focus_widget.attr_map[None]))
-            else:
-                self.setFeedArticles(focus_widget.attr_map[None])
+            try:
+                focus_widget, idx = self.get_focus()
+                if idx > 0:
+                    idx = idx - 1
+                    self.set_focus(idx)
+                focus_widget, idx = self.get_focus()
+                if self.isCategoryView:
+                    self.currentCategory = self.processAttrMap(focus_widget.attr_map)[0]
+                    self.setCategoryArticles(self.processAttrMap(focus_widget.attr_map))
+                else:
+                    self.setFeedArticles(focus_widget.attr_map[None])
+            except BaseException:
+                pass
             return
         elif key in ("l", "right"):
-            if self.isCategoryView and tui.fetcher.checkCategory(self.currentCategory):
+            self.stepInto()
+            return
+        elif key in ("h", "left"):
+            self.stepOut()
+            return
+        elif key in ("r"):
+            self.markAsRead()
+            return
+
+        return super().keypress(size, key)
+
+    def stepOut(self):
+        if not self.isCategoryView:
+            self.isCategoryView = True
+            self.fill(tui.categories, True)
+            tui.leftBox.set_title("Categories")
+            self.set_focus(self.categoryPosition)
+            focus_widget, idx = self.get_focus()
+            self.setCategoryArticles(self.processAttrMap(focus_widget.attr_map))
+
+    def stepInto(self):
+        try:
+            if self.isCategoryView and self.currentCategory != "Favorites":
                 self.isCategoryView = False
                 focus_widget, idx = self.get_focus()
                 self.categoryPosition = idx
-                categoryId = focus_widget.attr_map[None][0]
-                categoryName = focus_widget.attr_map[None][1]
-                feeds = tui.fetcher.feedsFromCategory(categoryId)
-                feedItems = [urwid.AttrMap(urwid.Columns([(16, urwid.Text(tui.fetcher.unreadCounts[feed["id"]][2])),
-                                                          urwid.Text(feed["title"]),
-                                                          (5, urwid.Text(str(tui.fetcher.unreadCounts[feed["id"]][0])))]),
-                                           (feed["id"], feed["title"]), "reveal focus") for feed in feeds]
-                walker = urwid.SimpleListWalker(feedItems)
-                self.body = walker
+                categoryId = self.processAttrMap(focus_widget.attr_map)[0]
+                categoryName = self.processAttrMap(focus_widget.attr_map)[1]
+                feeds = tui.cache.getFeeds(categoryId, tui.show_read)
+                self.fill(feeds, False)
                 focus_widget, idx = self.get_focus()
                 self.setFeedArticles(focus_widget.attr_map[None])
                 tui.leftBox.set_title(categoryName)
-            return
-        elif key in ("h", "left"):
-            if not self.isCategoryView:
-                self.isCategoryView = True
-                items = [
-                    urwid.AttrMap(
-                        urwid.Columns(
-                            [(16, urwid.Text(category["date"])),
-                             urwid.Text(category["name"]),
-                             (5, urwid.Text(str(category["count"])))]),
-                        (category["id"],
-                         category["name"],
-                         category["count"]),
-                        "reveal focus") for category in tui.fetcher.categories]
-                walker = urwid.SimpleFocusListWalker(items)
-                self.body = walker
-                tui.leftBox.set_title("Categories")
-                self.set_focus(self.categoryPosition)
-                focus_widget, idx = self.get_focus()
-                tui.articles = self.getArticlesFromCategory(focus_widget.attr_map[None][0])
-                tui.articleView.fill(tui.articles)
-                self.setArticlesPaneTitle(self.currentCategory[13:])
-            return
+        except BaseException:
+            pass
 
-        return super().keypress(size, key)
+    def markAsRead(self):
+        try:
+            focus_widget, idx = self.get_focus()
+            if self.processAttrMap(focus_widget.attr_map)[0] == "Favorites":
+                return
+            del self.body[idx]
+            if idx > 0:
+                idx -= 1
+                self.set_focus(idx)
+            tui.cache.markStreamAsRead(self.processAttrMap(focus_widget.attr_map)[0])
+            tui.categories = tui.cache.getCategories(tui.show_read)
+            focus_widget, idx = self.get_focus()
+            if focus_widget is None and not self.isCategoryView:
+                self.stepOut()
+                return
+            if self.isCategoryView:
+                self.currentCategory = self.processAttrMap(focus_widget.attr_map)[0]
+                self.setCategoryArticles(self.processAttrMap(focus_widget.attr_map))
+            else:
+                self.setFeedArticles(focus_widget.attr_map[None])
+        except BaseException:
+            pass
 
 
 class RightPane(urwid.ListBox):
@@ -136,15 +194,30 @@ class RightPane(urwid.ListBox):
         self.article = None
         self.chunkNumber = 0
 
-    def fill(self, articles):
+    def fill(self, articles, isCategoryView):
+
+        def makeColumns(article):
+            if tui.show_read:
+                style_text = "item_read"
+                style_feed = "feed_read"
+            else:
+                style_text = "item"
+                style_feed = "feed"
+            title = article[1]
+            if isCategoryView:
+                title = [(style_text, title), (style_feed, " — " + article[-1])]
+            cols = [
+                     (16, urwid.Text([(style_text, article[3])])),
+                     urwid.Text([(style_text, title)]),
+                   ]
+            return cols
+
         items = [
             urwid.AttrMap(
-                urwid.Columns(
-                    [(2, urwid.Text("")),
-                     (16, urwid.Text(Utils.timestampToDate(article["timestampUsec"]))),
-                     urwid.Text(article["title"])]),
-                article["id"],
-                "reveal focus") for article in articles]
+                urwid.Columns(makeColumns(article)),
+                article[0],
+                {"feed_read": "reveal focus", "feed": "reveal focus", None: "reveal focus", "item": "reveal focus",
+                 "item_read": "reveal focus"}) for article in articles]
         walker = urwid.SimpleListWalker(items)
         self.body = walker
 
@@ -163,40 +236,108 @@ class RightPane(urwid.ListBox):
                 self.body = walker
                 self.setArticleTitle()
             item_size = len(self.body)
-            focus_widget, idx = self.get_focus()
-            if idx < item_size - 1:
-                idx = idx + 1
-                self.set_focus(idx)
+            try:
+                focus_widget, idx = self.get_focus()
+                if idx < item_size - 1:
+                    idx = idx + 1
+                    self.set_focus(idx)
+            except BaseException:
+                pass
             return
         elif key in ("k", "up"):
             if not self.isList:
                 walker = urwid.SimpleListWalker([urwid.Text(self.article.scrollUp())])
                 self.body = walker
                 self.setArticleTitle()
-            focus_widget, idx = self.get_focus()
-            if idx > 0:
-                idx = idx - 1
-                self.set_focus(idx)
+            try:
+                focus_widget, idx = self.get_focus()
+                if idx > 0:
+                    idx = idx - 1
+                    self.set_focus(idx)
+            except BaseException:
+                pass
+            return
+        elif key in ("f"):
+            if self.isList is True:
+                article_widget, article_idx = self.get_focus()
+                articleId = article_widget.attr_map[None]
+                tui.cache.toggleArticleStarred(articleId)
+                tui.categories = tui.cache.getCategories(tui.show_read)
+                item_widget, item_idx = tui.feedView.get_focus()
+                if tui.feedView.isCategoryView:
+                    tui.feedView.fill(tui.categories, tui.feedView.isCategoryView)
+                    tui.feedView.set_focus(item_idx)
+                    if item_idx == 0:
+                        tui.feedView.setCategoryArticles(('Favorites', 'Favorites'))
+
+        elif key in ("r"):
+            if self.isList is True:
+                feeds = []
+                article_widget, article_idx = self.get_focus()
+                articleId = article_widget.attr_map[None]
+                tui.cache.toggleArticleStatus(articleId)
+                item_widget, item_idx = tui.feedView.get_focus()
+                itemAttrMap = item_widget.attr_map[None]
+                if tui.feedView.isCategoryView:
+                    tui.feedView.setCategoryArticles(itemAttrMap)
+                else:
+                    tui.feedView.setFeedArticles(itemAttrMap)
+                if article_idx > 0:
+                    article_idx -= 1
+                tui.categories = tui.cache.getCategories(tui.show_read)
+                if tui.feedView.isCategoryView:
+                    feeds = tui.categories
+                else:
+                    feeds = tui.cache.getFeeds(tui.feedView.currentCategory, tui.show_read)
+                tui.feedView.fill(feeds, tui.feedView.isCategoryView)
+                new_idx = tui.feedView.findById(itemAttrMap[0])
+                try:
+                    tui.feedView.set_focus(new_idx)
+                    focus_widget, idx = tui.feedView.get_focus()
+                    if tui.feedView.isCategoryView:
+                        tui.feedView.currentCategory = tui.feedView.processAttrMap(focus_widget.attr_map)[0]
+                        tui.feedView.setCategoryArticles(tui.feedView.processAttrMap(focus_widget.attr_map))
+                    else:
+                        tui.feedView.setFeedArticles(focus_widget.attr_map[None])
+                    try:
+                        self.set_focus(article_idx)
+                    except BaseException:
+                        pass
+                except BaseException:
+                    pass
             return
         elif key in ("l", "right"):
             if self.isList is True:
                 self.isList = False
                 self.chunkNumber = 0
-                focus_widget, idx = self.get_focus()
-                self.articlePosition = idx
-                articleId = focus_widget.attr_map[None]
-                self.article = Article(self.getArticle(articleId))
+                try:
+                    focus_widget, idx = self.get_focus()
+                    self.articlePosition = idx
+                    articleId = focus_widget.attr_map[None]
+                    self.article = Render(*tui.cache.getArticle(articleId, tui.feedView.currentCategory == "Favorites"),
+                                          tui.cache.getArticleLinks(articleId))
+                    walker = urwid.SimpleListWalker([urwid.Text(self.article.firstPage)])
+                    self.body = walker
+                    self.setArticleTitle()
+                except BaseException:
+                    pass
+            return
+        elif key in ("c"):
+            if self.isList is False:
+                self.article.loadComments()
                 walker = urwid.SimpleListWalker([urwid.Text(self.article.firstPage)])
                 self.body = walker
-                self.setArticleTitle()
             return
         elif key in ("h", "left"):
-            if self.isList is False:
-                self.isList = True
-                self.fill(tui.articles)
-                focusFeed, idx = tui.feedView.get_focus()
-                tui.rightBox.set_title(focusFeed.attr_map[None][1])
-                self.set_focus(self.articlePosition)
+            try:
+                if self.isList is False:
+                    self.isList = True
+                    self.fill(tui.articles, tui.feedView.isCategoryView)
+                    focusFeed, idx = tui.feedView.get_focus()
+                    tui.rightBox.set_title(tui.feedView.processAttrMap(focusFeed.attr_map)[1])
+                    self.set_focus(self.articlePosition)
+            except BaseException:
+                pass
             return
         elif key == "m":
             if self.isList is False and len(self.article.links) > 0:
@@ -219,9 +360,12 @@ class Links(urwid.ListBox):
         self.body = walker
 
     def parseLink(self, link):
-        ext = link.split(".")[-1]
-        if Utils.checkPic(ext.lower()):
-            os.system('nohup feh ' + link + ' </dev/null >/dev/null 2>&1 &')
+        ext = Utils.checkPic(link.split(".")[-1]).lower()
+        if ext:
+            fc = FileCache(link, ext)
+            fp = fc.fetch()
+            if fp is not None:
+                os.system('nohup feh ' + fp + ' </dev/null >/dev/null 2>&1 &')
         elif Utils.checkStreamingVideo(link):
             tui.destroyOverlay()
             os.system(
@@ -249,7 +393,7 @@ class Links(urwid.ListBox):
             focus_widget, idx = self.get_focus()
             link = focus_widget.attr_map[None]
             self.parseLink(link)
-        elif key == "q":
+        elif key in ("q", "esc"):
             tui.destroyOverlay()
             return
 
@@ -260,8 +404,12 @@ class TUI(urwid.Frame):
     def create(cls):
 
         tui = cls()
-        palette = [("linebox", "dark blue", "black"), ("text", "dark cyan", "dark cyan"),
-                   ('header', 'white', 'black'), ('reveal focus', 'black', 'dark cyan', 'standout')]
+        palette = [("linebox", "bold", "dark cyan", "standout"), ("text", "dark cyan", "dark cyan"),
+                   ("favorite", "dark green", "black"),
+                   ("feed", "bold", "black"),
+                   ("feed_read", "bold, dark blue", "black"),
+                   ("item", "white", "black"),
+                   ('item_read', 'dark blue', 'black'), ('reveal focus', 'black', 'dark cyan', 'standout')]
         loop = urwid.MainLoop(
             tui,
             palette,
@@ -279,14 +427,17 @@ class TUI(urwid.Frame):
             URL = config["server"]["URL"]
             token = config["server"]["token"]
 
+        self.show_read = 0
         self.overlay = None
         self.fetcher = Fetcher(URL, token)
+        self.cache = Cache(self.fetcher)
+        self.categories = self.cache.getCategories(self.show_read)
 
         self.leftPaneItems = {}
         self.activePane = False
         self.executor = ThreadPoolExecutor(max_workers=1)
         self.loop = None
-        self.feedView = LeftPane(self.fetcher.categories)
+        self.feedView = LeftPane([])
         self.articleView = RightPane([])
         self.leftBox = urwid.LineBox(self.feedView, title="Categories")
         self.rightBox = urwid.LineBox(self.articleView, title="Articles")
@@ -298,14 +449,24 @@ class TUI(urwid.Frame):
 
         super().__init__(self.body)
 
+    def initialize_panes(self):
+        try:
+            self.feedView.fill(self.cache.getCategories(self.show_read), True)
+            try:
+                self.feedView.set_focus(1)
+            except BaseException:
+                pass
+            focus_widget, idx = self.feedView.get_focus()
+            item = self.feedView.processAttrMap(focus_widget.attr_map)[0]
+            name = self.feedView.processAttrMap(focus_widget.attr_map)[1]
+            self.articles = self.cache.getArticlesFromCategory(item, self.show_read)
+            self.articleView.fill(self.articles, True)
+            self.feedView.setArticlesPaneTitle(name)
+        except BaseException:
+            pass
+
     def run(self):
-        focus_widget, idx = self.feedView.get_focus()
-        item = focus_widget.attr_map[None][0]
-        name = focus_widget.attr_map[None][1]
-        self.fetcher.articlesFromCategory(item, str(focus_widget.attr_map[None][2]))
-        self.articles = tui.fetcher.articles[item]
-        self.articleView.fill(self.articles)
-        self.feedView.setArticlesPaneTitle(name)
+        self.initialize_panes()
         self.loop.run()
         self.executor.shutdown(wait=False)
 
@@ -327,17 +488,40 @@ class TUI(urwid.Frame):
                 tui.articleView.isList = True
         elif key == "q":
             raise urwid.ExitMainLoop()
-        elif key == "r":
+        elif key == "S":
             olb = urwid.ListBox(urwid.SimpleListWalker([urwid.Text("")]))
             overlay = urwid.Overlay(
                 urwid.LineBox(olb),
                 self.body, align="center", width=("relative", 50), valign="middle", height=3)
             self.body = overlay
-            articles = tui.fetcher.fetch()
+            articles = tui.cache.refresh()
             for text in articles:
                 olb.body = urwid.SimpleListWalker([urwid.Text(text)])
                 tui.loop.entering_idle()
             self.body = overlay.bottom_w
+            self.initialize_panes()
+        elif key == "R":
+            self.show_read = int(not self.show_read)
+            focus_widget, idx = self.feedView.get_focus()
+            if self.feedView.isCategoryView:
+                if focus_widget.attr_map.get("attrs") is not None:
+                    attr_map = focus_widget.attr_map["attrs"]
+                else:
+                    attr_map = focus_widget.attr_map[None]
+                self.feedView.setCategoryArticles(attr_map)
+            else:
+                self.feedView.setFeedArticles(focus_widget.attr_map[None])
+            self.categories = tui.cache.getCategories(self.show_read)
+            feeds = tui.cache.getFeeds(self.feedView.currentCategory, self.show_read)
+            if self.feedView.isCategoryView:
+                self.feedView.fill(self.categories, True)
+            else:
+                self.feedView.fill(feeds, False)
+            try:
+                self.feedView.set_focus(idx)
+            except BaseException:
+                pass
+            return
 
 
 tui = TUI.create()

Cache.py

@@ -0,0 +1,230 @@
+import sqlite3
+from sqlite3 import Error
+import os
+import functools
+import operator
+import re
+from Render import Article
+import Utils
+
+
+class Cache:
+    def __init__(self, api):
+        self.conn = None
+        self.api = api
+        file_path = os.path.expanduser("~") + '/.config/inomnibus/cache.db'
+        create_categories = """create table if not exists categories (
+                                   id text primary key,
+                                   name text not null,
+                                   timestamp integer,
+                                   date text
+                               )
+                            """
+        create_feeds = """create table if not exists feeds (
+                              id text primary key,
+                              name text not null,
+                              timestamp integer,
+                              date text,
+                              category_id text,
+                              foreign key (category_id) references categories (id)
+                          )
+                       """
+        create_articles = """create table if not exists articles (
+                              id text primary key,
+                              title text not null,
+                              timestamp integer,
+                              date text,
+                              content text,
+                              url text,
+                              is_read boolean,
+                              origin text,
+                              category_id text,
+                              foreign key (category_id) references categories (id),
+                              foreign key (origin) references feeds (id)
+                          )
+                       """
+        create_favorites = """create table if not exists favorites (
+                              id text primary key,
+                              title text not null,
+                              timestamp integer,
+                              date text,
+                              content text,
+                              url text,
+                              origin text,
+                              foreign key (origin) references feeds (id)
+                          )
+                       """
+        create_links = """create table if not exists links (
+                              id text,
+                              url text not null,
+                              foreign key (id) references articles (id)
+                          )
+                       """
+        try:
+            self.conn = sqlite3.connect(file_path)
+            self.conn.cursor().execute(create_categories)
+            self.conn.cursor().execute(create_feeds)
+            self.conn.cursor().execute(create_articles)
+            self.conn.cursor().execute(create_links)
+            self.conn.cursor().execute(create_favorites)
+
+        except Error as e:
+            Utils.writeLog(e)
+
+    def markStreamAsRead(self, streamId):
+        cur = self.conn.cursor()
+        if re.search("^feed/[0-9]+", streamId):
+            table = "feeds"
+        else:
+            table = "categories"
+        cur.execute("""select timestamp from """ + table + """ where id = ?""", (streamId,))
+        ts = cur.fetchone()[0]
+        if self.api.markStreamAsRead(streamId, ts):
+            if table == "feeds":
+                cur.execute("""update articles set is_read = 1 where origin = ?""", (streamId,))
+            else:
+                cur.execute("""update articles set is_read = 1 where category_id = ?""", (streamId,))
+            self.conn.commit()
+
+    def getArticle(self, id, is_fav):
+        cur = self.conn.cursor()
+        if is_fav:
+            cur.execute("""select title,content,url from favorites where id = ?""", (id,))
+        else:
+            cur.execute("""select title,content,url from articles where id = ?""", (id,))
+        return cur.fetchone()
+
+    def toggleArticleStatus(self, id):
+        cur = self.conn.cursor()
+        cur.execute("""update articles set is_read = not is_read where id = ?""", (id,))
+        cur.execute("""select is_read from articles where id = ?""", (id,))
+        is_read = cur.fetchone()
+        if self.api.toggleArticleStatus(id, is_read):
+            self.conn.commit()
+        else:
+            self.conn.rollback()
+
+    def toggleArticleStarred(self, id):
+        cur = self.conn.cursor()
+        is_favorite = 1
+        cur.execute("""select id from favorites where id = ?""", (id,))
+        if bool(cur.fetchone()):
+            cur.execute("""delete from favorites where id = ?""", (id,))
+            is_favorite = 0
+        else:
+            cur.execute("""select title, timestamp, date, content, url, origin from articles where id = ?""", (id,))
+            title, timestamp, date, content, url, origin = cur.fetchone()
+            cur.execute("""insert into favorites(id, title, timestamp, date, content, url, origin) values(:id, :title, :timestamp, :date
+                        ,:content, :url, :origin)""", {"id": id, "title": title, "timestamp": timestamp, "date": date, "content": content,
+                                                       "url": url, "origin": origin})
+        if self.api.toggleArticleStarred(id, is_favorite):
+            self.conn.commit()
+        else:
+            self.conn.rollback()
+
+    def getArticleLinks(self, id):
+        cur = self.conn.cursor()
+        cur.execute("""select url from links where id = ?""", (id,))
+        links = functools.reduce(operator.iconcat, cur.fetchall(), [])
+        return links
+
+    def getArticlesFromFeed(self, feed_id, is_read):
+        cur = self.conn.cursor()
+        cur.execute("""select * from articles where origin = ? and is_read = ? order by timestamp desc""", (feed_id, is_read,))
+        return cur.fetchall()
+
+    def getArticlesFromCategory(self, category_id, is_read):
+        cur = self.conn.cursor()
+        if category_id == "Favorites":
+            cur.execute("""select *, (select name from feeds where id = origin) as origin_name from favorites order by timestamp desc""")
+        else:
+            cur.execute("""select *, (select name from feeds where id = origin) as origin_name from articles where
+                        category_id = ? and is_read = ? order by timestamp desc""", (category_id, is_read,))
+        return cur.fetchall()
+
+    def getCategories(self, show_read):
+        cur = self.conn.cursor()
+        # statement = """select * from categories where id = 'Favorites'"""
+        statement = """select id, name,
+                    (select count(*) from favorites) as unread_count, timestamp, date from categories
+                    where id = 'Favorites' order by timestamp desc"""
+        cur.execute(statement)
+        favorites = cur.fetchone()
+        statement = """select id, name,
+                    (select count(*) from articles where category_id = categories.id and is_read = ?) as unread_count, timestamp, date
+                    from categories
+                    where unread_count != 0 and id != 'Favorites' order by timestamp desc"""
+        cur.execute(statement, (show_read,))
+        return [favorites, *cur.fetchall()]
+
+    def getFeeds(self, category_id, show_read):
+        statement = """select id, name, (select count(*) from articles where origin = feeds.id and is_read = ?) as unread_count,
+        timestamp, date, category_id
+        from feeds
+        where category_id = ? and unread_count != 0 order by timestamp desc"""
+        cur = self.conn.cursor()
+        cur.execute(statement, (show_read, category_id))
+        return cur.fetchall()
+
+    def refresh(self):  # noqa
+        timestamps = {}
+        cur = self.conn.cursor()
+        cur.execute("""select id, timestamp from categories""")
+        for row in cur.fetchall():
+            timestamps[row[0]] = row[1]
+
+        self.api.refresh()
+        for item in self.api.unreadCounts.keys():
+            if item[0:13] != "user/-/label/":
+                continue
+            data = self.api.unreadCounts[item]
+            cur.execute("""insert into categories(id,name,timestamp,date) values(:id,:name,:ts,:d)
+                        on conflict(id) do update set timestamp = :ts, date = :d;
+                        """,
+                        {"id": item, "name": item[13:], "ts": data[1], "d": data[2]})
+        self.conn.commit()
+        for item in self.api.feeds:
+            data = self.api.unreadCounts[item["id"]]
+            cur.execute("""insert into feeds(id,name,timestamp,date,category_id) values(:id,:name,:ts,:d,:c_id)
+                        on conflict(id) do update set timestamp = :ts, date = :d,category_id = :c_id;
+                        """,
+                        {"id": item["id"], "name": item["title"], "ts": data[1], "d": data[2],
+                            "c_id": item["categories"][0]["id"]})
+        self.conn.commit()
+
+        yield "Fetching Favorites"
+        cur.execute("""select timestamp from categories where id = 'Favorites'""")
+        favoritesTimestampTuple = cur.fetchone()
+        if favoritesTimestampTuple is None:
+            cur.execute("""insert into categories(id,name,timestamp,date) values('Favorites','Favorites',0,'')""")
+            self.conn.commit()
+        last_updated, last_updated_date, favorites = self.api.getFavorites()
+        cur.execute("""update categories set timestamp = :ts, date = :d where id = 'Favorites'""",
+                    {"ts": last_updated, "d": last_updated_date})
+        self.conn.commit()
+        for favorite in favorites:
+            favObj = Article(favorite)
+            cur.execute("""insert or ignore into favorites(id,title,timestamp,date,content,url,origin) values(:id,:t,:ts,:d,:c,:u,:o)""",
+                        {"id": favObj.id, "t": favObj.title, "ts": favObj.timestamp, "d": favObj.date,
+                         "c": favObj.text, "u": favObj.url, "o": favObj.origin})
+            for link in favObj.links:
+                cur.execute("""insert or ignore into links(id,url) values(?,?)""", (favObj.id, link))
+        self.conn.commit()
+
+        cur.execute("""select id, name from categories where id != 'Favorites'""")
+        for row in cur.fetchall():
+            yield "Fetching category " + row[1]
+            articles = self.api.articlesFromCategory(row[0])
+            for article in articles:
+                articleObj = Article(article)
+                cur.execute("""insert into articles(id,title,timestamp,date,content,url,is_read,origin,category_id)
+                            values(:id,:t,:ts,:d,:c,:u,:r,:o,:c_id) on conflict(id) do update
+                            set id = :id, title = :t, timestamp = :ts, date = :d, content = :c, url = :u, is_read = :r, category_id = :c_id,
+                            origin = :o;
+                            """,
+                            {"id": articleObj.id, "t": articleObj.title, "ts": articleObj.timestamp, "d": articleObj.date,
+                             "c": articleObj.text, "u": articleObj.url, "r": articleObj.is_read, "o": articleObj.origin, "c_id": row[0]})
+                for link in articleObj.links:
+                    cur.execute("""insert into links(id,url) values(?,?)""", (articleObj.id, link))
+
+            self.conn.commit()

FileCache.py

@@ -0,0 +1,29 @@
+import httpx
+import os
+import Utils
+import hashlib
+
+
+class FileCache:
+
+    def __init__(self, link, ext):
+        self.link = link
+        path = os.path.expanduser("~") + "/.config/inomnibus/file_cache/"
+        h = hashlib.sha1()
+        h.update(self.link.encode())
+        self.file_path = path + h.hexdigest() + "." + ext
+
+    def download(self):
+        try:
+            file = httpx.get(self.link, follow_redirects=True)
+            open(self.file_path, "wb").write(file.content)
+            return self.file_path
+        except BaseException as e:
+            Utils.writeLog(str(e))
+            return None
+
+    def fetch(self):
+        if os.path.isfile(self.file_path):
+            return self.file_path
+        else:
+            return self.download()

RedditCommentsParser.py

@@ -1,28 +1,20 @@
-from bs4 import BeautifulSoup
-import httpx
+import praw
+from datetime import datetime
 
 
 class RedditComments:
     def __init__(self, link):
-        page = httpx.get(link)
-        content = page.text
-        self.soup = BeautifulSoup(content)
-        self.commentObjects = self.soup.find_all("div", "Comment")
-        self.comments = []
-
-    def getHeader(self, commentObj):
-        headers = commentObj.find_all("a")
-        username = headers[0]["href"].split("/")[2]
-        date = headers[1].text
-        return username + " " + date
+        self.reddit = praw.Reddit(client_id='unBoGZxgkQSk8KsKWr_jag',
+                                  client_secret=None,
+                                  redirect_uri='http://localhost:8888',
+                                  user_agent='agent',
+                                  check_for_async=False)
 
-    def getText(self, commentObj):
-        p = commentObj.find("p")
-        if p is not None:
-            return p.text
-        else:
-            return ""
+        self.comments = []
+        self.link = link
 
     def getComments(self):
-        for co in self.commentObjects:
-            self.comments.append(self.getHeader(co) + "\n" + self.getText(co) + "\n")
+        submission = self.reddit.submission(url=self.link)
+        for comment in submission.comments:
+            comment_date = datetime.utcfromtimestamp(comment.created_utc).strftime('%Y-%m-%d %H:%M:%S')
+            self.comments.append(str(comment.author) + " " + str(comment_date) + "\n" + str(comment.body) + "\n")

Render.py

@@ -5,31 +5,16 @@ from bs4 import BeautifulSoup
 from RedditCommentsParser import RedditComments
 
 
-class Article:
-    def __init__(self, articleObj):
-        content = articleObj["summary"]["content"]
-        soup = BeautifulSoup(content)
-        links = soup.find_all(href=True)
-        media = soup.find_all(src=True)
-        links_set = set()
-        for link in links:
-            links_set.add(link['href'])
-        for m in media:
-            links_set.add(m['src'])
-        self.links = list(links_set)
-        self.text = get_text(content)
-        self.title = articleObj["title"]
-        self.date = Utils.timestampToDate(articleObj["timestampUsec"])
-        self.url = articleObj["canonical"][0]["href"]
-        if Utils.checkStreamingVideo(self.url):
-            self.links.append(self.url)
-        elif Utils.checkReddit(self.url):
-            comments_link = Utils.checkRedditComments(self.links)
-            if comments_link:
-                commentsObj = RedditComments(comments_link)
-                commentsObj.getComments()
-                for comment in commentsObj.comments:
-                    self.text += "\n\n" + comment
+class Render:
+    def __init__(self, title, content, url, links):
+        self.title = title
+        self.links = links
+        self.content = content
+        self.splitByPages()
+        self.url = url
+        self.areCommentsLoaded = False
+
+    def splitByPages(self):
         self.currentPageNumber = 1
         terminal_width, terminal_height = os.get_terminal_size()
         terminal_width -= 76
@@ -39,7 +24,7 @@ class Article:
         self.chunks = []
         i = 0
         column_position = 0
-        for s in self.text:
+        for s in self.content:
             i += 1
             column_position += 1
             if column_position > terminal_width or s == "\n":
@@ -47,14 +32,26 @@ class Article:
                 column_position = 0
                 if rows_passed > terminal_height - 2:
                     end_of_chunk = i
-                    self.chunks.append(self.text[start_of_chunk:end_of_chunk])
+                    self.chunks.append(self.content[start_of_chunk:end_of_chunk])
                     start_of_chunk = end_of_chunk
                     rows_passed = 0
         if end_of_chunk <= i:
-            self.chunks.append(self.text[start_of_chunk:i])
+            self.chunks.append(self.content[start_of_chunk:i])
         self.firstPage = self.chunks[0]
         self.numberOfPages = len(self.chunks)
 
+    def loadComments(self):
+        if self.areCommentsLoaded is False:
+            self.areCommentsLoaded = True
+            if Utils.checkReddit(self.url):
+                comments_link = Utils.checkRedditComments(self.links)
+                if comments_link:
+                    commentsObj = RedditComments(comments_link)
+                    commentsObj.getComments()
+                    for comment in commentsObj.comments:
+                        self.content += "\n\n" + comment
+            self.splitByPages()
+
     def scrollDown(self):
         if self.currentPageNumber == self.numberOfPages:
             pass
@@ -68,3 +65,27 @@ class Article:
         else:
             self.currentPageNumber -= 1
         return self.chunks[self.currentPageNumber - 1]
+
+
+class Article:
+    def __init__(self, articleJSON):
+        content = articleJSON["summary"]["content"]
+        soup = BeautifulSoup(content)
+        links = soup.find_all(href=True)
+        media = soup.find_all(src=True)
+        links_set = set()
+        for link in links:
+            links_set.add(link['href'])
+        for m in media:
+            links_set.add(m['src'])
+        self.links = list(links_set)
+        self.text = get_text(content)
+        self.title = articleJSON["title"]
+        self.timestamp = articleJSON["timestampUsec"]
+        self.date = Utils.timestampToDate(self.timestamp)
+        self.url = articleJSON["canonical"][0]["href"]
+        self.is_read = "user/-/state/com.google/read" in articleJSON["categories"]
+        self.id = articleJSON["id"]
+        self.origin = articleJSON["origin"]["streamId"]
+        if Utils.checkStreamingVideo(self.url):
+            self.links.append(self.url)

Utils.py

@@ -12,7 +12,7 @@ def timestampToDate(ts):
 def checkPic(ext):
     for p in pics:
         if re.search(p, ext) is not None:
-            return True
+            return p[1:p.find("\\")]
     return False
 
 
@@ -36,4 +36,4 @@ def checkRedditComments(links):
 
 def writeLog(text):
     with open("debug.log", "a") as f:
-        f.write(str(text))
+        f.write(str(text)+"\n")

requirements.txt

@@ -0,0 +1,23 @@
+anyio==3.6.2
+beautifulsoup4==4.11.1
+certifi==2022.9.24
+charset-normalizer==2.1.1
+commonmark==0.9.1
+h11==0.14.0
+httpcore==0.16.2
+httpx==0.23.1
+idna==3.4
+importlib-metadata==4.13.0
+inscriptis==2.3.1
+lxml==4.9.1
+nanoid==2.0.0
+Pygments==2.13.0
+PyYAML==6.0
+rfc3986==1.5.0
+rich==12.6.0
+sniffio==1.3.0
+soupsieve==2.3.2.post1
+typing-extensions==4.4.0
+urllib3==1.26.12
+urwid==2.1.2
+zipp==3.10.0