API.py

@@ -7,17 +7,18 @@ class Fetcher:
         self.URL = URL
         self.token = token
         self.headers = {"Authorization": "GoogleLogin auth="+token}
-        self.articles = {}
-
-    def refresh(self):
         self.getUnreadCounts()
         self.getSubscriptions()
+        self.articles = {}
 
     def getUnreadCounts(self):
         response = httpx.get(self.URL+"/reader/api/0/unread-count?output=json", headers=self.headers)
         result = dict([(item["id"], (item["count"], item["newestItemTimestampUsec"], Utils.timestampToDate(
             item["newestItemTimestampUsec"]))) for item in response.json()["unreadcounts"]])
         self.unreadCounts = result
+        self.categories = sorted([{"id": item, "name": item[13:], "count": self.unreadCounts[item][0], "date": self.unreadCounts[item][2]}
+                                 for item in self.unreadCounts.keys() if item[0:13] == "user/-/label/"],
+                                 key=lambda item: item["date"], reverse=True)
 
     def getSubscriptions(self):
         response = httpx.get(self.URL+"/reader/api/0/subscription/list?output=json", headers=self.headers)
@@ -26,42 +27,28 @@ class Fetcher:
     def checkCategory(self, category):
         return category in self.articles.keys()
 
-    def articlesFromCategory(self, category):
-        response = httpx.get(self.URL+"/reader/api/0/stream/contents?n=1000"+"&s="+category, headers=self.headers)
-        return response.json()["items"]
-
-    def getFavorites(self):
-        response = httpx.get(self.URL+"/reader/api/0/stream/contents?s=user/-/state/com.google/starred&n=100", headers=self.headers)
-        json = response.json()
-        return (json["updated"], Utils.timestampToDate(json["updated"]*1000000), json["items"])
+    def feedsFromCategory(self, category):
+        return sorted([item for item in self.feeds if item["categories"][0]["id"] == category and self.unreadCounts[item["id"]][0] != 0],
+                      key=lambda item: self.unreadCounts[item["id"]][2], reverse=True)
 
-    def markStreamAsRead(self, streamId, ts):
-        try:
-            response = httpx.post(self.URL+"/reader/api/0/mark-all-as-read", data={"s": streamId, "ts": ts}, headers=self.headers)
-            if response.status_code == 200:
-                return True
-            else:
-                return False
-        except BaseException:
-            return False
+    def articlesFromCategory(self, category, number):
+        if category not in self.articles:
+            response = httpx.get(self.URL+"/reader/api/0/stream/contents?n="+number+"&s="+category, headers=self.headers)
+            self.articles[category] = response.json()["items"]
 
-    def toggleArticleStatus(self, articleId, is_read):
-        return self.toggleArticleTag(articleId, is_read, "user/-/state/com.google/read")
+    async def articlesFromCategoryAsync(self, category, number):
+        if category not in self.articles:
+            response = await httpx.AsyncClient().get(self.URL+"/reader/api/0/stream/contents?n="+number +
+                                                     "&s="+category, headers=self.headers)
+            self.articles[category] = response.json()["items"]
 
-    def toggleArticleStarred(self, articleId, is_favorite):
-        return self.toggleArticleTag(articleId, is_favorite, "user/-/state/com.google/starred")
-
-    def toggleArticleTag(self, articleId, is_set, tag):
-        if is_set == 0:
-            tag_op = 'r'
-        else:
-            tag_op = 'a'
+    def articlesFromFeed(self, feed, category):
         try:
-            response = httpx.post(self.URL+"/reader/api/0/edit-tag", data={"i": articleId, tag_op: tag},
-                                  headers=self.headers)
-            if response.status_code == 200:
-                return True
-            else:
-                return False
+            return [article for article in self.articles[category] if article["origin"]["streamId"] == feed]
         except BaseException:
-            return False
+            return None
+
+    def fetch(self):
+        for category in self.categories:
+            yield "Fetching category " + category["name"]
+            self.articlesFromCategory(category["id"], str(self.unreadCounts[category["id"]][0]))

App.py

@@ -1,13 +1,12 @@
 import urwid
 import yaml
+import asyncio
 import warnings
 import subprocess
 import os
 from concurrent.futures import ThreadPoolExecutor
 from API import Fetcher
-from Cache import Cache
-from Render import Render
-from FileCache import FileCache
+from Render import Article
 import Utils
 warnings.filterwarnings("ignore")
 
@@ -16,172 +15,115 @@ class LeftPane(urwid.ListBox):
     def __init__(self, categories):
         super().__init__(self)
         items = [urwid.AttrMap(urwid.Columns([
-            (16, urwid.Text(category[4])), urwid.Text(category[1]), (5, urwid.Text(str(category[2])))]),
-                               (category[0], category[1], category[2]), "reveal focus") for category in categories]
+            (16, urwid.Text(category["date"])), urwid.Text(category["name"]), (5, urwid.Text(str(category["count"])))]),
+                               (category["id"], category["name"], category["count"]), "reveal focus") for category in categories]
         walker = urwid.SimpleListWalker(items)
         self.body = walker
         self.categoryPosition = 0
         self.isCategoryView = True
         self.currentCategory = ""
-
-    def processAttrMap(self, attrMap):
-        res = attrMap[None]
-        if res == "favorite":
-            return attrMap["attrs"]
-        else:
-            return res
-
-    def fill(self, items, is_category_view):
-        def getAttrs(item):
-            if is_category_view is True:
-                return (item[0], item[1], item[2])
-            else:
-                return (item[0], item[1])
-        if is_category_view is True:
-            items = [
-                urwid.AttrMap(
-                    urwid.Columns(
-                        [(16, urwid.Text(items[0][4])),
-                         urwid.Text(items[0][1]),
-                         (5, urwid.Text(str(items[0][2])))]),
-                    {None: "favorite", "attrs": getAttrs(items[0])},
-                    "reveal focus"),
-                *
-                [urwid.AttrMap(
-                     urwid.Columns([(16, urwid.Text(item[4])),
-                                    urwid.Text(item[1]),
-                                    (5, urwid.Text(str(item[2])))]),
-                     getAttrs(item),
-                     "reveal focus") for item in items[1:]]]
-        else:
-            items = [urwid.AttrMap(urwid.Columns([
-                (16, urwid.Text(item[4])), urwid.Text(item[1]), (5, urwid.Text(str(item[2])))]),
-                                getAttrs(item), "reveal focus") for item in items]
-        walker = urwid.SimpleListWalker(items)
-        self.body = walker
         focus_widget, idx = self.get_focus()
         if self.isCategoryView:
-            try:
             self.currentCategory = focus_widget.attr_map[None][0]
-            except BaseException:
-                self.currentCategory = None
-
-    def findById(self, id):
-        idx = 0
-        for idx, item in zip(range(len(self.body)), self.body):
-            if item.attr_map[None][0] == id:
-                break
-        return idx
 
     def setArticlesPaneTitle(self, text):
         tui.rightBox.set_title(text)
 
-    def setCategoryArticles(self, attrMap):
-        Utils.writeLog(attrMap)
+    def getArticlesFromCategory(self, category, number=0):
+        tui.fetcher.articlesFromCategory(category, str(number))
+        articles = tui.fetcher.articles[category]
+        return articles
+
+    def getArticlesFromFeed(self, feed):
+        return tui.fetcher.articlesFromFeed(feed, self.currentCategory)
+
+    async def setCategoryArticles(self, attrMap):
         itemId = attrMap[0]
+        number = attrMap[2]
         name = attrMap[1]
-        tui.articles = tui.cache.getArticlesFromCategory(itemId, tui.show_read)
-        tui.articleView.fill(tui.articles, True)
+        await tui.fetcher.articlesFromCategoryAsync(itemId, str(number))
+        focus_widget, idx = self.get_focus()
+        currentCategory = focus_widget.attr_map[None][0]
+        if itemId == currentCategory:
+            tui.articles = tui.fetcher.articles[currentCategory]
+            tui.articleView.fill(tui.articles)
             self.setArticlesPaneTitle(name)
 
     def setFeedArticles(self, attrMap):
         itemId = attrMap[0]
-        tui.articles = tui.cache.getArticlesFromFeed(itemId, tui.show_read)
+        tui.articles = tui.fetcher.articlesFromFeed(itemId, self.currentCategory)
         if tui.articles is not None:
-            tui.articleView.fill(tui.articles, False)
+            tui.articleView.fill(tui.articles)
             self.setArticlesPaneTitle(attrMap[1])
 
-    def keypress(self, size, key):  # noqa
+    def keypress(self, size, key):
         if key in ("j", "down"):
             item_size = len(self.body)
-            try:
             focus_widget, idx = self.get_focus()
             if idx < item_size - 1:
                 idx = idx + 1
                 self.set_focus(idx)
             focus_widget, idx = self.get_focus()
             if self.isCategoryView:
-                    self.currentCategory = self.processAttrMap(focus_widget.attr_map)[0]
-                    self.setCategoryArticles(self.processAttrMap(focus_widget.attr_map))
+                self.currentCategory = focus_widget.attr_map[None][0]
+                asyncio.create_task(self.setCategoryArticles(focus_widget.attr_map[None]))
             else:
                 self.setFeedArticles(focus_widget.attr_map[None])
-            except BaseException:
-                pass
             return
         elif key in ("k", "up"):
-            try:
             focus_widget, idx = self.get_focus()
             if idx > 0:
                 idx = idx - 1
                 self.set_focus(idx)
             focus_widget, idx = self.get_focus()
             if self.isCategoryView:
-                    self.currentCategory = self.processAttrMap(focus_widget.attr_map)[0]
-                    self.setCategoryArticles(self.processAttrMap(focus_widget.attr_map))
+                self.currentCategory = focus_widget.attr_map[None][0]
+                asyncio.create_task(self.setCategoryArticles(focus_widget.attr_map[None]))
             else:
                 self.setFeedArticles(focus_widget.attr_map[None])
-            except BaseException:
-                pass
             return
         elif key in ("l", "right"):
-            self.stepInto()
-            return
-        elif key in ("h", "left"):
-            self.stepOut()
-            return
-        elif key in ("r"):
-            self.markAsRead()
-            return
-
-        return super().keypress(size, key)
-
-    def stepOut(self):
-        if not self.isCategoryView:
-            self.isCategoryView = True
-            self.fill(tui.categories, True)
-            tui.leftBox.set_title("Categories")
-            self.set_focus(self.categoryPosition)
-            focus_widget, idx = self.get_focus()
-            self.setCategoryArticles(self.processAttrMap(focus_widget.attr_map))
-
-    def stepInto(self):
-        try:
-            if self.isCategoryView and self.currentCategory != "Favorites":
+            if self.isCategoryView and tui.fetcher.checkCategory(self.currentCategory):
                 self.isCategoryView = False
                 focus_widget, idx = self.get_focus()
                 self.categoryPosition = idx
-                categoryId = self.processAttrMap(focus_widget.attr_map)[0]
-                categoryName = self.processAttrMap(focus_widget.attr_map)[1]
-                feeds = tui.cache.getFeeds(categoryId, tui.show_read)
-                self.fill(feeds, False)
+                categoryId = focus_widget.attr_map[None][0]
+                categoryName = focus_widget.attr_map[None][1]
+                feeds = tui.fetcher.feedsFromCategory(categoryId)
+                feedItems = [urwid.AttrMap(urwid.Columns([(16, urwid.Text(tui.fetcher.unreadCounts[feed["id"]][2])),
+                                                          urwid.Text(feed["title"]),
+                                                          (5, urwid.Text(str(tui.fetcher.unreadCounts[feed["id"]][0])))]),
+                                           (feed["id"], feed["title"]), "reveal focus") for feed in feeds]
+                walker = urwid.SimpleListWalker(feedItems)
+                self.body = walker
                 focus_widget, idx = self.get_focus()
                 self.setFeedArticles(focus_widget.attr_map[None])
                 tui.leftBox.set_title(categoryName)
-        except BaseException:
-            pass
-
-    def markAsRead(self):
-        try:
-            focus_widget, idx = self.get_focus()
-            if self.processAttrMap(focus_widget.attr_map)[0] == "Favorites":
             return
-            del self.body[idx]
-            if idx > 0:
-                idx -= 1
-                self.set_focus(idx)
-            tui.cache.markStreamAsRead(self.processAttrMap(focus_widget.attr_map)[0])
-            tui.categories = tui.cache.getCategories(tui.show_read)
+        elif key in ("h", "left"):
+            if not self.isCategoryView:
+                self.isCategoryView = True
+                items = [
+                    urwid.AttrMap(
+                        urwid.Columns(
+                            [(16, urwid.Text(category["date"])),
+                             urwid.Text(category["name"]),
+                             (5, urwid.Text(str(category["count"])))]),
+                        (category["id"],
+                         category["name"],
+                         category["count"]),
+                        "reveal focus") for category in tui.fetcher.categories]
+                walker = urwid.SimpleFocusListWalker(items)
+                self.body = walker
+                tui.leftBox.set_title("Categories")
+                self.set_focus(self.categoryPosition)
                 focus_widget, idx = self.get_focus()
-            if focus_widget is None and not self.isCategoryView:
-                self.stepOut()
+                tui.articles = self.getArticlesFromCategory(focus_widget.attr_map[None][0])
+                tui.articleView.fill(tui.articles)
+                self.setArticlesPaneTitle(self.currentCategory[13:])
             return
-            if self.isCategoryView:
-                self.currentCategory = self.processAttrMap(focus_widget.attr_map)[0]
-                self.setCategoryArticles(self.processAttrMap(focus_widget.attr_map))
-            else:
-                self.setFeedArticles(focus_widget.attr_map[None])
-        except BaseException:
-            pass
+
+        return super().keypress(size, key)
 
 
 class RightPane(urwid.ListBox):
@@ -194,30 +136,15 @@ class RightPane(urwid.ListBox):
         self.article = None
         self.chunkNumber = 0
 
-    def fill(self, articles, isCategoryView):
-
-        def makeColumns(article):
-            if tui.show_read:
-                style_text = "item_read"
-                style_feed = "feed_read"
-            else:
-                style_text = "item"
-                style_feed = "feed"
-            title = article[1]
-            if isCategoryView:
-                title = [(style_text, title), (style_feed, " — " + article[-1])]
-            cols = [
-                     (16, urwid.Text([(style_text, article[3])])),
-                     urwid.Text([(style_text, title)]),
-                   ]
-            return cols
-
+    def fill(self, articles):
         items = [
             urwid.AttrMap(
-                urwid.Columns(makeColumns(article)),
-                article[0],
-                {"feed_read": "reveal focus", "feed": "reveal focus", None: "reveal focus", "item": "reveal focus",
-                 "item_read": "reveal focus"}) for article in articles]
+                urwid.Columns(
+                    [(2, urwid.Text("")),
+                     (16, urwid.Text(Utils.timestampToDate(article["timestampUsec"]))),
+                     urwid.Text(article["title"])]),
+                article["id"],
+                "reveal focus") for article in articles]
         walker = urwid.SimpleListWalker(items)
         self.body = walker
 
@@ -236,108 +163,40 @@ class RightPane(urwid.ListBox):
                 self.body = walker
                 self.setArticleTitle()
             item_size = len(self.body)
-            try:
             focus_widget, idx = self.get_focus()
             if idx < item_size - 1:
                 idx = idx + 1
                 self.set_focus(idx)
-            except BaseException:
-                pass
             return
         elif key in ("k", "up"):
             if not self.isList:
                 walker = urwid.SimpleListWalker([urwid.Text(self.article.scrollUp())])
                 self.body = walker
                 self.setArticleTitle()
-            try:
             focus_widget, idx = self.get_focus()
             if idx > 0:
                 idx = idx - 1
                 self.set_focus(idx)
-            except BaseException:
-                pass
-            return
-        elif key in ("f"):
-            if self.isList is True:
-                article_widget, article_idx = self.get_focus()
-                articleId = article_widget.attr_map[None]
-                tui.cache.toggleArticleStarred(articleId)
-                tui.categories = tui.cache.getCategories(tui.show_read)
-                item_widget, item_idx = tui.feedView.get_focus()
-                if tui.feedView.isCategoryView:
-                    tui.feedView.fill(tui.categories, tui.feedView.isCategoryView)
-                    tui.feedView.set_focus(item_idx)
-                    if item_idx == 0:
-                        tui.feedView.setCategoryArticles(('Favorites', 'Favorites'))
-
-        elif key in ("r"):
-            if self.isList is True:
-                feeds = []
-                article_widget, article_idx = self.get_focus()
-                articleId = article_widget.attr_map[None]
-                tui.cache.toggleArticleStatus(articleId)
-                item_widget, item_idx = tui.feedView.get_focus()
-                itemAttrMap = item_widget.attr_map[None]
-                if tui.feedView.isCategoryView:
-                    tui.feedView.setCategoryArticles(itemAttrMap)
-                else:
-                    tui.feedView.setFeedArticles(itemAttrMap)
-                if article_idx > 0:
-                    article_idx -= 1
-                tui.categories = tui.cache.getCategories(tui.show_read)
-                if tui.feedView.isCategoryView:
-                    feeds = tui.categories
-                else:
-                    feeds = tui.cache.getFeeds(tui.feedView.currentCategory, tui.show_read)
-                tui.feedView.fill(feeds, tui.feedView.isCategoryView)
-                new_idx = tui.feedView.findById(itemAttrMap[0])
-                try:
-                    tui.feedView.set_focus(new_idx)
-                    focus_widget, idx = tui.feedView.get_focus()
-                    if tui.feedView.isCategoryView:
-                        tui.feedView.currentCategory = tui.feedView.processAttrMap(focus_widget.attr_map)[0]
-                        tui.feedView.setCategoryArticles(tui.feedView.processAttrMap(focus_widget.attr_map))
-                    else:
-                        tui.feedView.setFeedArticles(focus_widget.attr_map[None])
-                    try:
-                        self.set_focus(article_idx)
-                    except BaseException:
-                        pass
-                except BaseException:
-                    pass
             return
         elif key in ("l", "right"):
             if self.isList is True:
                 self.isList = False
                 self.chunkNumber = 0
-                try:
                 focus_widget, idx = self.get_focus()
                 self.articlePosition = idx
                 articleId = focus_widget.attr_map[None]
-                    self.article = Render(*tui.cache.getArticle(articleId, tui.feedView.currentCategory == "Favorites"),
-                                          tui.cache.getArticleLinks(articleId))
+                self.article = Article(self.getArticle(articleId))
                 walker = urwid.SimpleListWalker([urwid.Text(self.article.firstPage)])
                 self.body = walker
                 self.setArticleTitle()
-                except BaseException:
-                    pass
-            return
-        elif key in ("c"):
-            if self.isList is False:
-                self.article.loadComments()
-                walker = urwid.SimpleListWalker([urwid.Text(self.article.firstPage)])
-                self.body = walker
             return
         elif key in ("h", "left"):
-            try:
             if self.isList is False:
                 self.isList = True
-                    self.fill(tui.articles, tui.feedView.isCategoryView)
+                self.fill(tui.articles)
                 focusFeed, idx = tui.feedView.get_focus()
-                    tui.rightBox.set_title(tui.feedView.processAttrMap(focusFeed.attr_map)[1])
+                tui.rightBox.set_title(focusFeed.attr_map[None][1])
                 self.set_focus(self.articlePosition)
-            except BaseException:
-                pass
             return
         elif key == "m":
             if self.isList is False and len(self.article.links) > 0:
@@ -360,12 +219,9 @@ class Links(urwid.ListBox):
         self.body = walker
 
     def parseLink(self, link):
-        ext = Utils.checkPic(link.split(".")[-1]).lower()
-        if ext:
-            fc = FileCache(link, ext)
-            fp = fc.fetch()
-            if fp is not None:
-                os.system('nohup feh ' + fp + ' </dev/null >/dev/null 2>&1 &')
+        ext = link.split(".")[-1]
+        if Utils.checkPic(ext.lower()):
+            os.system('nohup feh ' + link + ' </dev/null >/dev/null 2>&1 &')
         elif Utils.checkStreamingVideo(link):
             tui.destroyOverlay()
             os.system(
@@ -393,7 +249,7 @@ class Links(urwid.ListBox):
             focus_widget, idx = self.get_focus()
             link = focus_widget.attr_map[None]
             self.parseLink(link)
-        elif key in ("q", "esc"):
+        elif key == "q":
             tui.destroyOverlay()
             return
 
@@ -404,12 +260,8 @@ class TUI(urwid.Frame):
     def create(cls):
 
         tui = cls()
-        palette = [("linebox", "bold", "dark cyan", "standout"), ("text", "dark cyan", "dark cyan"),
-                   ("favorite", "dark green", "black"),
-                   ("feed", "bold", "black"),
-                   ("feed_read", "bold, dark blue", "black"),
-                   ("item", "white", "black"),
-                   ('item_read', 'dark blue', 'black'), ('reveal focus', 'black', 'dark cyan', 'standout')]
+        palette = [("linebox", "dark blue", "black"), ("text", "dark cyan", "dark cyan"),
+                   ('header', 'white', 'black'), ('reveal focus', 'black', 'dark cyan', 'standout')]
         loop = urwid.MainLoop(
             tui,
             palette,
@@ -427,17 +279,14 @@ class TUI(urwid.Frame):
             URL = config["server"]["URL"]
             token = config["server"]["token"]
 
-        self.show_read = 0
         self.overlay = None
         self.fetcher = Fetcher(URL, token)
-        self.cache = Cache(self.fetcher)
-        self.categories = self.cache.getCategories(self.show_read)
 
         self.leftPaneItems = {}
         self.activePane = False
         self.executor = ThreadPoolExecutor(max_workers=1)
         self.loop = None
-        self.feedView = LeftPane([])
+        self.feedView = LeftPane(self.fetcher.categories)
         self.articleView = RightPane([])
         self.leftBox = urwid.LineBox(self.feedView, title="Categories")
         self.rightBox = urwid.LineBox(self.articleView, title="Articles")
@@ -449,24 +298,14 @@ class TUI(urwid.Frame):
 
         super().__init__(self.body)
 
-    def initialize_panes(self):
-        try:
-            self.feedView.fill(self.cache.getCategories(self.show_read), True)
-            try:
-                self.feedView.set_focus(1)
-            except BaseException:
-                pass
+    def run(self):
         focus_widget, idx = self.feedView.get_focus()
-            item = self.feedView.processAttrMap(focus_widget.attr_map)[0]
-            name = self.feedView.processAttrMap(focus_widget.attr_map)[1]
-            self.articles = self.cache.getArticlesFromCategory(item, self.show_read)
-            self.articleView.fill(self.articles, True)
+        item = focus_widget.attr_map[None][0]
+        name = focus_widget.attr_map[None][1]
+        self.fetcher.articlesFromCategory(item, str(focus_widget.attr_map[None][2]))
+        self.articles = tui.fetcher.articles[item]
+        self.articleView.fill(self.articles)
         self.feedView.setArticlesPaneTitle(name)
-        except BaseException:
-            pass
-
-    def run(self):
-        self.initialize_panes()
         self.loop.run()
         self.executor.shutdown(wait=False)
 
@@ -488,40 +327,17 @@ class TUI(urwid.Frame):
                 tui.articleView.isList = True
         elif key == "q":
             raise urwid.ExitMainLoop()
-        elif key == "S":
+        elif key == "r":
             olb = urwid.ListBox(urwid.SimpleListWalker([urwid.Text("")]))
             overlay = urwid.Overlay(
                 urwid.LineBox(olb),
                 self.body, align="center", width=("relative", 50), valign="middle", height=3)
             self.body = overlay
-            articles = tui.cache.refresh()
+            articles = tui.fetcher.fetch()
             for text in articles:
                 olb.body = urwid.SimpleListWalker([urwid.Text(text)])
                 tui.loop.entering_idle()
             self.body = overlay.bottom_w
-            self.initialize_panes()
-        elif key == "R":
-            self.show_read = int(not self.show_read)
-            focus_widget, idx = self.feedView.get_focus()
-            if self.feedView.isCategoryView:
-                if focus_widget.attr_map.get("attrs") is not None:
-                    attr_map = focus_widget.attr_map["attrs"]
-                else:
-                    attr_map = focus_widget.attr_map[None]
-                self.feedView.setCategoryArticles(attr_map)
-            else:
-                self.feedView.setFeedArticles(focus_widget.attr_map[None])
-            self.categories = tui.cache.getCategories(self.show_read)
-            feeds = tui.cache.getFeeds(self.feedView.currentCategory, self.show_read)
-            if self.feedView.isCategoryView:
-                self.feedView.fill(self.categories, True)
-            else:
-                self.feedView.fill(feeds, False)
-            try:
-                self.feedView.set_focus(idx)
-            except BaseException:
-                pass
-            return
 
 
 tui = TUI.create()

Cache.py

@@ -1,230 +0,0 @@
-import sqlite3
-from sqlite3 import Error
-import os
-import functools
-import operator
-import re
-from Render import Article
-import Utils
-
-
-class Cache:
-    def __init__(self, api):
-        self.conn = None
-        self.api = api
-        file_path = os.path.expanduser("~") + '/.config/inomnibus/cache.db'
-        create_categories = """create table if not exists categories (
-                                   id text primary key,
-                                   name text not null,
-                                   timestamp integer,
-                                   date text
-                               )
-                            """
-        create_feeds = """create table if not exists feeds (
-                              id text primary key,
-                              name text not null,
-                              timestamp integer,
-                              date text,
-                              category_id text,
-                              foreign key (category_id) references categories (id)
-                          )
-                       """
-        create_articles = """create table if not exists articles (
-                              id text primary key,
-                              title text not null,
-                              timestamp integer,
-                              date text,
-                              content text,
-                              url text,
-                              is_read boolean,
-                              origin text,
-                              category_id text,
-                              foreign key (category_id) references categories (id),
-                              foreign key (origin) references feeds (id)
-                          )
-                       """
-        create_favorites = """create table if not exists favorites (
-                              id text primary key,
-                              title text not null,
-                              timestamp integer,
-                              date text,
-                              content text,
-                              url text,
-                              origin text,
-                              foreign key (origin) references feeds (id)
-                          )
-                       """
-        create_links = """create table if not exists links (
-                              id text,
-                              url text not null,
-                              foreign key (id) references articles (id)
-                          )
-                       """
-        try:
-            self.conn = sqlite3.connect(file_path)
-            self.conn.cursor().execute(create_categories)
-            self.conn.cursor().execute(create_feeds)
-            self.conn.cursor().execute(create_articles)
-            self.conn.cursor().execute(create_links)
-            self.conn.cursor().execute(create_favorites)
-
-        except Error as e:
-            Utils.writeLog(e)
-
-    def markStreamAsRead(self, streamId):
-        cur = self.conn.cursor()
-        if re.search("^feed/[0-9]+", streamId):
-            table = "feeds"
-        else:
-            table = "categories"
-        cur.execute("""select timestamp from """ + table + """ where id = ?""", (streamId,))
-        ts = cur.fetchone()[0]
-        if self.api.markStreamAsRead(streamId, ts):
-            if table == "feeds":
-                cur.execute("""update articles set is_read = 1 where origin = ?""", (streamId,))
-            else:
-                cur.execute("""update articles set is_read = 1 where category_id = ?""", (streamId,))
-            self.conn.commit()
-
-    def getArticle(self, id, is_fav):
-        cur = self.conn.cursor()
-        if is_fav:
-            cur.execute("""select title,content,url from favorites where id = ?""", (id,))
-        else:
-            cur.execute("""select title,content,url from articles where id = ?""", (id,))
-        return cur.fetchone()
-
-    def toggleArticleStatus(self, id):
-        cur = self.conn.cursor()
-        cur.execute("""update articles set is_read = not is_read where id = ?""", (id,))
-        cur.execute("""select is_read from articles where id = ?""", (id,))
-        is_read = cur.fetchone()
-        if self.api.toggleArticleStatus(id, is_read):
-            self.conn.commit()
-        else:
-            self.conn.rollback()
-
-    def toggleArticleStarred(self, id):
-        cur = self.conn.cursor()
-        is_favorite = 1
-        cur.execute("""select id from favorites where id = ?""", (id,))
-        if bool(cur.fetchone()):
-            cur.execute("""delete from favorites where id = ?""", (id,))
-            is_favorite = 0
-        else:
-            cur.execute("""select title, timestamp, date, content, url, origin from articles where id = ?""", (id,))
-            title, timestamp, date, content, url, origin = cur.fetchone()
-            cur.execute("""insert into favorites(id, title, timestamp, date, content, url, origin) values(:id, :title, :timestamp, :date
-                        ,:content, :url, :origin)""", {"id": id, "title": title, "timestamp": timestamp, "date": date, "content": content,
-                                                       "url": url, "origin": origin})
-        if self.api.toggleArticleStarred(id, is_favorite):
-            self.conn.commit()
-        else:
-            self.conn.rollback()
-
-    def getArticleLinks(self, id):
-        cur = self.conn.cursor()
-        cur.execute("""select url from links where id = ?""", (id,))
-        links = functools.reduce(operator.iconcat, cur.fetchall(), [])
-        return links
-
-    def getArticlesFromFeed(self, feed_id, is_read):
-        cur = self.conn.cursor()
-        cur.execute("""select * from articles where origin = ? and is_read = ? order by timestamp desc""", (feed_id, is_read,))
-        return cur.fetchall()
-
-    def getArticlesFromCategory(self, category_id, is_read):
-        cur = self.conn.cursor()
-        if category_id == "Favorites":
-            cur.execute("""select *, (select name from feeds where id = origin) as origin_name from favorites order by timestamp desc""")
-        else:
-            cur.execute("""select *, (select name from feeds where id = origin) as origin_name from articles where
-                        category_id = ? and is_read = ? order by timestamp desc""", (category_id, is_read,))
-        return cur.fetchall()
-
-    def getCategories(self, show_read):
-        cur = self.conn.cursor()
-        # statement = """select * from categories where id = 'Favorites'"""
-        statement = """select id, name,
-                    (select count(*) from favorites) as unread_count, timestamp, date from categories
-                    where id = 'Favorites' order by timestamp desc"""
-        cur.execute(statement)
-        favorites = cur.fetchone()
-        statement = """select id, name,
-                    (select count(*) from articles where category_id = categories.id and is_read = ?) as unread_count, timestamp, date
-                    from categories
-                    where unread_count != 0 and id != 'Favorites' order by timestamp desc"""
-        cur.execute(statement, (show_read,))
-        return [favorites, *cur.fetchall()]
-
-    def getFeeds(self, category_id, show_read):
-        statement = """select id, name, (select count(*) from articles where origin = feeds.id and is_read = ?) as unread_count,
-        timestamp, date, category_id
-        from feeds
-        where category_id = ? and unread_count != 0 order by timestamp desc"""
-        cur = self.conn.cursor()
-        cur.execute(statement, (show_read, category_id))
-        return cur.fetchall()
-
-    def refresh(self):  # noqa
-        timestamps = {}
-        cur = self.conn.cursor()
-        cur.execute("""select id, timestamp from categories""")
-        for row in cur.fetchall():
-            timestamps[row[0]] = row[1]
-
-        self.api.refresh()
-        for item in self.api.unreadCounts.keys():
-            if item[0:13] != "user/-/label/":
-                continue
-            data = self.api.unreadCounts[item]
-            cur.execute("""insert into categories(id,name,timestamp,date) values(:id,:name,:ts,:d)
-                        on conflict(id) do update set timestamp = :ts, date = :d;
-                        """,
-                        {"id": item, "name": item[13:], "ts": data[1], "d": data[2]})
-        self.conn.commit()
-        for item in self.api.feeds:
-            data = self.api.unreadCounts[item["id"]]
-            cur.execute("""insert into feeds(id,name,timestamp,date,category_id) values(:id,:name,:ts,:d,:c_id)
-                        on conflict(id) do update set timestamp = :ts, date = :d,category_id = :c_id;
-                        """,
-                        {"id": item["id"], "name": item["title"], "ts": data[1], "d": data[2],
-                            "c_id": item["categories"][0]["id"]})
-        self.conn.commit()
-
-        yield "Fetching Favorites"
-        cur.execute("""select timestamp from categories where id = 'Favorites'""")
-        favoritesTimestampTuple = cur.fetchone()
-        if favoritesTimestampTuple is None:
-            cur.execute("""insert into categories(id,name,timestamp,date) values('Favorites','Favorites',0,'')""")
-            self.conn.commit()
-        last_updated, last_updated_date, favorites = self.api.getFavorites()
-        cur.execute("""update categories set timestamp = :ts, date = :d where id = 'Favorites'""",
-                    {"ts": last_updated, "d": last_updated_date})
-        self.conn.commit()
-        for favorite in favorites:
-            favObj = Article(favorite)
-            cur.execute("""insert or ignore into favorites(id,title,timestamp,date,content,url,origin) values(:id,:t,:ts,:d,:c,:u,:o)""",
-                        {"id": favObj.id, "t": favObj.title, "ts": favObj.timestamp, "d": favObj.date,
-                         "c": favObj.text, "u": favObj.url, "o": favObj.origin})
-            for link in favObj.links:
-                cur.execute("""insert or ignore into links(id,url) values(?,?)""", (favObj.id, link))
-        self.conn.commit()
-
-        cur.execute("""select id, name from categories where id != 'Favorites'""")
-        for row in cur.fetchall():
-            yield "Fetching category " + row[1]
-            articles = self.api.articlesFromCategory(row[0])
-            for article in articles:
-                articleObj = Article(article)
-                cur.execute("""insert into articles(id,title,timestamp,date,content,url,is_read,origin,category_id)
-                            values(:id,:t,:ts,:d,:c,:u,:r,:o,:c_id) on conflict(id) do update
-                            set id = :id, title = :t, timestamp = :ts, date = :d, content = :c, url = :u, is_read = :r, category_id = :c_id,
-                            origin = :o;
-                            """,
-                            {"id": articleObj.id, "t": articleObj.title, "ts": articleObj.timestamp, "d": articleObj.date,
-                             "c": articleObj.text, "u": articleObj.url, "r": articleObj.is_read, "o": articleObj.origin, "c_id": row[0]})
-                for link in articleObj.links:
-                    cur.execute("""insert into links(id,url) values(?,?)""", (articleObj.id, link))
-
-            self.conn.commit()

FileCache.py

@@ -1,29 +0,0 @@
-import httpx
-import os
-import Utils
-import hashlib
-
-
-class FileCache:
-
-    def __init__(self, link, ext):
-        self.link = link
-        path = os.path.expanduser("~") + "/.config/inomnibus/file_cache/"
-        h = hashlib.sha1()
-        h.update(self.link.encode())
-        self.file_path = path + h.hexdigest() + "." + ext
-
-    def download(self):
-        try:
-            file = httpx.get(self.link, follow_redirects=True)
-            open(self.file_path, "wb").write(file.content)
-            return self.file_path
-        except BaseException as e:
-            Utils.writeLog(str(e))
-            return None
-
-    def fetch(self):
-        if os.path.isfile(self.file_path):
-            return self.file_path
-        else:
-            return self.download()

RedditCommentsParser.py

@@ -1,20 +1,28 @@
-import praw
-from datetime import datetime
+from bs4 import BeautifulSoup
+import httpx
 
 
 class RedditComments:
     def __init__(self, link):
-        self.reddit = praw.Reddit(client_id='unBoGZxgkQSk8KsKWr_jag',
-                                  client_secret=None,
-                                  redirect_uri='http://localhost:8888',
-                                  user_agent='agent',
-                                  check_for_async=False)
-
+        page = httpx.get(link)
+        content = page.text
+        self.soup = BeautifulSoup(content)
+        self.commentObjects = self.soup.find_all("div", "Comment")
         self.comments = []
-        self.link = link
+
+    def getHeader(self, commentObj):
+        headers = commentObj.find_all("a")
+        username = headers[0]["href"].split("/")[2]
+        date = headers[1].text
+        return username + " " + date
+
+    def getText(self, commentObj):
+        p = commentObj.find("p")
+        if p is not None:
+            return p.text
+        else:
+            return ""
 
     def getComments(self):
-        submission = self.reddit.submission(url=self.link)
-        for comment in submission.comments:
-            comment_date = datetime.utcfromtimestamp(comment.created_utc).strftime('%Y-%m-%d %H:%M:%S')
-            self.comments.append(str(comment.author) + " " + str(comment_date) + "\n" + str(comment.body) + "\n")
+        for co in self.commentObjects:
+            self.comments.append(self.getHeader(co) + "\n" + self.getText(co) + "\n")

Render.py

@@ -5,16 +5,31 @@ from bs4 import BeautifulSoup
 from RedditCommentsParser import RedditComments
 
 
-class Render:
-    def __init__(self, title, content, url, links):
-        self.title = title
-        self.links = links
-        self.content = content
-        self.splitByPages()
-        self.url = url
-        self.areCommentsLoaded = False
-
-    def splitByPages(self):
+class Article:
+    def __init__(self, articleObj):
+        content = articleObj["summary"]["content"]
+        soup = BeautifulSoup(content)
+        links = soup.find_all(href=True)
+        media = soup.find_all(src=True)
+        links_set = set()
+        for link in links:
+            links_set.add(link['href'])
+        for m in media:
+            links_set.add(m['src'])
+        self.links = list(links_set)
+        self.text = get_text(content)
+        self.title = articleObj["title"]
+        self.date = Utils.timestampToDate(articleObj["timestampUsec"])
+        self.url = articleObj["canonical"][0]["href"]
+        if Utils.checkStreamingVideo(self.url):
+            self.links.append(self.url)
+        elif Utils.checkReddit(self.url):
+            comments_link = Utils.checkRedditComments(self.links)
+            if comments_link:
+                commentsObj = RedditComments(comments_link)
+                commentsObj.getComments()
+                for comment in commentsObj.comments:
+                    self.text += "\n\n" + comment
         self.currentPageNumber = 1
         terminal_width, terminal_height = os.get_terminal_size()
         terminal_width -= 76
@@ -24,7 +39,7 @@ class Render:
         self.chunks = []
         i = 0
         column_position = 0
-        for s in self.content:
+        for s in self.text:
             i += 1
             column_position += 1
             if column_position > terminal_width or s == "\n":
@@ -32,26 +47,14 @@ class Render:
                 column_position = 0
                 if rows_passed > terminal_height - 2:
                     end_of_chunk = i
-                    self.chunks.append(self.content[start_of_chunk:end_of_chunk])
+                    self.chunks.append(self.text[start_of_chunk:end_of_chunk])
                     start_of_chunk = end_of_chunk
                     rows_passed = 0
         if end_of_chunk <= i:
-            self.chunks.append(self.content[start_of_chunk:i])
+            self.chunks.append(self.text[start_of_chunk:i])
         self.firstPage = self.chunks[0]
         self.numberOfPages = len(self.chunks)
 
-    def loadComments(self):
-        if self.areCommentsLoaded is False:
-            self.areCommentsLoaded = True
-            if Utils.checkReddit(self.url):
-                comments_link = Utils.checkRedditComments(self.links)
-                if comments_link:
-                    commentsObj = RedditComments(comments_link)
-                    commentsObj.getComments()
-                    for comment in commentsObj.comments:
-                        self.content += "\n\n" + comment
-            self.splitByPages()
-
     def scrollDown(self):
         if self.currentPageNumber == self.numberOfPages:
             pass
@@ -65,27 +68,3 @@ class Render:
         else:
             self.currentPageNumber -= 1
         return self.chunks[self.currentPageNumber - 1]
-
-
-class Article:
-    def __init__(self, articleJSON):
-        content = articleJSON["summary"]["content"]
-        soup = BeautifulSoup(content)
-        links = soup.find_all(href=True)
-        media = soup.find_all(src=True)
-        links_set = set()
-        for link in links:
-            links_set.add(link['href'])
-        for m in media:
-            links_set.add(m['src'])
-        self.links = list(links_set)
-        self.text = get_text(content)
-        self.title = articleJSON["title"]
-        self.timestamp = articleJSON["timestampUsec"]
-        self.date = Utils.timestampToDate(self.timestamp)
-        self.url = articleJSON["canonical"][0]["href"]
-        self.is_read = "user/-/state/com.google/read" in articleJSON["categories"]
-        self.id = articleJSON["id"]
-        self.origin = articleJSON["origin"]["streamId"]
-        if Utils.checkStreamingVideo(self.url):
-            self.links.append(self.url)

Utils.py

@@ -12,7 +12,7 @@ def timestampToDate(ts):
 def checkPic(ext):
     for p in pics:
         if re.search(p, ext) is not None:
-            return p[1:p.find("\\")]
+            return True
     return False
 
 
@@ -36,4 +36,4 @@ def checkRedditComments(links):
 
 def writeLog(text):
     with open("debug.log", "a") as f:
-        f.write(str(text)+"\n")
+        f.write(str(text))

requirements.txt

@@ -1,23 +0,0 @@
-anyio==3.6.2
-beautifulsoup4==4.11.1
-certifi==2022.9.24
-charset-normalizer==2.1.1
-commonmark==0.9.1
-h11==0.14.0
-httpcore==0.16.2
-httpx==0.23.1
-idna==3.4
-importlib-metadata==4.13.0
-inscriptis==2.3.1
-lxml==4.9.1
-nanoid==2.0.0
-Pygments==2.13.0
-PyYAML==6.0
-rfc3986==1.5.0
-rich==12.6.0
-sniffio==1.3.0
-soupsieve==2.3.2.post1
-typing-extensions==4.4.0
-urllib3==1.26.12
-urwid==2.1.2
-zipp==3.10.0