diff --git a/API.py b/API.py index 4255835..f956b2f 100644 --- a/API.py +++ b/API.py @@ -18,9 +18,6 @@ class Fetcher: result = dict([(item["id"], (item["count"], item["newestItemTimestampUsec"], Utils.timestampToDate( item["newestItemTimestampUsec"]))) for item in response.json()["unreadcounts"]]) self.unreadCounts = result - self.categories = sorted([{"id": item, "name": item[13:], "count": self.unreadCounts[item][0], "date": self.unreadCounts[item][2]} - for item in self.unreadCounts.keys() if item[0:13] == "user/-/label/"], - key=lambda item: item["date"], reverse=True) def getSubscriptions(self): response = httpx.get(self.URL+"/reader/api/0/subscription/list?output=json", headers=self.headers) @@ -36,7 +33,8 @@ class Fetcher: def articlesFromCategory(self, category, number): if category not in self.articles: response = httpx.get(self.URL+"/reader/api/0/stream/contents?n="+number+"&s="+category, headers=self.headers) - self.articles[category] = response.json()["items"] + return response.json()["items"] + # self.articles[category] = response.json()["items"] async def articlesFromCategoryAsync(self, category, number): if category not in self.articles: diff --git a/App.py b/App.py index 469dd36..4d81570 100644 --- a/App.py +++ b/App.py @@ -6,8 +6,8 @@ import subprocess import os from concurrent.futures import ThreadPoolExecutor from API import Fetcher -from DB import DB -from Render import Article +from Cache import Cache +from Render import Article, Render import Utils warnings.filterwarnings("ignore") @@ -16,8 +16,8 @@ class LeftPane(urwid.ListBox): def __init__(self, categories): super().__init__(self) items = [urwid.AttrMap(urwid.Columns([ - (16, urwid.Text(category["date"])), urwid.Text(category["name"]), (5, urwid.Text(str(category["count"])))]), - (category["id"], category["name"], category["count"]), "reveal focus") for category in categories] + (16, urwid.Text(category[4])), urwid.Text(category[1]), (5, urwid.Text(str(category[2])))]), + (category[0], category[1], category[2]), "reveal focus") for category in categories] walker = urwid.SimpleListWalker(items) self.body = walker self.categoryPosition = 0 @@ -25,7 +25,22 @@ class LeftPane(urwid.ListBox): self.currentCategory = "" focus_widget, idx = self.get_focus() if self.isCategoryView: - self.currentCategory = focus_widget.attr_map[None][0] + try: + self.currentCategory = focus_widget.attr_map[None][0] + except BaseException: + self.currentCategory = None + + def fill(self, items, is_category_view): + def getAttrs(item): + if is_category_view is True: + return (item[0], item[1], item[2]) + else: + return (item[0], item[1]) + items = [urwid.AttrMap(urwid.Columns([ + (16, urwid.Text(item[4])), urwid.Text(item[1]), (5, urwid.Text(str(item[2])))]), + getAttrs(item), "reveal focus") for item in items] + walker = urwid.SimpleListWalker(items) + self.body = walker def setArticlesPaneTitle(self, text): tui.rightBox.set_title(text) @@ -40,19 +55,14 @@ class LeftPane(urwid.ListBox): async def setCategoryArticles(self, attrMap): itemId = attrMap[0] - number = attrMap[2] name = attrMap[1] - await tui.fetcher.articlesFromCategoryAsync(itemId, str(number)) - focus_widget, idx = self.get_focus() - currentCategory = focus_widget.attr_map[None][0] - if itemId == currentCategory: - tui.articles = tui.fetcher.articles[currentCategory] - tui.articleView.fill(tui.articles) - self.setArticlesPaneTitle(name) + tui.articles = tui.cache.getArticlesFromCategory(itemId) + tui.articleView.fill(tui.articles) + self.setArticlesPaneTitle(name) def setFeedArticles(self, attrMap): itemId = attrMap[0] - tui.articles = tui.fetcher.articlesFromFeed(itemId, self.currentCategory) + tui.articles = tui.cache.getArticlesFromFeed(itemId) if tui.articles is not None: tui.articleView.fill(tui.articles) self.setArticlesPaneTitle(attrMap[1]) @@ -84,19 +94,14 @@ class LeftPane(urwid.ListBox): self.setFeedArticles(focus_widget.attr_map[None]) return elif key in ("l", "right"): - if self.isCategoryView and tui.fetcher.checkCategory(self.currentCategory): + if self.isCategoryView: self.isCategoryView = False focus_widget, idx = self.get_focus() self.categoryPosition = idx categoryId = focus_widget.attr_map[None][0] categoryName = focus_widget.attr_map[None][1] - feeds = tui.fetcher.feedsFromCategory(categoryId) - feedItems = [urwid.AttrMap(urwid.Columns([(16, urwid.Text(tui.fetcher.unreadCounts[feed["id"]][2])), - urwid.Text(feed["title"]), - (5, urwid.Text(str(tui.fetcher.unreadCounts[feed["id"]][0])))]), - (feed["id"], feed["title"]), "reveal focus") for feed in feeds] - walker = urwid.SimpleListWalker(feedItems) - self.body = walker + feeds = tui.cache.getFeeds(categoryId) + self.fill(feeds, False) focus_widget, idx = self.get_focus() self.setFeedArticles(focus_widget.attr_map[None]) tui.leftBox.set_title(categoryName) @@ -104,24 +109,11 @@ class LeftPane(urwid.ListBox): elif key in ("h", "left"): if not self.isCategoryView: self.isCategoryView = True - items = [ - urwid.AttrMap( - urwid.Columns( - [(16, urwid.Text(category["date"])), - urwid.Text(category["name"]), - (5, urwid.Text(str(category["count"])))]), - (category["id"], - category["name"], - category["count"]), - "reveal focus") for category in tui.fetcher.categories] - walker = urwid.SimpleFocusListWalker(items) - self.body = walker + self.fill(tui.categories, True) tui.leftBox.set_title("Categories") self.set_focus(self.categoryPosition) focus_widget, idx = self.get_focus() - tui.articles = self.getArticlesFromCategory(focus_widget.attr_map[None][0]) - tui.articleView.fill(tui.articles) - self.setArticlesPaneTitle(self.currentCategory[13:]) + asyncio.create_task(self.setCategoryArticles(focus_widget.attr_map[None])) return return super().keypress(size, key) @@ -142,9 +134,9 @@ class RightPane(urwid.ListBox): urwid.AttrMap( urwid.Columns( [(2, urwid.Text("")), - (16, urwid.Text(Utils.timestampToDate(article["timestampUsec"]))), - urwid.Text(article["title"])]), - article["id"], + (16, urwid.Text(article[3])), + urwid.Text(article[1])]), + article[0], "reveal focus") for article in articles] walker = urwid.SimpleListWalker(items) self.body = walker @@ -282,14 +274,14 @@ class TUI(urwid.Frame): self.overlay = None self.fetcher = Fetcher(URL, token) - self.DB = DB(self.fetcher) - self.DB.refresh() + self.cache = Cache(self.fetcher) + self.categories = self.cache.getCategories() self.leftPaneItems = {} self.activePane = False self.executor = ThreadPoolExecutor(max_workers=1) self.loop = None - self.feedView = LeftPane(self.fetcher.categories) + self.feedView = LeftPane([]) self.articleView = RightPane([]) self.leftBox = urwid.LineBox(self.feedView, title="Categories") self.rightBox = urwid.LineBox(self.articleView, title="Articles") @@ -301,14 +293,20 @@ class TUI(urwid.Frame): super().__init__(self.body) + def initialize_panes(self): + try: + self.feedView.fill(self.cache.getCategories(), True) + focus_widget, idx = self.feedView.get_focus() + item = focus_widget.attr_map[None][0] + name = focus_widget.attr_map[None][1] + self.articles = self.cache.getArticlesFromCategory(item) + self.articleView.fill(self.articles) + self.feedView.setArticlesPaneTitle(name) + except BaseException: + pass + def run(self): - focus_widget, idx = self.feedView.get_focus() - item = focus_widget.attr_map[None][0] - name = focus_widget.attr_map[None][1] - self.fetcher.articlesFromCategory(item, str(focus_widget.attr_map[None][2])) - self.articles = tui.fetcher.articles[item] - self.articleView.fill(self.articles) - self.feedView.setArticlesPaneTitle(name) + self.initialize_panes() self.loop.run() self.executor.shutdown(wait=False) @@ -336,11 +334,12 @@ class TUI(urwid.Frame): urwid.LineBox(olb), self.body, align="center", width=("relative", 50), valign="middle", height=3) self.body = overlay - articles = tui.fetcher.fetch() + articles = tui.cache.refresh() for text in articles: olb.body = urwid.SimpleListWalker([urwid.Text(text)]) tui.loop.entering_idle() self.body = overlay.bottom_w + self.initialize_panes() tui = TUI.create() diff --git a/Cache.py b/Cache.py new file mode 100644 index 0000000..5145e0b --- /dev/null +++ b/Cache.py @@ -0,0 +1,115 @@ +import sqlite3 +from sqlite3 import Error +import os +from Render import Article + + +class Cache: + def __init__(self, api): + self.conn = None + self.api = api + create_categories = """create table if not exists categories ( + id text primary key, + name text not null, + unread_count integer, + timestamp integer, + date text + ) + """ + create_feeds = """create table if not exists feeds ( + id text primary key, + name text not null, + unread_count integer, + timestamp integer, + date text, + category_id text, + foreign key (category_id) references categories (id) + ) + """ + create_articles = """create table if not exists articles ( + id text primary key, + title text not null, + timestamp integer, + date text, + content text, + url text, + is_read boolean, + origin text, + category_id text, + foreign key (category_id) references categories (id), + foreign key (origin) references feeds (id) + ) + """ + create_links = """create table if not exists links ( + id text, + url text not null, + foreign key (id) references articles (id) + ) + """ + try: + self.conn = sqlite3.connect(os.path.expanduser("~") + '/.config/inomnibus/cache.db') + self.conn.cursor().execute(create_categories) + self.conn.cursor().execute(create_feeds) + self.conn.cursor().execute(create_articles) + self.conn.cursor().execute(create_links) + + except Error as e: + print(e) + + def getArticlesFromFeed(self, feed_id): + cur = self.conn.cursor() + cur.execute("""select * from articles where origin = ?""", (feed_id,)) + return cur.fetchall() + + def getArticlesFromCategory(self, category_id): + cur = self.conn.cursor() + cur.execute("""select * from articles where category_id = ?""", (category_id,)) + return cur.fetchall() + + def getCategories(self): + cur = self.conn.cursor() + cur.execute("""select * from categories where unread_count != 0 order by timestamp desc""") + return cur.fetchall() + + def getFeeds(self, category_id): + cur = self.conn.cursor() + cur.execute("""select * from feeds where category_id = ? and unread_count != 0 order by timestamp desc""", (category_id,)) + return cur.fetchall() + + def refresh(self): + self.api.refresh() + for item in self.api.unreadCounts.keys(): + if item[0:13] != "user/-/label/": + continue + data = self.api.unreadCounts[item] + cur = self.conn.cursor() + cur.execute("""insert into categories(id,name,unread_count,timestamp,date) values(:id,:name,:count,:ts,:d) + on conflict(id) do update set unread_count = :count, timestamp = :ts, date = :d; + """, + {"id": item, "name": item[13:], "count": data[0], "ts": data[1], "d": data[2]}) + self.conn.commit() + for item in self.api.feeds: + data = self.api.unreadCounts[item["id"]] + cur.execute("""insert into feeds(id,name,unread_count,timestamp,date,category_id) values(:id,:name,:count,:ts,:d,:c_id) + on conflict(id) do update set unread_count = :count, timestamp = :ts, date = :d,category_id = :c_id; + """, + {"id": item["id"], "name": item["title"], "count": data[0], "ts": data[1], "d": data[2], + "c_id": item["categories"][0]["id"]}) + self.conn.commit() + cur.execute("""select id, name, unread_count from categories""") + for row in cur.fetchall(): + yield "Fetching category " + row[1] + articles = self.api.articlesFromCategory(row[0], str(row[2])) + for article in articles: + articleObj = Article(article) + cur.execute("""insert into articles(id,title,timestamp,date,content,url,is_read,origin,category_id) + values(:id,:t,:ts,:d,:c,:u,:r,:o,:c_id) on conflict(id) do update + set id = :id, title = :t, timestamp = :ts, date = :d, content = :c, url = :u, is_read = :r, category_id = :c_id, + origin = :o; + """, + {"id": articleObj.id, "t": articleObj.title, "ts": articleObj.timestamp, "d": articleObj.date, + "c": articleObj.text, "u": articleObj.url, "r": articleObj.is_read, "o": articleObj.origin, "c_id": row[0]}) + for link in articleObj.links: + cur.execute("""insert into links(id,url) values(?,?)""", (articleObj.id, link)) + + self.conn.commit() diff --git a/DB.py b/DB.py deleted file mode 100644 index 7d5c8bb..0000000 --- a/DB.py +++ /dev/null @@ -1,55 +0,0 @@ -import sqlite3 -from sqlite3 import Error -import os - - -class DB: - def __init__(self, api): - self.conn = None - self.api = api - create_categories = """create table if not exists categories ( - id text primary key, - name text not null, - unread_count integer, - timestamp integer, - date text - ) - """ - create_feeds = """create table if not exists feeds ( - id text primary key, - name text not null, - unread_count integer, - timestamp integer, - date text, - category_id text, - foreign key (category_id) references categories (id) - ) - """ - try: - self.conn = sqlite3.connect(os.path.expanduser("~") + '/.config/inomnibus/cache.db') - self.conn.cursor().execute(create_categories) - self.conn.cursor().execute(create_feeds) - - except Error as e: - print(e) - - def refresh(self): - self.api.refresh() - for item in self.api.unreadCounts.keys(): - if item[0:13] != "user/-/label/": - continue - data = self.api.unreadCounts[item] - cur = self.conn.cursor() - cur.execute("""insert into categories(id,name,unread_count,timestamp,date) values(:id,:name,:count,:ts,:d) - on conflict(id) do update set unread_count = :count, timestamp = :ts, date = :d; - """, - {"id": item, "name": item[13:], "count": data[0], "ts": data[1], "d": data[2]}) - self.conn.commit() - for item in self.api.feeds: - data = self.api.unreadCounts[item["id"]] - cur.execute("""insert into feeds(id,name,unread_count,timestamp,date,category_id) values(:id,:name,:count,:ts,:d,:c_id) - on conflict(id) do update set unread_count = :count, timestamp = :ts, date = :d,category_id = :c_id; - """, - {"id": item["id"], "name": item["title"], "count": data[0], "ts": data[1], "d": data[2], - "c_id": item["categories"][0]["id"]}) - self.conn.commit() diff --git a/Render.py b/Render.py index d3ddbb4..37d1bc2 100644 --- a/Render.py +++ b/Render.py @@ -5,26 +5,10 @@ from bs4 import BeautifulSoup from RedditCommentsParser import RedditComments -class Article: - def __init__(self, articleObj): - content = articleObj["summary"]["content"] - soup = BeautifulSoup(content) - links = soup.find_all(href=True) - media = soup.find_all(src=True) - links_set = set() - for link in links: - links_set.add(link['href']) - for m in media: - links_set.add(m['src']) - self.links = list(links_set) - self.text = get_text(content) - self.title = articleObj["title"] - self.date = Utils.timestampToDate(articleObj["timestampUsec"]) - self.url = articleObj["canonical"][0]["href"] - if Utils.checkStreamingVideo(self.url): - self.links.append(self.url) - elif Utils.checkReddit(self.url): - comments_link = Utils.checkRedditComments(self.links) +class Render: + def __init__(self, id, content, url, links): + if Utils.checkReddit(self.url): + comments_link = Utils.checkRedditComments(links) if comments_link: commentsObj = RedditComments(comments_link) commentsObj.getComments() @@ -68,3 +52,27 @@ class Article: else: self.currentPageNumber -= 1 return self.chunks[self.currentPageNumber - 1] + + +class Article: + def __init__(self, articleJSON): + content = articleJSON["summary"]["content"] + soup = BeautifulSoup(content) + links = soup.find_all(href=True) + media = soup.find_all(src=True) + links_set = set() + for link in links: + links_set.add(link['href']) + for m in media: + links_set.add(m['src']) + self.links = list(links_set) + self.text = get_text(content) + self.title = articleJSON["title"] + self.timestamp = articleJSON["timestampUsec"] + self.date = Utils.timestampToDate(self.timestamp) + self.url = articleJSON["canonical"][0]["href"] + self.is_read = "user/-/state/com.google/read" in articleJSON["categories"] + self.id = articleJSON["id"] + self.origin = articleJSON["origin"]["streamId"] + if Utils.checkStreamingVideo(self.url): + self.links.append(self.url)