107 lines
2.8 KiB
Python
107 lines
2.8 KiB
Python
import feedparser
|
|
from readability import Document
|
|
import requests
|
|
import html2text
|
|
import re
|
|
import os
|
|
from time import mktime
|
|
from datetime import datetime, timedelta
|
|
|
|
feeds = [('News', 'Tagesschau', 'https://www.tagesschau.de/xml/rss2'),
|
|
('Linux', 'NixOS', 'https://nixos.org/blogs.xml'),
|
|
('News', 'Vice', 'https://www.vice.com/de/rss')
|
|
]
|
|
|
|
out_directory = './out'
|
|
delta = 365
|
|
|
|
|
|
# Get content of a webpage
|
|
def get_html_content(url):
|
|
response = requests.get(url)
|
|
doc = Document(response.text)
|
|
return doc.summary()
|
|
|
|
|
|
def html_to_markdown(html):
|
|
return html2text.html2text(html)
|
|
|
|
|
|
# Get articles of a RSS feed
|
|
def get_articles(url):
|
|
feed = feedparser.parse(url)
|
|
return feed.entries
|
|
|
|
|
|
def write_to_file(filename, text):
|
|
file = open(filename, 'w')
|
|
file.write(text)
|
|
file.close()
|
|
|
|
|
|
# Get filename from feedparser article
|
|
def get_filename(date, title):
|
|
|
|
# Get date as single block
|
|
date = date.strftime('%Y%m%d%H%M')
|
|
|
|
# Get title as lowercase words concatenated with underscores
|
|
title = re.sub('[^A-Za-z0-9 ]+', '', title.lower())
|
|
title = re.sub(' ', '_', title)
|
|
|
|
return '{}_{}.md'.format(date, title)
|
|
|
|
|
|
# Update feed
|
|
def update_feed(feed):
|
|
|
|
category = feed[0]
|
|
name = feed[1]
|
|
url = feed[2]
|
|
|
|
feedpath_new = os.path.join(out_directory, category, name, 'new')
|
|
feedpath_read = os.path.join(out_directory, category, name, 'read')
|
|
if not os.path.exists(feedpath_new):
|
|
os.makedirs(feedpath_new)
|
|
if not os.path.exists(feedpath_read):
|
|
os.makedirs(feedpath_read)
|
|
|
|
articles = get_articles(url)
|
|
threshold_date = datetime.now() - timedelta(days = delta)
|
|
for a in articles:
|
|
date = datetime.fromtimestamp(mktime(a.published_parsed))
|
|
if date > threshold_date:
|
|
filename = get_filename(date, a.title)
|
|
if not os.path.exists(os.path.join(feedpath_new, filename)) and not os.path.exists(os.path.join(feedpath_read, filename)):
|
|
text = html_to_markdown(get_html_content(a.link))
|
|
write_to_file(os.path.join(feedpath_new, filename), text)
|
|
|
|
|
|
# Delete articles older than day delta
|
|
def delete_old_articles():
|
|
|
|
threshold_date = datetime.now() - timedelta(days = delta)
|
|
for subdir, dirs, files in os.walk(out_directory):
|
|
|
|
# Skip 'loved' directory
|
|
if not os.path.join(out_directory, 'loved') in subdir:
|
|
for file in files:
|
|
date = datetime.strptime(file[:12], '%Y%m%d%H%M')
|
|
if threshold_date > date:
|
|
os.remove(os.path.join(subdir, file))
|
|
|
|
|
|
|
|
def main():
|
|
lovedpath = os.path.join(out_directory, 'loved')
|
|
if not os.path.exists(lovedpath):
|
|
os.makedirs(lovedpath)
|
|
for feed in feeds:
|
|
update_feed(feed)
|
|
delete_old_articles()
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|
|
|