182 lines
4.9 KiB
Python
Executable file
182 lines
4.9 KiB
Python
Executable file
#!/usr/bin/env python
|
|
|
|
import feedparser
|
|
from readability import Document
|
|
import requests
|
|
import html2text
|
|
import re
|
|
import os
|
|
import time
|
|
from time import mktime
|
|
from datetime import datetime, timedelta
|
|
from config import base_directory, update_interval, max_age, verbose, feeds
|
|
import logging
|
|
import sys, getopt
|
|
|
|
|
|
'''
|
|
Output functions
|
|
'''
|
|
|
|
def log(text):
|
|
if verbose:
|
|
#logging.info(text)
|
|
print('{} - {}'.format(datetime.now().strftime('%d.%m %H:%M'), text))
|
|
|
|
|
|
def error(text):
|
|
#logging.error(text)
|
|
print('{} - ERROR: {}'.format(datetime.now().strftime('%d.%m %H:%M'), text))
|
|
|
|
|
|
def print_logo():
|
|
logo = '''
|
|
;:
|
|
.N' ,K:
|
|
,O .0MWx' lk 0;
|
|
,kl;';O lx. .xc :k, .kMMXl.c .:x. .xl .dl :kl,'oo .xd:,,O.
|
|
kNOokOWc ;WMKccXMX .WMX. :XM,,OMMl oXMcNMd 'oMMk:XW:.OWddO0N, cKNlkONk
|
|
MMo 0c KMK :MM' XMO oMM. MMl cMM. ON: .MMl 'MM, .K' OMX oO
|
|
WMWxOWMN: KMK ;MM' XMO oMM. MMl cMM:c; .MMl .WMXx0MMX. xMMOkNMWk
|
|
'X; .MMo KMK ;MM' XMO oMM. MMl cMM, .MMl :X. ;MM, .0d 0MX
|
|
.Kdc:'MMo.oNMNl;lMW. .WM0 KMMk:'MMl dMM: .. cMMk.' ,Xlc;cMM, xOl:'KMX
|
|
;kddkNKl. XMNkWk, :N0; .'cOW0c. ,lOW0; .:0Nl. okddOW0:. .kdoxXNd,
|
|
WMX
|
|
;..cc
|
|
'''
|
|
|
|
print(logo)
|
|
|
|
'''
|
|
Utility functions
|
|
'''
|
|
|
|
# Get content of a webpage
|
|
def get_html_content(url):
|
|
response = requests.get(url)
|
|
doc = Document(response.text)
|
|
return doc.summary()
|
|
|
|
|
|
def html_to_markdown(html):
|
|
return html2text.html2text(html)
|
|
|
|
|
|
# Get articles of a feed
|
|
def get_articles(feed):
|
|
feed = feedparser.parse(feed[2])
|
|
return feed.entries
|
|
|
|
|
|
def write_to_file(filename, text):
|
|
file = open(filename, 'w')
|
|
file.write(text)
|
|
file.close()
|
|
|
|
|
|
# Get filename from feedparser article
|
|
def get_filename(date, title):
|
|
|
|
# Get date as single block
|
|
date = date.strftime('%Y%m%d%H%M')
|
|
|
|
# Get title as lowercase words concatenated with underscores
|
|
title = re.sub('[^A-Za-z0-9 ]+', '', title.lower())
|
|
title = re.sub(' ', '_', title)
|
|
|
|
return '{}_{}.md'.format(date, title)
|
|
|
|
|
|
# Get Markdown text from an article
|
|
def get_article_text(article):
|
|
head = '# {}\n\n[Link]({})'.format(article.title, article.link)
|
|
body = html_to_markdown(get_html_content(article.link))
|
|
return '{}\n\n{}'.format(head, body)
|
|
|
|
|
|
# Update feed
|
|
def update_feed(feed):
|
|
|
|
category = feed[0]
|
|
name = feed[1]
|
|
|
|
log('updating feed "{}"'.format(name))
|
|
|
|
feedpath_new = os.path.join(base_directory, category, name, 'new')
|
|
feedpath_read = os.path.join(base_directory, category, name, 'read')
|
|
if not os.path.exists(feedpath_new):
|
|
os.makedirs(feedpath_new)
|
|
if not os.path.exists(feedpath_read):
|
|
os.makedirs(feedpath_read)
|
|
|
|
articles = get_articles(feed)
|
|
threshold_date = datetime.now() - timedelta(days = max_age)
|
|
for a in articles:
|
|
date = datetime.fromtimestamp(mktime(a.published_parsed))
|
|
if date > threshold_date:
|
|
filename = get_filename(date, a.title)
|
|
if not os.path.exists(os.path.join(feedpath_new, filename)) and not os.path.exists(os.path.join(feedpath_read, filename)):
|
|
text = get_article_text(a)
|
|
write_to_file(os.path.join(feedpath_new, filename), text)
|
|
|
|
|
|
# Delete articles older than max_age
|
|
def delete_old_articles():
|
|
|
|
log('removing old articles')
|
|
|
|
threshold_date = datetime.now() - timedelta(days = max_age)
|
|
for subdir, dirs, files in os.walk(base_directory):
|
|
|
|
# Skip 'loved' directory
|
|
if not os.path.join(base_directory, 'loved') in subdir:
|
|
for file in files:
|
|
date = datetime.strptime(file[:12], '%Y%m%d%H%M')
|
|
if threshold_date > date:
|
|
os.remove(os.path.join(subdir, file))
|
|
|
|
|
|
def initialize():
|
|
|
|
# Create 'loved' directory if not existent
|
|
lovedpath = os.path.join(base_directory, 'loved')
|
|
if not os.path.exists(lovedpath):
|
|
os.makedirs(lovedpath)
|
|
|
|
|
|
def crawl():
|
|
|
|
# Main loop
|
|
while True:
|
|
log('starting crawl')
|
|
for feed in feeds:
|
|
update_feed(feed)
|
|
delete_old_articles()
|
|
time.sleep(update_interval * 60)
|
|
|
|
def get_help_message():
|
|
return 'spiderrss.py | run'
|
|
|
|
def main(argv):
|
|
|
|
print_logo()
|
|
|
|
## Get arguments
|
|
#try:
|
|
# opts, args = getopt,getopt(argv, 'h', ['ifile=', 'ofile='])
|
|
#except:
|
|
# print('spiderrss.py [ run | create_config <file> ]')
|
|
|
|
#for opt, arg in opts:
|
|
# if opt == '-h':
|
|
# print('spiderrss.py [ run | create_config <file> ]')
|
|
|
|
|
|
initialize()
|
|
crawl()
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main(sys.argv[1:])
|
|
|