170 lines
4.2 KiB
Python
170 lines
4.2 KiB
Python
import feedparser
|
|
from readability import Document
|
|
import requests
|
|
import html2text
|
|
import re
|
|
import os
|
|
import time
|
|
from time import mktime
|
|
from datetime import datetime, timedelta
|
|
from config import base_directory, update_interval, max_age, verbose, feeds
|
|
import logging
|
|
import sys, getopt
|
|
|
|
|
|
'''
|
|
Output functions
|
|
'''
|
|
|
|
def log(text):
|
|
if verbose:
|
|
#logging.info(text)
|
|
print('{} - {}'.format(datetime.now().strftime('%d.%m %H:%M'), text))
|
|
|
|
|
|
def error(text):
|
|
#logging.error(text)
|
|
print('{} - ERROR: {}'.format(datetime.now().strftime('%d.%m %H:%M'), text))
|
|
|
|
|
|
def print_logo():
|
|
logo = '''
|
|
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
|
###### ###### ## ##### ###### ###### ###### ###### ######
|
|
## ## ## ## ## ## ## ## ## ## ## ## ##
|
|
###### ###### ## ## ## ##### ###### ###### ###### ######
|
|
## ## ## ## ## ## ## ## ## ## ## ##
|
|
###### ## ## ##### ###### ## ## ## ## ###### ######
|
|
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
|
'''
|
|
|
|
print(logo)
|
|
|
|
'''
|
|
Utility functions
|
|
'''
|
|
|
|
# Get content of a webpage
|
|
def get_html_content(url):
|
|
response = requests.get(url)
|
|
doc = Document(response.text)
|
|
return doc.summary()
|
|
|
|
|
|
def html_to_markdown(html):
|
|
return html2text.html2text(html)
|
|
|
|
|
|
# Get articles of a feed
|
|
def get_articles(feed):
|
|
try:
|
|
feed = feedparser.parse(feed[2])
|
|
return feed.entries
|
|
except Exception as e:
|
|
error('failed to get feed "{}: {}"'.format(feed[1], e.msg))
|
|
return []
|
|
|
|
|
|
def write_to_file(filename, text):
|
|
file = open(filename, 'w')
|
|
file.write(text)
|
|
file.close()
|
|
|
|
|
|
# Get filename from feedparser article
|
|
def get_filename(date, title):
|
|
|
|
# Get date as single block
|
|
date = date.strftime('%Y%m%d%H%M')
|
|
|
|
# Get title as lowercase words concatenated with underscores
|
|
title = re.sub('[^A-Za-z0-9 ]+', '', title.lower())
|
|
title = re.sub(' ', '_', title)
|
|
|
|
return '{}_{}.md'.format(date, title)
|
|
|
|
|
|
# Update feed
|
|
def update_feed(feed):
|
|
|
|
category = feed[0]
|
|
name = feed[1]
|
|
|
|
log('updating feed "{}"'.format(name))
|
|
|
|
feedpath_new = os.path.join(base_directory, category, name, 'new')
|
|
feedpath_read = os.path.join(base_directory, category, name, 'read')
|
|
if not os.path.exists(feedpath_new):
|
|
os.makedirs(feedpath_new)
|
|
if not os.path.exists(feedpath_read):
|
|
os.makedirs(feedpath_read)
|
|
|
|
articles = get_articles(feed)
|
|
threshold_date = datetime.now() - timedelta(days = max_age)
|
|
for a in articles:
|
|
date = datetime.fromtimestamp(mktime(a.published_parsed))
|
|
if date > threshold_date:
|
|
filename = get_filename(date, a.title)
|
|
if not os.path.exists(os.path.join(feedpath_new, filename)) and not os.path.exists(os.path.join(feedpath_read, filename)):
|
|
text = html_to_markdown(get_html_content(a.link))
|
|
write_to_file(os.path.join(feedpath_new, filename), text)
|
|
|
|
|
|
# Delete articles older than max_age
|
|
def delete_old_articles():
|
|
|
|
threshold_date = datetime.now() - timedelta(days = max_age)
|
|
for subdir, dirs, files in os.walk(base_directory):
|
|
|
|
# Skip 'loved' directory
|
|
if not os.path.join(base_directory, 'loved') in subdir:
|
|
for file in files:
|
|
date = datetime.strptime(file[:12], '%Y%m%d%H%M')
|
|
if threshold_date > date:
|
|
os.remove(os.path.join(subdir, file))
|
|
log('deleted old articles')
|
|
|
|
|
|
def initialize():
|
|
|
|
# Create 'loved' directory if not existent
|
|
lovedpath = os.path.join(base_directory, 'loved')
|
|
if not os.path.exists(lovedpath):
|
|
os.makedirs(lovedpath)
|
|
|
|
|
|
def crawl():
|
|
|
|
# Main loop
|
|
while True:
|
|
for feed in feeds:
|
|
update_feed(feed)
|
|
delete_old_articles()
|
|
time.sleep(update_interval * 60)
|
|
|
|
def get_help_message():
|
|
return 'spiderrss.py | run'
|
|
|
|
def main(argv):
|
|
|
|
print_logo()
|
|
|
|
## Get arguments
|
|
#try:
|
|
# opts, args = getopt,getopt(argv, 'h', ['ifile=', 'ofile='])
|
|
#except:
|
|
# print('spiderrss.py [ run | create_config <file> ]')
|
|
|
|
#for opt, arg in opts:
|
|
# if opt == '-h':
|
|
# print('spiderrss.py [ run | create_config <file> ]')
|
|
|
|
|
|
#initialize()
|
|
#crawl()
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main(sys.argv[1:])
|
|
|