Du kannst nicht mehr als 25 Themen auswählen Themen müssen entweder mit einem Buchstaben oder einer Ziffer beginnen. Sie können Bindestriche („-“) enthalten und bis zu 35 Zeichen lang sein.

feedcake.py 8.3 KiB

vor 5 Jahren
vor 5 Jahren
vor 5 Jahren
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213
  1. # This script is intended for personal and scientific use only
  2. import os
  3. import sys
  4. import re
  5. import hashlib
  6. import json
  7. from time import sleep
  8. import feedparser
  9. import requests
  10. from bs4 import BeautifulSoup, Comment
  11. from bs4.element import CData
  12. # default config location is a 'config.json' next to the script.
  13. try:
  14. filedir = os.path.dirname(os.path.abspath(__file__))
  15. if len(sys.argv) < 2:
  16. configpath = filedir+'/config.json'
  17. print("Using default config location: ", configpath)
  18. config = json.load(open(configpath))
  19. else:
  20. configpath = sys.argv[1]
  21. config = json.load(open(configpath))
  22. except:
  23. print("Problem reading config file: ", configpath)
  24. print("ERROR: Config file not found or invalid!")
  25. sys.exit(1)
  26. print(filedir)
  27. public_path = filedir + '/public'
  28. assets_path = public_path + '/assets'
  29. # e.g. https://example.com/some-string
  30. base_url = config['base_url']
  31. # "I'm a robot which promises you clicks and $ ... Give me ALL your content!"
  32. requestheaders = {
  33. 'user-'+'age'+'nt' :
  34. 'Mo' + 'zill' + 'a/5.' + '0 (' + 'comp' + 'ati' + 'ble; '+'Go'
  35. + 'og'+'le'+ 'bo' + 't/' + '2.1; +http' + '://www.' + 'go'
  36. + 'og'+ 'le'+'.com/'+'bo'+'t.html)'
  37. }
  38. # need filname safe strings for storing images along html files
  39. def get_valid_filename(s):
  40. s = str(s).split('?')[0].strip().strip('/').strip('http://').strip('https://').replace(' ', '-')
  41. return re.sub(r'(?u)[^-\w.]', '-', s)
  42. # Get a unique and valid filename from URL (for images)
  43. def filename_from_url(url):
  44. # remove get attributes and path
  45. new_filename = url.split('?')[0].split('/')[-1]
  46. # Split filename
  47. new_filename = new_filename.split('.')
  48. # insert a hash before suffix
  49. new_filename.insert(1, str(hashlib.md5(url.encode('utf-8')).hexdigest()) )
  50. # convert back to string and extra validate
  51. new_filename = get_valid_filename('.'.join(new_filename))
  52. return new_filename
  53. # Download images and so on
  54. def download_image(url, entry_dir, filename):
  55. # take care of protocol relative URLs ... let's just assume that https works.
  56. if url.startswith('//'):
  57. url = 'https:'+url
  58. response = requests.get(url, headers=requestheaders)
  59. if response.status_code == 200:
  60. with open(assets_path + '/' + entry_dir + '/' + filename, 'wb') as f:
  61. #f.write(response.content)
  62. for chunk in response.iter_content(1024):
  63. f.write(chunk)
  64. def process_feed(feed_url, output_filename):
  65. # Get the feed
  66. r_feed = requests.get(feed_url, headers=requestheaders)
  67. # TODO: exceptions.(what if 404 or whatever?)
  68. # Store data of new articles
  69. for entry in feedparser.parse(r_feed.text).entries:
  70. print(entry.link)
  71. entry_dir = get_valid_filename(entry.link) # input e.g. https://orf.at/stories/3117136/
  72. entry_path = assets_path + '/'+ entry_dir
  73. if not os.path.exists(entry_path):
  74. r = requests.get(entry.link.split('?')[0], headers=requestheaders)
  75. online_soup = BeautifulSoup(r.text, 'html.parser')
  76. content_soup = BeautifulSoup('<article></article>', 'html.parser')
  77. # Remove all Comments
  78. for element in online_soup(text=lambda text: isinstance(text, Comment)):
  79. element.extract()
  80. # domain and path specific rules
  81. # ... split strings for (very simple) ob+fu+sca+tion
  82. if entry.link.startswith('https://or'+'f.a'+'t/sto'+'ries'):
  83. if entry.date:
  84. article_time = content_soup.new_tag('time', datetime=entry.date)
  85. content_soup.article.append(article_time)
  86. article_headline = online_soup.find('h1', attrs={'class': 'story-lead-headline'})
  87. content_soup.article.append(article_headline)
  88. article_body = online_soup.find('div', attrs={'class': 'story-content'})
  89. content_soup.article.append(article_body)
  90. article_link = content_soup.new_tag('a', href=entry.link)
  91. article_link['class'] = 'source';
  92. article_link.string = 'Quelle (' + entry.link + ')'
  93. content_soup.article.append(article_link)
  94. if entry.link.startswith('https://de'+'rst'+'and'+'ard'+'.a'+'t/20'): # url starts with number ... too lazy for regex :)
  95. if entry.published:
  96. article_time = content_soup.new_tag('time', datetime=entry.published)
  97. content_soup.article.append(article_time)
  98. article_headline = online_soup.find('h1', attrs={'itemprop': 'headline'})
  99. content_soup.article.append(article_headline)
  100. # images etc
  101. article_aside = online_soup.find('div', id="content-aside")
  102. content_soup.article.append(article_aside)
  103. article_body = online_soup.find('div', attrs={'itemprop': 'articleBody'})
  104. content_soup.article.append(article_body)
  105. article_link = content_soup.new_tag('a', href=entry.link)
  106. article_link['class'] = 'source';
  107. article_link.string = 'Quelle (' + entry.link.split('?')[0] + ')'
  108. content_soup.article.append(article_link)
  109. # modify original link -> mobile version and comment section
  110. link_to_comments = re.sub(r'(\/\/)', r'\1mobil.',entry.link.split('?')[0]) + '?_viewMode=forum#'
  111. article_comments_link = content_soup.new_tag('a', href=link_to_comments)
  112. article_comments_link['class'] = 'comments';
  113. article_comments_link.sting = 'Kommentare'
  114. content_soup.article.append(article_comments_link)
  115. article_link.string = 'Quelle (' + entry.link.split('?')[0] + ')'
  116. content_soup.article.append(article_link)
  117. # create directory for storing and serving html and images
  118. os.makedirs(entry_path)
  119. # download all article images and replace image source
  120. for img in content_soup.findAll('img'):
  121. print(img)
  122. if img.get('data-src'):
  123. old_url = img['data-src']
  124. if not old_url.startswith('data:'):
  125. new_filename = filename_from_url(old_url)
  126. img['data-src'] = base_url + '/' + entry_dir + '/' + new_filename
  127. download_image(old_url, entry_dir, new_filename)
  128. if img.get('src'):
  129. old_url = img['src']
  130. if not old_url.startswith('data:'):
  131. new_filename = filename_from_url(old_url)
  132. img['src'] = base_url + '/' + entry_dir + '/' + new_filename
  133. download_image(old_url, entry_dir, new_filename)
  134. if img.get('data-srcset'):
  135. srcset = img['data-srcset'].split(', ')
  136. new_srcset = []
  137. for src in srcset:
  138. old_url = src.split(' ')[0]
  139. src_res = src.split(' ')[1]
  140. new_filename = filename_from_url(old_url)
  141. download_image(old_url, entry_dir, new_filename)
  142. new_url = base_url + '/' + entry_dir + '/' + new_filename
  143. src = ' '.join([new_url, src_res])
  144. new_srcset.append(src)
  145. img['data-srcset'] = ', '.join(new_srcset)
  146. # TODO(?): HTML5 picture tag
  147. f = open(entry_path + '/index.html', 'w')
  148. f.write(str(content_soup.prettify()))
  149. f.close()
  150. sleep(1.3)
  151. # Create new feed
  152. # Maybe buiding a new feed from scretch using a template would be nicer but ...
  153. # let's just modify the original one!
  154. feed_soup = BeautifulSoup(r_feed.text, 'lxml-xml')
  155. for e in feed_soup.findAll('item'):
  156. entry_dir = get_valid_filename(e.link.text)
  157. f_content = open(assets_path + '/' + entry_dir + '/index.html', 'r')
  158. content_tag = feed_soup.new_tag('content:encoded')
  159. content_tag.string = CData(f_content.read())
  160. e.append(content_tag)
  161. f_content.close
  162. f = open(public_path + '/' + output_filename, 'w')
  163. f.write(str(feed_soup.prettify()))
  164. f.close()
  165. # Let's actually fetch the stuff!
  166. for feed in config['feeds']:
  167. process_feed(feed['source'], feed['destination'])