選択できるのは25トピックまでです。 トピックは、先頭が英数字で、英数字とダッシュ('-')を使用した35文字以内のものにしてください。

289 行
10 KiB

  1. import os.path
  2. import sys
  3. import feedparser
  4. from mastodon import Mastodon
  5. import json
  6. import requests
  7. import re
  8. import sqlite3
  9. import html2text
  10. from datetime import datetime, date, time, timedelta
  11. # default config location is a 'config.json' next to the script.
  12. try:
  13. filedir = os.path.dirname(os.path.abspath(__file__))
  14. if len(sys.argv) < 2:
  15. print("Using default config location: %s/config.json" % filedir)
  16. config = json.load(open(filedir+'/config.json'))
  17. else:
  18. config = json.load(open(sys.argv[1]))
  19. except:
  20. print("ERROR: Config file not found!")
  21. sys.exit(1)
  22. mastinstance = config['mastodon']['instance']
  23. mastuser = config['mastodon']['user']
  24. mastpasswd = config['mastodon']['password']
  25. twitteruser = config['sources']['twitter']['user']
  26. soupuser = config['sources']['soup']['user']
  27. dryrun = config['settings']['dryrun']
  28. days = config['settings']['days']
  29. # sqlite db to store processed tweets (and corresponding toots ids)
  30. sql = sqlite3.connect(config['settings']['databasefilepath'])
  31. db = sql.cursor()
  32. db.execute('''CREATE TABLE IF NOT EXISTS posts (srcpost text, srcuser text, mastpost text, mastuser text, mastinstance text)''')
  33. mastodon_api = None
  34. def register_app(mastuser,mastpasswd,mastinstance,mastodon_api):
  35. if mastodon_api is None:
  36. if not os.path.isfile(mastinstance+'.secret'):
  37. if Mastodon.create_app(
  38. 'metasyndicator',
  39. api_base_url='https://'+mastinstance,
  40. to_file = mastinstance+'.secret'
  41. ):
  42. print('app created on instance '+mastinstance)
  43. else:
  44. print('failed to create app on instance '+mastinstance)
  45. sys.exit(1)
  46. try:
  47. mastodon_api = Mastodon(
  48. client_id=mastinstance+'.secret',
  49. api_base_url='https://'+mastinstance
  50. )
  51. mastodon_api.log_in(
  52. username=mastuser,
  53. password=mastpasswd,
  54. scopes=['read', 'write'],
  55. to_file=mastuser+".secret"
  56. )
  57. return mastodon_api
  58. except:
  59. print("ERROR: First Login Failed!")
  60. sys.exit(1)
  61. # twitter section
  62. print('====== TWITTER ======')
  63. t = feedparser.parse('http://twitrss.me/twitter_user_to_rss/?user='+twitteruser)
  64. # start with oldest
  65. for p in reversed(t.entries):
  66. # check if this tweet has been processed
  67. db.execute(
  68. 'SELECT * FROM posts WHERE srcpost = ? AND srcuser = ? AND mastuser = ? AND mastinstance = ?',
  69. (p.id, twitteruser, mastuser, mastinstance)
  70. )
  71. last = db.fetchone()
  72. print('Processing: %s' % p.id)
  73. shouldpost = True
  74. posttime = datetime(p.published_parsed.tm_year, p.published_parsed.tm_mon, p.published_parsed.tm_mday, p.published_parsed.tm_hour, p.published_parsed.tm_min, p.published_parsed.tm_sec)
  75. if last is not None:
  76. shouldpost = False
  77. print("skip: already posted")
  78. # process only unprocessed tweets less than n days old
  79. age = datetime.now() - posttime
  80. if age > timedelta(days=days):
  81. shouldpost = False
  82. print("skip: Posting older than %s days (%s)" % (days, age) )
  83. # kill tweets with fb links with fire!
  84. if "https://www.facebook.com" in p.title or "https://m.facebook.com" in p.title:
  85. shouldpost = False
  86. print("skip: a Tweet that links to facebook? ... That's too much.")
  87. if shouldpost:
  88. print(posttime)
  89. # Create application if it does not exist
  90. mastodon_api = register_app(mastuser, mastpasswd, mastinstance, mastodon_api)
  91. c = p.title
  92. if p.author.lower() != '(@%s)' % twitteruser.lower():
  93. c = ("RT %s from Twitter:\n" % p.author[1:-1]) + c
  94. toot_media = []
  95. # get the pictures...
  96. for pic in re.finditer(r"https://pbs.twimg.com/[^ \xa0\"]*", p.summary):
  97. if (not dryrun):
  98. media = requests.get(pic.group(0))
  99. media_posted = mastodon_api.media_post(media.content, mime_type=media.headers.get('content-type'))
  100. toot_media.append(media_posted['id'])
  101. media = None
  102. else:
  103. print('Dryrun: not fetching ', pic.group(0), ' and not uploading it to mastodon')
  104. # replace t.co link by original URL
  105. m = re.search(r"http[^ \xa0]*", c)
  106. if m != None:
  107. l = m.group(0)
  108. r = requests.get(l, allow_redirects=False)
  109. if r.status_code in {301,302}:
  110. c = c.replace(l,r.headers.get('Location'))
  111. # remove pic.twitter.com links
  112. m = re.search(r"pic.twitter.com[^ \xa0]*", c)
  113. if m != None:
  114. l = m.group(0)
  115. c = c.replace(l,' ')
  116. # remove ellipsis
  117. c = c.replace('\xa0…',' ')
  118. c += '\n\nSource: %s' % p.link
  119. print(c)
  120. if (not dryrun):
  121. toot = mastodon_api.status_post(c, in_reply_to_id=None, media_ids=toot_media, sensitive=False, visibility=config['sources']['twitter']['visibility'], spoiler_text=None)
  122. print( '--> toot posted!')
  123. try:
  124. db.execute("INSERT INTO posts VALUES ( ? , ? , ? , ? , ? )", (p.id, twitteruser, toot.id, mastuser, mastinstance))
  125. sql.commit()
  126. except:
  127. print('database execution failed.')
  128. print('p.id: ', p.id)
  129. print('toot.id: ', toot.id)
  130. else:
  131. print('Dryrun: not posting toot and not adding it to database')
  132. print('------------------------')
  133. # soup.io section
  134. print('====== SOUP ======')
  135. h = html2text.HTML2Text()
  136. h.ignore_links = True
  137. h.ignore_images = True
  138. h.body_width = 0
  139. s = feedparser.parse('http://'+soupuser+'/rss')
  140. # start with oldest
  141. for p in reversed(s.entries):
  142. # check if this tweet has been processed
  143. db.execute(
  144. 'SELECT * FROM posts WHERE srcpost = ? AND srcuser = ? AND mastuser = ? AND mastinstance = ?',
  145. (p.id, soupuser, mastuser, mastinstance)
  146. )
  147. last = db.fetchone()
  148. print('Processing: %s' % p.id)
  149. if last is not None:
  150. shouldpost = False
  151. print("skip: already posted")
  152. # process only unprocessed tweets less than n days old
  153. shouldpost = True
  154. posttime = datetime(p.published_parsed.tm_year, p.published_parsed.tm_mon, p.published_parsed.tm_mday, p.published_parsed.tm_hour, p.published_parsed.tm_min, p.published_parsed.tm_sec)
  155. age = datetime.now() - posttime
  156. if age > timedelta(days=days):
  157. shouldpost = False
  158. print("skip: Posting older than %s days (%s)" % (days, age) )
  159. if shouldpost:
  160. # Create application if it does not exist
  161. mastodon_api = register_app(mastuser, mastpasswd, mastinstance, mastodon_api)
  162. print(p.link)
  163. j = json.loads(p.soup_attributes)
  164. # get status id and user if twitter is source
  165. tweet_id = None
  166. tweet_author = None
  167. if (isinstance(j['source'], str)):
  168. if ( j['source'].startswith('https://twitter.com/') or j['source'].startswith('https://mobile.twitter.com/')):
  169. twitterurl = j['source'].split('/')
  170. tweet_author = twitterurl[3]
  171. if ( twitterurl[4] == 'status'):
  172. tweet_id = twitterurl[5]
  173. # get all tweeted statuses
  174. print(twitteruser)
  175. db.execute('SELECT srcpost FROM posts where srcuser = ?', (twitteruser,))
  176. postedtweets = []
  177. for postedtweet in db.fetchall():
  178. postedtweets.append(postedtweet[0].split('/')[-1])
  179. # check if already tweeted
  180. if tweet_id in postedtweets:
  181. print('Already posted the Tweet: ', j['source'])
  182. else:
  183. # collect information about images
  184. pics = []
  185. accepted_filetypes = ('.jpg', '.jpeg', '.png', '.webm', '.JPG', '.JPEG', '.PNG', '.WEBM') # let's don't do mp4 for now.
  186. if (isinstance(j['source'], str) and j['source'].endswith(accepted_filetypes) ):
  187. pics.append(j['source'])
  188. elif ( 'url' in j and isinstance(j['url'], str) and j['url'].endswith(accepted_filetypes) ):
  189. pics.append(j['url'])
  190. # get the images and post them to mastadon ...
  191. toot_media = []
  192. for pic in pics:
  193. if (not dryrun):
  194. media = requests.get(pic)
  195. print(pic, ' has mimetype ', media.headers.get('content-type'))
  196. media_posted = mastodon_api.media_post(media.content, mime_type=media.headers.get('content-type'))
  197. toot_media.append(media_posted['id'])
  198. else:
  199. print('Dryrun: not fetching ', pic, ' and not uploading it to mastodon')
  200. # remove all html stuff - python module in use only supports markdown, not pure plaintext
  201. textsrc = h.handle(p.summary_detail.value.replace("<small>", "<br><small>"))
  202. # free text from lines without visible characters
  203. cleantextsrc = ''
  204. for line in textsrc.split('\n'):
  205. line = line.strip()
  206. cleantextsrc += line + '\n'
  207. # strip newlines, reduce newlines, remove markdown bold (i know, ugly), do some clean up
  208. text = cleantextsrc.strip('\n').replace('\n\n\n','\n\n').replace('**','').replace('\\--','')
  209. # link directly to source or use soup as source.
  210. if (isinstance(j['source'], str) and j['source'] not in text):
  211. source = '\n\nSource: ' + j['source']
  212. else:
  213. source = '\n\nSource: ' + p.link
  214. # shorten text if too long
  215. maximumlegth = 500 - 1 - len(source) - 50 # 50 ... just in case (if they also count attachement url and so on)
  216. text = (text[:maximumlegth] + '…') if len(text) > maximumlegth else text
  217. # add source
  218. text += source
  219. print(text)
  220. if (not dryrun):
  221. # post toot
  222. toot = mastodon_api.status_post(text, in_reply_to_id=None, media_ids=toot_media, sensitive=False, visibility=config['sources']['soup']['visibility'], spoiler_text=None)
  223. # add entry to database
  224. if "id" in toot:
  225. db.execute("INSERT INTO posts VALUES ( ? , ? , ? , ? , ? )", (p.id, soupuser, toot.id, mastuser, mastinstance))
  226. sql.commit()
  227. print( '--> ', p.id, ' posted!')
  228. else:
  229. print('Dryrun: not posting toot and not adding it to database')
  230. print('------------------------')