From 2f74791dd6931f34870cc0f8de120f935c32ed25 Mon Sep 17 00:00:00 2001 From: b3yond Date: Mon, 8 Jan 2018 00:17:19 +0100 Subject: [PATCH 01/20] renamed promotion directory on master, too --- {campaign => promotion}/campaign.md | 0 {campaign => promotion}/flyer.md | 0 {campaign => promotion}/flyer_polite.md | 0 {campaign => promotion}/hunderttausendAugenA5.png | Bin {campaign => promotion}/hunderttausendAugenA5.xcf | Bin {campaign => promotion}/logo.png | Bin {campaign => promotion}/reclaim_the_network.png | Bin {campaign => promotion}/reclaim_the_network_A5.png | Bin {campaign => promotion}/reclaim_the_network_A5.xcf | Bin {campaign => promotion}/sex_pistols.jpeg | Bin {campaign => promotion}/so fahren alle gut.png | Bin {campaign => promotion}/vag.xcf | Bin {campaign => promotion}/wallpaper.png | Bin 13 files changed, 0 insertions(+), 0 deletions(-) rename {campaign => promotion}/campaign.md (100%) rename {campaign => promotion}/flyer.md (100%) rename {campaign => promotion}/flyer_polite.md (100%) rename {campaign => promotion}/hunderttausendAugenA5.png (100%) rename {campaign => promotion}/hunderttausendAugenA5.xcf (100%) rename {campaign => promotion}/logo.png (100%) rename {campaign => promotion}/reclaim_the_network.png (100%) rename {campaign => promotion}/reclaim_the_network_A5.png (100%) rename {campaign => promotion}/reclaim_the_network_A5.xcf (100%) rename {campaign => promotion}/sex_pistols.jpeg (100%) rename {campaign => promotion}/so fahren alle gut.png (100%) rename {campaign => promotion}/vag.xcf (100%) rename {campaign => promotion}/wallpaper.png (100%) diff --git a/campaign/campaign.md b/promotion/campaign.md similarity index 100% rename from campaign/campaign.md rename to promotion/campaign.md diff --git a/campaign/flyer.md b/promotion/flyer.md similarity index 100% rename from campaign/flyer.md rename to promotion/flyer.md diff --git a/campaign/flyer_polite.md b/promotion/flyer_polite.md similarity index 100% rename from campaign/flyer_polite.md rename to promotion/flyer_polite.md diff --git a/campaign/hunderttausendAugenA5.png b/promotion/hunderttausendAugenA5.png similarity index 100% rename from campaign/hunderttausendAugenA5.png rename to promotion/hunderttausendAugenA5.png diff --git a/campaign/hunderttausendAugenA5.xcf b/promotion/hunderttausendAugenA5.xcf similarity index 100% rename from campaign/hunderttausendAugenA5.xcf rename to promotion/hunderttausendAugenA5.xcf diff --git a/campaign/logo.png b/promotion/logo.png similarity index 100% rename from campaign/logo.png rename to promotion/logo.png diff --git a/campaign/reclaim_the_network.png b/promotion/reclaim_the_network.png similarity index 100% rename from campaign/reclaim_the_network.png rename to promotion/reclaim_the_network.png diff --git a/campaign/reclaim_the_network_A5.png b/promotion/reclaim_the_network_A5.png similarity index 100% rename from campaign/reclaim_the_network_A5.png rename to promotion/reclaim_the_network_A5.png diff --git a/campaign/reclaim_the_network_A5.xcf b/promotion/reclaim_the_network_A5.xcf similarity index 100% rename from campaign/reclaim_the_network_A5.xcf rename to promotion/reclaim_the_network_A5.xcf diff --git a/campaign/sex_pistols.jpeg b/promotion/sex_pistols.jpeg similarity index 100% rename from campaign/sex_pistols.jpeg rename to promotion/sex_pistols.jpeg diff --git a/campaign/so fahren alle gut.png b/promotion/so fahren alle gut.png similarity index 100% rename from campaign/so fahren alle gut.png rename to promotion/so fahren alle gut.png diff --git a/campaign/vag.xcf b/promotion/vag.xcf similarity index 100% rename from campaign/vag.xcf rename to promotion/vag.xcf diff --git a/campaign/wallpaper.png b/promotion/wallpaper.png similarity index 100% rename from campaign/wallpaper.png rename to promotion/wallpaper.png From 9f060b405e671f784e151f53fac49cc30bf01ecb Mon Sep 17 00:00:00 2001 From: b3yond Date: Tue, 9 Jan 2018 23:01:01 +0100 Subject: [PATCH 02/20] added nice slogan! --- promotion/flyer_polite.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/promotion/flyer_polite.md b/promotion/flyer_polite.md index 7bfeeeb..01a80ad 100644 --- a/promotion/flyer_polite.md +++ b/promotion/flyer_polite.md @@ -96,6 +96,10 @@ SCHWARZFAHREN erwischt wirst, DANN nur weil Du noch nicht den Ticketfrei-Service deiner VGN nutzt. +WENN +ÜBERWACHUNG sich lohnt, +DANN weil die +VAG öffentlich ist # Eine Zukunft ohne Ticketautomaten und Kontrolleure From ff73c5dc21f4407a7057ac8580550eb3bd0cd57a Mon Sep 17 00:00:00 2001 From: b3yond Date: Thu, 18 Jan 2018 11:41:08 +0100 Subject: [PATCH 03/20] Standardized reports; moved flow() logic to crawl(), repost(), & post(); bots don't own Trigger anymore --- report.py | 36 ++++++++++++++++++++++ retootbot.py | 83 ++++++++++++++++++++++++++++++++++----------------- retweetbot.py | 72 ++++++++++++++++++++------------------------ ticketfrei.py | 9 +++--- 4 files changed, 127 insertions(+), 73 deletions(-) create mode 100644 report.py diff --git a/report.py b/report.py new file mode 100644 index 0000000..d58c25e --- /dev/null +++ b/report.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +class Report(object): + """ + A ticketfrei report object. + + Toots, Tweets, and E-Mails can be formed into ticketfrei reports. + + """ + + def __init__(self, author, source, text, id, timestamp): + """ + Constructor of a ticketfrei report + + :param author: username of the author + :param source: mastodon, twitter, or email + :param text: the text of the report + :param id: id in the network + :param timestamp: time of the report + """ + self.author = author + self.type = source + self.text = text + self.timestamp = timestamp + self.id = id + + def format(self): + """ + Format the report for bot.post() + + :rtype: string + :return: toot: text to be tooted, e.g. "_b3yond: There are + uniformed controllers in the U2 at Opernhaus." + """ + strng = self.author + ": " + self.text + return strng diff --git a/retootbot.py b/retootbot.py index be15756..78744e6 100755 --- a/retootbot.py +++ b/retootbot.py @@ -9,14 +9,14 @@ import time import trigger import logging import sendmail +import report logger = logging.getLogger(__name__) class RetootBot(object): - def __init__(self, config, trigger): + def __init__(self, config): self.config = config - self.trigger = trigger self.client_id = self.register() self.m = self.login() @@ -53,34 +53,61 @@ class RetootBot(object): ) return m - def retoot(self, toots=()): + def crawl(self): + """ + Crawl mentions from Mastodon. + + :return: list of statuses + """ + all = self.m.notifications() + mentions = [] + for status in all: + if (status['type'] == 'mention' and status['status']['id'] not in self.seen_toots): + # save state + self.seen_toots.add(status['status']['id']) + with os.fdopen(os.open('seen_toots.pickle.part', os.O_WRONLY | os.O_EXCL | os.O_CREAT), 'wb') as f: + pickle.dump(self.seen_toots, f) + os.rename('seen_toots.pickle.part', 'seen_toots.pickle') + # add mention to mentions + mentions.append(report.Report(status['account']['acct'], + "mastodon", + re.sub(r'<[^>]*>', '', status['status']['content']), + status['status']['id'], + status['status']['created_at'])) + return mentions + + def repost(self, mention): + """ + Retoots a mention. + + :param mention: (report.Report object) + """ + logger.info('Boosting toot from %s' % ( + mention.format())) + self.m.status_reblog(mention.id) + + + def post(self, report): + """ + Toots a report from other sources. + + :param report: (report.Report object) + """ + toot = report.format() + self.m.toot(toot) + + def flow(self, trigger, reports=()): # toot external provided messages - for toot in toots: - self.m.toot(toot) + for report in reports: + self.post(report) # boost mentions retoots = [] - for notification in self.m.notifications(): - if (notification['type'] == 'mention' - and notification['status']['id'] not in self.seen_toots): - self.seen_toots.add(notification['status']['id']) - text_content = re.sub(r'<[^>]*>', '', - notification['status']['content']) - if not self.trigger.is_ok(text_content): - continue - logger.info('Boosting toot from %s: %s' % ( - notification['status']['account']['acct'], - notification['status']['content'])) - self.m.status_reblog(notification['status']['id']) - retoots.append('%s: %s' % ( - notification['status']['account']['acct'], - re.sub(r'@\S*', '', text_content))) - # If the Mastodon instance returns interesting Errors, add them here: - - # save state - with os.fdopen(os.open('seen_toots.pickle.part', os.O_WRONLY | os.O_EXCL | os.O_CREAT), 'wb') as f: - pickle.dump(self.seen_toots, f) - os.rename('seen_toots.pickle.part', 'seen_toots.pickle') + for mention in self.crawl(): + if not trigger.is_ok(mention.text): + continue + self.repost(mention) + retoots.append(mention) # return mentions for mirroring return retoots @@ -96,11 +123,11 @@ if __name__ == '__main__': logger.addHandler(fh) trigger = trigger.Trigger(config) - bot = RetootBot(config, trigger) + bot = RetootBot(config) try: while True: - bot.retoot() + bot.flow(trigger) time.sleep(1) except KeyboardInterrupt: print("Good bye. Remember to restart the bot!") diff --git a/retweetbot.py b/retweetbot.py index 84f3b56..4a48da9 100755 --- a/retweetbot.py +++ b/retweetbot.py @@ -19,18 +19,14 @@ class RetweetBot(object): api: The api object, generated with your oAuth keys, responsible for communication with twitter rest API - triggers: a list of words, one of them has to be in a tweet for it to be - retweeted last_mention: the ID of the last tweet which mentioned you """ - def __init__(self, trigger, config, history_path="last_mention"): + def __init__(self, config, history_path="last_mention"): """ Initializes the bot and loads all the necessary data. - :param trigger: object of the trigger :param config: (dictionary) config.toml as a dictionary of dictionaries - :param logger: object of the logger :param history_path: Path to the file with ID of the last retweeted Tweet """ @@ -46,7 +42,6 @@ class RetweetBot(object): self.history_path = history_path self.last_mention = self.get_history(self.history_path) - self.trigger = trigger self.waitcounter = 0 def get_api_keys(self): @@ -101,19 +96,7 @@ class RetweetBot(object): self.waitcounter -= 1 return self.waitcounter - def format_mastodon(self, status): - """ - Bridge your Retweets to mastodon. - - :rtype: string - :param status: Object of a tweet. - :return: toot: text tooted on mastodon, e.g. "_b3yond: There are - uniformed controllers in the U2 at Opernhaus." - """ - toot = status.user.name + ": " + status.text - return toot - - def crawl_mentions(self): + def crawl(self): """ crawls all Tweets which mention the bot from the twitter rest API. @@ -134,20 +117,20 @@ class RetweetBot(object): self.waitcounter += 10 return [] - def retweet(self, status): + def repost(self, status): """ Retweets a given tweet. - :param status: A tweet object. + :param status: (report.Report object) :return: toot: string of the tweet, to toot on mastodon. """ while 1: try: self.api.retweet(status.id) - logger.info("Retweeted: " + self.format_mastodon(status)) + logger.info("Retweeted: " + status.format()) if status.id > self.last_mention: self.last_mention = status.id - return self.format_mastodon(status) + return status.format() except requests.exceptions.ConnectionError: logger.error("Twitter API Error: Bad Connection", exc_info=True) sleep(10) @@ -158,67 +141,76 @@ class RetweetBot(object): self.last_mention = status.id return None - def tweet(self, post): + def post(self, status): """ Tweet a post. - :param post: String with the text to tweet. + :param status: (report.Report object) """ - if len(post) > 280: - post = post[:280 - 4] + u' ...' + text = status.format() + if len(text) > 280: + text = status.text[:280 - 4] + u' ...' while 1: try: - self.api.update_status(status=post) + self.api.update_status(status=text) return except requests.exceptions.ConnectionError: logger.error("Twitter API Error: Bad Connection", exc_info=True) sleep(10) - def flow(self, to_tweet=()): + def flow(self, trigger, to_tweet=()): """ The flow of crawling mentions and retweeting them. :param to_tweet: list of strings to tweet :return list of retweeted tweets, to toot on mastodon """ - # Tweet the toots the Retootbot gives to us + # Tweet the reports from other sources for post in to_tweet: - self.tweet(post) + self.post(post) # Store all mentions in a list of Status Objects - mentions = self.crawl_mentions() - mastodon = [] + mentions = self.crawl() + + # initialise list of strings for other bots + all_tweets = [] for status in mentions: # Is the Text of the Tweet in the triggerlist? - if self.trigger.is_ok(status.text): + if trigger.is_ok(status.text): # Retweet status - toot = self.retweet(status) + toot = self.repost(status) if toot: - mastodon.append(toot) + all_tweets.append(toot) # save the id so it doesn't get crawled again if status.id > self.last_mention: self.last_mention = status.id self.save_last_mention() - # Return Retweets for tooting on mastodon - return mastodon + # Return Retweets for posting on other bots + return all_tweets if __name__ == "__main__": - # create an Api object + # get the config dict of dicts with open('config.toml') as configfile: config = toml.load(configfile) + # set log file fh = logging.FileHandler(config['logging']['logpath']) fh.setLevel(logging.DEBUG) logger.addHandler(fh) + # initialise trigger trigger = trigger.Trigger(config) + + # initialise twitter bot bot = RetweetBot(trigger, config) + try: while True: - bot.flow() + # :todo separate into small functions + bot.flow(trigger) sleep(60) except KeyboardInterrupt: print("Good bye. Remember to restart the bot!") diff --git a/ticketfrei.py b/ticketfrei.py index a796689..ad554ab 100755 --- a/ticketfrei.py +++ b/ticketfrei.py @@ -9,7 +9,6 @@ from retootbot import RetootBot from retweetbot import RetweetBot from trigger import Trigger - if __name__ == '__main__': # read config in TOML format (https://github.com/toml-lang/toml#toml) with open('config.toml') as configfile: @@ -21,14 +20,14 @@ if __name__ == '__main__': logger.addHandler(fh) trigger = Trigger(config) - mbot = RetootBot(config, trigger) - tbot = RetweetBot(trigger, config) + mbot = RetootBot(config) + tbot = RetweetBot(config) try: statuses = [] while True: - statuses = mbot.retoot(statuses) - statuses = tbot.flow(statuses) + statuses = mbot.flow(trigger, statuses) + statuses = tbot.flow(trigger, to_tweet=statuses) time.sleep(60) except KeyboardInterrupt: print("Good bye. Remember to restart the bot!") From cde5494de3242f5bd703f7a6b7b86a002e30b21e Mon Sep 17 00:00:00 2001 From: b3yond Date: Thu, 18 Jan 2018 12:42:37 +0100 Subject: [PATCH 04/20] mailbot uses reports now, and doesn't need to own trigger --- mailbot.py | 63 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/mailbot.py b/mailbot.py index b53cf14..39c1eb9 100644 --- a/mailbot.py +++ b/mailbot.py @@ -9,6 +9,7 @@ import email import logging import pytoml as toml import imaplib +import report logger = logging.getLogger(__name__) @@ -19,7 +20,7 @@ class Mailbot(object): other bots that it received mails. """ - def __init__(self, config, trigger, history_path="last_mail"): + def __init__(self, config, history_path="last_mail"): """ Creates a Bot who listens to mails and forwards them to other bots. @@ -27,7 +28,6 @@ class Mailbot(object): :param config: (dictionary) config.toml as a dictionary of dictionaries """ self.config = config - self.trigger = trigger self.history_path = history_path self.last_mail = self.get_history(self.history_path) @@ -55,10 +55,17 @@ class Mailbot(object): except: logger.error('Mail sending failed', exc_info=True) - def listen(self): + def repost(self): """ - listen for mails which contain goodwords but no badwords. - :return: + E-Mails don't have to be reposted - they already reached everyone on the mailing list. + The function still needs to be here because ticketfrei.py assumes it. + """ + pass + + def crawl(self): + """ + crawl for new mails. + :return: msgs: (list of report.Report objects) """ rv, data = self.mailbox.select("Inbox") msgs = [] @@ -81,11 +88,12 @@ class Mailbot(object): if date > self.get_history(self.history_path): self.last_mail = date self.save_last_mail() - msgs.append(msg) + msgs.append(self.make_report(msg)) return msgs def get_history(self, path): - """ This counter is needed to keep track of your mails, so you + """ + This counter is needed to keep track of your mails, so you don't double parse them :param path: string: contains path to the file where the ID of the @@ -106,21 +114,23 @@ class Mailbot(object): with open(self.history_path, "w") as f: f.write(str(self.last_mail)) - def send_report(self, statuses): + def post(self, statuses): """ - sends reports by twitter & mastodon to a mailing list. + sends reports by other sources to a mailing list. - :param statuses: (list) of status strings + :param statuses: (list of report.Report objects) """ for status in statuses: + status = status.format() mailer = sendmail.Mailer(self.config) mailer.send(status, self.mailinglist, "Warnung: Kontrolleure gesehen") - def to_social(self, msg): + def make_report(self, msg): """ - sends a report from the mailing list to social + generates a report out of a mail + :param msg: email.parser.Message object - :return: post: (string) of author + text + :return: post: report.Report object """ # get a comparable date out of the email date_tuple = email.utils.parsedate_tz(msg['Date']) @@ -131,26 +141,26 @@ class Mailbot(object): # :todo take only the part before the @ text = msg.get_payload() - post = author + ": " + text + post = report.Report(author, "mail", text, None, date) self.last_mail = date self.save_last_mail() return post - def flow(self, statuses): + def flow(self, trigger, statuses): """ - to be iterated + to be iterated. uses trigger to separate the sheep from the goats - :param statuses: (list) of statuses to send to mailinglist - :return: list of statuses to post in mastodon & twitter + :param statuses: (list of report.Report objects) + :return: statuses: (list of report.Report objects) """ - self.send_report(statuses) + self.post(statuses) - msgs = self.listen() + msgs = self.crawl() statuses = [] for msg in msgs: - if self.trigger.is_ok(msg.get_payload()): - statuses.append(self.to_social(msg)) + if trigger.is_ok(msg.get_payload()): + statuses.append(msg) return statuses @@ -159,16 +169,21 @@ if __name__ == "__main__": with open('config.toml') as configfile: config = toml.load(configfile) + # set log file fh = logging.FileHandler(config['logging']['logpath']) fh.setLevel(logging.DEBUG) logger.addHandler(fh) + # initialise trigger trigger = trigger.Trigger(config) - m = Mailbot(config, trigger) + + # initialise mail bot + m = Mailbot(config) + statuses = [] try: while 1: - print("Received Reports: " + str(m.flow(statuses))) + print("Received Reports: " + str(m.flow(trigger, statuses))) time.sleep(1) except KeyboardInterrupt: print("Good bye. Remember to restart the bot!") From d6a0c6d377ce190bef0d3e2eaac9ace16dc3a283 Mon Sep 17 00:00:00 2001 From: b3yond Date: Thu, 18 Jan 2018 13:06:53 +0100 Subject: [PATCH 05/20] changed ticketfrei flow logic, integrated mailbot!!! #11 --- mailbot.py | 12 ++++++------ retootbot.py | 8 ++++++-- retweetbot.py | 6 +++--- ticketfrei.py | 23 +++++++++++++++++------ 4 files changed, 32 insertions(+), 17 deletions(-) diff --git a/mailbot.py b/mailbot.py index 39c1eb9..fb6703d 100644 --- a/mailbot.py +++ b/mailbot.py @@ -87,7 +87,7 @@ class Mailbot(object): date = (date_tuple - datetime.datetime(1970, 1, 1)).total_seconds() if date > self.get_history(self.history_path): self.last_mail = date - self.save_last_mail() + self.save_last() msgs.append(self.make_report(msg)) return msgs @@ -109,7 +109,7 @@ class Mailbot(object): f.write(last_mail) return float(last_mail) - def save_last_mail(self): + def save_last(self): """ Saves the last retweeted tweet in last_mention. """ with open(self.history_path, "w") as f: f.write(str(self.last_mail)) @@ -121,9 +121,8 @@ class Mailbot(object): :param statuses: (list of report.Report objects) """ for status in statuses: - status = status.format() mailer = sendmail.Mailer(self.config) - mailer.send(status, self.mailinglist, "Warnung: Kontrolleure gesehen") + mailer.send(status.format(), self.mailinglist, "Warnung: Kontrolleure gesehen") def make_report(self, msg): """ @@ -143,7 +142,7 @@ class Mailbot(object): text = msg.get_payload() post = report.Report(author, "mail", text, None, date) self.last_mail = date - self.save_last_mail() + self.save_last() return post def flow(self, trigger, statuses): @@ -170,6 +169,7 @@ if __name__ == "__main__": config = toml.load(configfile) # set log file + logger = logging.getLogger() fh = logging.FileHandler(config['logging']['logpath']) fh.setLevel(logging.DEBUG) logger.addHandler(fh) @@ -189,7 +189,7 @@ if __name__ == "__main__": print("Good bye. Remember to restart the bot!") except: logger.error('Shutdown', exc_info=True) - m.save_last_mail() + m.save_last() try: mailer = sendmail.Mailer(config) mailer.send('', config['mail']['contact'], diff --git a/retootbot.py b/retootbot.py index 78744e6..8caaedb 100755 --- a/retootbot.py +++ b/retootbot.py @@ -53,6 +53,11 @@ class RetootBot(object): ) return m + def save_last(self): + """ save the last seen toot """ + with os.fdopen(os.open('seen_toots.pickle.part', os.O_WRONLY | os.O_EXCL | os.O_CREAT), 'wb') as f: + pickle.dump(self.seen_toots, f) + def crawl(self): """ Crawl mentions from Mastodon. @@ -65,8 +70,7 @@ class RetootBot(object): if (status['type'] == 'mention' and status['status']['id'] not in self.seen_toots): # save state self.seen_toots.add(status['status']['id']) - with os.fdopen(os.open('seen_toots.pickle.part', os.O_WRONLY | os.O_EXCL | os.O_CREAT), 'wb') as f: - pickle.dump(self.seen_toots, f) + self.save_last() os.rename('seen_toots.pickle.part', 'seen_toots.pickle') # add mention to mentions mentions.append(report.Report(status['account']['acct'], diff --git a/retweetbot.py b/retweetbot.py index 4a48da9..8dd3e10 100755 --- a/retweetbot.py +++ b/retweetbot.py @@ -80,7 +80,7 @@ class RetweetBot(object): f.write(last_mention) return int(last_mention) - def save_last_mention(self): + def save_last(self): """ Saves the last retweeted tweet in last_mention. """ with open(self.history_path, "w") as f: f.write(str(self.last_mention)) @@ -186,7 +186,7 @@ class RetweetBot(object): # save the id so it doesn't get crawled again if status.id > self.last_mention: self.last_mention = status.id - self.save_last_mention() + self.save_last() # Return Retweets for posting on other bots return all_tweets @@ -216,7 +216,7 @@ if __name__ == "__main__": print("Good bye. Remember to restart the bot!") except: logger.error('Shutdown', exc_info=True) - bot.save_last_mention() + bot.save_last() try: mailer = sendmail.Mailer(config) mailer.send('', config['mail']['contact'], diff --git a/ticketfrei.py b/ticketfrei.py index ad554ab..d6097a3 100755 --- a/ticketfrei.py +++ b/ticketfrei.py @@ -7,6 +7,7 @@ import sendmail from retootbot import RetootBot from retweetbot import RetweetBot +from mailbot import Mailbot from trigger import Trigger if __name__ == '__main__': @@ -14,26 +15,36 @@ if __name__ == '__main__': with open('config.toml') as configfile: config = toml.load(configfile) + # set log file logger = logging.getLogger() fh = logging.FileHandler(config['logging']['logpath']) fh.setLevel(logging.DEBUG) logger.addHandler(fh) trigger = Trigger(config) - mbot = RetootBot(config) - tbot = RetweetBot(config) + + bots = [RetootBot(config), RetweetBot(config), Mailbot(config)] try: statuses = [] while True: - statuses = mbot.flow(trigger, statuses) - statuses = tbot.flow(trigger, to_tweet=statuses) - time.sleep(60) + for bot in bots: + reports = bot.crawl() + for status in reports: + if not trigger.is_ok(status.text): + continue + for bot2 in bots: + if bot == bot2: + bot2.repost(status) + else: + bot2.post(status) + time.sleep(60) # twitter rate limit >.< except KeyboardInterrupt: print("Good bye. Remember to restart the bot!") except: logger.error('Shutdown', exc_info=True) - tbot.save_last_mention() + for bot in bots: + bot.save_last() try: mailer = sendmail.Mailer(config) mailer.send('', config['mail']['contact'], From 75e1ff902cd9ba256c5fadb2d6d012dd455a0ff9 Mon Sep 17 00:00:00 2001 From: b3yond Date: Thu, 18 Jan 2018 13:19:11 +0100 Subject: [PATCH 06/20] function needs to take an argument --- mailbot.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mailbot.py b/mailbot.py index fb6703d..098fc24 100644 --- a/mailbot.py +++ b/mailbot.py @@ -55,10 +55,13 @@ class Mailbot(object): except: logger.error('Mail sending failed', exc_info=True) - def repost(self): + def repost(self, status): """ E-Mails don't have to be reposted - they already reached everyone on the mailing list. - The function still needs to be here because ticketfrei.py assumes it. + The function still needs to be here because ticketfrei.py assumes it, and take the + report object they want to give us. + + :param status: (report.Report object) """ pass From b5288f341ce67a37630315a68adf9a1842922e7e Mon Sep 17 00:00:00 2001 From: b3yond Date: Thu, 18 Jan 2018 13:40:07 +0100 Subject: [PATCH 07/20] bugfix: FileExistsError --- retootbot.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/retootbot.py b/retootbot.py index 8caaedb..c39f8cb 100755 --- a/retootbot.py +++ b/retootbot.py @@ -55,8 +55,12 @@ class RetootBot(object): def save_last(self): """ save the last seen toot """ - with os.fdopen(os.open('seen_toots.pickle.part', os.O_WRONLY | os.O_EXCL | os.O_CREAT), 'wb') as f: - pickle.dump(self.seen_toots, f) + try: + with os.fdopen(os.open('seen_toots.pickle.part', os.O_WRONLY | os.O_EXCL | os.O_CREAT), 'wb') as f: + pickle.dump(self.seen_toots, f) + except FileExistsError: + with os.fdopen(os.open('seen_toots.pickle.part', os.O_WRONLY), 'wb') as f: + pickle.dump(self.seen_toots, f) def crawl(self): """ From 048bad181bbc746cc81dca5e996df393954b7123 Mon Sep 17 00:00:00 2001 From: b3yond Date: Thu, 18 Jan 2018 13:42:23 +0100 Subject: [PATCH 08/20] only send one status at a time --- mailbot.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mailbot.py b/mailbot.py index 098fc24..20216d1 100644 --- a/mailbot.py +++ b/mailbot.py @@ -117,15 +117,14 @@ class Mailbot(object): with open(self.history_path, "w") as f: f.write(str(self.last_mail)) - def post(self, statuses): + def post(self, status): """ sends reports by other sources to a mailing list. - :param statuses: (list of report.Report objects) + :param status: (report.Report object) """ - for status in statuses: - mailer = sendmail.Mailer(self.config) - mailer.send(status.format(), self.mailinglist, "Warnung: Kontrolleure gesehen") + mailer = sendmail.Mailer(self.config) + mailer.send(status.format(), self.mailinglist, "Warnung: Kontrolleure gesehen") def make_report(self, msg): """ From b174db3cfeed2a07222d8efe3dd1c3ff4af60338 Mon Sep 17 00:00:00 2001 From: b3yond Date: Thu, 18 Jan 2018 13:54:32 +0100 Subject: [PATCH 09/20] twitterbot.crawl() returns reports now, not statuses --- retweetbot.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/retweetbot.py b/retweetbot.py index 8dd3e10..9b0e731 100755 --- a/retweetbot.py +++ b/retweetbot.py @@ -5,6 +5,7 @@ import requests import pytoml as toml import trigger from time import sleep +import report import logging import sendmail @@ -100,15 +101,18 @@ class RetweetBot(object): """ crawls all Tweets which mention the bot from the twitter rest API. - :return: list of Status objects + :return: reports: (list of report.Report objects) """ + reports = [] try: if not self.waiting(): if self.last_mention == 0: mentions = self.api.mentions_timeline() else: mentions = self.api.mentions_timeline(since_id=self.last_mention) - return mentions + for status in mentions: + reports.append(report.Report(status.author, "twitter", status.text, status.id, status.created_at)) + return reports except tweepy.RateLimitError: logger.error("Twitter API Error: Rate Limit Exceeded", exc_info=True) self.waitcounter += 60*15 + 1 From 72d0acb20ace4aca86c27c274c0cf66195424299 Mon Sep 17 00:00:00 2001 From: b3yond Date: Thu, 18 Jan 2018 13:59:37 +0100 Subject: [PATCH 10/20] bugfix: gave Report.__init__() twitter User object, not screen_name --- retweetbot.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/retweetbot.py b/retweetbot.py index 9b0e731..a585633 100755 --- a/retweetbot.py +++ b/retweetbot.py @@ -111,7 +111,11 @@ class RetweetBot(object): else: mentions = self.api.mentions_timeline(since_id=self.last_mention) for status in mentions: - reports.append(report.Report(status.author, "twitter", status.text, status.id, status.created_at)) + reports.append(report.Report(status.author.screen_name, + "twitter", + status.text, + status.id, + status.created_at)) return reports except tweepy.RateLimitError: logger.error("Twitter API Error: Rate Limit Exceeded", exc_info=True) From ee61ba19e6c7a6131b06c5c05d605754a5fad095 Mon Sep 17 00:00:00 2001 From: b3yond Date: Thu, 18 Jan 2018 14:23:11 +0100 Subject: [PATCH 11/20] mailbot doesn't crawl mails which it wrote itself anymore --- mailbot.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/mailbot.py b/mailbot.py index 20216d1..a513359 100644 --- a/mailbot.py +++ b/mailbot.py @@ -84,14 +84,16 @@ class Mailbot(object): return msgs msg = email.message_from_bytes(data[0][1]) - # get a comparable date out of the email - date_tuple = email.utils.parsedate_tz(msg['Date']) - date_tuple = datetime.datetime.fromtimestamp(email.utils.mktime_tz(date_tuple)) - date = (date_tuple - datetime.datetime(1970, 1, 1)).total_seconds() - if date > self.get_history(self.history_path): - self.last_mail = date - self.save_last() - msgs.append(self.make_report(msg)) + if not self.config['mail']['user'] + "@" + \ + self.config["mail"]["mailserver"].partition(".")[2] in msg['From']: + # get a comparable date out of the email + date_tuple = email.utils.parsedate_tz(msg['Date']) + date_tuple = datetime.datetime.fromtimestamp(email.utils.mktime_tz(date_tuple)) + date = (date_tuple - datetime.datetime(1970, 1, 1)).total_seconds() + if date > self.get_history(self.history_path): + self.last_mail = date + self.save_last() + msgs.append(self.make_report(msg)) return msgs def get_history(self, path): @@ -154,7 +156,8 @@ class Mailbot(object): :param statuses: (list of report.Report objects) :return: statuses: (list of report.Report objects) """ - self.post(statuses) + for status in statuses: + self.post(status) msgs = self.crawl() From 10de40549c19b11735b94e5c1e41e98ec94e3f2f Mon Sep 17 00:00:00 2001 From: b3yond Date: Thu, 18 Jan 2018 14:48:53 +0100 Subject: [PATCH 12/20] added regex magic so twitter & masto don't mention themselves by accident --- retootbot.py | 3 ++- retweetbot.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/retootbot.py b/retootbot.py index c39f8cb..8b4ffdc 100755 --- a/retootbot.py +++ b/retootbot.py @@ -77,9 +77,10 @@ class RetootBot(object): self.save_last() os.rename('seen_toots.pickle.part', 'seen_toots.pickle') # add mention to mentions + text = re.sub("(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)", "", status['status']['content']) mentions.append(report.Report(status['account']['acct'], "mastodon", - re.sub(r'<[^>]*>', '', status['status']['content']), + re.sub(r'<[^>]*>', '', text), status['status']['id'], status['status']['created_at'])) return mentions diff --git a/retweetbot.py b/retweetbot.py index a585633..b120a4f 100755 --- a/retweetbot.py +++ b/retweetbot.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import tweepy +import re import requests import pytoml as toml import trigger @@ -111,9 +112,10 @@ class RetweetBot(object): else: mentions = self.api.mentions_timeline(since_id=self.last_mention) for status in mentions: + text = re.sub("(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)", "", status.text) reports.append(report.Report(status.author.screen_name, "twitter", - status.text, + text, status.id, status.created_at)) return reports From c8e67d193769e9683756936d3b57d84616572b9f Mon Sep 17 00:00:00 2001 From: b3yond Date: Thu, 18 Jan 2018 15:10:05 +0100 Subject: [PATCH 13/20] better error handling of FileExistsError, fixed regex for mastobot --- retootbot.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/retootbot.py b/retootbot.py index 8b4ffdc..31e0ab4 100755 --- a/retootbot.py +++ b/retootbot.py @@ -59,8 +59,10 @@ class RetootBot(object): with os.fdopen(os.open('seen_toots.pickle.part', os.O_WRONLY | os.O_EXCL | os.O_CREAT), 'wb') as f: pickle.dump(self.seen_toots, f) except FileExistsError: - with os.fdopen(os.open('seen_toots.pickle.part', os.O_WRONLY), 'wb') as f: + os.unlink('seen_toots.pickle.part') + with os.fdopen(os.open('seen_toots.pickle.part', os.O_WRONLY | os.O_EXCL | os.O_CREAT), 'wb') as f: pickle.dump(self.seen_toots, f) + os.rename('seen_toots.pickle.part', 'seen_toots.pickle') def crawl(self): """ @@ -75,12 +77,12 @@ class RetootBot(object): # save state self.seen_toots.add(status['status']['id']) self.save_last() - os.rename('seen_toots.pickle.part', 'seen_toots.pickle') # add mention to mentions - text = re.sub("(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)", "", status['status']['content']) + text = re.sub(r'<[^>]*>', '', status['status']['content']) + text = re.sub("(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)", "", text) mentions.append(report.Report(status['account']['acct'], "mastodon", - re.sub(r'<[^>]*>', '', text), + text, status['status']['id'], status['status']['created_at'])) return mentions From 21e4af6fa9b76d17001cbd94bbd7177521cf39ee Mon Sep 17 00:00:00 2001 From: b3yond Date: Thu, 18 Jan 2018 15:14:04 +0100 Subject: [PATCH 14/20] twitter accidentially crawled too many tweets --- retweetbot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/retweetbot.py b/retweetbot.py index b120a4f..315f0ed 100755 --- a/retweetbot.py +++ b/retweetbot.py @@ -118,6 +118,7 @@ class RetweetBot(object): text, status.id, status.created_at)) + self.save_last() return reports except tweepy.RateLimitError: logger.error("Twitter API Error: Rate Limit Exceeded", exc_info=True) From 12fbbde79c2d46b3f04900d8fef5f0e5cd111010 Mon Sep 17 00:00:00 2001 From: b3yond Date: Thu, 18 Jan 2018 15:18:20 +0100 Subject: [PATCH 15/20] bots don't own trigger anymore --- retweetbot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/retweetbot.py b/retweetbot.py index 315f0ed..8d99196 100755 --- a/retweetbot.py +++ b/retweetbot.py @@ -216,7 +216,7 @@ if __name__ == "__main__": trigger = trigger.Trigger(config) # initialise twitter bot - bot = RetweetBot(trigger, config) + bot = RetweetBot(config) try: while True: From 9305a32eb7ae694b77d4e3bd5a395d68b05b4bf0 Mon Sep 17 00:00:00 2001 From: b3yond Date: Thu, 18 Jan 2018 20:15:41 +0100 Subject: [PATCH 16/20] added more save_last(), schadet nicht --- retweetbot.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/retweetbot.py b/retweetbot.py index 8d99196..88fd2fb 100755 --- a/retweetbot.py +++ b/retweetbot.py @@ -141,6 +141,7 @@ class RetweetBot(object): logger.info("Retweeted: " + status.format()) if status.id > self.last_mention: self.last_mention = status.id + self.save_last() return status.format() except requests.exceptions.ConnectionError: logger.error("Twitter API Error: Bad Connection", exc_info=True) @@ -150,6 +151,7 @@ class RetweetBot(object): logger.error("Twitter Error", exc_info=True) if status.id > self.last_mention: self.last_mention = status.id + self.save_last() return None def post(self, status): @@ -194,10 +196,6 @@ class RetweetBot(object): if toot: all_tweets.append(toot) - # save the id so it doesn't get crawled again - if status.id > self.last_mention: - self.last_mention = status.id - self.save_last() # Return Retweets for posting on other bots return all_tweets From 37b2706a3b18490810c72e3244e72598f8beb140 Mon Sep 17 00:00:00 2001 From: b3yond Date: Thu, 18 Jan 2018 21:48:36 +0100 Subject: [PATCH 17/20] excepted TweepError that was raised without an explanation further than 503 --- retweetbot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/retweetbot.py b/retweetbot.py index 88fd2fb..e196ca6 100755 --- a/retweetbot.py +++ b/retweetbot.py @@ -126,6 +126,8 @@ class RetweetBot(object): except requests.exceptions.ConnectionError: logger.error("Twitter API Error: Bad Connection", exc_info=True) self.waitcounter += 10 + except tweepy.TweepError: + logger.error("Twitter API Error: General Error", exc_info=True) return [] def repost(self, status): From 04e05ee8ca754815d35b83c7d3679daacb723dc0 Mon Sep 17 00:00:00 2001 From: b3yond Date: Fri, 19 Jan 2018 00:17:09 +0100 Subject: [PATCH 18/20] excepted Mastodon API Error with a too broad exception --- retootbot.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/retootbot.py b/retootbot.py index 31e0ab4..b545cef 100755 --- a/retootbot.py +++ b/retootbot.py @@ -70,8 +70,12 @@ class RetootBot(object): :return: list of statuses """ - all = self.m.notifications() mentions = [] + try: + all = self.m.notifications() + except: # mastodon.Mastodon.MastodonAPIError is unfortunately not in __init__.py + logger.error("Unknown Mastodon API Error.", exc_info=True) + return mentions for status in all: if (status['type'] == 'mention' and status['status']['id'] not in self.seen_toots): # save state From c9e6a353728f5e9a260fb209b990d7fad75b5986 Mon Sep 17 00:00:00 2001 From: b3yond Date: Fri, 19 Jan 2018 16:00:36 +0100 Subject: [PATCH 19/20] updated README to version 1.0. you can disable accounts now --- README.md | 88 +++++++++++++++++++++++++-------------------- config.toml.example | 7 ++-- ticketfrei.py | 9 ++++- 3 files changed, 63 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index da93933..bbe781a 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,37 @@ # Ticketfrei micro messaging bot +Version: 1.0 + -The functionality is simple: it retweets every tweet where it is mentioned. +The functionality is simple: it retweets every tweet where it is +mentioned. -This leads to a community which evolves around it; if you see ticket controllers, you tweet their location and mention the bot. The bot then retweets your tweet and others can read the info and think twice if they want to buy a ticket. If enough people, a critical mass, participate for the bot to become reliable, you have positive self-reinforcing dynamics. +This leads to a community which evolves around it; if you see ticket +controllers, you tweet their location and mention the bot. The bot +then retweets your tweet and others can read the info and think twice +if they want to buy a ticket. If enough people, a critical mass, +participate for the bot to become reliable, you have positive +self-reinforcing dynamics. -There is one security hole: people could start mentioning the bot with useless information, turning it into a spammer. That's why it has to be maintained; if someone spams the bot, mute them and undo the retweet. So it won't retweet their future tweets and the useless retweet is deleted if someone tries to check if something was retweeted in the last hour or something. -To this date, we have never heard of this happening though. +In the promotion folder, you will find some promotion material you +can use to build up such a community in your city. It is in german +though =/ -Website: https://wiki.links-it.de/IT/Ticketfrei +Website: https://wiki.links-tech.org/IT/Ticketfrei ## Install -Setting up a ticketfrei bot for your city is quite easy. Here are the few steps: +Setting up a ticketfrei bot for your city is quite easy. Here are the +few steps: + +First you need to install python3 and virtualenv with your favourite +package manager. -First you need to install python and virtualenv with your favourite package manager. Create and activate virtualenv: ```shell -sudo apt install python virtualenv +sudo apt install python3 virtualenv virtualenv -p python3 . . bin/activate ``` @@ -34,19 +46,44 @@ Configure the bot: cp config.toml.example config.toml vim config.toml ``` -Edit the account credentials, so your bot can use your accounts. + +You can use a Twitter, a Mastodon, and Mail with the account. They +will communicate with each other; if someone warns others via Mail, +Twitter and Mastodon users will also see the message. And vice versa. + +You have to configure all of the accounts via config.toml; it should +be fairly intuitive to enter the right values. + +## Maintaining + +There is one security hole: people could start mentioning the bot +with useless information, turning it into a spammer. That's why it +has to be maintained; if someone spams the bot, mute them and undo +the retweet. So it won't retweet their future tweets and the useless +retweet is deleted if someone tries to check if something was +retweeted in the last hour or something. + +To this date, we have never heard of this happening though. ### blacklisting -Also add the words to the goodlist, which you want to require. A tweet is only retweeted, if it contains at least one of them. If you want to RT everything, just add your account name. +You also need to edit the goodlist and the blacklist. They are in the +"goodlists" and "blacklists" folders. All text files in those +directories will be used, so you should delete our templates; but +feel free to use them as an orientation. -There is also a blacklist, which you can use to automatically sort out malicious tweets. Be careful though, our filter can't read the intention with which a word was used. Maybe you wanted it there. +Just add the words to the goodlist, which you want to require. A +report is only spread, if it contains at least one of them. If you +want to RT everything, just add a ```\*```. -Note that atm the good- & blacklist are still outside of config.toml, in separate files. we will repare this soon. +There is also a blacklist, which you can use to automatically sort +out malicious tweets. Be careful though, our filter can't read the +intention with which a word was used. Maybe you wanted it there. ### screen -To keep the bots running when you are logged out of the shell, you can use screen: +To keep the bots running when you are logged out of the shell, you +can use screen: ```shell sudo apt-get install screen @@ -57,28 +94,3 @@ python3 ticketfrei.py To log out of the screen session, press "ctrl+a", and then "d". -## ideas - -* You can only use the twitter API if you have confirmed a phone number and sacrificed a penguin in a blood ritual. So we should build it in a way that it uses the twitter web GUI. It's difficult, but maybe it works. We had another twitter bot that worked similarly, years ago: https://github.com/b3yond/twitter-bot -* Build a tool that deletes wrong toots/tweets on both platforms, would work nicely with a web UI. -* write the muted people to the db, to easily undo the mutes if necessary. - -## to do -Desktop/pycharm-community-2017.1.4/bin/pycharm.sh -- [x] Twitter: Crawl mentions -- [x] Mastodon: Crawl mentions -- [ ] Write toots/tweets to database/log -- [x] Twitter: retweet people -- [x] Mastodon: boost people -- [x] Twitter: access the API -- [ ] Web UI that lets you easily delete toots/tweets per db id and/or mute the tweet author -- [x] Write Bots as Classes to be easier implemented -- [x] Create extra Class for the filter -- [x] Put as much as possible into config.toml -- [x] Make both bots run on their own *and* next to each other - - [x] implement trigger class in retootbot - - [x] read config in retweetbot -- [x] put shutdown contact in config.toml -- [x] document what you have to configure if you setup the bot in another city -- [ ] write a script to setup the bot easily. ask the admin for the necessary information -- [ ] ongoing: solve issues diff --git a/config.toml.example b/config.toml.example index ab040ce..235653f 100644 --- a/config.toml.example +++ b/config.toml.example @@ -3,6 +3,7 @@ name = 'yourcity_ticketfrei' # What you want the app to be called [muser] +enabled = 'false' # set to true if you want to use Mastodon email = 'youremail@server.tld' # E-mail address of your Mastodon account password = 'yourpassword' # Password of your Mastodon account server = 'yourmastodoninstance' # Instance where you have your Mastodon account @@ -14,12 +15,14 @@ consumer_key = "your_consumer_key" consumer_secret = "your_consumer_secret" [tuser] +enabled = 'false' # set to true if you want to use Twitter # You get those keys when you follow these steps: # https://developer.twitter.com/en/docs/basics/authentication/guides/access-tokens access_token_key = "your_access_token_key" access_token_secret = "your_acces_token_secret" [mail] +enabled = 'false' # set to true if you want to use Mail notifications # This is the mail the bot uses to send emails. mailserver = "smtp.riseup.net" user = "ticketfrei" @@ -28,11 +31,11 @@ passphrase = "sup3rs3cur3" # when it breaks down), you should specify a contact email address: #contact = "your_mail@riseup.net" # Mailing list where you want to send warnings to -#list = "nbg_ticketfrei@lists.links-tech.org" +list = "yourcity_ticketfrei@lists.links-tech.org" [logging] # The directory where logs should be stored. -logpath = "logs" +logpath = "logs/ticketfrei.log" # [trigger] # goodlists are one regex per line. diff --git a/ticketfrei.py b/ticketfrei.py index d6097a3..f9f33ba 100755 --- a/ticketfrei.py +++ b/ticketfrei.py @@ -23,7 +23,14 @@ if __name__ == '__main__': trigger = Trigger(config) - bots = [RetootBot(config), RetweetBot(config), Mailbot(config)] + bots = [] + + if config["muser"]["enabled"] != "false": + bots.append(RetootBot(config)) + if config["tuser"]["enabled"] != "false": + bots.append(RetweetBot(config)) + if config["mail"]["enabled"] != "false": + bots.append(Mailbot(config)) try: statuses = [] From c0328be3a479b38820923b4aed2847e7a2bc300f Mon Sep 17 00:00:00 2001 From: b3yond Date: Fri, 19 Jan 2018 16:27:30 +0100 Subject: [PATCH 20/20] one \ to much lol --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bbe781a..43a0426 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ feel free to use them as an orientation. Just add the words to the goodlist, which you want to require. A report is only spread, if it contains at least one of them. If you -want to RT everything, just add a ```\*```. +want to RT everything, just add a ```*```. There is also a blacklist, which you can use to automatically sort out malicious tweets. Be careful though, our filter can't read the