From ec3053a0ab4433c3dc740cd3840fc9a02369091f Mon Sep 17 00:00:00 2001 From: b3yond Date: Wed, 28 Mar 2018 20:24:21 +0200 Subject: [PATCH 1/3] small bugfixes --- active_bots/mailbot.py | 7 ++- active_bots/twitterbot.py | 7 +-- mall bugfixes | 104 ++++++++++++++++++++++++++++++++++++++ sendmail.py | 10 ++-- session.py | 2 +- user.py | 2 +- 6 files changed, 114 insertions(+), 18 deletions(-) create mode 100644 mall bugfixes diff --git a/active_bots/mailbot.py b/active_bots/mailbot.py index a98872a..a11231f 100644 --- a/active_bots/mailbot.py +++ b/active_bots/mailbot.py @@ -20,14 +20,13 @@ class Mailbot(object): other bots that it received mails. """ - def __init__(self, uid, db): + def __init__(self, uid): """ Creates a Bot who listens to mails and forwards them to other bots. - :param config: (dictionary) config.toml as a dictionary of dictionaries """ - self.user = User(db, uid) + self.user = User(uid) try: self.last_mail = self.user.get_seen_mail() @@ -115,7 +114,7 @@ class Mailbot(object): :param status: (report.Report object) """ - mailer = sendmail.Mailer(config) + mailer = sendmail.Mailer() mailer.send(status.format(), self.mailinglist, "Warnung: Kontrolleure gesehen") diff --git a/active_bots/twitterbot.py b/active_bots/twitterbot.py index 787cdfb..065da45 100755 --- a/active_bots/twitterbot.py +++ b/active_bots/twitterbot.py @@ -24,16 +24,13 @@ class TwitterBot(object): last_mention: the ID of the last tweet which mentioned you """ - def __init__(self, uid, db): + def __init__(self, uid): """ Initializes the bot and loads all the necessary data. - :param config: (dictionary) config.toml as a dictionary of dictionaries - :param history_path: Path to the file with ID of the last retweeted Tweet """ - self.db = db - self.user = User(db, uid) + self.user = User(uid) # initialize API access keys = self.get_api_keys() diff --git a/mall bugfixes b/mall bugfixes new file mode 100644 index 0000000..3ab1dcf --- /dev/null +++ b/mall bugfixes @@ -0,0 +1,104 @@ +diff --git a/active_bots/mailbot.py b/active_bots/mailbot.py +index a98872a..a11231f 100644 +--- a/active_bots/mailbot.py ++++ b/active_bots/mailbot.py +@@ -20,14 +20,13 @@ class Mailbot(object): + other bots that it received mails. + """ +  +- def __init__(self, uid, db): ++ def __init__(self, uid): + """ + Creates a Bot who listens to mails and forwards them to other + bots. +  +- :param config: (dictionary) config.toml as a dictionary of dictionaries + """ +- self.user = User(db, uid) ++ self.user = User(uid) +  + try: + self.last_mail = self.user.get_seen_mail() +@@ -115,7 +114,7 @@ class Mailbot(object): +  + :param status: (report.Report object) + """ +- mailer = sendmail.Mailer(config) ++ mailer = sendmail.Mailer() + mailer.send(status.format(), self.mailinglist, + "Warnung: Kontrolleure gesehen") +  +diff --git a/active_bots/twitterbot.py b/active_bots/twitterbot.py +index 787cdfb..065da45 100755 +--- a/active_bots/twitterbot.py ++++ b/active_bots/twitterbot.py +@@ -24,16 +24,13 @@ class TwitterBot(object): + last_mention: the ID of the last tweet which mentioned you + """ +  +- def __init__(self, uid, db): ++ def __init__(self, uid): + """ + Initializes the bot and loads all the necessary data. +  +- :param config: (dictionary) config.toml as a dictionary of dictionaries +- :param history_path: Path to the file with ID of the last retweeted + Tweet + """ +- self.db = db +- self.user = User(db, uid) ++ self.user = User(uid) +  + # initialize API access + keys = self.get_api_keys() +diff --git a/sendmail.py b/sendmail.py +index df91d1d..93028d9 100755 +--- a/sendmail.py ++++ b/sendmail.py +@@ -2,6 +2,7 @@ +  + import smtplib + import ssl ++from config import config + from email.mime.text import MIMEText + from email.mime.application import MIMEApplication + from email.mime.multipart import MIMEMultipart +@@ -12,13 +13,12 @@ class Mailer(object): + Maintains the connection to the mailserver and sends text to users. + """ +  +- def __init__(self, config): ++ def __init__(self): + """ + Creates an SMTP client to send a mail. Is called only once + when you actually want to send a mail. After you sent the + mail, the SMTP client is shut down again. +  +- :param config: The config file generated from config.toml + """ + # This generates the From address by stripping the part until the first + # period from the mail server address and won't work always. +@@ -65,9 +65,5 @@ class Mailer(object): +  + # For testing: + if __name__ == '__main__': +- import prepare +- +- config = prepare.get_config() +- +- m = Mailer(config) ++ m = Mailer() + print(m.send("This is a test mail.", m.fromaddr, "Test")) +diff --git a/user.py b/user.py +index c4e99e4..ce95cd3 100644 +--- a/user.py ++++ b/user.py +@@ -53,7 +53,7 @@ class User(object): + return jwt.encode({ + 'email': email, + 'uid': self.uid +- }, self.secret).decode('ascii') ++ }, db.secret).decode('ascii') +  + def is_appropriate(self, report): + db.execute("SELECT pattern FROM triggerpatterns WHERE user_id=?;", diff --git a/sendmail.py b/sendmail.py index df91d1d..93028d9 100755 --- a/sendmail.py +++ b/sendmail.py @@ -2,6 +2,7 @@ import smtplib import ssl +from config import config from email.mime.text import MIMEText from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart @@ -12,13 +13,12 @@ class Mailer(object): Maintains the connection to the mailserver and sends text to users. """ - def __init__(self, config): + def __init__(self): """ Creates an SMTP client to send a mail. Is called only once when you actually want to send a mail. After you sent the mail, the SMTP client is shut down again. - :param config: The config file generated from config.toml """ # This generates the From address by stripping the part until the first # period from the mail server address and won't work always. @@ -65,9 +65,5 @@ class Mailer(object): # For testing: if __name__ == '__main__': - import prepare - - config = prepare.get_config() - - m = Mailer(config) + m = Mailer() print(m.send("This is a test mail.", m.fromaddr, "Test")) diff --git a/session.py b/session.py index 5c68cfd..5d0c263 100644 --- a/session.py +++ b/session.py @@ -20,7 +20,7 @@ class SessionPlugin(object): uid = request.get_cookie('uid', secret=db.secret) if uid is None: return redirect(self.loginpage) - kwargs[self.keyword] = User(db, uid) + kwargs[self.keyword] = User(uid) return callback(*args, **kwargs) return wrapper diff --git a/user.py b/user.py index c4e99e4..ce95cd3 100644 --- a/user.py +++ b/user.py @@ -53,7 +53,7 @@ class User(object): return jwt.encode({ 'email': email, 'uid': self.uid - }, self.secret).decode('ascii') + }, db.secret).decode('ascii') def is_appropriate(self, report): db.execute("SELECT pattern FROM triggerpatterns WHERE user_id=?;", From 49bd00fba34cd5c15cddc3947001fd5ae14dd88a Mon Sep 17 00:00:00 2001 From: b3yond Date: Wed, 28 Mar 2018 22:12:57 +0200 Subject: [PATCH 2/3] clean up after refactor --- active_bots/mastodonbot.py | 5 ++--- backend.py | 2 +- bot.py | 4 ++-- promotion/README.md | 1 + 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/active_bots/mastodonbot.py b/active_bots/mastodonbot.py index ad741f5..c0e0240 100755 --- a/active_bots/mastodonbot.py +++ b/active_bots/mastodonbot.py @@ -26,10 +26,9 @@ class MastodonBot(Bot): return mentions for status in notifications: if (status['type'] == 'mention' and - status['status']['id'] > self.seen_toots): + status['status']['id'] > user.get_seen_toot()): # save state - self.seen_toots = status['status']['id'] - self.save_last() + user.save_seen_toot(status['status']['id']) # add mention to mentions text = re.sub(r'<[^>]*>', '', status['status']['content']) text = re.sub( diff --git a/backend.py b/backend.py index 5ed1efd..bba591a 100755 --- a/backend.py +++ b/backend.py @@ -33,7 +33,7 @@ if __name__ == '__main__': time.sleep(60) # twitter rate limit >.< except: logger.error('Shutdown', exc_info=True) - mailer = sendmail.Mailer(config) + mailer = sendmail.Mailer() try: mailer.send('', config['web']['contact'], 'Ticketfrei Crash Report', diff --git a/bot.py b/bot.py index 0f135f7..b003ab8 100644 --- a/bot.py +++ b/bot.py @@ -1,8 +1,8 @@ class Bot(object): # returns a list of Report objects - def crawl(user): + def crawl(self, user): pass # post/boost Report object - def post(user, report): + def post(self, user, report): pass diff --git a/promotion/README.md b/promotion/README.md index 2c69555..5b89d99 100644 --- a/promotion/README.md +++ b/promotion/README.md @@ -8,6 +8,7 @@ Students: usually already have a ticket, but may be solidaric Leftist scene * Flyers in alternative centers * Graffitis in alternative neighbourhoods +* Posters in social centers Schools: * especially trade schools From 66bb1f86a388ea8839b258f4909d3ec170678c21 Mon Sep 17 00:00:00 2001 From: b3yond Date: Wed, 28 Mar 2018 23:33:04 +0200 Subject: [PATCH 3/3] reworked twitterbot according to new scheme --- active_bots/twitterbot.py | 195 ++++++++------------------------------ frontend.py | 9 +- user.py | 9 ++ 3 files changed, 53 insertions(+), 160 deletions(-) diff --git a/active_bots/twitterbot.py b/active_bots/twitterbot.py index 065da45..051502b 100755 --- a/active_bots/twitterbot.py +++ b/active_bots/twitterbot.py @@ -1,190 +1,71 @@ #!/usr/bin/env python3 -from config import config import logging import tweepy import re import requests -from time import sleep import report -from user import User +from bot import Bot logger = logging.getLogger(__name__) -class TwitterBot(object): - """ - This bot retweets all tweets which - 1) mention him, - 2) contain at least one of the triggerwords provided. - - api: The api object, generated with your oAuth keys, responsible for - communication with twitter rest API - last_mention: the ID of the last tweet which mentioned you - """ - - def __init__(self, uid): - """ - Initializes the bot and loads all the necessary data. - - Tweet - """ - self.user = User(uid) - - # initialize API access - keys = self.get_api_keys() +class TwitterBot(Bot): + def get_api(self, user): + keys = user.get_api_keys() auth = tweepy.OAuthHandler(consumer_key=keys[0], consumer_secret=keys[1]) auth.set_access_token(keys[2], # access_token_key keys[3]) # access_token_secret - self.api = tweepy.API(auth) + return tweepy.API(auth) - self.last_mention = self.user.get_seen_tweet() - self.waitcounter = 0 - - def get_api_keys(self): - """ - How to get these keys is described in doc/twitter_api.md - - After you received keys, store them in your config.toml like this: - [tapp] - consumer_key = "..." - consumer_secret = "..." - - [tuser] - access_token_key = "..." - access_token_secret = "..." - - :return: keys: list of these 4 strings. - """ - keys = [config['twitter']['consumer_key'], - config['twitter']['consumer_secret']] - row = self.user.get_twitter_token() - keys.append(row[0]) - keys.append(row[1]) - return keys - - def save_last(self): - """ Saves the last retweeted tweet in last_mention. """ - self.user.save_seen_tweet(self.last_mention) - - def waiting(self): - """ - If the counter is not 0, you should be waiting instead. - - :return: self.waitcounter(int): if 0, do smth. - """ - if self.waitcounter > 0: - sleep(1) - self.waitcounter -= 1 - return self.waitcounter - - def crawl(self): + def crawl(self, user): """ crawls all Tweets which mention the bot from the twitter rest API. :return: reports: (list of report.Report objects) """ reports = [] + api = self.get_api(user) + last_mention = user.get_seen_tweet() try: - if not self.waiting(): - if self.last_mention == 0: - mentions = self.api.mentions_timeline() - else: - mentions = self.api.mentions_timeline( - since_id=self.last_mention) - for status in mentions: - text = re.sub( - "(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)", - "", status.text) - reports.append(report.Report(status.author.screen_name, - "twitter", - text, - status.id, - status.created_at)) - self.save_last() - return reports + if last_mention == 0: + mentions = api.mentions_timeline() + else: + mentions = api.mentions_timeline( + since_id=last_mention) + for status in mentions: + text = re.sub( + "(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)", + "", status.text) + reports.append(report.Report(status.author.screen_name, + "twitter", + text, + status.id, + status.created_at)) + user.save_seen_tweet(last_mention) + return reports except tweepy.RateLimitError: logger.error("Twitter API Error: Rate Limit Exceeded", exc_info=True) - self.waitcounter += 60*15 + 1 + # :todo implement rate limiting except requests.exceptions.ConnectionError: logger.error("Twitter API Error: Bad Connection", exc_info=True) - self.waitcounter += 10 except tweepy.TweepError: logger.error("Twitter API Error: General Error", exc_info=True) return [] - def repost(self, status): - """ - Retweets a given tweet. - - :param status: (report.Report object) - :return: toot: string of the tweet, to toot on mastodon. - """ - while 1: - try: - self.api.retweet(status.id) - logger.info("Retweeted: " + status.format()) - if status.id > self.last_mention: - self.last_mention = status.id - self.save_last() - return status.format() - except requests.exceptions.ConnectionError: - logger.error("Twitter API Error: Bad Connection", - exc_info=True) - sleep(10) - # maybe one day we get rid of this error: - except tweepy.TweepError: - logger.error("Twitter Error", exc_info=True) - if status.id > self.last_mention: - self.last_mention = status.id - self.save_last() - return None - - def post(self, status): - """ - Tweet a post. - - :param status: (report.Report object) - """ - text = status.format() - if len(text) > 280: - text = status.text[:280 - 4] + u' ...' - while 1: - try: - self.api.update_status(status=text) - return - except requests.exceptions.ConnectionError: - logger.error("Twitter API Error: Bad Connection", - exc_info=True) - sleep(10) - - def flow(self, trigger, to_tweet=()): - """ The flow of crawling mentions and retweeting them. - - :param to_tweet: list of strings to tweet - :return list of retweeted tweets, to toot on mastodon - """ - - # Tweet the reports from other sources - for post in to_tweet: - self.post(post) - - # Store all mentions in a list of Status Objects - mentions = self.crawl() - - # initialise list of strings for other bots - all_tweets = [] - - for status in mentions: - # Is the Text of the Tweet in the triggerlist? - if trigger.is_ok(status.text): - # Retweet status - toot = self.repost(status) - if toot: - all_tweets.append(toot) - - # Return Retweets for posting on other bots - return all_tweets + def post(self, user, report): + api = self.get_api(user) + try: + if report.source == self: + api.retweet(report.id) + else: + # text = report.format() + if len(report.text) > 280: + text = report.text[:280 - 4] + u' ...' + except requests.exceptions.ConnectionError: + logger.error("Twitter API Error: Bad Connection", + exc_info=True) + # :todo implement rate limiting diff --git a/frontend.py b/frontend.py index c13c999..b11fd55 100755 --- a/frontend.py +++ b/frontend.py @@ -10,9 +10,6 @@ import smtplib from mastodon import Mastodon -logger = logging.getLogger(__name__) - - @get('/') @view('template/propaganda.tpl') def propaganda(): @@ -153,6 +150,12 @@ def login_mastodon(user): return dict(error='Login to Mastodon failed.') +logpath = config['logging']['logpath'] +logger = logging.getLogger() +fh = logging.FileHandler(logpath) +fh.setLevel(logging.DEBUG) +logger.addHandler(fh) + application = bottle.default_app() bottle.install(SessionPlugin('/')) diff --git a/user.py b/user.py index ce95cd3..e4edf10 100644 --- a/user.py +++ b/user.py @@ -1,3 +1,4 @@ +from config import config from bottle import response from db import db import jwt @@ -81,6 +82,14 @@ class User(object): instance = db.cur.fetchone() return instance[1], instance[2], row[0], instance[0] + def get_twitter_credentials(self): + keys = [config['twitter']['consumer_key'], + config['twitter']['consumer_secret']] + row = self.get_twitter_token() + keys.append(row[0]) + keys.append(row[1]) + return keys + def get_seen_toot(self): db.execute("SELECT toot_id FROM seen_toots WHERE user_id = ?;", (self.uid, ))