From daf6fe831f45c7defc70db006874b0f36bd37ce7 Mon Sep 17 00:00:00 2001 From: Thomas L Date: Sat, 24 Mar 2018 16:26:35 +0100 Subject: [PATCH] cleanup --- backend.py | 25 ++++++++--------- config.py | 5 ++++ db.py | 7 ++--- frontend.py | 46 ++++++++++++++++++------------- mailbot.py | 73 +++++++++++++++++++------------------------------- mastodonbot.py | 57 ++++++++++++++------------------------- prepare.py | 19 ------------- twitterbot.py | 65 ++++++++++++++++---------------------------- 8 files changed, 119 insertions(+), 178 deletions(-) create mode 100644 config.py delete mode 100644 prepare.py diff --git a/backend.py b/backend.py index 19d23db..ddce2c3 100755 --- a/backend.py +++ b/backend.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 -import prepare +import logging import time import sendmail from db import DB +from config import config from mastodonbot import MastodonBot from twitterbot import TwitterBot @@ -20,18 +21,22 @@ def get_users(db): return users -def init_bots(config, logger, db, users): +def init_bots(config, db, users): for uid in users: users[uid].append(Trigger(config, uid, db)) - users[uid].append(MastodonBot(config, logger, uid, db)) - users[uid].append(TwitterBot(config, logger, uid, db)) - users[uid].append(Mailbot(config, logger, uid, db)) + users[uid].append(MastodonBot(config, uid, db)) + users[uid].append(TwitterBot(config, uid, db)) + users[uid].append(Mailbot(config, uid, db)) return users -def run(): - config = prepare.get_config() - logger = prepare.get_logger(config) +if __name__ == '__main__': + logpath = config['logging']['logpath'] + logger = logging.getLogger() + fh = logging.FileHandler(logpath) + fh.setLevel(logging.DEBUG) + logger.addHandler(fh) + db = DB() while True: @@ -68,7 +73,3 @@ def run(): attachment=config['logging']['logpath']) except: logger.error('Mail sending failed', exc_info=True) - - -if __name__ == '__main__': - run() diff --git a/config.py b/config.py new file mode 100644 index 0000000..fa01302 --- /dev/null +++ b/config.py @@ -0,0 +1,5 @@ +import pytoml as toml + +# read config in TOML format (https://github.com/toml-lang/toml#toml) +with open('config.toml') as configfile: + config = toml.load(configfile) diff --git a/db.py b/db.py index 49fffc8..d9a8fde 100644 --- a/db.py +++ b/db.py @@ -2,17 +2,18 @@ from bottle import redirect, request from functools import wraps from inspect import Signature import jwt +import logging from os import urandom from pylibscrypt import scrypt_mcf, scrypt_mcf_check import sqlite3 -import prepare from user import User +logger = logging.getLogger(__name__) + + class DB(object): def __init__(self, dbfile): - self.config = prepare.get_config() - self.logger = prepare.get_logger(self.config) self.conn = sqlite3.connect(dbfile) self.cur = self.conn.cursor() self.create() diff --git a/frontend.py b/frontend.py index 21c6be9..685e83a 100755 --- a/frontend.py +++ b/frontend.py @@ -1,11 +1,15 @@ import bottle from bottle import get, post, redirect, request, response, view +from config import config from db import DBPlugin +import logging import tweepy import sendmail import smtplib from mastodon import Mastodon -import prepare + + +logger = logging.getLogger(__name__) @get('/') @@ -26,14 +30,15 @@ def register_post(db): return dict(error='Email address already in use.') # send confirmation mail confirm_link = request.url + "/../confirm/" + db.token(email, password) - send_confirmation_mail(db.config, confirm_link, email) + send_confirmation_mail(confirm_link, email) return dict(info='Confirmation mail sent.') -def send_confirmation_mail(config, confirm_link, email): - m = sendmail.Mailer(config) +def send_confirmation_mail(confirm_link, email): + m = sendmail.Mailer() try: - m.send("Complete your registration here: " + confirm_link, email, "[Ticketfrei] Confirm your account") + m.send("Complete your registration here: " + confirm_link, email, + "[Ticketfrei] Confirm your account") except smtplib.SMTPRecipientsRefused: return "Please enter a valid E-Mail address." @@ -88,14 +93,15 @@ def login_twitter(user): Starts the twitter OAuth authentication process. :return: redirect to twitter. """ - consumer_key = user.db.config["tapp"]["consumer_key"] - consumer_secret = user.db.config["tapp"]["consumer_secret"] - callback_url = bottle.request.get_header('host') + "/login/twitter/callback" + consumer_key = config["tapp"]["consumer_key"] + consumer_secret = config["tapp"]["consumer_secret"] + callback_url = request.get_header('host') + "/login/twitter/callback" auth = tweepy.OAuthHandler(consumer_key, consumer_secret, callback_url) try: redirect_url = auth.get_authorization_url() except tweepy.TweepError: - user.db.logger.error('Twitter OAuth Error: Failed to get request token.', exc_info=True) + logger.error('Twitter OAuth Error: Failed to get request token.', + exc_info=True) return dict(error="Failed to get request token.") user.save_request_token(auth.request_token) return bottle.redirect(redirect_url) @@ -108,9 +114,9 @@ def twitter_callback(user): :return: """ # twitter passes the verifier/oauth token secret in a GET request. - verifier = bottle.request.query('oauth_verifier') - consumer_key = user.db.config["tapp"]["consumer_key"] - consumer_secret = user.db.config["tapp"]["consumer_secret"] + verifier = request.query('oauth_verifier') + consumer_key = config["tapp"]["consumer_key"] + consumer_secret = config["tapp"]["consumer_secret"] auth = tweepy.OAuthHandler(consumer_key, consumer_secret) request_token = user.get_request_token auth.request_token = {"oauth_token": request_token, @@ -127,19 +133,22 @@ def login_mastodon(user): :return: redirect to twitter. """ # get app tokens - instance_url = bottle.request.forms.get('instance_url') - masto_email = bottle.request.forms.get('email') + instance_url = request.forms.get('instance_url') + masto_email = request.forms.get('email') print(masto_email) - masto_pass = bottle.request.forms.get('pass') + masto_pass = request.forms.get('pass') print(masto_pass) client_id, client_secret = user.get_mastodon_app_keys(instance_url) - m = Mastodon(client_id=client_id, client_secret=client_secret, api_base_url=instance_url) + m = Mastodon(client_id=client_id, client_secret=client_secret, + api_base_url=instance_url) try: access_token = m.log_in(masto_email, masto_pass) user.save_masto_token(access_token, instance_url) - return dict(info='Thanks for supporting decentralized social networks!') + return dict( + info='Thanks for supporting decentralized social networks!' + ) except: - user.db.logger.error('Login to Mastodon failed.', exc_info=True) + logger.error('Login to Mastodon failed.', exc_info=True) return dict(error='Login to Mastodon failed.') @@ -148,6 +157,5 @@ if __name__ == '__main__': bottle.install(DBPlugin(':memory:', '/')) bottle.run(host='localhost', port=8080) else: - config = prepare.get_config() bottle.install(DBPlugin(config['database']['db_path'], '/')) application = bottle.default_app() diff --git a/mailbot.py b/mailbot.py index 80d8a8d..a98872a 100644 --- a/mailbot.py +++ b/mailbot.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +from config import config +import logging import sendmail import ssl import datetime @@ -9,21 +11,22 @@ import report from user import User +logger = logging.getLogger(__name__) + + class Mailbot(object): """ Bot which sends Mails if mentioned via twitter/mastodon, and tells other bots that it received mails. """ - def __init__(self, config, logger, uid, db): + def __init__(self, uid, db): """ Creates a Bot who listens to mails and forwards them to other bots. :param config: (dictionary) config.toml as a dictionary of dictionaries """ - self.config = config - self.logger = logger self.user = User(db, uid) try: @@ -36,23 +39,24 @@ class Mailbot(object): except TypeError: self.mailinglist = None - self.mailbox = imaplib.IMAP4_SSL(self.config["mail"]["imapserver"]) + self.mailbox = imaplib.IMAP4_SSL(config["mail"]["imapserver"]) context = ssl.create_default_context() try: self.mailbox.starttls(ssl_context=context) except: - self.logger.error('StartTLS failed', exc_info=True) + logger.error('StartTLS failed', exc_info=True) try: - self.mailbox.login(self.config["mail"]["user"], self.config["mail"]["passphrase"]) + self.mailbox.login(config["mail"]["user"], + config["mail"]["passphrase"]) except imaplib.IMAP4.error: - self.logger.error("Login to mail server failed", exc_info=True) + logger.error("Login to mail server failed", exc_info=True) try: - mailer = sendmail.Mailer(config) + mailer = sendmail.Mailer() mailer.send('', config['web']['contact'], 'Ticketfrei Crash Report', attachment=config['logging']['logpath']) except: - self.logger.error('Mail sending failed', exc_info=True) + logger.error('Mail sending failed', exc_info=True) def repost(self, status): """ @@ -72,7 +76,7 @@ class Mailbot(object): try: rv, data = self.mailbox.select("Inbox") except imaplib.IMAP4.abort: - self.logger.error("Crawling Mail failed", exc_info=True) + logger.error("Crawling Mail failed", exc_info=True) rv = False msgs = [] if rv == 'OK': @@ -83,15 +87,18 @@ class Mailbot(object): for num in data[0].split(): rv, data = self.mailbox.fetch(num, '(RFC822)') if rv != 'OK': - self.logger.error("Couldn't fetch mail %s %s" % (rv, str(data))) + logger.error("Couldn't fetch mail %s %s" % (rv, str(data))) return msgs msg = email.message_from_bytes(data[0][1]) if not self.user.get_mail() in msg['From']: # get a comparable date out of the email date_tuple = email.utils.parsedate_tz(msg['Date']) - date_tuple = datetime.datetime.fromtimestamp(email.utils.mktime_tz(date_tuple)) - date = int((date_tuple - datetime.datetime(1970, 1, 1)).total_seconds()) + date_tuple = datetime.datetime.fromtimestamp( + email.utils.mktime_tz(date_tuple) + ) + date = int((date_tuple - + datetime.datetime(1970, 1, 1)).total_seconds()) if date > self.user.get_seen_mail(): self.last_mail = date self.save_last() @@ -108,8 +115,9 @@ class Mailbot(object): :param status: (report.Report object) """ - mailer = sendmail.Mailer(self.config) - mailer.send(status.format(), self.mailinglist, "Warnung: Kontrolleure gesehen") + mailer = sendmail.Mailer(config) + mailer.send(status.format(), self.mailinglist, + "Warnung: Kontrolleure gesehen") def make_report(self, msg): """ @@ -120,8 +128,10 @@ class Mailbot(object): """ # get a comparable date out of the email date_tuple = email.utils.parsedate_tz(msg['Date']) - date_tuple = datetime.datetime.fromtimestamp(email.utils.mktime_tz(date_tuple)) - date = (date_tuple-datetime.datetime(1970,1,1)).total_seconds() + date_tuple = datetime.datetime.fromtimestamp( + email.utils.mktime_tz(date_tuple) + ) + date = (date_tuple - datetime.datetime(1970, 1, 1)).total_seconds() author = msg.get("From") # get mail author from email header # :todo take only the part before the @ @@ -149,32 +159,3 @@ class Mailbot(object): if trigger.is_ok(msg.get_payload()): statuses.append(msg) return statuses - -""" -if __name__ == "__main__": - config = prepare.get_config() - - # initialise trigger - trigger = trigger.Trigger(config) - - # initialise mail bot - m = Mailbot(config) - - statuses = [] - try: - while 1: - print("Received Reports: " + str(m.flow(trigger, statuses))) - time.sleep(1) - except KeyboardInterrupt: - print("Good bye. Remember to restart the bot!") - except: - m.logger.error('Shutdown', exc_info=True) - m.save_last() - try: - mailer = sendmail.Mailer(config) - mailer.send('', config['mail']['contact'], - 'Ticketfrei Crash Report', - attachment=config['logging']['logpath']) - except: - m.logger.error('Mail sending failed', exc_info=True) -""" \ No newline at end of file diff --git a/mastodonbot.py b/mastodonbot.py index 152f01e..561a7fb 100755 --- a/mastodonbot.py +++ b/mastodonbot.py @@ -1,22 +1,26 @@ #!/usr/bin/env python3 +import logging import mastodon import re -# import time -# import trigger -# import sendmail import report from user import User +logger = logging.getLogger(__name__) + + class MastodonBot(object): - def __init__(self, config, logger, uid, db): - self.config = config - self.logger = logger + def __init__(self, uid, db): self.user = User(db, uid) - client_id, client_secret, access_token, instance_url = self.user.get_masto_credentials() - self.m = mastodon.Mastodon(client_id=client_id, client_secret=client_secret, - access_token=access_token, api_base_url=instance_url) + client_id, client_secret, access_token, instance_url = \ + self.user.get_masto_credentials() + self.m = mastodon.Mastodon( + client_id=client_id, + client_secret=client_secret, + access_token=access_token, + api_base_url=instance_url + ) # load state try: @@ -37,16 +41,19 @@ class MastodonBot(object): try: notifications = self.m.notifications() except: # mastodon.Mastodon.MastodonAPIError is unfortunately not in __init__.py - self.logger.error("Unknown Mastodon API Error.", exc_info=True) + logger.error("Unknown Mastodon API Error.", exc_info=True) return mentions for status in notifications: - if status['type'] == 'mention' and status['status']['id'] > self.seen_toots: + if (status['type'] == 'mention' and + status['status']['id'] > self.seen_toots): # save state self.seen_toots = status['status']['id'] self.save_last() # add mention to mentions text = re.sub(r'<[^>]*>', '', status['status']['content']) - text = re.sub("(?<=^|(?<=[^a-zA-Z0-9-_.]))@([A-Za-z]+[A-Za-z0-9-_]+)", "", text) + text = re.sub( + "(?<=^|(?<=[^a-zA-Z0-9-_.]))@([A-Za-z]+[A-Za-z0-9-_]+)", + "", text) mentions.append(report.Report(status['account']['acct'], "mastodon", text, @@ -60,7 +67,7 @@ class MastodonBot(object): :param mention: (report.Report object) """ - self.logger.info('Boosting toot from %s' % ( + logger.info('Boosting toot from %s' % ( mention.format())) self.m.status_reblog(mention.id) @@ -88,27 +95,3 @@ class MastodonBot(object): # return mentions for mirroring return retoots - -""" -if __name__ == '__main__': - config = backend.get_config() - - trigger = trigger.Trigger(config) - bot = MastodonBot(config) - - try: - while True: - bot.flow(trigger) - time.sleep(1) - except KeyboardInterrupt: - print("Good bye. Remember to restart the bot!") - except: - bot.logger.error('Shutdown', exc_info=True) - try: - mailer = sendmail.Mailer(config) - mailer.send('', config['mail']['contact'], - 'Ticketfrei Crash Report', - attachment=config['logging']['logpath']) - except: - bot.logger.error('Mail sending failed', exc_info=True) -""" \ No newline at end of file diff --git a/prepare.py b/prepare.py deleted file mode 100644 index 58c482a..0000000 --- a/prepare.py +++ /dev/null @@ -1,19 +0,0 @@ -import logging -import pytoml as toml - -def get_logger(config): - logpath = config['logging']['logpath'] - logger = logging.getLogger() - fh = logging.FileHandler(logpath) - fh.setLevel(logging.DEBUG) - logger.addHandler(fh) - return logger - - -def get_config(): - # read config in TOML format (https://github.com/toml-lang/toml#toml) - with open('config.toml') as configfile: - config = toml.load(configfile) - return config - - diff --git a/twitterbot.py b/twitterbot.py index 9eeb87c..787cdfb 100755 --- a/twitterbot.py +++ b/twitterbot.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +from config import config +import logging import tweepy import re import requests @@ -8,6 +10,9 @@ import report from user import User +logger = logging.getLogger(__name__) + + class TwitterBot(object): """ This bot retweets all tweets which @@ -19,7 +24,7 @@ class TwitterBot(object): last_mention: the ID of the last tweet which mentioned you """ - def __init__(self, config, logger, uid, db): + def __init__(self, uid, db): """ Initializes the bot and loads all the necessary data. @@ -27,8 +32,6 @@ class TwitterBot(object): :param history_path: Path to the file with ID of the last retweeted Tweet """ - self.config = config - self.logger = logger self.db = db self.user = User(db, uid) @@ -58,7 +61,8 @@ class TwitterBot(object): :return: keys: list of these 4 strings. """ - keys = [self.config['twitter']['consumer_key'], self.config['twitter']['consumer_secret']] + keys = [config['twitter']['consumer_key'], + config['twitter']['consumer_secret']] row = self.user.get_twitter_token() keys.append(row[0]) keys.append(row[1]) @@ -91,9 +95,12 @@ class TwitterBot(object): if self.last_mention == 0: mentions = self.api.mentions_timeline() else: - mentions = self.api.mentions_timeline(since_id=self.last_mention) + mentions = self.api.mentions_timeline( + since_id=self.last_mention) for status in mentions: - text = re.sub("(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)", "", status.text) + text = re.sub( + "(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)", + "", status.text) reports.append(report.Report(status.author.screen_name, "twitter", text, @@ -102,13 +109,14 @@ class TwitterBot(object): self.save_last() return reports except tweepy.RateLimitError: - self.logger.error("Twitter API Error: Rate Limit Exceeded", exc_info=True) + logger.error("Twitter API Error: Rate Limit Exceeded", + exc_info=True) self.waitcounter += 60*15 + 1 except requests.exceptions.ConnectionError: - self.logger.error("Twitter API Error: Bad Connection", exc_info=True) + logger.error("Twitter API Error: Bad Connection", exc_info=True) self.waitcounter += 10 except tweepy.TweepError: - self.logger.error("Twitter API Error: General Error", exc_info=True) + logger.error("Twitter API Error: General Error", exc_info=True) return [] def repost(self, status): @@ -121,17 +129,18 @@ class TwitterBot(object): while 1: try: self.api.retweet(status.id) - self.logger.info("Retweeted: " + status.format()) + logger.info("Retweeted: " + status.format()) if status.id > self.last_mention: self.last_mention = status.id self.save_last() return status.format() except requests.exceptions.ConnectionError: - self.logger.error("Twitter API Error: Bad Connection", exc_info=True) + logger.error("Twitter API Error: Bad Connection", + exc_info=True) sleep(10) # maybe one day we get rid of this error: except tweepy.TweepError: - self.logger.error("Twitter Error", exc_info=True) + logger.error("Twitter Error", exc_info=True) if status.id > self.last_mention: self.last_mention = status.id self.save_last() @@ -151,7 +160,8 @@ class TwitterBot(object): self.api.update_status(status=text) return except requests.exceptions.ConnectionError: - self.logger.error("Twitter API Error: Bad Connection", exc_info=True) + logger.error("Twitter API Error: Bad Connection", + exc_info=True) sleep(10) def flow(self, trigger, to_tweet=()): @@ -181,32 +191,3 @@ class TwitterBot(object): # Return Retweets for posting on other bots return all_tweets - -""" -if __name__ == "__main__": - config = backend.get_config() - - # initialise trigger - trigger = trigger.Trigger(config) - - # initialise twitter bot - bot = TwitterBot(config) - - try: - while True: - # :todo separate into small functions - bot.flow(trigger) - sleep(60) - except KeyboardInterrupt: - print("Good bye. Remember to restart the bot!") - except: - bot.logger.error('Shutdown', exc_info=True) - bot.save_last() - try: - mailer = sendmail.Mailer(config) - mailer.send('', config['mail']['contact'], - 'Ticketfrei Crash Report', - attachment=config['logging']['logpath']) - except: - bot.logger.error('Mail sending failed', exc_info=True) -""" \ No newline at end of file