juergen/ticketfrei
1
0
Fork 0
Code Issues15 Pull requests Releases Wiki Activity

cleanup

Browse source
This commit is contained in:
Thomas L 2018-03-24 16:26:35 +01:00
parent 751f9154cc
commit daf6fe831f
8 changed files with 119 additions and 178 deletions

25
backend.py
View file

@ -1,10 +1,11 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import prepare import logging
import time import time
import sendmail import sendmail
from db import DB from db import DB
from config import config
from mastodonbot import MastodonBot from mastodonbot import MastodonBot
from twitterbot import TwitterBot from twitterbot import TwitterBot
@ -20,18 +21,22 @@ def get_users(db):
return users return users
def init_bots(config, logger, db, users): def init_bots(config, db, users):
for uid in users: for uid in users:
users[uid].append(Trigger(config, uid, db)) users[uid].append(Trigger(config, uid, db))
users[uid].append(MastodonBot(config, logger, uid, db)) users[uid].append(MastodonBot(config, uid, db))
users[uid].append(TwitterBot(config, logger, uid, db)) users[uid].append(TwitterBot(config, uid, db))
users[uid].append(Mailbot(config, logger, uid, db)) users[uid].append(Mailbot(config, uid, db))
return users return users
def run(): if __name__ == '__main__':
config = prepare.get_config() logpath = config['logging']['logpath']
logger = prepare.get_logger(config) logger = logging.getLogger()
fh = logging.FileHandler(logpath)
fh.setLevel(logging.DEBUG)
logger.addHandler(fh)
db = DB() db = DB()
while True: while True:
@ -68,7 +73,3 @@ def run():
attachment=config['logging']['logpath']) attachment=config['logging']['logpath'])
except: except:
logger.error('Mail sending failed', exc_info=True) logger.error('Mail sending failed', exc_info=True)
if __name__ == '__main__':
run()

5
config.py Normal file
View file

@ -0,0 +1,5 @@
import pytoml as toml
# read config in TOML format (https://github.com/toml-lang/toml#toml)
with open('config.toml') as configfile:
config = toml.load(configfile)

7
db.py
View file

@ -2,17 +2,18 @@ from bottle import redirect, request
from functools import wraps from functools import wraps
from inspect import Signature from inspect import Signature
import jwt import jwt
import logging
from os import urandom from os import urandom
from pylibscrypt import scrypt_mcf, scrypt_mcf_check from pylibscrypt import scrypt_mcf, scrypt_mcf_check
import sqlite3 import sqlite3
import prepare
from user import User from user import User
logger = logging.getLogger(__name__)
class DB(object): class DB(object):
def __init__(self, dbfile): def __init__(self, dbfile):
self.config = prepare.get_config()
self.logger = prepare.get_logger(self.config)
self.conn = sqlite3.connect(dbfile) self.conn = sqlite3.connect(dbfile)
self.cur = self.conn.cursor() self.cur = self.conn.cursor()
self.create() self.create()

46
frontend.py
View file

@ -1,11 +1,15 @@
import bottle import bottle
from bottle import get, post, redirect, request, response, view from bottle import get, post, redirect, request, response, view
from config import config
from db import DBPlugin from db import DBPlugin
import logging
import tweepy import tweepy
import sendmail import sendmail
import smtplib import smtplib
from mastodon import Mastodon from mastodon import Mastodon
import prepare
logger = logging.getLogger(__name__)
@get('/') @get('/')
@ -26,14 +30,15 @@ def register_post(db):
return dict(error='Email address already in use.') return dict(error='Email address already in use.')
# send confirmation mail # send confirmation mail
confirm_link = request.url + "/../confirm/" + db.token(email, password) confirm_link = request.url + "/../confirm/" + db.token(email, password)
send_confirmation_mail(db.config, confirm_link, email) send_confirmation_mail(confirm_link, email)
return dict(info='Confirmation mail sent.') return dict(info='Confirmation mail sent.')
def send_confirmation_mail(config, confirm_link, email): def send_confirmation_mail(confirm_link, email):
m = sendmail.Mailer(config) m = sendmail.Mailer()
try: try:
m.send("Complete your registration here: " + confirm_link, email, "[Ticketfrei] Confirm your account") m.send("Complete your registration here: " + confirm_link, email,
"[Ticketfrei] Confirm your account")
except smtplib.SMTPRecipientsRefused: except smtplib.SMTPRecipientsRefused:
return "Please enter a valid E-Mail address." return "Please enter a valid E-Mail address."
@ -88,14 +93,15 @@ def login_twitter(user):
Starts the twitter OAuth authentication process. Starts the twitter OAuth authentication process.
:return: redirect to twitter. :return: redirect to twitter.
""" """
consumer_key = user.db.config["tapp"]["consumer_key"] consumer_key = config["tapp"]["consumer_key"]
consumer_secret = user.db.config["tapp"]["consumer_secret"] consumer_secret = config["tapp"]["consumer_secret"]
callback_url = bottle.request.get_header('host') + "/login/twitter/callback" callback_url = request.get_header('host') + "/login/twitter/callback"
auth = tweepy.OAuthHandler(consumer_key, consumer_secret, callback_url) auth = tweepy.OAuthHandler(consumer_key, consumer_secret, callback_url)
try: try:
redirect_url = auth.get_authorization_url() redirect_url = auth.get_authorization_url()
except tweepy.TweepError: except tweepy.TweepError:
user.db.logger.error('Twitter OAuth Error: Failed to get request token.', exc_info=True) logger.error('Twitter OAuth Error: Failed to get request token.',
exc_info=True)
return dict(error="Failed to get request token.") return dict(error="Failed to get request token.")
user.save_request_token(auth.request_token) user.save_request_token(auth.request_token)
return bottle.redirect(redirect_url) return bottle.redirect(redirect_url)
@ -108,9 +114,9 @@ def twitter_callback(user):
:return: :return:
""" """
# twitter passes the verifier/oauth token secret in a GET request. # twitter passes the verifier/oauth token secret in a GET request.
verifier = bottle.request.query('oauth_verifier') verifier = request.query('oauth_verifier')
consumer_key = user.db.config["tapp"]["consumer_key"] consumer_key = config["tapp"]["consumer_key"]
consumer_secret = user.db.config["tapp"]["consumer_secret"] consumer_secret = config["tapp"]["consumer_secret"]
auth = tweepy.OAuthHandler(consumer_key, consumer_secret) auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
request_token = user.get_request_token request_token = user.get_request_token
auth.request_token = {"oauth_token": request_token, auth.request_token = {"oauth_token": request_token,
@ -127,19 +133,22 @@ def login_mastodon(user):
:return: redirect to twitter. :return: redirect to twitter.
""" """
# get app tokens # get app tokens
instance_url = bottle.request.forms.get('instance_url') instance_url = request.forms.get('instance_url')
masto_email = bottle.request.forms.get('email') masto_email = request.forms.get('email')
print(masto_email) print(masto_email)
masto_pass = bottle.request.forms.get('pass') masto_pass = request.forms.get('pass')
print(masto_pass) print(masto_pass)
client_id, client_secret = user.get_mastodon_app_keys(instance_url) client_id, client_secret = user.get_mastodon_app_keys(instance_url)
m = Mastodon(client_id=client_id, client_secret=client_secret, api_base_url=instance_url) m = Mastodon(client_id=client_id, client_secret=client_secret,
api_base_url=instance_url)
try: try:
access_token = m.log_in(masto_email, masto_pass) access_token = m.log_in(masto_email, masto_pass)
user.save_masto_token(access_token, instance_url) user.save_masto_token(access_token, instance_url)
return dict(info='Thanks for supporting decentralized social networks!') return dict(
info='Thanks for supporting decentralized social networks!'
)
except: except:
user.db.logger.error('Login to Mastodon failed.', exc_info=True) logger.error('Login to Mastodon failed.', exc_info=True)
return dict(error='Login to Mastodon failed.') return dict(error='Login to Mastodon failed.')
@ -148,6 +157,5 @@ if __name__ == '__main__':
bottle.install(DBPlugin(':memory:', '/')) bottle.install(DBPlugin(':memory:', '/'))
bottle.run(host='localhost', port=8080) bottle.run(host='localhost', port=8080)
else: else:
config = prepare.get_config()
bottle.install(DBPlugin(config['database']['db_path'], '/')) bottle.install(DBPlugin(config['database']['db_path'], '/'))
application = bottle.default_app() application = bottle.default_app()

73
mailbot.py
View file

@ -1,5 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
from config import config
import logging
import sendmail import sendmail
import ssl import ssl
import datetime import datetime
@ -9,21 +11,22 @@ import report
from user import User from user import User
logger = logging.getLogger(__name__)
class Mailbot(object): class Mailbot(object):
""" """
Bot which sends Mails if mentioned via twitter/mastodon, and tells Bot which sends Mails if mentioned via twitter/mastodon, and tells
other bots that it received mails. other bots that it received mails.
""" """
def __init__(self, config, logger, uid, db): def __init__(self, uid, db):
""" """
Creates a Bot who listens to mails and forwards them to other Creates a Bot who listens to mails and forwards them to other
bots. bots.
:param config: (dictionary) config.toml as a dictionary of dictionaries :param config: (dictionary) config.toml as a dictionary of dictionaries
""" """
self.config = config
self.logger = logger
self.user = User(db, uid) self.user = User(db, uid)
try: try:
@ -36,23 +39,24 @@ class Mailbot(object):
except TypeError: except TypeError:
self.mailinglist = None self.mailinglist = None
self.mailbox = imaplib.IMAP4_SSL(self.config["mail"]["imapserver"]) self.mailbox = imaplib.IMAP4_SSL(config["mail"]["imapserver"])
context = ssl.create_default_context() context = ssl.create_default_context()
try: try:
self.mailbox.starttls(ssl_context=context) self.mailbox.starttls(ssl_context=context)
except: except:
self.logger.error('StartTLS failed', exc_info=True) logger.error('StartTLS failed', exc_info=True)
try: try:
self.mailbox.login(self.config["mail"]["user"], self.config["mail"]["passphrase"]) self.mailbox.login(config["mail"]["user"],
config["mail"]["passphrase"])
except imaplib.IMAP4.error: except imaplib.IMAP4.error:
self.logger.error("Login to mail server failed", exc_info=True) logger.error("Login to mail server failed", exc_info=True)
try: try:
mailer = sendmail.Mailer(config) mailer = sendmail.Mailer()
mailer.send('', config['web']['contact'], mailer.send('', config['web']['contact'],
'Ticketfrei Crash Report', 'Ticketfrei Crash Report',
attachment=config['logging']['logpath']) attachment=config['logging']['logpath'])
except: except:
self.logger.error('Mail sending failed', exc_info=True) logger.error('Mail sending failed', exc_info=True)
def repost(self, status): def repost(self, status):
""" """
@ -72,7 +76,7 @@ class Mailbot(object):
try: try:
rv, data = self.mailbox.select("Inbox") rv, data = self.mailbox.select("Inbox")
except imaplib.IMAP4.abort: except imaplib.IMAP4.abort:
self.logger.error("Crawling Mail failed", exc_info=True) logger.error("Crawling Mail failed", exc_info=True)
rv = False rv = False
msgs = [] msgs = []
if rv == 'OK': if rv == 'OK':
@ -83,15 +87,18 @@ class Mailbot(object):
for num in data[0].split(): for num in data[0].split():
rv, data = self.mailbox.fetch(num, '(RFC822)') rv, data = self.mailbox.fetch(num, '(RFC822)')
if rv != 'OK': if rv != 'OK':
self.logger.error("Couldn't fetch mail %s %s" % (rv, str(data))) logger.error("Couldn't fetch mail %s %s" % (rv, str(data)))
return msgs return msgs
msg = email.message_from_bytes(data[0][1]) msg = email.message_from_bytes(data[0][1])
if not self.user.get_mail() in msg['From']: if not self.user.get_mail() in msg['From']:
# get a comparable date out of the email # get a comparable date out of the email
date_tuple = email.utils.parsedate_tz(msg['Date']) date_tuple = email.utils.parsedate_tz(msg['Date'])
date_tuple = datetime.datetime.fromtimestamp(email.utils.mktime_tz(date_tuple)) date_tuple = datetime.datetime.fromtimestamp(
date = int((date_tuple - datetime.datetime(1970, 1, 1)).total_seconds()) email.utils.mktime_tz(date_tuple)
)
date = int((date_tuple -
datetime.datetime(1970, 1, 1)).total_seconds())
if date > self.user.get_seen_mail(): if date > self.user.get_seen_mail():
self.last_mail = date self.last_mail = date
self.save_last() self.save_last()
@ -108,8 +115,9 @@ class Mailbot(object):
:param status: (report.Report object) :param status: (report.Report object)
""" """
mailer = sendmail.Mailer(self.config) mailer = sendmail.Mailer(config)
mailer.send(status.format(), self.mailinglist, "Warnung: Kontrolleure gesehen") mailer.send(status.format(), self.mailinglist,
"Warnung: Kontrolleure gesehen")
def make_report(self, msg): def make_report(self, msg):
""" """
@ -120,8 +128,10 @@ class Mailbot(object):
""" """
# get a comparable date out of the email # get a comparable date out of the email
date_tuple = email.utils.parsedate_tz(msg['Date']) date_tuple = email.utils.parsedate_tz(msg['Date'])
date_tuple = datetime.datetime.fromtimestamp(email.utils.mktime_tz(date_tuple)) date_tuple = datetime.datetime.fromtimestamp(
date = (date_tuple-datetime.datetime(1970,1,1)).total_seconds() email.utils.mktime_tz(date_tuple)
)
date = (date_tuple - datetime.datetime(1970, 1, 1)).total_seconds()
author = msg.get("From") # get mail author from email header author = msg.get("From") # get mail author from email header
# :todo take only the part before the @ # :todo take only the part before the @
@ -149,32 +159,3 @@ class Mailbot(object):
if trigger.is_ok(msg.get_payload()): if trigger.is_ok(msg.get_payload()):
statuses.append(msg) statuses.append(msg)
return statuses return statuses
"""
if __name__ == "__main__":
config = prepare.get_config()
# initialise trigger
trigger = trigger.Trigger(config)
# initialise mail bot
m = Mailbot(config)
statuses = []
try:
while 1:
print("Received Reports: " + str(m.flow(trigger, statuses)))
time.sleep(1)
except KeyboardInterrupt:
print("Good bye. Remember to restart the bot!")
except:
m.logger.error('Shutdown', exc_info=True)
m.save_last()
try:
mailer = sendmail.Mailer(config)
mailer.send('', config['mail']['contact'],
'Ticketfrei Crash Report',
attachment=config['logging']['logpath'])
except:
m.logger.error('Mail sending failed', exc_info=True)
"""

57
mastodonbot.py
View file

@ -1,22 +1,26 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import logging
import mastodon import mastodon
import re import re
# import time
# import trigger
# import sendmail
import report import report
from user import User from user import User
logger = logging.getLogger(__name__)
class MastodonBot(object): class MastodonBot(object):
def __init__(self, config, logger, uid, db): def __init__(self, uid, db):
self.config = config
self.logger = logger
self.user = User(db, uid) self.user = User(db, uid)
client_id, client_secret, access_token, instance_url = self.user.get_masto_credentials() client_id, client_secret, access_token, instance_url = \
self.m = mastodon.Mastodon(client_id=client_id, client_secret=client_secret, self.user.get_masto_credentials()
access_token=access_token, api_base_url=instance_url) self.m = mastodon.Mastodon(
client_id=client_id,
client_secret=client_secret,
access_token=access_token,
api_base_url=instance_url
)
# load state # load state
try: try:
@ -37,16 +41,19 @@ class MastodonBot(object):
try: try:
notifications = self.m.notifications() notifications = self.m.notifications()
except: # mastodon.Mastodon.MastodonAPIError is unfortunately not in __init__.py except: # mastodon.Mastodon.MastodonAPIError is unfortunately not in __init__.py
self.logger.error("Unknown Mastodon API Error.", exc_info=True) logger.error("Unknown Mastodon API Error.", exc_info=True)
return mentions return mentions
for status in notifications: for status in notifications:
if status['type'] == 'mention' and status['status']['id'] > self.seen_toots: if (status['type'] == 'mention' and
status['status']['id'] > self.seen_toots):
# save state # save state
self.seen_toots = status['status']['id'] self.seen_toots = status['status']['id']
self.save_last() self.save_last()
# add mention to mentions # add mention to mentions
text = re.sub(r'<[^>]*>', '', status['status']['content']) text = re.sub(r'<[^>]*>', '', status['status']['content'])
text = re.sub("(?<=^|(?<=[^a-zA-Z0-9-_.]))@([A-Za-z]+[A-Za-z0-9-_]+)", "", text) text = re.sub(
"(?<=^|(?<=[^a-zA-Z0-9-_.]))@([A-Za-z]+[A-Za-z0-9-_]+)",
"", text)
mentions.append(report.Report(status['account']['acct'], mentions.append(report.Report(status['account']['acct'],
"mastodon", "mastodon",
text, text,
@ -60,7 +67,7 @@ class MastodonBot(object):
:param mention: (report.Report object) :param mention: (report.Report object)
""" """
self.logger.info('Boosting toot from %s' % ( logger.info('Boosting toot from %s' % (
mention.format())) mention.format()))
self.m.status_reblog(mention.id) self.m.status_reblog(mention.id)
@ -88,27 +95,3 @@ class MastodonBot(object):
# return mentions for mirroring # return mentions for mirroring
return retoots return retoots
"""
if __name__ == '__main__':
config = backend.get_config()
trigger = trigger.Trigger(config)
bot = MastodonBot(config)
try:
while True:
bot.flow(trigger)
time.sleep(1)
except KeyboardInterrupt:
print("Good bye. Remember to restart the bot!")
except:
bot.logger.error('Shutdown', exc_info=True)
try:
mailer = sendmail.Mailer(config)
mailer.send('', config['mail']['contact'],
'Ticketfrei Crash Report',
attachment=config['logging']['logpath'])
except:
bot.logger.error('Mail sending failed', exc_info=True)
"""

19
prepare.py
View file

@ -1,19 +0,0 @@
import logging
import pytoml as toml
def get_logger(config):
logpath = config['logging']['logpath']
logger = logging.getLogger()
fh = logging.FileHandler(logpath)
fh.setLevel(logging.DEBUG)
logger.addHandler(fh)
return logger
def get_config():
# read config in TOML format (https://github.com/toml-lang/toml#toml)
with open('config.toml') as configfile:
config = toml.load(configfile)
return config

65
twitterbot.py
View file

@ -1,5 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
from config import config
import logging
import tweepy import tweepy
import re import re
import requests import requests
@ -8,6 +10,9 @@ import report
from user import User from user import User
logger = logging.getLogger(__name__)
class TwitterBot(object): class TwitterBot(object):
""" """
This bot retweets all tweets which This bot retweets all tweets which
@ -19,7 +24,7 @@ class TwitterBot(object):
last_mention: the ID of the last tweet which mentioned you last_mention: the ID of the last tweet which mentioned you
""" """
def __init__(self, config, logger, uid, db): def __init__(self, uid, db):
""" """
Initializes the bot and loads all the necessary data. Initializes the bot and loads all the necessary data.
@ -27,8 +32,6 @@ class TwitterBot(object):
:param history_path: Path to the file with ID of the last retweeted :param history_path: Path to the file with ID of the last retweeted
Tweet Tweet
""" """
self.config = config
self.logger = logger
self.db = db self.db = db
self.user = User(db, uid) self.user = User(db, uid)
@ -58,7 +61,8 @@ class TwitterBot(object):
:return: keys: list of these 4 strings. :return: keys: list of these 4 strings.
""" """
keys = [self.config['twitter']['consumer_key'], self.config['twitter']['consumer_secret']] keys = [config['twitter']['consumer_key'],
config['twitter']['consumer_secret']]
row = self.user.get_twitter_token() row = self.user.get_twitter_token()
keys.append(row[0]) keys.append(row[0])
keys.append(row[1]) keys.append(row[1])
@ -91,9 +95,12 @@ class TwitterBot(object):
if self.last_mention == 0: if self.last_mention == 0:
mentions = self.api.mentions_timeline() mentions = self.api.mentions_timeline()
else: else:
mentions = self.api.mentions_timeline(since_id=self.last_mention) mentions = self.api.mentions_timeline(
since_id=self.last_mention)
for status in mentions: for status in mentions:
text = re.sub("(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)", "", status.text) text = re.sub(
"(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)",
"", status.text)
reports.append(report.Report(status.author.screen_name, reports.append(report.Report(status.author.screen_name,
"twitter", "twitter",
text, text,
@ -102,13 +109,14 @@ class TwitterBot(object):
self.save_last() self.save_last()
return reports return reports
except tweepy.RateLimitError: except tweepy.RateLimitError:
self.logger.error("Twitter API Error: Rate Limit Exceeded", exc_info=True) logger.error("Twitter API Error: Rate Limit Exceeded",
exc_info=True)
self.waitcounter += 60*15 + 1 self.waitcounter += 60*15 + 1
except requests.exceptions.ConnectionError: except requests.exceptions.ConnectionError:
self.logger.error("Twitter API Error: Bad Connection", exc_info=True) logger.error("Twitter API Error: Bad Connection", exc_info=True)
self.waitcounter += 10 self.waitcounter += 10
except tweepy.TweepError: except tweepy.TweepError:
self.logger.error("Twitter API Error: General Error", exc_info=True) logger.error("Twitter API Error: General Error", exc_info=True)
return [] return []
def repost(self, status): def repost(self, status):
@ -121,17 +129,18 @@ class TwitterBot(object):
while 1: while 1:
try: try:
self.api.retweet(status.id) self.api.retweet(status.id)
self.logger.info("Retweeted: " + status.format()) logger.info("Retweeted: " + status.format())
if status.id > self.last_mention: if status.id > self.last_mention:
self.last_mention = status.id self.last_mention = status.id
self.save_last() self.save_last()
return status.format() return status.format()
except requests.exceptions.ConnectionError: except requests.exceptions.ConnectionError:
self.logger.error("Twitter API Error: Bad Connection", exc_info=True) logger.error("Twitter API Error: Bad Connection",
exc_info=True)
sleep(10) sleep(10)
# maybe one day we get rid of this error: # maybe one day we get rid of this error:
except tweepy.TweepError: except tweepy.TweepError:
self.logger.error("Twitter Error", exc_info=True) logger.error("Twitter Error", exc_info=True)
if status.id > self.last_mention: if status.id > self.last_mention:
self.last_mention = status.id self.last_mention = status.id
self.save_last() self.save_last()
@ -151,7 +160,8 @@ class TwitterBot(object):
self.api.update_status(status=text) self.api.update_status(status=text)
return return
except requests.exceptions.ConnectionError: except requests.exceptions.ConnectionError:
self.logger.error("Twitter API Error: Bad Connection", exc_info=True) logger.error("Twitter API Error: Bad Connection",
exc_info=True)
sleep(10) sleep(10)
def flow(self, trigger, to_tweet=()): def flow(self, trigger, to_tweet=()):
@ -181,32 +191,3 @@ class TwitterBot(object):
# Return Retweets for posting on other bots # Return Retweets for posting on other bots
return all_tweets return all_tweets
"""
if __name__ == "__main__":
config = backend.get_config()
# initialise trigger
trigger = trigger.Trigger(config)
# initialise twitter bot
bot = TwitterBot(config)
try:
while True:
# :todo separate into small functions
bot.flow(trigger)
sleep(60)
except KeyboardInterrupt:
print("Good bye. Remember to restart the bot!")
except:
bot.logger.error('Shutdown', exc_info=True)
bot.save_last()
try:
mailer = sendmail.Mailer(config)
mailer.send('', config['mail']['contact'],
'Ticketfrei Crash Report',
attachment=config['logging']['logpath'])
except:
bot.logger.error('Mail sending failed', exc_info=True)
"""