From 0acb89ebf044bfcd6098d10fce34e965221c2535 Mon Sep 17 00:00:00 2001 From: b3yond Date: Tue, 23 Jan 2018 09:18:59 +0100 Subject: [PATCH 1/8] excepted IMAP4 error with unknown cause --- mailbot.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mailbot.py b/mailbot.py index a513359..d74e3b8 100644 --- a/mailbot.py +++ b/mailbot.py @@ -70,7 +70,11 @@ class Mailbot(object): crawl for new mails. :return: msgs: (list of report.Report objects) """ - rv, data = self.mailbox.select("Inbox") + try: + rv, data = self.mailbox.select("Inbox") + except imaplib.IMAP4.abort: + logger.error("Crawling Mail failed", exc_info=True) + rv = False msgs = [] if rv == 'OK': rv, data = self.mailbox.search(None, "ALL") From 9e221ed29047ece677d00a668becc8b970d634a2 Mon Sep 17 00:00:00 2001 From: b3yond Date: Tue, 30 Jan 2018 16:09:29 +0100 Subject: [PATCH 2/8] Excepted IMAP connection Error --- .gitignore | 1 + mailbot.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 8b2b385..3ccf5f8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ __pycache__/ last_mention last_mail ticketfrei.cfg +ticketfrei.sqlite seen_toots.pickle seen_toots.pickle.part pip-selfcheck.json diff --git a/mailbot.py b/mailbot.py index d74e3b8..7ccc92e 100644 --- a/mailbot.py +++ b/mailbot.py @@ -73,8 +73,11 @@ class Mailbot(object): try: rv, data = self.mailbox.select("Inbox") except imaplib.IMAP4.abort: - logger.error("Crawling Mail failed", exc_info=True) - rv = False + rv = "Crawling Mail failed" + logger.error(rv, exc_info=True) + except TimeoutError: + rv = "No Connection" + logger.error(rv, exc_info=True) msgs = [] if rv == 'OK': rv, data = self.mailbox.search(None, "ALL") From 9b3efd7bd237688302ea11f319478ac3d9bff617 Mon Sep 17 00:00:00 2001 From: b3yond Date: Fri, 5 Oct 2018 23:40:41 +0200 Subject: [PATCH 3/8] fixed #41 - mention has to be in status text now --- active_bots/twitterbot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/active_bots/twitterbot.py b/active_bots/twitterbot.py index 7b56523..081fc07 100755 --- a/active_bots/twitterbot.py +++ b/active_bots/twitterbot.py @@ -50,7 +50,7 @@ class TwitterBot(Bot): text = re.sub( "(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)", "", status.text) - username = api.me() + username = "@" + api.me().screen_name if username in status.text: reports.append(report.Report(status.author.screen_name, self, From 304d83ffad51429fae42ab961f899f06be4d1690 Mon Sep 17 00:00:00 2001 From: b3yond Date: Sat, 6 Oct 2018 00:56:12 +0200 Subject: [PATCH 4/8] fixing #39 - saving last request in global var, not db. --- active_bots/twitterbot.py | 11 +++++------ backend.py | 5 ++++- db.py | 6 ------ frontend.py | 4 ++-- user.py | 10 ---------- 5 files changed, 11 insertions(+), 25 deletions(-) diff --git a/active_bots/twitterbot.py b/active_bots/twitterbot.py index 081fc07..2bda8fb 100755 --- a/active_bots/twitterbot.py +++ b/active_bots/twitterbot.py @@ -7,6 +7,7 @@ import requests from time import time import report from bot import Bot +from backend import last_twitter_request logger = logging.getLogger(__name__) @@ -29,11 +30,9 @@ class TwitterBot(Bot): :return: reports: (list of report.Report objects) """ reports = [] - try: - if user.get_last_twitter_request() + 60 > time(): - return reports - except TypeError: - user.set_last_twitter_request(time()) + global last_twitter_request + if last_twitter_request + 60 > time(): + return reports try: api = self.get_api(user) except Exception: @@ -45,7 +44,7 @@ class TwitterBot(Bot): mentions = api.mentions_timeline() else: mentions = api.mentions_timeline(since_id=last_mention) - user.set_last_twitter_request(time()) + last_twitter_request = time() for status in mentions: text = re.sub( "(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)", diff --git a/backend.py b/backend.py index b9c60dc..13dcfe7 100755 --- a/backend.py +++ b/backend.py @@ -5,7 +5,7 @@ from config import config from db import db import logging from sendmail import sendmail -import time +from time import time def shutdown(): @@ -16,12 +16,15 @@ def shutdown(): exit(1) +last_twitter_request = time() + if __name__ == '__main__': logger = logging.getLogger() fh = logging.FileHandler('/var/log/ticketfrei/backend.log') fh.setLevel(logging.DEBUG) logger.addHandler(fh) + bots = [] for ActiveBot in active_bots.__dict__.values(): if isinstance(ActiveBot, type) and issubclass(ActiveBot, Bot): diff --git a/db.py b/db.py index 2e0306e..117cd8a 100644 --- a/db.py +++ b/db.py @@ -93,12 +93,6 @@ class DB(object): active INTEGER, FOREIGN KEY(user_id) REFERENCES user(id) ); - CREATE TABLE IF NOT EXISTS twitter_last_request ( - id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, - user_id INTEGER, - date INTEGER, - FOREIGN KEY(user_id) REFERENCES user(id) - ); CREATE TABLE IF NOT EXISTS telegram_accounts ( id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, user_id INTEGER, diff --git a/frontend.py b/frontend.py index 7a1df7a..8ae5273 100755 --- a/frontend.py +++ b/frontend.py @@ -231,8 +231,8 @@ def twitter_callback(user): @post('/login/mastodon') def login_mastodon(user): """ - Starts the mastodon OAuth authentication process. - :return: redirect to twitter. + Mastodon OAuth authentication process. + :return: redirect to city page. """ # get app tokens instance_url = request.forms.get('instance_url') diff --git a/user.py b/user.py index b2da2e6..a2e8821 100644 --- a/user.py +++ b/user.py @@ -144,16 +144,6 @@ schlitz keys.append(row[1]) return keys - def get_last_twitter_request(self): - db.execute("SELECT date FROM twitter_last_request WHERE user_id = ?;", - (self.uid,)) - return db.cur.fetchone()[0] - - def set_last_twitter_request(self, date): - db.execute("UPDATE twitter_last_request SET date = ? WHERE user_id = ?;", - (date, self.uid)) - db.commit() - def toot_is_seen(self, toot_uri): db.execute("SELECT COUNT(*) FROM seen_toots WHERE user_id = ? AND toot_uri = ?;", (self.uid, toot_uri)) From b9a4899981a1aa37f4e0e019dfc78273062292b1 Mon Sep 17 00:00:00 2001 From: b3yond Date: Sat, 6 Oct 2018 02:46:54 +0200 Subject: [PATCH 5/8] fixing #38: putting the city into the From address of report mails --- active_bots/mailbot.py | 5 +++-- sendmail.py | 7 +++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/active_bots/mailbot.py b/active_bots/mailbot.py index 8125137..c10229e 100644 --- a/active_bots/mailbot.py +++ b/active_bots/mailbot.py @@ -36,8 +36,9 @@ class Mailbot(Bot): + db.mail_subscription_token(rec, user.get_city()) if report.author != rec: try: - sendmail.sendmail(rec, "Ticketfrei " + user.get_city() + - " Report", body=body) + city = user.get_city() + sendmail.sendmail(rec, "Ticketfrei " + city + " Report", + city=city, body=body) except Exception: logger.error("Sending Mail failed.", exc_info=True) diff --git a/sendmail.py b/sendmail.py index d739d78..9772836 100755 --- a/sendmail.py +++ b/sendmail.py @@ -71,9 +71,12 @@ class Mailer(object): return "Sent mail to " + recipient + ": " + subject -def sendmail(to, subject, body=''): +def sendmail(to, subject, city=None, body=''): msg = MIMEMultipart() - msg['From'] = 'Ticketfrei <%s@%s>' % (getuser(), getfqdn()) + if city: + msg['From'] = 'Ticketfrei <%s@%s>' % (city, getfqdn()) + else: + msg['From'] = 'Ticketfrei <%s@%s>' % (getuser(), getfqdn()) msg['To'] = to msg['Subject'] = '[Ticketfrei] %s' % (subject, ) msg.attach(MIMEText(body)) From 2068b99b87f03c4acd4a69391716981bb93a268e Mon Sep 17 00:00:00 2001 From: b3yond Date: Sat, 6 Oct 2018 10:20:37 +0200 Subject: [PATCH 6/8] fixing #44 - refactoring how mails are sent --- active_bots/mailbot.py | 4 +-- db.py | 2 +- sendmail.py | 64 ++---------------------------------------- 3 files changed, 5 insertions(+), 65 deletions(-) diff --git a/active_bots/mailbot.py b/active_bots/mailbot.py index c10229e..9488fba 100644 --- a/active_bots/mailbot.py +++ b/active_bots/mailbot.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import logging -import sendmail +from sendmail import sendmail import datetime import mailbox import email @@ -37,7 +37,7 @@ class Mailbot(Bot): if report.author != rec: try: city = user.get_city() - sendmail.sendmail(rec, "Ticketfrei " + city + " Report", + sendmail(rec, "Ticketfrei " + city + " Report", city=city, body=body) except Exception: logger.error("Sending Mail failed.", exc_info=True) diff --git a/db.py b/db.py index 117cd8a..cd6b99b 100644 --- a/db.py +++ b/db.py @@ -1,7 +1,7 @@ from config import config import jwt import logging -from os import urandom, system +from os import urandom from pylibscrypt import scrypt_mcf import sqlite3 diff --git a/sendmail.py b/sendmail.py index 9772836..dec722d 100755 --- a/sendmail.py +++ b/sendmail.py @@ -1,76 +1,16 @@ #!/usr/bin/env python3 from config import config from email.mime.text import MIMEText -from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart import logging from getpass import getuser import smtplib from socket import getfqdn -import ssl logger = logging.getLogger(__name__) -class Mailer(object): - """ - Maintains the connection to the mailserver and sends text to users. - """ - - def __init__(self): - """ - Creates an SMTP client to send a mail. Is called only once - when you actually want to send a mail. After you sent the - mail, the SMTP client is shut down again. - - """ - # This generates the From address by stripping the part until the first - # period from the mail server address and won't work always. - self.fromaddr = config["mail"]["user"] + "@" + config["mail"]["mailserver"].partition(".")[2] - - # starts a client session with the SMTP server - self.s = smtplib.SMTP(config["mail"]["mailserver"]) - try: - context = ssl.create_default_context() - self.s.starttls(context=context) - except BaseException: # TODO: Amend specific exception - logger.error('StartTLS failed.', exc_info=True) - self.s.login(config["mail"]["user"], config["mail"]["passphrase"]) - - def send(self, text, recipient, subject, attachment=None): - """ - - :param text: (string) the content of the mail - :param recipient: (string) the recipient of the mail - :param subject: (string) the subject of the mail - :param attachment: (string) the path to the logfile - :return: string for logging purposes, contains recipient & subject - """ - msg = MIMEMultipart() - msg.attach(MIMEText(text)) - - msg["From"] = self.fromaddr - msg["To"] = recipient - msg["Subject"] = subject - - # attach logfile - if attachment: - with open(attachment, "rb") as fil: - part = MIMEApplication( - fil.read(), - Name="logfile" - ) - # After the file is closed - part['Content-Disposition'] = 'attachment; filename="logfile"' - msg.attach(part) - - self.s.send_message(msg) - self.s.close() - - return "Sent mail to " + recipient + ": " + subject - - def sendmail(to, subject, city=None, body=''): msg = MIMEMultipart() if city: @@ -87,5 +27,5 @@ def sendmail(to, subject, city=None, body=''): # For testing: if __name__ == '__main__': - m = Mailer() - print(m.send("This is a test mail.", m.fromaddr, "Test")) + sendmail(config['mail']['contact'], "Test Mail", + body="This is a test mail.") From 5119c6bfbbd5f8824438a64653757f9eeab326eb Mon Sep 17 00:00:00 2001 From: b3yond Date: Sat, 6 Oct 2018 10:44:07 +0200 Subject: [PATCH 7/8] globals are in a separate python file now #39 #45 --- active_bots/twitterDMs.py | 5 +++++ active_bots/twitterbot.py | 12 +++++++----- tfglobals.py | 3 +++ user.py | 16 ++++++++-------- 4 files changed, 23 insertions(+), 13 deletions(-) create mode 100644 tfglobals.py diff --git a/active_bots/twitterDMs.py b/active_bots/twitterDMs.py index 36b2562..a9bd5e4 100644 --- a/active_bots/twitterDMs.py +++ b/active_bots/twitterDMs.py @@ -5,6 +5,8 @@ import tweepy import re import requests import report +import tfglobals +from time import time from bot import Bot @@ -27,6 +29,8 @@ class TwitterBot(Bot): :return: reports: (list of report.Report objects) """ reports = [] + if tfglobals.last_twitter_request + 60 > time(): + return reports try: api = self.get_api(user) except IndexError: @@ -37,6 +41,7 @@ class TwitterBot(Bot): mentions = api.direct_messages() else: mentions = api.mentions_timeline(since_id=last_dm[0]) + tfglobals.last_twitter_request = time() for status in mentions: text = re.sub( "(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)", diff --git a/active_bots/twitterbot.py b/active_bots/twitterbot.py index 2bda8fb..24ea012 100755 --- a/active_bots/twitterbot.py +++ b/active_bots/twitterbot.py @@ -7,7 +7,7 @@ import requests from time import time import report from bot import Bot -from backend import last_twitter_request +import tfglobals logger = logging.getLogger(__name__) @@ -30,12 +30,14 @@ class TwitterBot(Bot): :return: reports: (list of report.Report objects) """ reports = [] - global last_twitter_request - if last_twitter_request + 60 > time(): + #global last_twitter_request + if tfglobals.last_twitter_request + 60 > time(): return reports try: api = self.get_api(user) - except Exception: + except TypeError: + # When there is no twitter account for this bot, we want to + # seamlessly continue. #logger.error("Error Authenticating Twitter", exc_info=True) return reports last_mention = user.get_seen_tweet() @@ -44,7 +46,7 @@ class TwitterBot(Bot): mentions = api.mentions_timeline() else: mentions = api.mentions_timeline(since_id=last_mention) - last_twitter_request = time() + tfglobals.last_twitter_request = time() for status in mentions: text = re.sub( "(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)", diff --git a/tfglobals.py b/tfglobals.py new file mode 100644 index 0000000..ded3bb1 --- /dev/null +++ b/tfglobals.py @@ -0,0 +1,3 @@ +from time import time + +last_twitter_request = time() diff --git a/user.py b/user.py index a2e8821..9074b5a 100644 --- a/user.py +++ b/user.py @@ -136,14 +136,6 @@ schlitz instance = db.cur.fetchone() return instance[1], instance[2], row[0], instance[0] - def get_twitter_credentials(self): - keys = [config['twitter']['consumer_key'], - config['twitter']['consumer_secret']] - row = self.get_twitter_token() - keys.append(row[0]) - keys.append(row[1]) - return keys - def toot_is_seen(self, toot_uri): db.execute("SELECT COUNT(*) FROM seen_toots WHERE user_id = ? AND toot_uri = ?;", (self.uid, toot_uri)) @@ -276,6 +268,14 @@ schlitz (self.uid, )) return db.cur.fetchone() + def get_twitter_credentials(self): + keys = [config['twitter']['consumer_key'], + config['twitter']['consumer_secret']] + row = self.get_twitter_token() + keys.append(row[0]) + keys.append(row[1]) + return keys + def update_telegram_key(self, apikey): db.execute("UPDATE telegram_accounts SET apikey = ? WHERE user_id = ?;", (apikey, self.uid)) db.commit() From 942f19fefef996a1c11d7da985c39ef2409bbb33 Mon Sep 17 00:00:00 2001 From: b3yond Date: Sat, 6 Oct 2018 11:58:08 +0200 Subject: [PATCH 8/8] comments, because the use of this file is not obvious. --- tfglobals.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tfglobals.py b/tfglobals.py index ded3bb1..32282b5 100644 --- a/tfglobals.py +++ b/tfglobals.py @@ -1,3 +1,10 @@ from time import time +""" +This file is for shared global variables. They only stay during runtime. + +For reference: +https://stackoverflow.com/questions/15959534/visibility-of-global-variables-in-imported-modules +""" + last_twitter_request = time()