Merge remote-tracking branch 'origin/multi-deployment' into multi-deployment

# Conflicts:
#	template/settings.tpl

LICENSE

@@ -1,5 +1,6 @@
 Copyright (c) 2017 Thomas L <tom@dl6tom.de>
 Copyright (c) 2017 b3yond <b3yond@riseup.net>
+Copyright (c) 2018 sid
 
 Permission to use, copy, modify, and distribute this software for any
 purpose with or without fee is hereby granted, provided that the above

README.md

@@ -50,16 +50,16 @@ to check if something was retweeted in the last hour or something.
 
 To this date, we have never heard of this happening though.
 
-### blacklisting
+### blockisting
 
-You also need to edit the goodlist and the blacklist. You can do this on the
+You also need to edit the goodlist and the blocklist. You can do this on the
 website, in the settings of your bot.
 
 Just add the words to the goodlist, which you want to require. A report is only
 spread if it contains at least one of them. If you want to RT everything, just
 add a ```*```.
 
-There is also a blacklist, which you can use to automatically sort out
+There is also a blocklist, which you can use to automatically sort out
 malicious messages. Be careful though, our filter can't read the intention with
 which a word was used. Maybe you wanted it there.
 
@@ -91,7 +91,7 @@ virtualenv -p python3 .
 Install the dependencies:
 
 ```shell
-pip install tweepy pytoml Mastodon.py bottle pyjwt pylibscrypt Markdown
+pip install tweepy pytoml Mastodon.py bottle pyjwt pylibscrypt Markdown twx
 ```
 
 Configure the bot:
@@ -179,7 +179,7 @@ virtualenv -p python3 .
 Install the dependencies:
 
 ```shell
-pip install tweepy pytoml Mastodon.py bottle pyjwt pylibscrypt Markdown
+pip install tweepy pytoml Mastodon.py bottle pyjwt pylibscrypt Markdown twx
 ```
 
 Configure the bot:

active_bots/__init__.py

@@ -12,4 +12,3 @@ for loader, name, is_pkg in pkgutil.walk_packages(__path__):
 
         globals()[name] = value
         __all__.append(name)
-

active_bots/mailbot.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+
+import logging
+import sendmail
+import datetime
+import mailbox
+import email
+import report
+from bot import Bot
+from config import config
+from db import db
+
+logger = logging.getLogger(__name__)
+
+
+class Mailbot(Bot):
+
+    # returns a list of Report objects
+    def crawl(self, user):
+        reports = []
+        # todo: adjust to actual mailbox
+        mails = mailbox.mbox('/var/mail/test')
+        for msg in mails:
+            if get_date_from_header(msg['Date']) > user.get_seen_mail():
+                reports.append(make_report(msg, user))
+        return reports
+
+    # post/boost Report object
+    def post(self, user, report):
+        recipients = user.get_mailinglist()
+        for rec in recipients:
+            rec = rec[0]
+            unsubscribe_text = "\n_______\nYou don't want to receive those messages? Unsubscribe with this link: "
+            body = report.text + unsubscribe_text + config['web']['host'] + "/city/mail/unsubscribe/" \
+                   + db.mail_subscription_token(rec, user.get_city())
+            if report.author != rec:
+                try:
+                    sendmail.sendmail(rec, "Ticketfrei " + user.get_city() +
+                                      " Report", body=body)
+                except Exception:
+                    logger.error("Sending Mail failed.", exc_info=True)
+
+
+def make_report(msg, user):
+    """
+    generates a report out of a mail
+
+    :param msg: email.parser.Message object
+    :return: post: report.Report object
+    """
+    # get a comparable date out of the email
+    date = get_date_from_header(msg['Date'])
+
+    author = msg['From']  # get mail author from email header
+    # :todo take only the part in between the < >
+
+    text = msg.get_payload()
+    post = report.Report(author, "mail", text, None, date)
+    user.save_seen_mail(date)
+    return post
+
+
+def get_date_from_header(header):
+    """
+    :param header: msg['Date']
+    :return: float: total seconds
+    """
+    date_tuple = email.utils.parsedate_tz(header)
+    date_tuple = datetime.datetime.fromtimestamp(
+        email.utils.mktime_tz(date_tuple)
+    )
+    return (date_tuple - datetime.datetime(1970, 1, 1)).total_seconds()

active_bots/mastodonbot.py

@@ -21,7 +21,7 @@ class MastodonBot(Bot):
         try:
             m = Mastodon(*user.get_masto_credentials())
         except TypeError:
-            #logger.error("No Mastodon Credentials in database.", exc_info=True)
+            # logger.error("No Mastodon Credentials in database.", exc_info=True)
             return mentions
         try:
             notifications = m.notifications()
@@ -29,6 +29,8 @@ class MastodonBot(Bot):
             logger.error("Unknown Mastodon API Error.", exc_info=True)
             return mentions
         for status in notifications:
+            if user.get_seen_toot() is None:
+                user.init_seen_toot(m.instance()['uri'])
             if (status['type'] == 'mention' and
                     status['status']['id'] > user.get_seen_toot()):
                 # save state
@@ -36,8 +38,8 @@ class MastodonBot(Bot):
                 # add mention to mentions
                 text = re.sub(r'<[^>]*>', '', status['status']['content'])
                 text = re.sub(
-                        "(?<=^|(?<=[^a-zA-Z0-9-_.]))@([A-Za-z]+[A-Za-z0-9-_]+)",
+                    "(?<=^|(?<=[^a-zA-Z0-9-_.]))@([A-Za-z]+[A-Za-z0-9-_]+)",
-                        "", text)
+                    "", text)
                 if status['status']['visibility'] == 'public':
                     mentions.append(Report(status['account']['acct'],
                                            self,
@@ -53,7 +55,10 @@ class MastodonBot(Bot):
         return mentions
 
     def post(self, user, report):
-        m = Mastodon(*user.get_masto_credentials())
+        try:
+            m = Mastodon(*user.get_masto_credentials())
+        except TypeError:
+            return  # no mastodon account for this user.
         if report.source == self:
             try:
                 m.status_reblog(report.id)
@@ -66,4 +71,5 @@ class MastodonBot(Bot):
             try:
                 m.toot(text)
             except Exception:
-                logger.error('Error tooting: ' + user.get_city() + ': ' + report.id, exc_info=True)
+                logger.error('Error tooting: ' + user.get_city() + ': ' +
+                             report.id, exc_info=True)

active_bots/telegrambot.py

@@ -0,0 +1,59 @@
+from bot import Bot
+import logging
+from report import Report
+from twx.botapi import TelegramBot as Telegram
+
+
+logger = logging.getLogger(__name__)
+
+
+class TelegramBot(Bot):
+    def crawl(self, user):
+        tb = Telegram(user.get_telegram_credentials())
+        seen_tg = user.get_seen_tg()
+        try:
+            updates = tb.get_updates(offset=seen_tg + 1,
+                                     allowed_updates="message").wait()
+        except TypeError:
+            updates = tb.get_updates().wait()
+        reports = []
+        if updates == None:
+            return reports
+        for update in updates:
+            if update == 404:
+                return reports
+            user.save_seen_tg(update.update_id)
+            if update.message.text.lower() == "/start":
+                user.add_telegram_subscribers(update.message.sender.id)
+                tb.send_message(
+                    update.message.sender.id,
+                    "You are now subscribed to report notifications.")
+                # TODO: /start message should be set in frontend
+            elif update.message.text.lower() == "/stop":
+                user.remove_telegram_subscribers(update.message.sender.id)
+                tb.send_message(
+                    update.message.sender.id,
+                    "You are now unsubscribed from report notifications.")
+                # TODO: /stop message should be set in frontend
+            elif update.message.text.lower() == "/help":
+                tb.send_message(
+                    update.message.sender.id,
+                    "Send reports here to share them with other users. Use /start and /stop to get reports or not.")
+                # TODO: /help message should be set in frontend
+            else:
+                reports.append(Report(update.message.sender.username, self,
+                                      update.message.text, None,
+                                      update.message.date))
+        return reports
+
+    def post(self, user, report):
+        tb = Telegram(user.get_telegram_credentials())
+        text = report.text
+        if len(text) > 4096:
+            text = text[:4096 - 4] + u' ...'
+        try:
+            for subscriber_id in user.get_telegram_subscribers():
+                tb.send_message(subscriber_id, text).wait()
+        except Exception:
+            logger.error('Error telegramming: ' + user.get_city() + ': '
+                         + str(report.id), exc_info=True)

active_bots/twitterDMs.py

@@ -27,17 +27,20 @@ class TwitterBot(Bot):
         :return: reports: (list of report.Report objects)
         """
         reports = []
-        api = self.get_api(user)
+        try:
+            api = self.get_api(user)
+        except IndexError:
+            return reports  # no twitter account for this user.
         last_dm = user.get_seen_dm()
         try:
-            if last_dm == None:
+            if last_dm is None:
                 mentions = api.direct_messages()
             else:
                 mentions = api.mentions_timeline(since_id=last_dm[0])
             for status in mentions:
                 text = re.sub(
-                        "(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)",
+                       "(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)",
-                        "", status.text)
+                       "", status.text)
                 reports.append(report.Report(status.author.screen_name,
                                              "twitterDM",
                                              text,

active_bots/twitterbot.py

@@ -4,6 +4,7 @@ import logging
 import tweepy
 import re
 import requests
+from time import time
 import report
 from bot import Bot
 
@@ -12,6 +13,7 @@ logger = logging.getLogger(__name__)
 
 
 class TwitterBot(Bot):
+
     def get_api(self, user):
         keys = user.get_twitter_credentials()
         auth = tweepy.OAuthHandler(consumer_key=keys[0],
@@ -27,6 +29,11 @@ class TwitterBot(Bot):
         :return: reports: (list of report.Report objects)
         """
         reports = []
+        try:
+            if user.get_last_twitter_request() + 60 > time():
+                return reports
+        except TypeError:
+            user.set_last_twitter_request(time())
         try:
             api = self.get_api(user)
         except Exception:
@@ -38,10 +45,11 @@ class TwitterBot(Bot):
                 mentions = api.mentions_timeline()
             else:
                 mentions = api.mentions_timeline(since_id=last_mention)
+            user.set_last_twitter_request(time())
             for status in mentions:
                 text = re.sub(
-                        "(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)",
+                    "(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)",
-                        "", status.text)
+                    "", status.text)
                 reports.append(report.Report(status.author.screen_name,
                                              self,
                                              text,
@@ -60,7 +68,10 @@ class TwitterBot(Bot):
         return []
 
     def post(self, user, report):
-        api = self.get_api(user)
+        try:
+            api = self.get_api(user)
+        except IndexError:
+            return  # no twitter account for this user.
         try:
             if report.source == self:
                 api.retweet(report.id)

backend.py

@@ -37,7 +37,6 @@ if __name__ == '__main__':
                             continue
                         for bot2 in bots:
                             bot2.post(user, status)
-            time.sleep(60)  # twitter rate limit >.<
     except Exception:
         logger.error("Shutdown.", exc_info=True)
         shutdown()

config.toml.example

@@ -6,6 +6,7 @@ consumer_secret = "your_consumer_secret"
 
 [web]
 host = "0.0.0.0"  # will be used by bottle as a host.
+port = 80
 contact = "b3yond@riseup.net"
 
 [mail]

db.py

@@ -14,7 +14,7 @@ class DB(object):
         self.conn = sqlite3.connect(dbfile)
         self.cur = self.conn.cursor()
         self.create()
-        self.secret = urandom(32)
+        self.secret = self.get_secret()
 
     def execute(self, *args, **kwargs):
         return self.cur.execute(*args, **kwargs)
@@ -70,11 +70,17 @@ class DB(object):
                 id         INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
                 user_id            INTEGER,
                 mastodon_accounts_id    INTEGER,
-                toot_id            TEXT,
+                toot_id            INTEGER,
                 FOREIGN KEY(user_id) REFERENCES user(id),
                 FOREIGN KEY(mastodon_accounts_id)
                     REFERENCES mastodon_accounts(id)
             );
+            CREATE TABLE IF NOT EXISTS seen_telegrams (
+                id                 INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
+                user_id            INTEGER,
+                tg_id              INTEGER,
+                FOREIGN KEY(user_id) REFERENCES user(id)
+            );
             CREATE TABLE IF NOT EXISTS twitter_request_tokens (
                 id          INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
                 user_id     INTEGER,
@@ -90,6 +96,12 @@ class DB(object):
                 active      INTEGER,
                 FOREIGN KEY(user_id) REFERENCES user(id)
             );
+            CREATE TABLE IF NOT EXISTS twitter_last_request (
+                id          INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
+                user_id     INTEGER,
+                date        INTEGER,
+                FOREIGN KEY(user_id) REFERENCES user(id)
+            );
             CREATE TABLE IF NOT EXISTS telegram_accounts (
                 id          INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
                 user_id     INTEGER,
@@ -115,11 +127,30 @@ class DB(object):
                 FOREIGN KEY(twitter_accounts_id)
                     REFERENCES twitter_accounts(id)
             );
+            CREATE TABLE IF NOT EXISTS telegram_accounts (
+                id          INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
+                user_id     INTEGER,
+                api_token   TEXT,
+                active      INTEGER,
+                FOREIGN KEY(user_id) REFERENCES user(id)
+            );
+            CREATE TABLE IF NOT EXISTS telegram_subscribers (
+                id          INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
+                user_id     INTEGER,
+                subscriber_id   INTEGER,
+                FOREIGN KEY(user_id) REFERENCES user(id),
+                UNIQUE(user_id, subscriber_id) ON CONFLICT IGNORE
+            );
             CREATE TABLE IF NOT EXISTS mailinglist (
                 id          INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
                 user_id     INTEGER,
                 email       TEXT,
-                active      INTEGER,
+                FOREIGN KEY(user_id) REFERENCES user(id)
+            );
+            CREATE TABLE IF NOT EXISTS seen_mail (
+                id          INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
+                user_id     INTEGER,
+                mail_date   REAL,
                 FOREIGN KEY(user_id) REFERENCES user(id)
             );
             CREATE TABLE IF NOT EXISTS cities (
@@ -127,13 +158,36 @@ class DB(object):
                 user_id     INTEGER,
                 city        TEXT,
                 markdown    TEXT,
+                mail_md     TEXT,
                 masto_link  TEXT,
                 twit_link   TEXT,
                 FOREIGN KEY(user_id) REFERENCES user(id),
                 UNIQUE(user_id, city) ON CONFLICT IGNORE
             );
+            CREATE TABLE IF NOT EXISTS secret (
+                id          INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
+                secret      BLOB
+            );
         ''')
 
+    def get_secret(self):
+        """
+        At __init__(), the db needs a secret. It tries to fetch it from the db,
+        and if it fails, it generates a new one.
+
+        :return:
+        """
+        # select only the newest secret. should be only one row anyway.
+        self.execute("SELECT secret FROM secret ORDER BY id DESC LIMIT 1")
+        try:
+            return self.cur.fetchone()[0]
+        except TypeError:
+            new_secret = urandom(32)
+            self.execute("INSERT INTO secret (secret) VALUES (?);",
+                         (new_secret, ))
+            self.commit()
+            return new_secret
+
     def user_token(self, email, password):
         """
         This function is called by the register confirmation process. It wants
@@ -144,11 +198,11 @@ class DB(object):
         :return:
         """
         return jwt.encode({
-                'email': email,
+            'email': email,
-                'passhash': scrypt_mcf(
+            'passhash': scrypt_mcf(
-                        password.encode('utf-8')
+                password.encode('utf-8')
-                    ).decode('ascii')
+            ).decode('ascii')
-            }, self.secret).decode('ascii')
+        }, self.secret).decode('ascii')
 
     def mail_subscription_token(self, email, city):
         """
@@ -159,16 +213,16 @@ class DB(object):
         :param city: string
         :return: a token with an encoded json dict { email: x, city: y }
         """
-        return jwt.encode({
+        token = jwt.encode({
             'email': email,
             'city': city
         }, self.secret).decode('ascii')
+        return token
 
     def confirm_subscription(self, token):
         json = jwt.decode(token, self.secret)
         return json['email'], json['city']
 
-
     def confirm(self, token, city):
         from user import User
         try:
@@ -180,8 +234,7 @@ class DB(object):
             self.execute("INSERT INTO user (passhash) VALUES(?);",
                          (json['passhash'], ))
             uid = self.cur.lastrowid
-            default_triggerpatterns = """
+            default_triggerpatterns = """kontroll?e
-kontroll?e
 konti
 db
 vgn
@@ -197,8 +250,7 @@ linie
 nuernberg
 nürnberg
 s\d
-u\d\d?            
+u\d\d?"""
-            """
             self.execute("""INSERT INTO triggerpatterns (user_id, patterns)
                                 VALUES(?, ?); """, (uid, default_triggerpatterns))
             self.execute("INSERT INTO badwords (user_id, words) VALUES(?, ?);",
@@ -209,6 +261,10 @@ u\d\d?
                      (uid, json['email']))
         self.execute("""INSERT INTO telegram_accounts (user_id, apikey,
                         active) VALUES(?, ?, ?);""", (uid, "", 1))
+        self.execute(
+            "INSERT INTO seen_telegrams (user_id, tg_id) VALUES (?,?);", (uid, 0))
+        self.execute(
+            "INSERT INTO seen_mail (user_id, mail_date) VALUES (?,?);", (uid, 0))
         self.commit()
         user = User(uid)
         user.set_city(city)
@@ -233,13 +289,14 @@ u\d\d?
         return User(uid)
 
     def user_facing_properties(self, city):
-        self.execute("""SELECT city, markdown, masto_link, twit_link 
+        self.execute("""SELECT city, markdown, mail_md, masto_link, twit_link
                             FROM cities
                             WHERE city=?;""", (city, ))
         try:
-            city, markdown, masto_link, twit_link = self.cur.fetchone()
+            city, markdown, mail_md, masto_link, twit_link = self.cur.fetchone()
             return dict(city=city,
                         markdown=markdown,
+                        mail_md=mail_md,
                         masto_link=masto_link,
                         twit_link=twit_link,
                         mailinglist=city + "@" + config["web"]["host"])

frontend.py

@@ -9,6 +9,7 @@ from sendmail import sendmail
 from session import SessionPlugin
 from mastodon import Mastodon
 
+
 def url(route):
     return '%s://%s/%s' % (
             request.urlparts.scheme,
@@ -38,13 +39,13 @@ def register_post():
         return dict(error='Email address already in use.')
     # send confirmation mail
     try:
-        print(url('confirm/' + city + '/%s' % db.user_token(email, password)))  # only for local testing
+        link = url('confirm/' + city + '/%s' % db.user_token(email, password))
+        print(link)  # only for local testing
+        logger.error('confirmation link to ' + email + ": " + link)
         sendmail(
                 email,
                 "Confirm your account",
-                "Complete your registration here: %s" % (
+                "Complete your registration here: %s" % (link)
-                        url('confirm/' + city + '/%s' % db.user_token(email, password))
-                    )
             )
         return dict(info='Confirmation mail sent.')
     except Exception:
@@ -78,13 +79,14 @@ def login_post():
 
 
 @get('/city/<city>')
-@view('template/city.tpl')
+def city_page(city, info=None):
-def city_page(city):
     citydict = db.user_facing_properties(city)
     if citydict is not None:
-        return citydict
+        citydict['info'] = info
-    redirect('/')
+        return bottle.template('template/city.tpl', **citydict)
-    return dict(info='There is no Ticketfrei bot in your city yet. Create one yourself!')
+    return bottle.template('template/propaganda.tpl',
+                           **dict(info='There is no Ticketfrei bot in your city'
+                                       ' yet. Create one yourself!'))
 
 
 @get('/city/mail/<city>')
@@ -102,18 +104,26 @@ def subscribe_mail(city):
     print(confirm_link)  # only for local testing
     # send mail with code to email
     sendmail(email, "Subscribe to Ticketfrei " + city + " Mail Notifications",
-             body="To subscribe to the mail notifications for Ticketfrei " + city + ", click on this link: " + token)
+             body="To subscribe to the mail notifications for Ticketfrei " +
+                  city + ", click on this link: " + token)
+    return city_page(city, info="Thanks! You will receive a confirmation mail.")
 
 
 @get('/city/mail/confirm/<token>')
-@view('template/city.tpl')
 def confirm_subscribe(token):
     email, city = db.confirm_subscription(token)
-    print(email)  # debug
-    print(city)  # debug
     user = db.by_city(city)
     user.add_subscriber(email)
-    redirect('/city/' + city)
+    return city_page(city, info="Thanks for subscribing to mail notifications!")
+
+
+@get('/city/mail/unsubscribe/<token>')
+def unsubscribe(token):
+    email, city = db.confirm_subscription(token)
+    user = db.by_city(city)
+    user.remove_subscriber(email)
+    return city_page(city, info="You successfully unsubscribed " + email +
+                     " from the mail notifications.")
 
 
 @get('/settings')
@@ -129,6 +139,13 @@ def update_markdown(user):
     return user.state()
 
 
+@post('/settings/mail_md')
+@view('template/settings.tpl')
+def update_mail_md(user):
+    user.set_mail_md(request.forms['mail_md'])
+    return user.state()
+
+
 @post('/settings/goodlist')
 @view('template/settings.tpl')
 def update_trigger_patterns(user):
@@ -144,11 +161,10 @@ def update_badwords(user):
 
 
 @post('/settings/telegram')
-@view('template/settings.tpl')
 def register_telegram(user):
     apikey = request.forms['apikey']
-    user.set_telegram_key(apikey)
+    user.update_telegram_key(apikey)
-    return user.state()
+    return city_page(user.get_city(), info="Thanks for registering Telegram!")
 
 
 @get('/api/state')
@@ -221,18 +237,19 @@ def login_mastodon(user):
     # get app tokens
     instance_url = request.forms.get('instance_url')
     masto_email = request.forms.get('email')
-    print(masto_email)
     masto_pass = request.forms.get('pass')
-    print(masto_pass)
     client_id, client_secret = user.get_mastodon_app_keys(instance_url)
     m = Mastodon(client_id=client_id, client_secret=client_secret,
                  api_base_url=instance_url)
     try:
         access_token = m.log_in(masto_email, masto_pass)
         user.save_masto_token(access_token, instance_url)
-        return dict(
+
-                info='Thanks for supporting decentralized social networks!'
+        # Trying to set the seen_toot to 0, thereby initializing it.
-            )
+        # It should work now, but has default values. Not sure if I need them.
+        user.init_seen_toot(instance_url)
+
+        return city_page(user.get_city(), info='Thanks for supporting decentralized social networks!')
     except Exception:
         logger.error('Login to Mastodon failed.', exc_info=True)
         return dict(error='Login to Mastodon failed.')
@@ -248,6 +265,6 @@ bottle.install(SessionPlugin('/'))
 
 if __name__ == '__main__':
     # testing only
-    bottle.run(host='localhost', port=8080)
+    bottle.run(host=config["web"]["host"], port=config["web"]["port"])
 else:
     application.catchall = False

report.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+
 class Report(object):
     """
     A ticketfrei report object.
@@ -19,7 +20,7 @@ class Report(object):
         :param timestamp: time of the report
         """
         self.author = author
-        self.type = source
+        self.source = source
         self.text = text
         self.timestamp = timestamp
         self.id = id

sendmail.py

@@ -34,7 +34,7 @@ class Mailer(object):
         try:
             context = ssl.create_default_context()
             self.s.starttls(context=context)
-        except:
+        except BaseException:  # TODO: Amend specific exception
             logger.error('StartTLS failed.', exc_info=True)
         self.s.login(config["mail"]["user"], config["mail"]["passphrase"])
 

static/bot.html

@@ -83,16 +83,16 @@
             </form>
         </div>
 
-        <!-- blacklist entry field -->
+        <!-- blocklist entry field -->
         <div style="float:right; padding: 1.5em;">
             <p>
                 These words are not allowed in reports.
                 If you encounter spam, you can add more here - the bot will ignore reports which use such words.
-                <!-- There are words which you can't exclude from the blacklist, e.g. certain racist, sexist, or antisemitic slurs. (to be implemented) -->
+                <!-- There are words which you can't exclude from the blocklist, e.g. certain racist, sexist, or antisemitic slurs. (to be implemented) -->
             </p>
-            <form action="/settings/blacklist" method="post">
+            <form action="/settings/blocklist" method="post">
-                <!-- find a way to display current blacklist. js which reads from a cookie? template? -->
+                <!-- find a way to display current blocklist. js which reads from a cookie? template? -->
-                <textarea id="blacklist" rows="8" cols="70" name="blacklist" wrap="physical"></textarea>
+                <textarea id="blocklist" rows="8" cols="70" name="blocklist" wrap="physical"></textarea>
                 <input name='confirm' value='Submit' type='submit'/>
             </form>
         </div>

static/js/functions.js

@@ -36,4 +36,4 @@ document.getElementById("enablebutton").innerHTML = enableButton();
 
 document.getElementById("goodlist").innerHTML = listformat(getCookie("goodlist"));
 
-document.getElementById("blacklist").innerHTML = listformat(getCookie("blacklist"));
+document.getElementById("blocklist").innerHTML = listformat(getCookie("blocklist"));

template/city.tpl

@@ -6,4 +6,12 @@ import markdown as md
 html = md.markdown(markdown)
 %>
 
+% if info is not None:
+<div class="ui-widget">
+    <div class="ui-state-highlight ui-corner-all" style="padding: 0.7em;">
+        <p><span class="ui-icon ui-icon-info" style="float: left; margin-right: .3em;"></span>{{!info}}</p>
+    </div>
+</div>
+% end
+
 {{!html}}

template/mail.tpl

@@ -3,13 +3,14 @@
 <%
 import markdown as md
 
-html = md.markdown(markdown)
+html = md.markdown(mail_md)
 %>
 
+{{!html}}
+
 <form action="/city/mail/submit/{{!city}}" method="post">
     <input type="text" name="mailaddress" placeholder="E-Mail address" id="mailaddress">
     <input name='confirm' value='Subscribe to E-Mail notifications' type='submit'/>
 </form>
-
+<br>
-
+<p style="text-align: center;"><a href="/city/{{!city}}">Back to Ticketfrei {{!city}} overview</a></p>
-{{!html}}

template/propaganda.tpl

@@ -1,4 +1,12 @@
 % rebase('template/wrapper.tpl')
+% if defined('info'):
+<div class="ui-widget">
+    <div class="ui-state-highlight ui-corner-all" style="padding: 0.7em;">
+        <p><span class="ui-icon ui-icon-info" style="float: left; margin-right: .3em;"></span>{{!info}}</p>
+    </div>
+</div>
+<br>
+% end
 % include('template/login-plain.tpl')
 <h1>Features</h1>
 <p>

template/settings.tpl

@@ -87,10 +87,20 @@
 <div>
     <h2>Edit your city page</h2>
     <p>
-        With your bot, we generated you a page, which you can use for promotion: <a href="/city/{{city}}"
+        With your bot, we generated you a page, which you can use for promotion:
-        target="_blank">Ticketfrei {{city}}</a> You can change what your users will read there, and adjust it to your
+        <a href="/city/{{city}}" target="_blank">Ticketfrei {{city}}</a> You
-        needs. <b>You should definitely adjust the Social Media profile links.</b> This is just the default text we
+        can change what your users will read there, and adjust it to your
-        suggest:
+        needs.
+    </p>
+    <p>
+        <b>You should definitely adjust the Social Media, E-Mail, and Telegram
+        profile links.</b>
+        Also consider adding this link to the text: <a href="/city/mail/{{city}}"
+        target="_blank">Link to the mail subscription page</a>. Your readers
+        can use this to subscribe to mail notifications.
+    </p>
+    <p>
+        So this is the default text we suggest:
     </p>
     <form action="/settings/markdown" method="post">
         <textarea id="markdown" rows="20" cols="70" name="markdown" wrap="physical">{{markdown}}</textarea>
@@ -98,12 +108,30 @@
     </form>
 </div>
 
+<div>
+    <h2>Edit your mail subscription page</h2>
+    <p>
+        There is also a page where users can subscribe to mail notifications:
+        <a href="/city/mail/{{city}}" target="_blank">Ticketfrei {{city}}</a>.
+        You can change what your users will read there, and adjust it to your
+        needs.
+    </p>
+    <p>
+        So this is the default text we suggest:
+    </p>
+    <form action="/settings/mail_md" method="post">
+        <textarea id="mail_md" rows="20" cols="70" name="mail_md" wrap="physical">{{mail_md}}</textarea>
+        <input name='confirm' value='Save' type='submit'/>
+    </form>
+</div>
+
 <div>
     <h2>Edit your trigger patterns</h2>
     <p>
-        These words have to be contained in a report.
+        These words have to be contained in a report. If none of these
-        If none of these expressions is in the report, it will be ignored by the bot.
+        expressions is in the report, it will be ignored by the bot. You can
-        You can use the defaults, or enter some expressions specific to your city and language.
+        use the defaults, or enter some expressions specific to your city and
+        language.
     </p>
     <form action="/settings/goodlist" method="post">
         <!-- find a way to display current good list. js which reads from a cookie? template? -->
@@ -115,11 +143,12 @@
 <div>
     <h2>Edit the blocklist</h2>
     <p>
-        These words are not allowed in reports.
+        These words are not allowed in reports. If you encounter spam, you can
-        If you encounter spam, you can add more here - the bot will ignore reports which use such words.
+        add more here - the bot will ignore reports which use such words.
-        There are words which you can't exclude from the blocklist, e.g. certain racist, sexist, or antisemitic slurs.
+        <!-- There are words which you can't exclude from the blocklist, e.g. certain racist, sexist, or antisemitic slurs.
     </p>
     <form action="/settings/blocklist" method="post">
+        <!-- find a way to display current blocklist. js which reads from a cookie? template? -->
         <textarea id="blocklist" rows="8" cols="70" name="blocklist" wrap="physical">{{badwords}}</textarea>
         <input name='confirm' value='Submit' type='submit'/>
     </form>

user.py

@@ -13,7 +13,7 @@ class User(object):
         self.uid = uid
 
     def check_password(self, password):
-        db.execute("SELECT passhash FROM user WHERE id=?;", (self.uid, ))
+        db.execute("SELECT passhash FROM user WHERE id=?;", (self.uid,))
         passhash, = db.cur.fetchone()
         return scrypt_mcf_check(passhash.encode('ascii'),
                                 password.encode('utf-8'))
@@ -23,6 +23,7 @@ class User(object):
         db.execute("UPDATE user SET passhash=? WHERE id=?;",
                    (passhash, self.uid))
         db.commit()
+
     password = property(None, password)  # setter only, can't read back
 
     @property
@@ -38,11 +39,11 @@ class User(object):
 
     @property
     def emails(self):
-        db.execute("SELECT email FROM email WHERE user_id=?;", (self.uid, ))
+        db.execute("SELECT email FROM email WHERE user_id=?;", (self.uid,))
-        return (*db.cur.fetchall(), )
+        return (*db.cur.fetchall(),)
 
     def delete_email(self, email):
-        db.execute("SELECT COUNT(*) FROM email WHERE user_id=?", (self.uid, ))
+        db.execute("SELECT COUNT(*) FROM email WHERE user_id=?", (self.uid,))
         if db.cur.fetchone()[0] == 1:
             return False  # don't allow to delete last email
         db.execute("DELETE FROM email WHERE user_id=? AND email=?;",
@@ -52,16 +53,16 @@ class User(object):
 
     def email_token(self, email):
         return jwt.encode({
-                'email': email,
+            'email': email,
-                'uid': self.uid
+            'uid': self.uid
-            }, db.secret).decode('ascii')
+        }, db.secret).decode('ascii')
 
     def is_appropriate(self, report):
-        db.execute("SELECT pattern FROM triggerpatterns WHERE user_id=?;",
+        db.execute("SELECT patterns FROM triggerpatterns WHERE user_id=?;",
                    (self.uid, ))
-        patterns = db.cur.fetchone()
+        patterns = db.cur.fetchone()[0]
         for pattern in patterns.splitlines():
-            if pattern.search(report.text) is not None:
+            if pattern in report.text.lower():
                 break
         else:
             # no pattern matched
@@ -81,7 +82,7 @@ nigger
 neger
 schlitz
         """
-        db.execute("SELECT word FROM badwords WHERE user_id=?;",
+        db.execute("SELECT words FROM badwords WHERE user_id=?;",
                    (self.uid, ))
         badwords = db.cur.fetchone()
         for word in report.text.lower().splitlines():
@@ -92,12 +93,46 @@ schlitz
                 return False
         return True
 
-    def get_masto_credentials(self):
+    def get_telegram_credentials(self):
-        db.execute("SELECT access_token, instance_id FROM mastodon_accounts WHERE user_id = ? AND active = 1;",
+        db.execute("""SELECT apikey 
-                   (self.uid, ))
+                          FROM telegram_accounts 
+                          WHERE user_id = ? AND active = 1;""",
+                   (self.uid,))
         row = db.cur.fetchone()
-        db.execute("SELECT instance, client_id, client_secret FROM mastodon_instances WHERE id = ?;",
+        return row[0]
-                   (row[1], ))
+
+    def get_telegram_subscribers(self):
+        db.execute("""SELECT subscriber_id 
+                          FROM telegram_subscribers 
+                          WHERE user_id = ?;""",
+                   (self.uid,))
+        rows = db.cur.fetchall()
+        return rows
+
+    def add_telegram_subscribers(self, subscriber_id):
+        db.execute("""INSERT INTO telegram_subscribers (
+                            user_id, subscriber_id) VALUES(?, ?);""",
+                   (self.uid, subscriber_id))
+        db.commit()
+
+    def remove_telegram_subscribers(self, subscriber_id):
+        db.execute("""DELETE 
+                          FROM telegram_subscribers 
+                          WHERE user_id = ?
+                          AND subscriber_id = ?;""",
+                   (self.uid, subscriber_id))
+        db.commit()
+
+    def get_masto_credentials(self):
+        db.execute("""SELECT access_token, instance_id 
+                          FROM mastodon_accounts 
+                          WHERE user_id = ? AND active = 1;""",
+                   (self.uid,))
+        row = db.cur.fetchone()
+        db.execute("""SELECT instance, client_id, client_secret 
+                          FROM mastodon_instances 
+                          WHERE id = ?;""",
+                   (row[1],))
         instance = db.cur.fetchone()
         return instance[1], instance[2], row[0], instance[0]
 
@@ -109,10 +144,32 @@ schlitz
         keys.append(row[1])
         return keys
 
+    def get_last_twitter_request(self):
+        db.execute("SELECT date FROM twitter_last_request WHERE user_id = ?;",
+                   (self.uid,))
+        return db.cur.fetchone()[0]
+
+    def set_last_twitter_request(self, date):
+        db.execute("UPDATE twitter_last_request SET date = ? WHERE user_id = ?;",
+                   (date, self.uid))
+        db.commit()
+
+    def init_seen_toot(self, instance_url):
+        db.execute("SELECT id FROM mastodon_instances WHERE instance = ?;",
+                   (instance_url,))
+        masto_instance = db.cur.fetchone()[0]
+        db.execute("INSERT INTO seen_toots (user_id, mastodon_accounts_id, toot_id) VALUES (?,?,?);",
+            (self.uid, masto_instance, 0))
+        db.conn.commit()
+        return
+
     def get_seen_toot(self):
         db.execute("SELECT toot_id FROM seen_toots WHERE user_id = ?;",
-                   (self.uid, ))
+                   (self.uid,))
-        return db.cur.fetchone()[0]
+        try:
+            return db.cur.fetchone()[0]
+        except TypeError:
+            return None
 
     def save_seen_toot(self, toot_id):
         db.execute("UPDATE seen_toots SET toot_id = ? WHERE user_id = ?;",
@@ -121,7 +178,7 @@ schlitz
 
     def get_seen_tweet(self):
         db.execute("SELECT tweet_id FROM seen_tweets WHERE user_id = ?;",
-                   (self.uid, ))
+                   (self.uid,))
         return db.cur.fetchone()[0]
 
     def save_seen_tweet(self, tweet_id):
@@ -131,7 +188,7 @@ schlitz
 
     def get_seen_dm(self):
         db.execute("SELECT message_id FROM seen_dms WHERE user_id = ?;",
-                   (self.uid, ))
+                   (self.uid,))
         return db.cur.fetchone()
 
     def save_seen_dm(self, tweet_id):
@@ -139,12 +196,22 @@ schlitz
                    (tweet_id, self.uid))
         db.commit()
 
-    def get_mailinglist(self):
+    def get_seen_tg(self):
-        db.execute("SELECT email FROM mailinglist WHERE user_id = ? AND active = 1;", (self.uid, ))
+        db.execute("SELECT tg_id FROM seen_telegrams WHERE user_id = ?;",
+                   (self.uid,))
         return db.cur.fetchone()[0]
 
+    def save_seen_tg(self, tg_id):
+        db.execute("UPDATE seen_telegrams SET tg_id = ? WHERE user_id = ?;",
+                   (tg_id, self.uid))
+        db.commit()
+
+    def get_mailinglist(self):
+        db.execute("SELECT email FROM mailinglist WHERE user_id = ?;", (self.uid, ))
+        return db.cur.fetchall()
+
     def get_seen_mail(self):
-        db.execute("SELECT mail_date FROM seen_mails WHERE user_id = ?;", (self.uid, ))
+        db.execute("SELECT mail_date FROM seen_mail WHERE user_id = ?;", (self.uid, ))
         return db.cur.fetchone()[0]
 
     def save_seen_mail(self, mail_date):
@@ -163,7 +230,11 @@ schlitz
         return db.cur.fetchone()[0]
 
     def add_subscriber(self, email):
-        db.execute("INSERT INTO mailinglist(user_id, email, active) VALUES(?, ?, ?);", (self.uid, email, 1))
+        db.execute("INSERT INTO mailinglist(user_id, email) VALUES(?, ?);", (self.uid, email))
+        db.commit()
+
+    def remove_subscriber(self, email):
+        db.execute("DELETE FROM mailinglist WHERE email = ? AND user_id = ?;", (email, self.uid))
         db.commit()
 
     def set_badwords(self, words):
@@ -180,6 +251,7 @@ schlitz
         # necessary:
         # - city
         # - markdown
+        # - mail_md
         # - goodlist
         # - blocklist
         # - logged in with twitter?
@@ -188,27 +260,36 @@ schlitz
         citydict = db.user_facing_properties(self.get_city())
         return dict(city=citydict['city'],
                     markdown=citydict['markdown'],
+                    mail_md=citydict['mail_md'],
                     triggerwords=self.get_trigger_words(),
                     badwords=self.get_badwords(),
                     enabled=self.enabled)
 
     def save_request_token(self, token):
-        db.execute("INSERT INTO twitter_request_tokens(user_id, request_token, request_token_secret) VALUES(?, ?, ?);",
+        db.execute("""INSERT INTO
-                   (self.uid, token["oauth_token"], token["oauth_token_secret"]))
+                          twitter_request_tokens(
+                              user_id, request_token, request_token_secret
+                          ) VALUES(?, ?, ?);""",
+                   (self.uid, token["oauth_token"],
+                    token["oauth_token_secret"]))
         db.commit()
 
     def get_request_token(self):
-        db.execute("SELECT request_token, request_token_secret FROM twitter_request_tokens WHERE user_id = ?;", (self.uid,))
+        db.execute("""SELECT request_token, request_token_secret 
+                          FROM twitter_request_tokens 
+                          WHERE user_id = ?;""", (self.uid,))
         request_token = db.cur.fetchone()
-        db.execute("DELETE FROM twitter_request_tokens WHERE user_id = ?;", (self.uid,))
+        db.execute("""DELETE FROM twitter_request_tokens 
+                          WHERE user_id = ?;""", (self.uid,))
         db.commit()
-        return {"oauth_token" : request_token[0],
+        return {"oauth_token": request_token[0],
-                "oauth_token_secret" : request_token[1]}
+                "oauth_token_secret": request_token[1]}
 
     def save_twitter_token(self, access_token, access_token_secret):
-        db.execute(
+        db.execute(""""INSERT INTO twitter_accounts(
-            "INSERT INTO twitter_accounts(user_id, client_id, client_secret) VALUES(?, ?, ?);",
+                           user_id, client_id, client_secret
-            (self.uid, access_token, access_token_secret))
+                           ) VALUES(?, ?, ?);""",
+                   (self.uid, access_token, access_token_secret))
         db.commit()
 
     def get_twitter_token(self):
@@ -216,12 +297,14 @@ schlitz
                    (self.uid, ))
         return db.cur.fetchall()
 
-    def set_telegram_key(self, apikey):
+    def update_telegram_key(self, apikey):
         db.execute("UPDATE telegram_accounts SET apikey = ? WHERE user_id = ?;", (apikey, self.uid))
         db.commit()
 
     def get_mastodon_app_keys(self, instance):
-        db.execute("SELECT client_id, client_secret FROM mastodon_instances WHERE instance = ?;", (instance, ))
+        db.execute("""SELECT client_id, client_secret 
+                          FROM mastodon_instances 
+                          WHERE instance = ?;""", (instance,))
         try:
             row = db.cur.fetchone()
             client_id = row[0]
@@ -229,14 +312,19 @@ schlitz
             return client_id, client_secret
         except TypeError:
             app_name = "ticketfrei" + str(db.secret)[0:4]
-            client_id, client_secret = Mastodon.create_app(app_name, api_base_url=instance)
+            client_id, client_secret \
-            db.execute("INSERT INTO mastodon_instances(instance, client_id, client_secret) VALUES(?, ?, ?);",
+                = Mastodon.create_app(app_name, api_base_url=instance)
+            db.execute("""INSERT INTO mastodon_instances(
+                              instance, client_id, client_secret
+                              ) VALUES(?, ?, ?);""",
                        (instance, client_id, client_secret))
             db.commit()
             return client_id, client_secret
 
     def save_masto_token(self, access_token, instance):
-        db.execute("SELECT id FROM mastodon_instances WHERE instance = ?;", (instance, ))
+        db.execute("""SELECT id 
+                          FROM mastodon_instances 
+                          WHERE instance = ?;""", (instance,))
         instance_id = db.cur.fetchone()[0]
         db.execute("INSERT INTO mastodon_accounts(user_id, access_token, instance_id, active) "
                    "VALUES(?, ?, ?, ?);", (self.uid, access_token, instance_id, 1))
@@ -247,13 +335,18 @@ schlitz
                    (markdown, self.uid))
         db.commit()
 
+    def set_mail_md(self, mail_md):
+        db.execute("UPDATE cities SET mail_md = ? WHERE user_id = ?;",
+                   (mail_md, self.uid))
+        db.commit()
+
     def get_city(self):
         db.execute("SELECT city FROM cities WHERE user_id == ?;", (self.uid, ))
         return db.cur.fetchone()[0]
 
     def set_city(self, city):
-        masto_link = "https://example.mastodon.social/@" + city # get masto_link
+        masto_link = "https://example.mastodon.social/@" + city  # get masto_link
-        twit_link = "https://example.twitter.com/" + city # get twit_link
+        twit_link = "https://example.twitter.com/" + city  # get twit_link
         mailinglist = city + "@" + config['web']['host']
         markdown = """# Wie funktioniert Ticketfrei?
 
@@ -278,6 +371,12 @@ also pass trotzdem auf, wer auf dem Bahnsteig steht.
 Aber je mehr Leute mitmachen, desto eher kannst du dir sicher 
 sein, dass wir sie finden, bevor sie uns finden.
 
+Wenn du immer direkt gewarnt werden willst, kannst du auch die
+Benachrichtigungen über E-Mail oder Telegram aktivieren. Gib 
+einfach <a href="/city/mail/""" + city + """"/">hier</a> deine 
+E-Mail-Adresse an oder subscribe dem Telegram-Bot [@ticketfrei_""" + city + \
+                   "_bot](https://t.me/ticketfrei_" + city + """_bot) 
+
 Also, wenn du weniger Glück hast, und der erste bist, der einen 
 Kontrolleur sieht:
 
@@ -287,6 +386,8 @@ Ganz einfach, du schreibst es den anderen. Das geht entweder
 
 * mit Mastodon [Link zu unserem Profil](""" + masto_link + """)
 * über Twitter: [Link zu unserem Profil](""" + twit_link + """)
+* über Telegram an [@ticketfrei_""" + city + "_bot](https://t.me/ticketfrei_" \
+                   + city + """_bot)
 * Oder per Mail an [""" + mailinglist + "](mailto:" + mailinglist + """), wenn 
   ihr kein Social Media benutzen wollt.
 
@@ -342,7 +443,24 @@ sicher vor Zensur.
 Um Mastodon zu benutzen, besucht diese Seite: 
 [https://joinmastodon.org/](https://joinmastodon.org/)
         """
-        db.execute("""INSERT INTO cities(user_id, city, markdown, masto_link, 
+        mail_md = """# Immer up-to-date
-                        twit_link) VALUES(?,?,?,?,?)""",
+
-                   (self.uid, city, markdown, masto_link, twit_link))
+Du bist viel unterwegs und hast keine Lust, jedes Mal auf das Profil des Bots
+zu schauen? Kein Problem. Unsere Mail Notifications benachrichtigen dich, wenn
+irgendwo Kontis gesehen werden.
+
+Wenn du uns deine E-Mail-Adresse gibst, kriegst du bei jedem Konti-Report eine
+Mail. Wenn du eine Mail-App auf dem Handy hast, so wie 
+[K9Mail](https://k9mail.github.io/), kriegst du sogar eine Push Notification. So
+bist du immer Up-to-date über alles, was im Verkehrsnetz passiert.
+
+## Keine Sorge
+
+Wir benutzen deine E-Mail-Adresse selbstverständlich für nichts anderes. Du 
+kannst die Benachrichtigungen jederzeit deaktivieren, mit jeder Mail wird ein
+unsubscribe-link mitgeschickt. 
+        """
+        db.execute("""INSERT INTO cities(user_id, city, markdown, mail_md,
+                        masto_link, twit_link) VALUES(?,?,?,?,?,?)""",
+                   (self.uid, city, markdown, mail_md, masto_link, twit_link))
         db.commit()