fixed telegram api bug (from -> sender)

.gitignore

@@ -11,6 +11,7 @@ seen_toots.pickle
 seen_toots.pickle.part
 pip-selfcheck.json
 config.toml
+venv/
 bin/
 include/
 lib/

LICENSE

@@ -1,5 +1,6 @@
 Copyright (c) 2017 Thomas L <tom@dl6tom.de>
 Copyright (c) 2017 b3yond <b3yond@riseup.net>
+Copyright (c) 2018 sid
 
 Permission to use, copy, modify, and distribute this software for any
 purpose with or without fee is hereby granted, provided that the above

README.md

@@ -133,6 +133,11 @@ sudo service nginx restart
 sudo cp deployment/ticketfrei-web.service /etc/systemd/system/
 sudo systemctl daemon-reload
 sudo systemctl start ticketfrei-web.service
+
+# create and start the backend systemd service
+sudo cp deployment/ticketfrei-backend.service /etc/systemd/system
+sudo systemctl daemon-reload
+sudo systemctl start ticketfrei-backend.service
 ```
 
 ### Logs
@@ -152,3 +157,51 @@ less /var/log/syslog
 # for the nginx web server:
 less /var/log/nginx/example.org_error.log
 ```
+
+### Development Install
+
+If you want to install it locally to develop on it:
+
+```shell
+sudo apt install python3 virtualenv uwsgi uwsgi-plugin-python3 nginx git
+sudo git clone https://github.com/b3yond/ticketfrei
+cd ticketfrei
+git checkout multi-deployment
+```
+
+Install the necessary packages, create and activate virtualenv:
+
+```shell
+virtualenv -p python3 .
+. bin/activate
+```
+
+Install the dependencies:
+
+```shell
+pip install tweepy pytoml Mastodon.py bottle pyjwt pylibscrypt Markdown
+```
+
+Configure the bot:
+
+```shell
+cp config.toml.example config.toml
+vim config.toml
+```
+
+This configuration is only for the admin. Users can log into
+twitter/mastodon/mail and configure their personal bot on the settings page.
+
+```shell
+# create folder for socket & database
+sudo mkdir /var/ticketfrei
+sudo chown $USER:$USER -R /var/ticketfrei
+
+# create folder for logs
+sudo mkdir /var/log/ticketfrei
+sudo chown $USER:$USER -R /var/log/ticketfrei
+
+# start Ticketfrei
+./frontend.py & ./backend.py &
+```
+

active_bots/__init__.py

@@ -0,0 +1,15 @@
+__all__ = []
+
+import pkgutil
+import inspect
+
+for loader, name, is_pkg in pkgutil.walk_packages(__path__):
+    module = loader.find_module(name).load_module(name)
+
+    for name, value in inspect.getmembers(module):
+        if name.startswith('__'):
+            continue
+
+        globals()[name] = value
+        __all__.append(name)
+

active_bots/mailbot.py

@@ -1,97 +1,46 @@
 #!/usr/bin/env python3
 
-from config import config
 import logging
 import sendmail
-import ssl
 import datetime
+import mailbox
 import email
-import imaplib
 import report
 from bot import Bot
-
+from config import config
+from db import db
 
 logger = logging.getLogger(__name__)
 
 
 class Mailbot(Bot):
-    """
-    Bot which sends Mails if mentioned via twitter/mastodon, and tells
-    other bots that it received mails.
-    """
-
-    def login(self, user):
-        # change to use mailbox of local system
-        mailinglist = user.get_mailinglist()
-        mailbox = imaplib.IMAP4_SSL(mailinglist)
-        context = ssl.create_default_context()
-        try:
-            mailbox.starttls(ssl_context=context)
-        except:
-            logger.error('StartTLS failed', exc_info=True)
-        try:
-            mailbox.login(config["mail"]["user"],
-                               config["mail"]["passphrase"])
-        except imaplib.IMAP4.error:
-            logger.error("Login to mail server failed", exc_info=True)
-            try:
-                mailer = sendmail.Mailer()
-                mailer.send('', config['web']['contact'],
-                            'Ticketfrei Crash Report',
-                            attachment=config['logging']['logpath'])
-            except:
-                logger.error('Mail sending failed', exc_info=True)
 
+    # returns a list of Report objects
     def crawl(self, user):
-        """
+        reports = []
-        crawl for new mails.
+        mails = mailbox.mbox('/var/mail/test')  # todo: adjust to actual mailbox
-        :return: msgs: (list of report.Report objects)
+        for msg in mails:
-        """
+            if get_date_from_header(msg['Date']) > user.get_seen_mail():
-        mailbox = self.login(user)
+                reports.append(make_report(msg, user))
-        try:
+        return reports
-            rv, data = mailbox.select("Inbox")
-        except imaplib.IMAP4.abort:
-            logger.error("Crawling Mail failed", exc_info=True)
-            rv = False
-        msgs = []
-        if rv == 'OK':
-            rv, data = mailbox.search(None, "ALL")
-            if rv != 'OK':
-                return msgs
-
-            for num in data[0].split():
-                rv, data = mailbox.fetch(num, '(RFC822)')
-                if rv != 'OK':
-                    logger.error("Couldn't fetch mail %s %s" % (rv, str(data)))
-                    return msgs
-                msg = email.message_from_bytes(data[0][1])
-
-                # check if email was sent by the bot itself. Different solution?
-                if not user.get_mailinglist() in msg['From']:
-                    # get a comparable date out of the email
-                    date_tuple = email.utils.parsedate_tz(msg['Date'])
-                    date_tuple = datetime.datetime.fromtimestamp(
-                            email.utils.mktime_tz(date_tuple)
-                        )
-                    date = int((date_tuple -
-                                datetime.datetime(1970, 1, 1)).total_seconds())
-                    if date > user.get_seen_mail():
-                        msgs.append(self.make_report(msg))
-        return msgs
 
+    # post/boost Report object
     def post(self, user, report):
-        """
+        recipients = user.get_mailinglist()
-        sends reports by other sources to a mailing list.
+        for rec in recipients:
+            rec = rec[0]
+            unsubscribe_text = "\n_______\nYou don't want to receive those messages? Unsubscribe with this link: "
+            body = report.text + unsubscribe_text + config['web']['host'] + "/city/mail/unsubscribe/" \
+                   + db.mail_subscription_token(rec, user.get_city())
+            if report.author != rec:
+                try:
+                    sendmail.sendmail(rec, "Ticketfrei " + user.get_city() +
+                                      " Report", body=body)
+                except Exception:
+                    logger.error("Sending Mail failed.", exc_info=True)
 
-        :param report: (report.Report object)
-        """
-        mailinglist = self.login(user)
-        if report.source is not self:
-            mailer = sendmail.Mailer()
-            mailer.send(report.text, mailinglist,
-                        "Warnung: Kontrolleure gesehen")
 
-    def make_report(self, msg):
+def make_report(msg, user):
     """
     generates a report out of a mail
 
@@ -99,16 +48,24 @@ class Mailbot(Bot):
     :return: post: report.Report object
     """
     # get a comparable date out of the email
-        date_tuple = email.utils.parsedate_tz(msg['Date'])
+    date = get_date_from_header(msg['Date'])
-        date_tuple = datetime.datetime.fromtimestamp(
-                email.utils.mktime_tz(date_tuple)
-            )
-        date = (date_tuple - datetime.datetime(1970, 1, 1)).total_seconds()
 
-        author = msg.get("From")  # get mail author from email header
+    author = msg['From']  # get mail author from email header
-        # :todo take only the part before the @
+    # :todo take only the part in between the < >
 
     text = msg.get_payload()
     post = report.Report(author, "mail", text, None, date)
-        self.last_mail = date
+    user.save_seen_mail(date)
     return post
+
+
+def get_date_from_header(header):
+    """
+    :param header: msg['Date']
+    :return: float: total seconds
+    """
+    date_tuple = email.utils.parsedate_tz(header)
+    date_tuple = datetime.datetime.fromtimestamp(
+        email.utils.mktime_tz(date_tuple)
+    )
+    return (date_tuple - datetime.datetime(1970, 1, 1)).total_seconds()

active_bots/mastodonbot.py

@@ -18,7 +18,11 @@ class MastodonBot(Bot):
         :return: list of statuses
         """
         mentions = []
+        try:
             m = Mastodon(*user.get_masto_credentials())
+        except TypeError:
+            #logger.error("No Mastodon Credentials in database.", exc_info=True)
+            return mentions
         try:
             notifications = m.notifications()
         except Exception:
@@ -49,7 +53,10 @@ class MastodonBot(Bot):
         return mentions
 
     def post(self, user, report):
+        try:
             m = Mastodon(*user.get_masto_credentials())
+        except TypeError:
+            return  # no mastodon account for this user.
         if report.source == self:
             try:
                 m.status_reblog(report.id)

active_bots/telegrambot.py

@@ -14,22 +14,22 @@ class TelegramBot(Bot):
         reports = []
         for update in updates:
             if update.message.text.lower() == "/start":
-                user.add_telegram_subscribers(update.message.from.id)
+                user.add_telegram_subscribers(update.message.sender.id)
-                tb.send_message(update.message.from.id, "You are now \
+                tb.send_message(update.message.sender.id, "You are now \
                         subscribed to report notifications.") 
                 #TODO: /start message should be set in frontend
             elif update.message.text.lower() == "/stop":
-                user.remove_telegram_subscribers(update.message.from.id)
+                user.remove_telegram_subscribers(update.message.sender.id)
-                tb.send_message(update.message.from.id, "You are now \
+                tb.send_message(update.message.sender.id, "You are now \
                         unsubscribed from report notifications.") 
                 #TODO: /stop message should be set in frontend
             elif update.message.text.lower() == "/help":
-                tb.send_message(update.message.from.id, "Send reports here to \
+                tb.send_message(update.message.sender.id, "Send reports here to \
                         share them with other users. Use /start and /stop to \
                         be included/excluded.") 
                 #TODO: /help message should be set in frontend
             else:
-                reports.append(Report(update.message.from.username,self,
+                reports.append(Report(update.message.sender.username, self,
                                update.message.text, None, update.message.date))
         return reports
 

active_bots/twitterDMs.py

@@ -18,7 +18,7 @@ class TwitterBot(Bot):
                                    consumer_secret=keys[1])
         auth.set_access_token(keys[2],  # access_token_key
                               keys[3])  # access_token_secret
-        return tweepy.API(auth)
+        return tweepy.API(auth, wait_on_rate_limit=True)
 
     def crawl(self, user):
         """
@@ -27,7 +27,10 @@ class TwitterBot(Bot):
         :return: reports: (list of report.Report objects)
         """
         reports = []
+        try:
             api = self.get_api(user)
+        except IndexError:
+            return reports  # no twitter account for this user.
         last_dm = user.get_seen_dm()
         try:
             if last_dm == None:

active_bots/twitterbot.py

@@ -12,13 +12,14 @@ logger = logging.getLogger(__name__)
 
 
 class TwitterBot(Bot):
+
     def get_api(self, user):
-        keys = user.get_api_keys()
+        keys = user.get_twitter_credentials()
         auth = tweepy.OAuthHandler(consumer_key=keys[0],
                                    consumer_secret=keys[1])
         auth.set_access_token(keys[2],  # access_token_key
                               keys[3])  # access_token_secret
-        return tweepy.API(auth)
+        return tweepy.API(auth, wait_on_rate_limit=True)
 
     def crawl(self, user):
         """
@@ -27,7 +28,11 @@ class TwitterBot(Bot):
         :return: reports: (list of report.Report objects)
         """
         reports = []
+        try:
             api = self.get_api(user)
+        except Exception:
+            #logger.error("Error Authenticating Twitter", exc_info=True)
+            return reports
         last_mention = user.get_seen_tweet()
         try:
             if last_mention == 0:
@@ -56,7 +61,10 @@ class TwitterBot(Bot):
         return []
 
     def post(self, user, report):
+        try:
             api = self.get_api(user)
+        except IndexError:
+            return  # no twitter account for this user.
         try:
             if report.source == self:
                 api.retweet(report.id)

backend.py

@@ -39,5 +39,5 @@ if __name__ == '__main__':
                             bot2.post(user, status)
             time.sleep(60)  # twitter rate limit >.<
     except Exception:
-        logger.error('Shutdown.', exc_info=True)
+        logger.error("Shutdown.", exc_info=True)
         shutdown()

bot.py

@@ -1,7 +1,8 @@
 class Bot(object):
     # returns a list of Report objects
     def crawl(self, user):
-        pass
+        reports = []
+        return reports
 
     # post/boost Report object
     def post(self, user, report):

config.toml.example

@@ -6,6 +6,7 @@ consumer_secret = "your_consumer_secret"
 
 [web]
 host = "0.0.0.0"  # will be used by bottle as a host.
+port = 80
 contact = "b3yond@riseup.net"
 
 [mail]

db.py

@@ -14,7 +14,7 @@ class DB(object):
         self.conn = sqlite3.connect(dbfile)
         self.cur = self.conn.cursor()
         self.create()
-        self.secret = urandom(32)
+        self.secret = self.get_secret()
 
     def execute(self, *args, **kwargs):
         return self.cur.execute(*args, **kwargs)
@@ -90,6 +90,13 @@ class DB(object):
                 active      INTEGER,
                 FOREIGN KEY(user_id) REFERENCES user(id)
             );
+            CREATE TABLE IF NOT EXISTS telegram_accounts (
+                id          INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
+                user_id     INTEGER,
+                apikey      TEXT,
+                active      INTEGER,
+                FOREIGN KEY(user_id) REFERENCES user(id)
+            );
             CREATE TABLE IF NOT EXISTS seen_tweets (
                 id         INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
                 user_id            INTEGER,
@@ -127,7 +134,12 @@ class DB(object):
                 id          INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
                 user_id     INTEGER,
                 email       TEXT,
-                active      INTEGER,
+                FOREIGN KEY(user_id) REFERENCES user(id)
+            );
+            CREATE TABLE IF NOT EXISTS seen_mail (
+                id          INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
+                user_id     INTEGER,
+                mail_date   REAL,
                 FOREIGN KEY(user_id) REFERENCES user(id)
             );
             CREATE TABLE IF NOT EXISTS cities (
@@ -135,14 +147,45 @@ class DB(object):
                 user_id     INTEGER,
                 city        TEXT,
                 markdown    TEXT,
+                mail_md     TEXT,
                 masto_link  TEXT,
                 twit_link   TEXT,
                 FOREIGN KEY(user_id) REFERENCES user(id),
                 UNIQUE(user_id, city) ON CONFLICT IGNORE 
             );
+            CREATE TABLE IF NOT EXISTS secret (
+                id          INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
+                secret      BLOB
+            );
         ''')
 
+    def get_secret(self):
+        """
+        At __init__(), the db needs a secret. It tries to fetch it from the db,
+        and if it fails, it generates a new one.
+
+        :return:
+        """
+        # select only the newest secret. should be only one row anyway.
+        self.execute("SELECT secret FROM secret ORDER BY id DESC LIMIT 1")
+        try:
+            return self.cur.fetchone()[0]
+        except TypeError:
+            new_secret = urandom(32)
+            self.execute("INSERT INTO secret (secret) VALUES (?);",
+                         (new_secret, ))
+            self.commit()
+            return new_secret
+
     def user_token(self, email, password):
+        """
+        This function is called by the register confirmation process. It wants
+        to write an email to the email table and a passhash to the user table.
+
+        :param email: a string with an E-Mail address.
+        :param password: a string with a passhash.
+        :return:
+        """
         return jwt.encode({
                 'email': email,
                 'passhash': scrypt_mcf(
@@ -150,6 +193,26 @@ class DB(object):
                     ).decode('ascii')
             }, self.secret).decode('ascii')
 
+    def mail_subscription_token(self, email, city):
+        """
+        This function is called by the mail subscription process. It wants
+        to write an email to the mailinglist table.
+
+        :param email: string
+        :param city: string
+        :return: a token with an encoded json dict { email: x, city: y }
+        """
+        token = jwt.encode({
+            'email': email,
+            'city': city
+        }, self.secret).decode('ascii')
+        return token
+
+    def confirm_subscription(self, token):
+        json = jwt.decode(token, self.secret)
+        return json['email'], json['city']
+
+
     def confirm(self, token, city):
         from user import User
         try:
@@ -188,8 +251,12 @@ u\d\d?
             uid = json['uid']
         self.execute("INSERT INTO email (user_id, email) VALUES(?, ?);",
                      (uid, json['email']))
+        self.execute("""INSERT INTO telegram_accounts (user_id, apikey,
+                        active) VALUES(?, ?, ?);""", (uid, "", 1))
         self.commit()
         user = User(uid)
+        self.execute("INSERT INTO seen_mail (user_id, mail_date) VALUES (?,?)",
+                     (uid, 0))
         user.set_city(city)
         return user
 
@@ -202,14 +269,24 @@ u\d\d?
             return None
         return User(uid)
 
+    def by_city(self, city):
+        from user import User
+        self.execute("SELECT user_id FROM cities WHERE city=?", (city, ))
+        try:
+            uid, = self.cur.fetchone()
+        except TypeError:
+            return None
+        return User(uid)
+
     def user_facing_properties(self, city):
-        self.execute("""SELECT city, markdown, masto_link, twit_link 
+        self.execute("""SELECT city, markdown, mail_md, masto_link, twit_link 
                             FROM cities
                             WHERE city=?;""", (city, ))
         try:
-            city, markdown, masto_link, twit_link = self.cur.fetchone()
+            city, markdown, mail_md, masto_link, twit_link = self.cur.fetchone()
             return dict(city=city,
                         markdown=markdown,
+                        mail_md=mail_md,
                         masto_link=masto_link,
                         twit_link=twit_link,
                         mailinglist=city + "@" + config["web"]["host"])

deployment/ticketfrei-backend.service

@@ -9,7 +9,7 @@ ExecStart=/srv/ticketfrei/bin/python3 backend.py
 #RuntimeDirectory=uwsgi
 Restart=always
 KillSignal=SIGQUIT
-Type=notify
+Type=simple
 StandardError=syslog
 NotifyAccess=all
 

frontend.py

@@ -9,6 +9,7 @@ from sendmail import sendmail
 from session import SessionPlugin
 from mastodon import Mastodon
 
+
 def url(route):
     return '%s://%s/%s' % (
             request.urlparts.scheme,
@@ -78,13 +79,51 @@ def login_post():
 
 
 @get('/city/<city>')
-@view('template/city.tpl')
+def city_page(city, info=None):
-def city_page(city):
     citydict = db.user_facing_properties(city)
     if citydict is not None:
-        return citydict
+        citydict['info'] = info
-    redirect('/')
+        return bottle.template('template/city.tpl', **citydict)
-    return dict(info='There is no Ticketfrei bot in your city yet. Create one yourself!')
+    return bottle.template('template/propaganda.tpl',
+                           **dict(info='There is no Ticketfrei bot in your city'
+                                       ' yet. Create one yourself!'))
+
+
+@get('/city/mail/<city>')
+@view('template/mail.tpl')
+def display_mail_page(city):
+    user = db.by_city(city)
+    return user.state()
+
+
+@post('/city/mail/submit/<city>')
+def subscribe_mail(city):
+    email = request.forms['mailaddress']
+    token = db.mail_subscription_token(email, city)
+    confirm_link = url('city/mail/confirm/' + token)
+    print(confirm_link)  # only for local testing
+    # send mail with code to email
+    sendmail(email, "Subscribe to Ticketfrei " + city + " Mail Notifications",
+             body="To subscribe to the mail notifications for Ticketfrei " +
+                  city + ", click on this link: " + token)
+    return city_page(city, info="Thanks! You will receive a confirmation mail.")
+
+
+@get('/city/mail/confirm/<token>')
+def confirm_subscribe(token):
+    email, city = db.confirm_subscription(token)
+    user = db.by_city(city)
+    user.add_subscriber(email)
+    return city_page(city, info="Thanks for subscribing to mail notifications!")
+
+
+@get('/city/mail/unsubscribe/<token>')
+def unsubscribe(token):
+    email, city = db.confirm_subscription(token)
+    user = db.by_city(city)
+    user.remove_subscriber(email)
+    return city_page(city, info="You successfully unsubscribed " + email +
+                         " from the mail notifications.")
 
 
 @get('/settings')
@@ -100,6 +139,12 @@ def update_markdown(user):
     return user.state()
 
 
+@post('/settings/mail_md')
+@view('template/settings.tpl')
+def update_mail_md(user):
+    user.set_mail_md(request.forms['mail_md'])
+    return user.state()
+
 @post('/settings/goodlist')
 @view('template/settings.tpl')
 def update_trigger_patterns(user):
@@ -114,6 +159,14 @@ def update_badwords(user):
     return user.state()
 
 
+@post('/settings/telegram')
+@view('template/settings.tpl')
+def register_telegram(user):
+    apikey = request.forms['apikey']
+    user.set_telegram_key(apikey)
+    return user.state()
+
+
 @get('/api/state')
 def api_enable(user):
     return user.state()
@@ -211,6 +264,6 @@ bottle.install(SessionPlugin('/'))
 
 if __name__ == '__main__':
     # testing only
-    bottle.run(host='localhost', port=8080)
+    bottle.run(host=config["web"]["host"], port=config["web"]["port"])
 else:
     application.catchall = False

report.py

@@ -19,7 +19,7 @@ class Report(object):
         :param timestamp: time of the report
         """
         self.author = author
-        self.type = source
+        self.source = source
         self.text = text
         self.timestamp = timestamp
         self.id = id

static/css/style.css

@@ -41,3 +41,7 @@ input[type=text], input[type=password] {
     display: inline-block;
     border: 1px solid #ccc;
 }
+
+h2 {
+    padding-top: 1em;
+}

template/city.tpl

@@ -6,4 +6,12 @@ import markdown as md
 html = md.markdown(markdown)
 %>
 
+% if info is not None:
+<div class="ui-widget">
+    <div class="ui-state-highlight ui-corner-all" style="padding: 0.7em;">
+        <p><span class="ui-icon ui-icon-info" style="float: left; margin-right: .3em;"></span>{{!info}}</p>
+    </div>
+</div>
+% end
+
 {{!html}}

template/mail.tpl

@@ -0,0 +1,16 @@
+% rebase('template/wrapper.tpl')
+
+<%
+import markdown as md
+
+html = md.markdown(mail_md)
+%>
+
+{{!html}}
+
+<form action="/city/mail/submit/{{!city}}" method="post">
+    <input type="text" name="mailaddress" placeholder="E-Mail address" id="mailaddress">
+    <input name='confirm' value='Subscribe to E-Mail notifications' type='submit'/>
+</form>
+<br>
+<p style="text-align: center;"><a href="/city/{{!city}}">Back to Ticketfrei {{!city}} overview</a></p>

template/propaganda.tpl

@@ -1,4 +1,12 @@
 % rebase('template/wrapper.tpl')
+% if defined('info'):
+<div class="ui-widget">
+    <div class="ui-state-highlight ui-corner-all" style="padding: 0.7em;">
+        <p><span class="ui-icon ui-icon-info" style="float: left; margin-right: .3em;"></span>{{!info}}</p>
+    </div>
+</div>
+<br>
+% end
 % include('template/login-plain.tpl')
 <h1>Features</h1>
 <p>

template/settings.tpl

@@ -16,9 +16,8 @@
     Log in with Twitter
 </a>
 
-<section style="padding: 1.5em;">
+<section>
     <h2>Log in with Mastodon</h2>
-    <p>
         <form action="/login/mastodon" method='post'>
             <label for="email">E-Mail of your Mastodon-Account</label>
             <input type="text" placeholder="Enter Email" name="email" id="email" required>
@@ -64,16 +63,43 @@
             </datalist>
             <input name='confirm' value='Log in' type='submit'/>
         </form>
-    </p>
 </section>
 
-<div style="float: left; padding: 1.5em;">
+<%
+# todo: hide this part, if there is already a telegram bot connected.
+%>
+<div>
+    <h2>Connect with Telegram</h2>
+    <p>
+        If you have a Telegram account, you can register a bot there. Just write to @botfather. You can find detailed
+        instructions <a href="https://botsfortelegram.com/project/the-bot-father/" target="_blank">on Bots for
+        Telegram</a>.
+    </p>
+    <p>
+        The botfather will give you an API key - with the API key, Ticketfrei can use the Telegram bot. Enter it here:
+    </p>
+    <form action="/settings/telegram" method="post">
+        <input type="text" name="apikey" placeholder="Telegram bot API key" id="apikey">
+        <input name='confirm' value='Login with Telegram' type='submit'/>
+    </form>
+</div>
+
+<div>
     <h2>Edit your city page</h2>
     <p>
-        With your bot, we generated you a page, which you can use for promotion: <a href="/city/{{city}}"
+        With your bot, we generated you a page, which you can use for promotion:
-        target="_blank">Ticketfrei {{city}}</a> You can change what your users will read there, and adjust it to your
+        <a href="/city/{{city}}" target="_blank">Ticketfrei {{city}}</a> You
-        needs. <b>You should definitely adjust the Social Media profile links.</b> This is just the default text we
+        can change what your users will read there, and adjust it to your
-        suggest:
+        needs.
+    </p>
+    <p>
+        <b>You should definitely adjust the Social Media profile links.</b>
+        Also consider adding this link to the text: <a href="/city/mail/{{city}}"
+        target="_blank">Link to the mail subscription page</a>. Your readers
+        can use this to subscribe to mail notifications.
+    </p>
+    <p>
+        So this is the default text we suggest:
     </p>
     <form action="/settings/markdown" method="post">
         <textarea id="markdown" rows="20" cols="70" name="markdown" wrap="physical">{{markdown}}</textarea>
@@ -81,12 +107,30 @@
     </form>
 </div>
 
-<div style="float: left; padding: 1.5em;">
+<div>
+    <h2>Edit your mail subscription page</h2>
+    <p>
+        There is also a page where users can subscribe to mail notifications:
+        <a href="/city/mail/{{city}}" target="_blank">Ticketfrei {{city}}</a>.
+        You can change what your users will read there, and adjust it to your
+        needs.
+    </p>
+    <p>
+        So this is the default text we suggest:
+    </p>
+    <form action="/settings/mail_md" method="post">
+        <textarea id="mail_md" rows="20" cols="70" name="mail_md" wrap="physical">{{mail_md}}</textarea>
+        <input name='confirm' value='Save' type='submit'/>
+    </form>
+</div>
+
+<div>
     <h2>Edit your trigger patterns</h2>
     <p>
-        These words have to be contained in a report.
+        These words have to be contained in a report. If none of these
-        If none of these expressions is in the report, it will be ignored by the bot.
+        expressions is in the report, it will be ignored by the bot. You can
-        You can use the defaults, or enter some expressions specific to your city and language.
+        use the defaults, or enter some expressions specific to your city and
+        language.
     </p>
     <form action="/settings/goodlist" method="post">
         <!-- find a way to display current good list. js which reads from a cookie? template? -->
@@ -95,12 +139,12 @@
     </form>
 </div>
 
-<div style="float:right; padding: 1.5em;">
+<div>
     <h2>Edit the blacklist</h2>
     <p>
-        These words are not allowed in reports.
+        These words are not allowed in reports. If you encounter spam, you can
-        If you encounter spam, you can add more here - the bot will ignore reports which use such words.
+        add more here - the bot will ignore reports which use such words.
-        <!-- There are words which you can't exclude from the blacklist, e.g. certain racist, sexist, or antisemitic slurs. (to be implemented) -->
+        <!-- There are words which you can't exclude from the blacklist, e.g. certain racist, sexist, or antisemitic slurs.
     </p>
     <form action="/settings/blacklist" method="post">
         <!-- find a way to display current blacklist. js which reads from a cookie? template? -->

template/wrapper.tpl

@@ -1,3 +1,5 @@
+<!DOCTYPE html>
+<html lang="en">
 <head>
     <title>Ticketfrei - {{get('title', 'A bot against control society!')}}</title>
     <meta name='og:title' content='Ticketfrei'/>
@@ -25,3 +27,4 @@
         <p>Contribute on <a href="https://github.com/b3yond/ticketfrei">GitHub!</a></p>
     </div>
 </body>
+</html>

user.py

@@ -58,11 +58,11 @@ class User(object):
         }, db.secret).decode('ascii')
 
     def is_appropriate(self, report):
-        db.execute("SELECT pattern FROM triggerpatterns WHERE user_id=?;",
+        db.execute("SELECT patterns FROM triggerpatterns WHERE user_id=?;",
                    (self.uid, ))
-        patterns = db.cur.fetchone()
+        patterns = db.cur.fetchone()[0]
         for pattern in patterns.splitlines():
-            if pattern.search(report.text) is not None:
+            if pattern in report.text:
                 break
         else:
             # no pattern matched
@@ -75,13 +75,14 @@ slut
 hure
 jude
 schwuchtel
+schlampe
 fag
 faggot
 nigger
 neger
 schlitz           
         """
-        db.execute("SELECT word FROM badwords WHERE user_id=?;",
+        db.execute("SELECT words FROM badwords WHERE user_id=?;",
                    (self.uid, ))
         badwords = db.cur.fetchone()
         for word in report.text.lower().splitlines():
@@ -158,6 +159,7 @@ schlitz
     def save_seen_toot(self, toot_id):
         db.execute("UPDATE seen_toots SET toot_id = ? WHERE user_id = ?;",
                    (toot_id, self.uid))
+        db.commit()
 
     def get_seen_tweet(self):
         db.execute("SELECT tweet_id FROM seen_tweets WHERE user_id = ?;",
@@ -167,6 +169,7 @@ schlitz
     def save_seen_tweet(self, tweet_id):
         db.execute("UPDATE seen_tweets SET tweet_id = ? WHERE user_id = ?;",
                    (tweet_id, self.uid))
+        db.commit()
 
     def get_seen_dm(self):
         db.execute("SELECT message_id FROM seen_dms WHERE user_id = ?;",
@@ -176,34 +179,43 @@ schlitz
     def save_seen_dm(self, tweet_id):
         db.execute("UPDATE seen_dms SET message_id = ? WHERE user_id = ?;",
                    (tweet_id, self.uid))
+        db.commit()
 
     def get_mailinglist(self):
-        db.execute("""SELECT email 
+        db.execute("SELECT email FROM mailinglist WHERE user_id = ?;", (self.uid, ))
-                          FROM mailinglist 
+        return db.cur.fetchall()
-                          WHERE user_id = ? AND active = 1;""", (self.uid,))
-        return db.cur.fetchone()[0]
 
     def get_seen_mail(self):
-        db.execute("""SELECT mail_date 
+        db.execute("SELECT mail_date FROM seen_mail WHERE user_id = ?;", (self.uid, ))
-                          FROM seen_mails WHERE user_id = ?;""", (self.uid,))
         return db.cur.fetchone()[0]
 
     def save_seen_mail(self, mail_date):
         db.execute("UPDATE seen_mail SET mail_date = ? WHERE user_id = ?;",
                    (mail_date, self.uid))
+        db.commit()
 
     def set_trigger_words(self, patterns):
         db.execute("UPDATE triggerpatterns SET patterns = ? WHERE user_id = ?;",
                    (patterns, self.uid))
+        db.commit()
 
     def get_trigger_words(self):
         db.execute("SELECT patterns FROM triggerpatterns WHERE user_id = ?;",
                    (self.uid,))
         return db.cur.fetchone()[0]
 
+    def add_subscriber(self, email):
+        db.execute("INSERT INTO mailinglist(user_id, email) VALUES(?, ?);", (self.uid, email))
+        db.commit()
+
+    def remove_subscriber(self, email):
+        db.execute("DELETE FROM mailinglist WHERE email = ? AND user_id = ?;", (email, self.uid))
+        db.commit()
+
     def set_badwords(self, words):
         db.execute("UPDATE badwords SET words = ? WHERE user_id = ?;",
                    (words, self.uid))
+        db.commit()
 
     def get_badwords(self):
         db.execute("SELECT words FROM badwords WHERE user_id = ?;",
@@ -214,6 +226,7 @@ schlitz
         # necessary:
         # - city
         # - markdown
+        # - mail_md
         # - goodlist
         # - blacklist
         # - logged in with twitter?
@@ -222,6 +235,7 @@ schlitz
         citydict = db.user_facing_properties(self.get_city())
         return dict(city=citydict['city'],
                     markdown=citydict['markdown'],
+                    mail_md=citydict['mail_md'],
                     triggerwords=self.get_trigger_words(),
                     badwords=self.get_badwords(),
                     enabled=self.enabled)
@@ -254,11 +268,14 @@ schlitz
         db.commit()
 
     def get_twitter_token(self):
-        db.execute("""SELECT access_token, access_token_secret 
+        db.execute("SELECT client_id, client_secret FROM twitter_accounts WHERE user_id = ?;",
-                          FROM twitter_accouts WHERE user_id = ?;""",
                    (self.uid, ))
         return db.cur.fetchall()
 
+    def set_telegram_key(self, apikey):
+        db.execute("UPDATE telegram_accounts SET apikey = ? WHERE user_id = ?;", (apikey, self.uid))
+        db.commit()
+
     def get_mastodon_app_keys(self, instance):
         db.execute("""SELECT client_id, client_secret 
                           FROM mastodon_instances 
@@ -291,6 +308,12 @@ schlitz
     def set_markdown(self, markdown):
         db.execute("UPDATE cities SET markdown = ? WHERE user_id = ?;",
                    (markdown, self.uid))
+        db.commit()
+
+    def set_mail_md(self, mail_md):
+        db.execute("UPDATE cities SET mail_md = ? WHERE user_id = ?;",
+                   (mail_md, self.uid))
+        db.commit()
 
     def get_city(self):
         db.execute("SELECT city FROM cities WHERE user_id == ?;", (self.uid, ))
@@ -310,6 +333,7 @@ Willst du einen Fahrscheinfreien ÖPNV erkämpfen?
 Schau einfach auf das Profil unseres Bots: """ + twit_link + """
 
 Hat jemand vor kurzem etwas über Kontrolleur\*innen gepostet?
+
 * Wenn ja, dann kauf dir vllt lieber ein Ticket. In Nürnberg 
   haben wir die Erfahrung gemacht, dass Kontis normalerweile
   ungefähr ne Woche aktiv sind, ein paar Stunden am Tag. Wenn es 
@@ -322,6 +346,11 @@ also pass trotzdem auf, wer auf dem Bahnsteig steht.
 Aber je mehr Leute mitmachen, desto eher kannst du dir sicher 
 sein, dass wir sie finden, bevor sie uns finden.
 
+Wenn du immer direkt gewarnt werden willst, kannst du auch die
+E-Mail-Benachrichtigungen aktivieren. Gib einfach 
+<a href="/city/mail/""" + city + """"/">hier</a> deine 
+E-Mail-Adresse an.
+
 Also, wenn du weniger Glück hast, und der erste bist, der einen 
 Kontrolleur sieht:
 
@@ -386,7 +415,25 @@ sicher vor Zensur.
 Um Mastodon zu benutzen, besucht diese Seite: 
 [https://joinmastodon.org/](https://joinmastodon.org/)
         """
-        db.execute("""INSERT INTO cities(user_id, city, markdown, masto_link, 
+        mail_md = """# Immer up-to-date
-                        twit_link) VALUES(?,?,?,?,?)""",
+
-                   (self.uid, city, markdown, masto_link, twit_link))
+Du bist viel unterwegs und hast keine Lust, jedes Mal auf das Profil des Bots
+zu schauen? Kein Problem. Unsere Mail Notifications benachrichtigen dich, wenn
+irgendwo Kontis gesehen werden.
+
+Wenn du uns deine E-Mail-Adresse gibst, kriegst du bei jedem Konti-Report eine
+Mail. Wenn du eine Mail-App auf dem Handy hast, so wie 
+[K9Mail](https://k9mail.github.io/), kriegst du sogar eine Push Notification. So
+bist du immer Up-to-date über alles, was im Verkehrsnetz passiert.
+
+## Keine Sorge
+
+Wir benutzen deine E-Mail-Adresse selbstverständlich für nichts anderes. Du 
+kannst die Benachrichtigungen jederzeit deaktivieren, mit jeder Mail wird ein
+unsubscribe-link mitgeschickt. 
+        """
+        db.execute("""INSERT INTO cities(user_id, city, markdown, mail_md,
+                        masto_link, twit_link) VALUES(?,?,?,?,?,?)""",
+                   (self.uid, city, markdown, mail_md, masto_link, twit_link))
         db.commit()
+