Merge pull request #32 from b3yond/mailbot

Rewriting the Mailbot, included subscription mechanism

active_bots/mailbot.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+import logging
+import sendmail
+import datetime
+import mailbox
+import email
+import report
+from bot import Bot
+from config import config
+from db import db
+
+logger = logging.getLogger(__name__)
+
+
+class Mailbot(Bot):
+
+    # returns a list of Report objects
+    def crawl(self, user):
+        reports = []
+        mails = mailbox.mbox('/var/mail/test')  # todo: adjust to actual mailbox
+        for msg in mails:
+            if get_date_from_header(msg['Date']) > user.get_seen_mail():
+                reports.append(make_report(msg, user))
+        return reports
+
+    # post/boost Report object
+    def post(self, user, report):
+        recipients = user.get_mailinglist()
+        for rec in recipients:
+            rec = rec[0]
+            unsubscribe_text = "\n_______\nYou don't want to receive those messages? Unsubscribe with this link: "
+            body = report.text + unsubscribe_text + config['web']['host'] + "/city/mail/unsubscribe/" \
+                   + db.mail_subscription_token(rec, user.get_city())
+            if report.author != rec:
+                try:
+                    sendmail.sendmail(rec, "Ticketfrei " + user.get_city() +
+                                      " Report", body=body)
+                except Exception:
+                    logger.error("Sending Mail failed.", exc_info=True)
+
+
+def make_report(msg, user):
+    """
+    generates a report out of a mail
+
+    :param msg: email.parser.Message object
+    :return: post: report.Report object
+    """
+    # get a comparable date out of the email
+    date = get_date_from_header(msg['Date'])
+
+    author = msg['From']  # get mail author from email header
+    # :todo take only the part in between the < >
+
+    text = msg.get_payload()
+    post = report.Report(author, "mail", text, None, date)
+    user.save_seen_mail(date)
+    return post
+
+
+def get_date_from_header(header):
+    """
+    :param header: msg['Date']
+    :return: float: total seconds
+    """
+    date_tuple = email.utils.parsedate_tz(header)
+    date_tuple = datetime.datetime.fromtimestamp(
+        email.utils.mktime_tz(date_tuple)
+    )
+    return (date_tuple - datetime.datetime(1970, 1, 1)).total_seconds()

active_bots/mastodonbot.py

@@ -53,7 +53,10 @@ class MastodonBot(Bot):
         return mentions
 
     def post(self, user, report):
-        m = Mastodon(*user.get_masto_credentials())
+        try:
+            m = Mastodon(*user.get_masto_credentials())
+        except TypeError:
+            return  # no mastodon account for this user.
         if report.source == self:
             try:
                 m.status_reblog(report.id)

active_bots/twitterDMs.py

@@ -27,7 +27,10 @@ class TwitterBot(Bot):
         :return: reports: (list of report.Report objects)
         """
         reports = []
-        api = self.get_api(user)
+        try:
+            api = self.get_api(user)
+        except IndexError:
+            return reports  # no twitter account for this user.
         last_dm = user.get_seen_dm()
         try:
             if last_dm == None:

active_bots/twitterbot.py

@@ -12,6 +12,7 @@ logger = logging.getLogger(__name__)
 
 
 class TwitterBot(Bot):
+
     def get_api(self, user):
         keys = user.get_twitter_credentials()
         auth = tweepy.OAuthHandler(consumer_key=keys[0],
@@ -60,7 +61,10 @@ class TwitterBot(Bot):
         return []
 
     def post(self, user, report):
-        api = self.get_api(user)
+        try:
+            api = self.get_api(user)
+        except IndexError:
+            return  # no twitter account for this user.
         try:
             if report.source == self:
                 api.retweet(report.id)

db.py

@@ -14,7 +14,7 @@ class DB(object):
         self.conn = sqlite3.connect(dbfile)
         self.cur = self.conn.cursor()
         self.create()
-        self.secret = urandom(32)
+        self.secret = self.get_secret()
 
     def execute(self, *args, **kwargs):
         return self.cur.execute(*args, **kwargs)
@@ -119,7 +119,12 @@ class DB(object):
                 id          INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
                 user_id     INTEGER,
                 email       TEXT,
-                active      INTEGER,
+                FOREIGN KEY(user_id) REFERENCES user(id)
+            );
+            CREATE TABLE IF NOT EXISTS seen_mail (
+                id          INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
+                user_id     INTEGER,
+                mail_date   REAL,
                 FOREIGN KEY(user_id) REFERENCES user(id)
             );
             CREATE TABLE IF NOT EXISTS cities (
@@ -127,13 +132,36 @@ class DB(object):
                 user_id     INTEGER,
                 city        TEXT,
                 markdown    TEXT,
+                mail_md     TEXT,
                 masto_link  TEXT,
                 twit_link   TEXT,
                 FOREIGN KEY(user_id) REFERENCES user(id),
                 UNIQUE(user_id, city) ON CONFLICT IGNORE 
             );
+            CREATE TABLE IF NOT EXISTS secret (
+                id          INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
+                secret      BLOB
+            );
         ''')
 
+    def get_secret(self):
+        """
+        At __init__(), the db needs a secret. It tries to fetch it from the db,
+        and if it fails, it generates a new one.
+
+        :return:
+        """
+        # select only the newest secret. should be only one row anyway.
+        self.execute("SELECT secret FROM secret ORDER BY id DESC LIMIT 1")
+        try:
+            return self.cur.fetchone()[0]
+        except TypeError:
+            new_secret = urandom(32)
+            self.execute("INSERT INTO secret (secret) VALUES (?);",
+                         (new_secret, ))
+            self.commit()
+            return new_secret
+
     def user_token(self, email, password):
         """
         This function is called by the register confirmation process. It wants
@@ -159,10 +187,11 @@ class DB(object):
         :param city: string
         :return: a token with an encoded json dict { email: x, city: y }
         """
-        return jwt.encode({
+        token = jwt.encode({
             'email': email,
             'city': city
         }, self.secret).decode('ascii')
+        return token
 
     def confirm_subscription(self, token):
         json = jwt.decode(token, self.secret)
@@ -211,6 +240,8 @@ u\d\d?
                         active) VALUES(?, ?, ?);""", (uid, "", 1))
         self.commit()
         user = User(uid)
+        self.execute("INSERT INTO seen_mail (user_id, mail_date) VALUES (?,?)",
+                     (uid, 0))
         user.set_city(city)
         return user
 
@@ -233,13 +264,14 @@ u\d\d?
         return User(uid)
 
     def user_facing_properties(self, city):
-        self.execute("""SELECT city, markdown, masto_link, twit_link 
+        self.execute("""SELECT city, markdown, mail_md, masto_link, twit_link 
                             FROM cities
                             WHERE city=?;""", (city, ))
         try:
-            city, markdown, masto_link, twit_link = self.cur.fetchone()
+            city, markdown, mail_md, masto_link, twit_link = self.cur.fetchone()
             return dict(city=city,
                         markdown=markdown,
+                        mail_md=mail_md,
                         masto_link=masto_link,
                         twit_link=twit_link,
                         mailinglist=city + "@" + config["web"]["host"])

frontend.py

@@ -9,6 +9,7 @@ from sendmail import sendmail
 from session import SessionPlugin
 from mastodon import Mastodon
 
+
 def url(route):
     return '%s://%s/%s' % (
             request.urlparts.scheme,
@@ -78,13 +79,14 @@ def login_post():
 
 
 @get('/city/<city>')
-@view('template/city.tpl')
+def city_page(city, info=None):
-def city_page(city):
     citydict = db.user_facing_properties(city)
     if citydict is not None:
-        return citydict
+        citydict['info'] = info
-    redirect('/')
+        return bottle.template('template/city.tpl', **citydict)
-    return dict(info='There is no Ticketfrei bot in your city yet. Create one yourself!')
+    return bottle.template('template/propaganda.tpl',
+                           **dict(info='There is no Ticketfrei bot in your city'
+                                       ' yet. Create one yourself!'))
 
 
 @get('/city/mail/<city>')
@@ -102,18 +104,26 @@ def subscribe_mail(city):
     print(confirm_link)  # only for local testing
     # send mail with code to email
     sendmail(email, "Subscribe to Ticketfrei " + city + " Mail Notifications",
-             body="To subscribe to the mail notifications for Ticketfrei " + city + ", click on this link: " + token)
+             body="To subscribe to the mail notifications for Ticketfrei " +
+                  city + ", click on this link: " + token)
+    return city_page(city, info="Thanks! You will receive a confirmation mail.")
 
 
 @get('/city/mail/confirm/<token>')
-@view('template/city.tpl')
 def confirm_subscribe(token):
     email, city = db.confirm_subscription(token)
-    print(email)  # debug
-    print(city)  # debug
     user = db.by_city(city)
     user.add_subscriber(email)
-    redirect('/city/' + city)
+    return city_page(city, info="Thanks for subscribing to mail notifications!")
+
+
+@get('/city/mail/unsubscribe/<token>')
+def unsubscribe(token):
+    email, city = db.confirm_subscription(token)
+    user = db.by_city(city)
+    user.remove_subscriber(email)
+    return city_page(city, info="You successfully unsubscribed " + email +
+                         " from the mail notifications.")
 
 
 @get('/settings')
@@ -129,6 +139,12 @@ def update_markdown(user):
     return user.state()
 
 
+@post('/settings/mail_md')
+@view('template/settings.tpl')
+def update_mail_md(user):
+    user.set_mail_md(request.forms['mail_md'])
+    return user.state()
+
 @post('/settings/goodlist')
 @view('template/settings.tpl')
 def update_trigger_patterns(user):

report.py

@@ -19,7 +19,7 @@ class Report(object):
         :param timestamp: time of the report
         """
         self.author = author
-        self.type = source
+        self.source = source
         self.text = text
         self.timestamp = timestamp
         self.id = id

template/city.tpl

@@ -6,4 +6,12 @@ import markdown as md
 html = md.markdown(markdown)
 %>
 
+% if info is not None:
+<div class="ui-widget">
+    <div class="ui-state-highlight ui-corner-all" style="padding: 0.7em;">
+        <p><span class="ui-icon ui-icon-info" style="float: left; margin-right: .3em;"></span>{{!info}}</p>
+    </div>
+</div>
+% end
+
 {{!html}}

template/mail.tpl

@@ -3,13 +3,14 @@
 <%
 import markdown as md
 
-html = md.markdown(markdown)
+html = md.markdown(mail_md)
 %>
 
+{{!html}}
+
 <form action="/city/mail/submit/{{!city}}" method="post">
     <input type="text" name="mailaddress" placeholder="E-Mail address" id="mailaddress">
     <input name='confirm' value='Subscribe to E-Mail notifications' type='submit'/>
 </form>
-
+<br>
-
+<p style="text-align: center;"><a href="/city/{{!city}}">Back to Ticketfrei {{!city}} overview</a></p>
-{{!html}}

template/propaganda.tpl

@@ -1,4 +1,12 @@
 % rebase('template/wrapper.tpl')
+% if defined('info'):
+<div class="ui-widget">
+    <div class="ui-state-highlight ui-corner-all" style="padding: 0.7em;">
+        <p><span class="ui-icon ui-icon-info" style="float: left; margin-right: .3em;"></span>{{!info}}</p>
+    </div>
+</div>
+<br>
+% end
 % include('template/login-plain.tpl')
 <h1>Features</h1>
 <p>

template/settings.tpl

@@ -87,10 +87,19 @@
 <div>
     <h2>Edit your city page</h2>
     <p>
-        With your bot, we generated you a page, which you can use for promotion: <a href="/city/{{city}}"
+        With your bot, we generated you a page, which you can use for promotion:
-        target="_blank">Ticketfrei {{city}}</a> You can change what your users will read there, and adjust it to your
+        <a href="/city/{{city}}" target="_blank">Ticketfrei {{city}}</a> You
-        needs. <b>You should definitely adjust the Social Media profile links.</b> This is just the default text we
+        can change what your users will read there, and adjust it to your
-        suggest:
+        needs.
+    </p>
+    <p>
+        <b>You should definitely adjust the Social Media profile links.</b>
+        Also consider adding this link to the text: <a href="/city/mail/{{city}}"
+        target="_blank">Link to the mail subscription page</a>. Your readers
+        can use this to subscribe to mail notifications.
+    </p>
+    <p>
+        So this is the default text we suggest:
     </p>
     <form action="/settings/markdown" method="post">
         <textarea id="markdown" rows="20" cols="70" name="markdown" wrap="physical">{{markdown}}</textarea>
@@ -98,12 +107,30 @@
     </form>
 </div>
 
+<div>
+    <h2>Edit your mail subscription page</h2>
+    <p>
+        There is also a page where users can subscribe to mail notifications:
+        <a href="/city/mail/{{city}}" target="_blank">Ticketfrei {{city}}</a>.
+        You can change what your users will read there, and adjust it to your
+        needs.
+    </p>
+    <p>
+        So this is the default text we suggest:
+    </p>
+    <form action="/settings/mail_md" method="post">
+        <textarea id="mail_md" rows="20" cols="70" name="mail_md" wrap="physical">{{mail_md}}</textarea>
+        <input name='confirm' value='Save' type='submit'/>
+    </form>
+</div>
+
 <div>
     <h2>Edit your trigger patterns</h2>
     <p>
-        These words have to be contained in a report.
+        These words have to be contained in a report. If none of these
-        If none of these expressions is in the report, it will be ignored by the bot.
+        expressions is in the report, it will be ignored by the bot. You can
-        You can use the defaults, or enter some expressions specific to your city and language.
+        use the defaults, or enter some expressions specific to your city and
+        language.
     </p>
     <form action="/settings/goodlist" method="post">
         <!-- find a way to display current good list. js which reads from a cookie? template? -->
@@ -115,9 +142,9 @@
 <div>
     <h2>Edit the blacklist</h2>
     <p>
-        These words are not allowed in reports.
+        These words are not allowed in reports. If you encounter spam, you can
-        If you encounter spam, you can add more here - the bot will ignore reports which use such words.
+        add more here - the bot will ignore reports which use such words.
-        <!-- There are words which you can't exclude from the blacklist, e.g. certain racist, sexist, or antisemitic slurs. (to be implemented) -->
+        <!-- There are words which you can't exclude from the blacklist, e.g. certain racist, sexist, or antisemitic slurs.
     </p>
     <form action="/settings/blacklist" method="post">
         <!-- find a way to display current blacklist. js which reads from a cookie? template? -->

user.py

@@ -57,11 +57,11 @@ class User(object):
             }, db.secret).decode('ascii')
 
     def is_appropriate(self, report):
-        db.execute("SELECT pattern FROM triggerpatterns WHERE user_id=?;",
+        db.execute("SELECT patterns FROM triggerpatterns WHERE user_id=?;",
                    (self.uid, ))
-        patterns = db.cur.fetchone()
+        patterns = db.cur.fetchone()[0]
         for pattern in patterns.splitlines():
-            if pattern.search(report.text) is not None:
+            if pattern in report.text:
                 break
         else:
             # no pattern matched
@@ -81,7 +81,7 @@ nigger
 neger
 schlitz           
         """
-        db.execute("SELECT word FROM badwords WHERE user_id=?;",
+        db.execute("SELECT words FROM badwords WHERE user_id=?;",
                    (self.uid, ))
         badwords = db.cur.fetchone()
         for word in report.text.lower().splitlines():
@@ -140,11 +140,11 @@ schlitz
         db.commit()
 
     def get_mailinglist(self):
-        db.execute("SELECT email FROM mailinglist WHERE user_id = ? AND active = 1;", (self.uid, ))
+        db.execute("SELECT email FROM mailinglist WHERE user_id = ?;", (self.uid, ))
-        return db.cur.fetchone()[0]
+        return db.cur.fetchall()
 
     def get_seen_mail(self):
-        db.execute("SELECT mail_date FROM seen_mails WHERE user_id = ?;", (self.uid, ))
+        db.execute("SELECT mail_date FROM seen_mail WHERE user_id = ?;", (self.uid, ))
         return db.cur.fetchone()[0]
 
     def save_seen_mail(self, mail_date):
@@ -163,7 +163,11 @@ schlitz
         return db.cur.fetchone()[0]
 
     def add_subscriber(self, email):
-        db.execute("INSERT INTO mailinglist(user_id, email, active) VALUES(?, ?, ?);", (self.uid, email, 1))
+        db.execute("INSERT INTO mailinglist(user_id, email) VALUES(?, ?);", (self.uid, email))
+        db.commit()
+
+    def remove_subscriber(self, email):
+        db.execute("DELETE FROM mailinglist WHERE email = ? AND user_id = ?;", (email, self.uid))
         db.commit()
 
     def set_badwords(self, words):
@@ -180,6 +184,7 @@ schlitz
         # necessary:
         # - city
         # - markdown
+        # - mail_md
         # - goodlist
         # - blacklist
         # - logged in with twitter?
@@ -188,6 +193,7 @@ schlitz
         citydict = db.user_facing_properties(self.get_city())
         return dict(city=citydict['city'],
                     markdown=citydict['markdown'],
+                    mail_md=citydict['mail_md'],
                     triggerwords=self.get_trigger_words(),
                     badwords=self.get_badwords(),
                     enabled=self.enabled)
@@ -247,6 +253,11 @@ schlitz
                    (markdown, self.uid))
         db.commit()
 
+    def set_mail_md(self, mail_md):
+        db.execute("UPDATE cities SET mail_md = ? WHERE user_id = ?;",
+                   (mail_md, self.uid))
+        db.commit()
+
     def get_city(self):
         db.execute("SELECT city FROM cities WHERE user_id == ?;", (self.uid, ))
         return db.cur.fetchone()[0]
@@ -278,6 +289,11 @@ also pass trotzdem auf, wer auf dem Bahnsteig steht.
 Aber je mehr Leute mitmachen, desto eher kannst du dir sicher 
 sein, dass wir sie finden, bevor sie uns finden.
 
+Wenn du immer direkt gewarnt werden willst, kannst du auch die
+E-Mail-Benachrichtigungen aktivieren. Gib einfach 
+<a href="/city/mail/""" + city + """"/">hier</a> deine 
+E-Mail-Adresse an.
+
 Also, wenn du weniger Glück hast, und der erste bist, der einen 
 Kontrolleur sieht:
 
@@ -342,7 +358,25 @@ sicher vor Zensur.
 Um Mastodon zu benutzen, besucht diese Seite: 
 [https://joinmastodon.org/](https://joinmastodon.org/)
         """
-        db.execute("""INSERT INTO cities(user_id, city, markdown, masto_link, 
+        mail_md = """# Immer up-to-date
-                        twit_link) VALUES(?,?,?,?,?)""",
+
-                   (self.uid, city, markdown, masto_link, twit_link))
+Du bist viel unterwegs und hast keine Lust, jedes Mal auf das Profil des Bots
+zu schauen? Kein Problem. Unsere Mail Notifications benachrichtigen dich, wenn
+irgendwo Kontis gesehen werden.
+
+Wenn du uns deine E-Mail-Adresse gibst, kriegst du bei jedem Konti-Report eine
+Mail. Wenn du eine Mail-App auf dem Handy hast, so wie 
+[K9Mail](https://k9mail.github.io/), kriegst du sogar eine Push Notification. So
+bist du immer Up-to-date über alles, was im Verkehrsnetz passiert.
+
+## Keine Sorge
+
+Wir benutzen deine E-Mail-Adresse selbstverständlich für nichts anderes. Du 
+kannst die Benachrichtigungen jederzeit deaktivieren, mit jeder Mail wird ein
+unsubscribe-link mitgeschickt. 
+        """
+        db.execute("""INSERT INTO cities(user_id, city, markdown, mail_md,
+                        masto_link, twit_link) VALUES(?,?,?,?,?,?)""",
+                   (self.uid, city, markdown, mail_md, masto_link, twit_link))
         db.commit()
+