Merge pull request #20 from git-sid/multi-deployment

added basic telegram backend support & smaller changes

LICENSE

@@ -1,5 +1,6 @@
 Copyright (c) 2017 Thomas L <tom@dl6tom.de>
 Copyright (c) 2017 b3yond <b3yond@riseup.net>
+Copyright (c) 2018 sid
 
 Permission to use, copy, modify, and distribute this software for any
 purpose with or without fee is hereby granted, provided that the above

README.md

@@ -91,7 +91,7 @@ virtualenv -p python3 .
 Install the dependencies:
 
 ```shell
-pip install tweepy pytoml Mastodon.py bottle pyjwt pylibscrypt Markdown
+pip install tweepy pytoml Mastodon.py bottle pyjwt pylibscrypt Markdown twx
 ```
 
 Configure the bot:
@@ -179,7 +179,7 @@ virtualenv -p python3 .
 Install the dependencies:
 
 ```shell
-pip install tweepy pytoml Mastodon.py bottle pyjwt pylibscrypt Markdown
+pip install tweepy pytoml Mastodon.py bottle pyjwt pylibscrypt Markdown twx
 ```
 
 Configure the bot:

active_bots/mastodonbot.py

@@ -29,6 +29,8 @@ class MastodonBot(Bot):
             logger.error("Unknown Mastodon API Error.", exc_info=True)
             return mentions
         for status in notifications:
+            if user.get_seen_toot() == None:
+                user.init_seen_toot(m.instance()['uri'])
             if (status['type'] == 'mention' and
                     status['status']['id'] > user.get_seen_toot()):
                 # save state

active_bots/telegrambot.py

@@ -0,0 +1,50 @@
+from bot import Bot
+import logging
+from report import Report
+from twx.botapi import TelegramBot as Telegram
+
+
+logger = logging.getLogger(__name__)
+
+
+class TelegramBot(Bot):
+    def crawl(self, user):
+        tb = Telegram(user.get_telegram_credentials())
+        seen_tg = user.get_seen_tg()
+        try:
+            updates = tb.get_updates(offset=seen_tg + 1,
+                                     allowed_updates="message").wait()
+        except TypeError:
+            updates = tb.get_updates().wait()
+        reports = []
+        for update in updates:
+            if update == 404:
+                return reports
+            user.save_seen_tg(update.update_id)
+            if update.message.text.lower() == "/start":
+                user.add_telegram_subscribers(update.message.sender.id)
+                tb.send_message(update.message.sender.id, "You are now subscribed to report notifications.")
+                # TODO: /start message should be set in frontend
+            elif update.message.text.lower() == "/stop":
+                user.remove_telegram_subscribers(update.message.sender.id)
+                tb.send_message(update.message.sender.id, "You are now unsubscribed from report notifications.")
+                # TODO: /stop message should be set in frontend
+            elif update.message.text.lower() == "/help":
+                tb.send_message(update.message.sender.id, "Send reports here to share them with other users. Use /start and /stop to get reports or not.")
+                # TODO: /help message should be set in frontend
+            else:
+                reports.append(Report(update.message.sender.username, self,
+                                      update.message.text, None, update.message.date))
+        return reports
+
+    def post(self, user, report):
+        tb = Telegram(user.get_telegram_credentials())
+        text = report.text
+        if len(text) > 4096:
+            text = text[:4096 - 4] + u' ...'
+        try:
+            for subscriber_id in user.get_telegram_subscribers():
+                tb.send_message(subscriber_id, text).wait()
+        except Exception:
+            logger.error('Error telegramming: ' + user.get_city() + ': '
+                         + str(report.id), exc_info=True)

active_bots/twitterbot.py

@@ -4,6 +4,7 @@ import logging
 import tweepy
 import re
 import requests
+from time import time
 import report
 from bot import Bot
 
@@ -28,6 +29,11 @@ class TwitterBot(Bot):
         :return: reports: (list of report.Report objects)
         """
         reports = []
+        try:
+            if user.get_last_twitter_request() + 60 > time():
+                return reports
+        except TypeError:
+            user.set_last_twitter_request(time())
         try:
             api = self.get_api(user)
         except Exception:
@@ -39,6 +45,7 @@ class TwitterBot(Bot):
                 mentions = api.mentions_timeline()
             else:
                 mentions = api.mentions_timeline(since_id=last_mention)
+            user.set_last_twitter_request(time())
             for status in mentions:
                 text = re.sub(
                         "(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)",

backend.py

@@ -37,7 +37,6 @@ if __name__ == '__main__':
                             continue
                         for bot2 in bots:
                             bot2.post(user, status)
-            time.sleep(60)  # twitter rate limit >.<
     except Exception:
         logger.error("Shutdown.", exc_info=True)
         shutdown()

db.py

@@ -70,11 +70,17 @@ class DB(object):
                 id         INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
                 user_id            INTEGER,
                 mastodon_accounts_id    INTEGER,
-                toot_id            TEXT,
+                toot_id            INTEGER,
                 FOREIGN KEY(user_id) REFERENCES user(id),
                 FOREIGN KEY(mastodon_accounts_id)
                     REFERENCES mastodon_accounts(id)
             );
+            CREATE TABLE IF NOT EXISTS seen_telegrams (
+                id                 INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
+                user_id            INTEGER,
+                tg_id              INTEGER,
+                FOREIGN KEY(user_id) REFERENCES user(id)
+            );
             CREATE TABLE IF NOT EXISTS twitter_request_tokens (
                 id          INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
                 user_id     INTEGER,
@@ -90,6 +96,12 @@ class DB(object):
                 active      INTEGER,
                 FOREIGN KEY(user_id) REFERENCES user(id)
             );
+            CREATE TABLE IF NOT EXISTS twitter_last_request (
+                id          INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
+                user_id     INTEGER,
+                date        INTEGER,
+                FOREIGN KEY(user_id) REFERENCES user(id)
+            );
             CREATE TABLE IF NOT EXISTS telegram_accounts (
                 id          INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
                 user_id     INTEGER,
@@ -115,6 +127,20 @@ class DB(object):
                 FOREIGN KEY(twitter_accounts_id)
                     REFERENCES twitter_accounts(id)
             );
+            CREATE TABLE IF NOT EXISTS telegram_accounts (
+                id          INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
+                user_id     INTEGER,
+                api_token   TEXT,
+                active      INTEGER,
+                FOREIGN KEY(user_id) REFERENCES user(id)
+            );
+            CREATE TABLE IF NOT EXISTS telegram_subscribers (
+                id          INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
+                user_id     INTEGER,
+                subscriber_id   INTEGER,
+                FOREIGN KEY(user_id) REFERENCES user(id),
+                UNIQUE(user_id, subscriber_id) ON CONFLICT IGNORE
+            );
             CREATE TABLE IF NOT EXISTS mailinglist (
                 id          INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
                 user_id     INTEGER,
@@ -209,8 +235,7 @@ class DB(object):
             self.execute("INSERT INTO user (passhash) VALUES(?);",
                          (json['passhash'], ))
             uid = self.cur.lastrowid
-            default_triggerpatterns = """
+            default_triggerpatterns = """kontroll?e
-kontroll?e
 konti
 db
 vgn
@@ -226,8 +251,7 @@ linie
 nuernberg
 nürnberg
 s\d
-u\d\d?            
+u\d\d?"""
-            """
             self.execute("""INSERT INTO triggerpatterns (user_id, patterns)
                                 VALUES(?, ?); """, (uid, default_triggerpatterns))
             self.execute("INSERT INTO badwords (user_id, words) VALUES(?, ?);",
@@ -238,10 +262,12 @@ u\d\d?
                      (uid, json['email']))
         self.execute("""INSERT INTO telegram_accounts (user_id, apikey,
                         active) VALUES(?, ?, ?);""", (uid, "", 1))
+        self.execute("INSERT INTO seen_telegrams (user_id, tg_id) VALUES (?,?);",
+                     (uid, 0))
+        self.execute("INSERT INTO seen_mail (user_id, mail_date) VALUES (?,?);",
+                     (uid, 0))
         self.commit()
         user = User(uid)
-        self.execute("INSERT INTO seen_mail (user_id, mail_date) VALUES (?,?)",
-                     (uid, 0))
         user.set_city(city)
         return user
 

frontend.py

@@ -39,13 +39,13 @@ def register_post():
         return dict(error='Email address already in use.')
     # send confirmation mail
     try:
-        print(url('confirm/' + city + '/%s' % db.user_token(email, password)))  # only for local testing
+        link = url('confirm/' + city + '/%s' % db.user_token(email, password))
+        print(link)  # only for local testing
+        logger.error('confirmation link to ' + email + ": " + link)
         sendmail(
                 email,
                 "Confirm your account",
-                "Complete your registration here: %s" % (
+                "Complete your registration here: %s" % (link)
-                        url('confirm/' + city + '/%s' % db.user_token(email, password))
-                    )
             )
         return dict(info='Confirmation mail sent.')
     except Exception:
@@ -160,11 +160,10 @@ def update_badwords(user):
 
 
 @post('/settings/telegram')
-@view('template/settings.tpl')
 def register_telegram(user):
     apikey = request.forms['apikey']
-    user.set_telegram_key(apikey)
+    user.update_telegram_key(apikey)
-    return user.state()
+    return city_page(user.get_city(), info="Thanks for registering Telegram!")
 
 
 @get('/api/state')
@@ -237,18 +236,19 @@ def login_mastodon(user):
     # get app tokens
     instance_url = request.forms.get('instance_url')
     masto_email = request.forms.get('email')
-    print(masto_email)
     masto_pass = request.forms.get('pass')
-    print(masto_pass)
     client_id, client_secret = user.get_mastodon_app_keys(instance_url)
     m = Mastodon(client_id=client_id, client_secret=client_secret,
                  api_base_url=instance_url)
     try:
         access_token = m.log_in(masto_email, masto_pass)
         user.save_masto_token(access_token, instance_url)
-        return dict(
+
-                info='Thanks for supporting decentralized social networks!'
+        # Trying to set the seen_toot to 0, thereby initializing it.
-            )
+        # It should work now, but has default values. Not sure if I need them.
+        user.init_seen_toot(instance_url)
+
+        return city_page(user.get_city(), info='Thanks for supporting decentralized social networks!')
     except Exception:
         logger.error('Login to Mastodon failed.', exc_info=True)
         return dict(error='Login to Mastodon failed.')

user.py

@@ -13,7 +13,7 @@ class User(object):
         self.uid = uid
 
     def check_password(self, password):
-        db.execute("SELECT passhash FROM user WHERE id=?;", (self.uid, ))
+        db.execute("SELECT passhash FROM user WHERE id=?;", (self.uid,))
         passhash, = db.cur.fetchone()
         return scrypt_mcf_check(passhash.encode('ascii'),
                                 password.encode('utf-8'))
@@ -23,6 +23,7 @@ class User(object):
         db.execute("UPDATE user SET passhash=? WHERE id=?;",
                    (passhash, self.uid))
         db.commit()
+
     password = property(None, password)  # setter only, can't read back
 
     @property
@@ -38,11 +39,11 @@ class User(object):
 
     @property
     def emails(self):
-        db.execute("SELECT email FROM email WHERE user_id=?;", (self.uid, ))
+        db.execute("SELECT email FROM email WHERE user_id=?;", (self.uid,))
-        return (*db.cur.fetchall(), )
+        return (*db.cur.fetchall(),)
 
     def delete_email(self, email):
-        db.execute("SELECT COUNT(*) FROM email WHERE user_id=?", (self.uid, ))
+        db.execute("SELECT COUNT(*) FROM email WHERE user_id=?", (self.uid,))
         if db.cur.fetchone()[0] == 1:
             return False  # don't allow to delete last email
         db.execute("DELETE FROM email WHERE user_id=? AND email=?;",
@@ -92,12 +93,46 @@ schlitz
                 return False
         return True
 
-    def get_masto_credentials(self):
+    def get_telegram_credentials(self):
-        db.execute("SELECT access_token, instance_id FROM mastodon_accounts WHERE user_id = ? AND active = 1;",
+        db.execute("""SELECT apikey 
-                   (self.uid, ))
+                          FROM telegram_accounts 
+                          WHERE user_id = ? AND active = 1;""",
+                   (self.uid,))
         row = db.cur.fetchone()
-        db.execute("SELECT instance, client_id, client_secret FROM mastodon_instances WHERE id = ?;",
+        return row[0]
-                   (row[1], ))
+
+    def get_telegram_subscribers(self):
+        db.execute("""SELECT subscriber_id 
+                          FROM telegram_subscribers 
+                          WHERE user_id = ?;""",
+                   (self.uid,))
+        rows = db.cur.fetchall()
+        return rows
+
+    def add_telegram_subscribers(self, subscriber_id):
+        db.execute("""INSERT INTO telegram_subscribers (
+                            user_id, subscriber_id) VALUES(?, ?);""",
+                   (self.uid, subscriber_id))
+        db.commit()
+
+    def remove_telegram_subscribers(self, subscriber_id):
+        db.execute("""DELETE 
+                          FROM telegram_subscribers 
+                          WHERE user_id = ?
+                          AND subscriber_id = ?;""",
+                   (self.uid, subscriber_id))
+        db.commit()
+
+    def get_masto_credentials(self):
+        db.execute("""SELECT access_token, instance_id 
+                          FROM mastodon_accounts 
+                          WHERE user_id = ? AND active = 1;""",
+                   (self.uid,))
+        row = db.cur.fetchone()
+        db.execute("""SELECT instance, client_id, client_secret 
+                          FROM mastodon_instances 
+                          WHERE id = ?;""",
+                   (row[1],))
         instance = db.cur.fetchone()
         return instance[1], instance[2], row[0], instance[0]
 
@@ -109,10 +144,32 @@ schlitz
         keys.append(row[1])
         return keys
 
+    def get_last_twitter_request(self):
+        db.execute("SELECT date FROM twitter_last_request WHERE user_id = ?;",
+                   (self.uid,))
+        return db.cur.fetchone()[0]
+
+    def set_last_twitter_request(self, date):
+        db.execute("UPDATE twitter_last_request SET date = ? WHERE user_id = ?;",
+                   (date, self.uid))
+        db.commit()
+
+    def init_seen_toot(self, instance_url):
+        db.execute("SELECT id FROM mastodon_instances WHERE instance = ?;",
+                   (instance_url,))
+        masto_instance = db.cur.fetchone()[0]
+        db.execute("INSERT INTO seen_toots (user_id, mastodon_accounts_id, toot_id) VALUES (?,?,?);",
+            (self.uid, masto_instance, 0))
+        db.conn.commit()
+        return
+
     def get_seen_toot(self):
         db.execute("SELECT toot_id FROM seen_toots WHERE user_id = ?;",
-                   (self.uid, ))
+                   (self.uid,))
+        try:
             return db.cur.fetchone()[0]
+        except TypeError:
+            return None
 
     def save_seen_toot(self, toot_id):
         db.execute("UPDATE seen_toots SET toot_id = ? WHERE user_id = ?;",
@@ -121,7 +178,7 @@ schlitz
 
     def get_seen_tweet(self):
         db.execute("SELECT tweet_id FROM seen_tweets WHERE user_id = ?;",
-                   (self.uid, ))
+                   (self.uid,))
         return db.cur.fetchone()[0]
 
     def save_seen_tweet(self, tweet_id):
@@ -131,7 +188,7 @@ schlitz
 
     def get_seen_dm(self):
         db.execute("SELECT message_id FROM seen_dms WHERE user_id = ?;",
-                   (self.uid, ))
+                   (self.uid,))
         return db.cur.fetchone()
 
     def save_seen_dm(self, tweet_id):
@@ -139,6 +196,16 @@ schlitz
                    (tweet_id, self.uid))
         db.commit()
 
+    def get_seen_tg(self):
+        db.execute("SELECT tg_id FROM seen_telegrams WHERE user_id = ?;",
+                   (self.uid,))
+        return db.cur.fetchone()[0]
+
+    def save_seen_tg(self, tg_id):
+        db.execute("UPDATE seen_telegrams SET tg_id = ? WHERE user_id = ?;",
+                   (tg_id, self.uid))
+        db.commit()
+
     def get_mailinglist(self):
         db.execute("SELECT email FROM mailinglist WHERE user_id = ?;", (self.uid, ))
         return db.cur.fetchall()
@@ -199,21 +266,29 @@ schlitz
                     enabled=self.enabled)
 
     def save_request_token(self, token):
-        db.execute("INSERT INTO twitter_request_tokens(user_id, request_token, request_token_secret) VALUES(?, ?, ?);",
+        db.execute("""INSERT INTO
-                   (self.uid, token["oauth_token"], token["oauth_token_secret"]))
+                          twitter_request_tokens(
+                              user_id, request_token, request_token_secret
+                          ) VALUES(?, ?, ?);""",
+                   (self.uid, token["oauth_token"],
+                    token["oauth_token_secret"]))
         db.commit()
 
     def get_request_token(self):
-        db.execute("SELECT request_token, request_token_secret FROM twitter_request_tokens WHERE user_id = ?;", (self.uid,))
+        db.execute("""SELECT request_token, request_token_secret 
+                          FROM twitter_request_tokens 
+                          WHERE user_id = ?;""", (self.uid,))
         request_token = db.cur.fetchone()
-        db.execute("DELETE FROM twitter_request_tokens WHERE user_id = ?;", (self.uid,))
+        db.execute("""DELETE FROM twitter_request_tokens 
+                          WHERE user_id = ?;""", (self.uid,))
         db.commit()
-        return {"oauth_token" : request_token[0],
+        return {"oauth_token": request_token[0],
-                "oauth_token_secret" : request_token[1]}
+                "oauth_token_secret": request_token[1]}
 
     def save_twitter_token(self, access_token, access_token_secret):
-        db.execute(
+        db.execute(""""INSERT INTO twitter_accounts(
-            "INSERT INTO twitter_accounts(user_id, client_id, client_secret) VALUES(?, ?, ?);",
+                           user_id, client_id, client_secret
+                           ) VALUES(?, ?, ?);""",
                    (self.uid, access_token, access_token_secret))
         db.commit()
 
@@ -222,12 +297,14 @@ schlitz
                    (self.uid, ))
         return db.cur.fetchall()
 
-    def set_telegram_key(self, apikey):
+    def update_telegram_key(self, apikey):
         db.execute("UPDATE telegram_accounts SET apikey = ? WHERE user_id = ?;", (apikey, self.uid))
         db.commit()
 
     def get_mastodon_app_keys(self, instance):
-        db.execute("SELECT client_id, client_secret FROM mastodon_instances WHERE instance = ?;", (instance, ))
+        db.execute("""SELECT client_id, client_secret 
+                          FROM mastodon_instances 
+                          WHERE instance = ?;""", (instance,))
         try:
             row = db.cur.fetchone()
             client_id = row[0]
@@ -235,14 +312,19 @@ schlitz
             return client_id, client_secret
         except TypeError:
             app_name = "ticketfrei" + str(db.secret)[0:4]
-            client_id, client_secret = Mastodon.create_app(app_name, api_base_url=instance)
+            client_id, client_secret \
-            db.execute("INSERT INTO mastodon_instances(instance, client_id, client_secret) VALUES(?, ?, ?);",
+                = Mastodon.create_app(app_name, api_base_url=instance)
+            db.execute("""INSERT INTO mastodon_instances(
+                              instance, client_id, client_secret
+                              ) VALUES(?, ?, ?);""",
                        (instance, client_id, client_secret))
             db.commit()
             return client_id, client_secret
 
     def save_masto_token(self, access_token, instance):
-        db.execute("SELECT id FROM mastodon_instances WHERE instance = ?;", (instance, ))
+        db.execute("""SELECT id 
+                          FROM mastodon_instances 
+                          WHERE instance = ?;""", (instance,))
         instance_id = db.cur.fetchone()[0]
         db.execute("INSERT INTO mastodon_accounts(user_id, access_token, instance_id, active) "
                    "VALUES(?, ?, ?, ?);", (self.uid, access_token, instance_id, 1))