diff --git a/LICENSE b/LICENSE index 723632a..6b24afd 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,6 @@ Copyright (c) 2017 Thomas L Copyright (c) 2017 b3yond +Copyright (c) 2018 sid Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above diff --git a/README.md b/README.md index 46f5110..da364b7 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ virtualenv -p python3 . Install the dependencies: ```shell -pip install tweepy pytoml Mastodon.py bottle pyjwt pylibscrypt Markdown +pip install tweepy pytoml Mastodon.py bottle pyjwt pylibscrypt Markdown twx ``` Configure the bot: @@ -179,7 +179,7 @@ virtualenv -p python3 . Install the dependencies: ```shell -pip install tweepy pytoml Mastodon.py bottle pyjwt pylibscrypt Markdown +pip install tweepy pytoml Mastodon.py bottle pyjwt pylibscrypt Markdown twx ``` Configure the bot: diff --git a/active_bots/mastodonbot.py b/active_bots/mastodonbot.py index a7f735b..5127eab 100755 --- a/active_bots/mastodonbot.py +++ b/active_bots/mastodonbot.py @@ -29,6 +29,8 @@ class MastodonBot(Bot): logger.error("Unknown Mastodon API Error.", exc_info=True) return mentions for status in notifications: + if user.get_seen_toot() == None: + user.init_seen_toot(m.instance()['uri']) if (status['type'] == 'mention' and status['status']['id'] > user.get_seen_toot()): # save state diff --git a/active_bots/telegrambot.py b/active_bots/telegrambot.py new file mode 100644 index 0000000..e916943 --- /dev/null +++ b/active_bots/telegrambot.py @@ -0,0 +1,50 @@ +from bot import Bot +import logging +from report import Report +from twx.botapi import TelegramBot as Telegram + + +logger = logging.getLogger(__name__) + + +class TelegramBot(Bot): + def crawl(self, user): + tb = Telegram(user.get_telegram_credentials()) + seen_tg = user.get_seen_tg() + try: + updates = tb.get_updates(offset=seen_tg + 1, + allowed_updates="message").wait() + except TypeError: + updates = tb.get_updates().wait() + reports = [] + for update in updates: + if update == 404: + return reports + user.save_seen_tg(update.update_id) + if update.message.text.lower() == "/start": + user.add_telegram_subscribers(update.message.sender.id) + tb.send_message(update.message.sender.id, "You are now subscribed to report notifications.") + # TODO: /start message should be set in frontend + elif update.message.text.lower() == "/stop": + user.remove_telegram_subscribers(update.message.sender.id) + tb.send_message(update.message.sender.id, "You are now unsubscribed from report notifications.") + # TODO: /stop message should be set in frontend + elif update.message.text.lower() == "/help": + tb.send_message(update.message.sender.id, "Send reports here to share them with other users. Use /start and /stop to get reports or not.") + # TODO: /help message should be set in frontend + else: + reports.append(Report(update.message.sender.username, self, + update.message.text, None, update.message.date)) + return reports + + def post(self, user, report): + tb = Telegram(user.get_telegram_credentials()) + text = report.text + if len(text) > 4096: + text = text[:4096 - 4] + u' ...' + try: + for subscriber_id in user.get_telegram_subscribers(): + tb.send_message(subscriber_id, text).wait() + except Exception: + logger.error('Error telegramming: ' + user.get_city() + ': ' + + str(report.id), exc_info=True) diff --git a/active_bots/twitterbot.py b/active_bots/twitterbot.py index 37b7029..bf127a2 100755 --- a/active_bots/twitterbot.py +++ b/active_bots/twitterbot.py @@ -4,6 +4,7 @@ import logging import tweepy import re import requests +from time import time import report from bot import Bot @@ -28,6 +29,11 @@ class TwitterBot(Bot): :return: reports: (list of report.Report objects) """ reports = [] + try: + if user.get_last_twitter_request() + 60 > time(): + return reports + except TypeError: + user.set_last_twitter_request(time()) try: api = self.get_api(user) except Exception: @@ -39,6 +45,7 @@ class TwitterBot(Bot): mentions = api.mentions_timeline() else: mentions = api.mentions_timeline(since_id=last_mention) + user.set_last_twitter_request(time()) for status in mentions: text = re.sub( "(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)", diff --git a/backend.py b/backend.py index 77189f3..47997a2 100755 --- a/backend.py +++ b/backend.py @@ -37,7 +37,6 @@ if __name__ == '__main__': continue for bot2 in bots: bot2.post(user, status) - time.sleep(60) # twitter rate limit >.< except Exception: logger.error("Shutdown.", exc_info=True) shutdown() diff --git a/db.py b/db.py index c379a0f..7eb938b 100644 --- a/db.py +++ b/db.py @@ -70,11 +70,17 @@ class DB(object): id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, user_id INTEGER, mastodon_accounts_id INTEGER, - toot_id TEXT, + toot_id INTEGER, FOREIGN KEY(user_id) REFERENCES user(id), FOREIGN KEY(mastodon_accounts_id) REFERENCES mastodon_accounts(id) ); + CREATE TABLE IF NOT EXISTS seen_telegrams ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, + user_id INTEGER, + tg_id INTEGER, + FOREIGN KEY(user_id) REFERENCES user(id) + ); CREATE TABLE IF NOT EXISTS twitter_request_tokens ( id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, user_id INTEGER, @@ -90,6 +96,12 @@ class DB(object): active INTEGER, FOREIGN KEY(user_id) REFERENCES user(id) ); + CREATE TABLE IF NOT EXISTS twitter_last_request ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, + user_id INTEGER, + date INTEGER, + FOREIGN KEY(user_id) REFERENCES user(id) + ); CREATE TABLE IF NOT EXISTS telegram_accounts ( id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, user_id INTEGER, @@ -115,6 +127,20 @@ class DB(object): FOREIGN KEY(twitter_accounts_id) REFERENCES twitter_accounts(id) ); + CREATE TABLE IF NOT EXISTS telegram_accounts ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, + user_id INTEGER, + api_token TEXT, + active INTEGER, + FOREIGN KEY(user_id) REFERENCES user(id) + ); + CREATE TABLE IF NOT EXISTS telegram_subscribers ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, + user_id INTEGER, + subscriber_id INTEGER, + FOREIGN KEY(user_id) REFERENCES user(id), + UNIQUE(user_id, subscriber_id) ON CONFLICT IGNORE + ); CREATE TABLE IF NOT EXISTS mailinglist ( id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, user_id INTEGER, @@ -209,8 +235,7 @@ class DB(object): self.execute("INSERT INTO user (passhash) VALUES(?);", (json['passhash'], )) uid = self.cur.lastrowid - default_triggerpatterns = """ -kontroll?e + default_triggerpatterns = """kontroll?e konti db vgn @@ -226,8 +251,7 @@ linie nuernberg nürnberg s\d -u\d\d? - """ +u\d\d?""" self.execute("""INSERT INTO triggerpatterns (user_id, patterns) VALUES(?, ?); """, (uid, default_triggerpatterns)) self.execute("INSERT INTO badwords (user_id, words) VALUES(?, ?);", @@ -238,10 +262,12 @@ u\d\d? (uid, json['email'])) self.execute("""INSERT INTO telegram_accounts (user_id, apikey, active) VALUES(?, ?, ?);""", (uid, "", 1)) + self.execute("INSERT INTO seen_telegrams (user_id, tg_id) VALUES (?,?);", + (uid, 0)) + self.execute("INSERT INTO seen_mail (user_id, mail_date) VALUES (?,?);", + (uid, 0)) self.commit() user = User(uid) - self.execute("INSERT INTO seen_mail (user_id, mail_date) VALUES (?,?)", - (uid, 0)) user.set_city(city) return user diff --git a/frontend.py b/frontend.py index 82eebf8..7b71ee6 100755 --- a/frontend.py +++ b/frontend.py @@ -39,13 +39,13 @@ def register_post(): return dict(error='Email address already in use.') # send confirmation mail try: - print(url('confirm/' + city + '/%s' % db.user_token(email, password))) # only for local testing + link = url('confirm/' + city + '/%s' % db.user_token(email, password)) + print(link) # only for local testing + logger.error('confirmation link to ' + email + ": " + link) sendmail( email, "Confirm your account", - "Complete your registration here: %s" % ( - url('confirm/' + city + '/%s' % db.user_token(email, password)) - ) + "Complete your registration here: %s" % (link) ) return dict(info='Confirmation mail sent.') except Exception: @@ -160,11 +160,10 @@ def update_badwords(user): @post('/settings/telegram') -@view('template/settings.tpl') def register_telegram(user): apikey = request.forms['apikey'] - user.set_telegram_key(apikey) - return user.state() + user.update_telegram_key(apikey) + return city_page(user.get_city(), info="Thanks for registering Telegram!") @get('/api/state') @@ -237,18 +236,19 @@ def login_mastodon(user): # get app tokens instance_url = request.forms.get('instance_url') masto_email = request.forms.get('email') - print(masto_email) masto_pass = request.forms.get('pass') - print(masto_pass) client_id, client_secret = user.get_mastodon_app_keys(instance_url) m = Mastodon(client_id=client_id, client_secret=client_secret, api_base_url=instance_url) try: access_token = m.log_in(masto_email, masto_pass) user.save_masto_token(access_token, instance_url) - return dict( - info='Thanks for supporting decentralized social networks!' - ) + + # Trying to set the seen_toot to 0, thereby initializing it. + # It should work now, but has default values. Not sure if I need them. + user.init_seen_toot(instance_url) + + return city_page(user.get_city(), info='Thanks for supporting decentralized social networks!') except Exception: logger.error('Login to Mastodon failed.', exc_info=True) return dict(error='Login to Mastodon failed.') diff --git a/user.py b/user.py index c5be91c..2e1a550 100644 --- a/user.py +++ b/user.py @@ -13,7 +13,7 @@ class User(object): self.uid = uid def check_password(self, password): - db.execute("SELECT passhash FROM user WHERE id=?;", (self.uid, )) + db.execute("SELECT passhash FROM user WHERE id=?;", (self.uid,)) passhash, = db.cur.fetchone() return scrypt_mcf_check(passhash.encode('ascii'), password.encode('utf-8')) @@ -23,6 +23,7 @@ class User(object): db.execute("UPDATE user SET passhash=? WHERE id=?;", (passhash, self.uid)) db.commit() + password = property(None, password) # setter only, can't read back @property @@ -38,11 +39,11 @@ class User(object): @property def emails(self): - db.execute("SELECT email FROM email WHERE user_id=?;", (self.uid, )) - return (*db.cur.fetchall(), ) + db.execute("SELECT email FROM email WHERE user_id=?;", (self.uid,)) + return (*db.cur.fetchall(),) def delete_email(self, email): - db.execute("SELECT COUNT(*) FROM email WHERE user_id=?", (self.uid, )) + db.execute("SELECT COUNT(*) FROM email WHERE user_id=?", (self.uid,)) if db.cur.fetchone()[0] == 1: return False # don't allow to delete last email db.execute("DELETE FROM email WHERE user_id=? AND email=?;", @@ -52,9 +53,9 @@ class User(object): def email_token(self, email): return jwt.encode({ - 'email': email, - 'uid': self.uid - }, db.secret).decode('ascii') + 'email': email, + 'uid': self.uid + }, db.secret).decode('ascii') def is_appropriate(self, report): db.execute("SELECT patterns FROM triggerpatterns WHERE user_id=?;", @@ -92,12 +93,46 @@ schlitz return False return True - def get_masto_credentials(self): - db.execute("SELECT access_token, instance_id FROM mastodon_accounts WHERE user_id = ? AND active = 1;", - (self.uid, )) + def get_telegram_credentials(self): + db.execute("""SELECT apikey + FROM telegram_accounts + WHERE user_id = ? AND active = 1;""", + (self.uid,)) row = db.cur.fetchone() - db.execute("SELECT instance, client_id, client_secret FROM mastodon_instances WHERE id = ?;", - (row[1], )) + return row[0] + + def get_telegram_subscribers(self): + db.execute("""SELECT subscriber_id + FROM telegram_subscribers + WHERE user_id = ?;""", + (self.uid,)) + rows = db.cur.fetchall() + return rows + + def add_telegram_subscribers(self, subscriber_id): + db.execute("""INSERT INTO telegram_subscribers ( + user_id, subscriber_id) VALUES(?, ?);""", + (self.uid, subscriber_id)) + db.commit() + + def remove_telegram_subscribers(self, subscriber_id): + db.execute("""DELETE + FROM telegram_subscribers + WHERE user_id = ? + AND subscriber_id = ?;""", + (self.uid, subscriber_id)) + db.commit() + + def get_masto_credentials(self): + db.execute("""SELECT access_token, instance_id + FROM mastodon_accounts + WHERE user_id = ? AND active = 1;""", + (self.uid,)) + row = db.cur.fetchone() + db.execute("""SELECT instance, client_id, client_secret + FROM mastodon_instances + WHERE id = ?;""", + (row[1],)) instance = db.cur.fetchone() return instance[1], instance[2], row[0], instance[0] @@ -109,10 +144,32 @@ schlitz keys.append(row[1]) return keys + def get_last_twitter_request(self): + db.execute("SELECT date FROM twitter_last_request WHERE user_id = ?;", + (self.uid,)) + return db.cur.fetchone()[0] + + def set_last_twitter_request(self, date): + db.execute("UPDATE twitter_last_request SET date = ? WHERE user_id = ?;", + (date, self.uid)) + db.commit() + + def init_seen_toot(self, instance_url): + db.execute("SELECT id FROM mastodon_instances WHERE instance = ?;", + (instance_url,)) + masto_instance = db.cur.fetchone()[0] + db.execute("INSERT INTO seen_toots (user_id, mastodon_accounts_id, toot_id) VALUES (?,?,?);", + (self.uid, masto_instance, 0)) + db.conn.commit() + return + def get_seen_toot(self): db.execute("SELECT toot_id FROM seen_toots WHERE user_id = ?;", - (self.uid, )) - return db.cur.fetchone()[0] + (self.uid,)) + try: + return db.cur.fetchone()[0] + except TypeError: + return None def save_seen_toot(self, toot_id): db.execute("UPDATE seen_toots SET toot_id = ? WHERE user_id = ?;", @@ -121,7 +178,7 @@ schlitz def get_seen_tweet(self): db.execute("SELECT tweet_id FROM seen_tweets WHERE user_id = ?;", - (self.uid, )) + (self.uid,)) return db.cur.fetchone()[0] def save_seen_tweet(self, tweet_id): @@ -131,7 +188,7 @@ schlitz def get_seen_dm(self): db.execute("SELECT message_id FROM seen_dms WHERE user_id = ?;", - (self.uid, )) + (self.uid,)) return db.cur.fetchone() def save_seen_dm(self, tweet_id): @@ -139,6 +196,16 @@ schlitz (tweet_id, self.uid)) db.commit() + def get_seen_tg(self): + db.execute("SELECT tg_id FROM seen_telegrams WHERE user_id = ?;", + (self.uid,)) + return db.cur.fetchone()[0] + + def save_seen_tg(self, tg_id): + db.execute("UPDATE seen_telegrams SET tg_id = ? WHERE user_id = ?;", + (tg_id, self.uid)) + db.commit() + def get_mailinglist(self): db.execute("SELECT email FROM mailinglist WHERE user_id = ?;", (self.uid, )) return db.cur.fetchall() @@ -199,22 +266,30 @@ schlitz enabled=self.enabled) def save_request_token(self, token): - db.execute("INSERT INTO twitter_request_tokens(user_id, request_token, request_token_secret) VALUES(?, ?, ?);", - (self.uid, token["oauth_token"], token["oauth_token_secret"])) + db.execute("""INSERT INTO + twitter_request_tokens( + user_id, request_token, request_token_secret + ) VALUES(?, ?, ?);""", + (self.uid, token["oauth_token"], + token["oauth_token_secret"])) db.commit() def get_request_token(self): - db.execute("SELECT request_token, request_token_secret FROM twitter_request_tokens WHERE user_id = ?;", (self.uid,)) + db.execute("""SELECT request_token, request_token_secret + FROM twitter_request_tokens + WHERE user_id = ?;""", (self.uid,)) request_token = db.cur.fetchone() - db.execute("DELETE FROM twitter_request_tokens WHERE user_id = ?;", (self.uid,)) + db.execute("""DELETE FROM twitter_request_tokens + WHERE user_id = ?;""", (self.uid,)) db.commit() - return {"oauth_token" : request_token[0], - "oauth_token_secret" : request_token[1]} + return {"oauth_token": request_token[0], + "oauth_token_secret": request_token[1]} def save_twitter_token(self, access_token, access_token_secret): - db.execute( - "INSERT INTO twitter_accounts(user_id, client_id, client_secret) VALUES(?, ?, ?);", - (self.uid, access_token, access_token_secret)) + db.execute(""""INSERT INTO twitter_accounts( + user_id, client_id, client_secret + ) VALUES(?, ?, ?);""", + (self.uid, access_token, access_token_secret)) db.commit() def get_twitter_token(self): @@ -222,12 +297,14 @@ schlitz (self.uid, )) return db.cur.fetchall() - def set_telegram_key(self, apikey): + def update_telegram_key(self, apikey): db.execute("UPDATE telegram_accounts SET apikey = ? WHERE user_id = ?;", (apikey, self.uid)) db.commit() def get_mastodon_app_keys(self, instance): - db.execute("SELECT client_id, client_secret FROM mastodon_instances WHERE instance = ?;", (instance, )) + db.execute("""SELECT client_id, client_secret + FROM mastodon_instances + WHERE instance = ?;""", (instance,)) try: row = db.cur.fetchone() client_id = row[0] @@ -235,14 +312,19 @@ schlitz return client_id, client_secret except TypeError: app_name = "ticketfrei" + str(db.secret)[0:4] - client_id, client_secret = Mastodon.create_app(app_name, api_base_url=instance) - db.execute("INSERT INTO mastodon_instances(instance, client_id, client_secret) VALUES(?, ?, ?);", + client_id, client_secret \ + = Mastodon.create_app(app_name, api_base_url=instance) + db.execute("""INSERT INTO mastodon_instances( + instance, client_id, client_secret + ) VALUES(?, ?, ?);""", (instance, client_id, client_secret)) db.commit() return client_id, client_secret def save_masto_token(self, access_token, instance): - db.execute("SELECT id FROM mastodon_instances WHERE instance = ?;", (instance, )) + db.execute("""SELECT id + FROM mastodon_instances + WHERE instance = ?;""", (instance,)) instance_id = db.cur.fetchone()[0] db.execute("INSERT INTO mastodon_accounts(user_id, access_token, instance_id, active) " "VALUES(?, ?, ?, ?);", (self.uid, access_token, instance_id, 1))