first try on database schema

don't log Mastodon 502 errors.

README.md

@@ -32,13 +32,15 @@ Today, you can use a Twitter, Mastodon, Telegram, and Mail with the account.
 They will communicate with each other; if someone warns others via Mail,
 Telegram, Twitter and Mastodon users will also see the message. And vice versa.
 
-In version 2, this repository contains a web application. On this website,
+In version 3, this repository contains a web application. On this website,
 people can register an own bot for their city - the website manages multiple
-bots for multiple citys. This way, you do not have to host it yourself.
+bots for multiple citys, which run in parallel. This way, you do not have to
+host it yourself, if you lack the know-how. But it is easily possible to do so
+with docker: http://github.com/ticketfrei/docker-ticketfrei/
 
-In the promotion folder, you'll find some promotion material you can use to
-build up such a community in your city. Unfortunately it is in german - but
-it's editable, feel free to translate it!
+In https://github.com/ticketfrei/promotion, you'll find some promotion material
+you can use to build up such a community in your city. Unfortunately it is in
+german - but it's editable, feel free to translate it!
 
 Website (our flagship instance): https://ticketfrei.links-tech.org
 
@@ -108,7 +110,7 @@ virtualenv -p python3 .
 Install the dependencies:
 
 ```shell
-pip install tweepy pytoml Mastodon.py bottle pyjwt pylibscrypt Markdown twx gitpython
+pip install tweepy pytoml Mastodon.py bottle pyjwt pylibscrypt Markdown twx gitpython SQLAlchemy
 ```
 
 Configure the bot:

active_bots/mastodonbot.py

@@ -2,7 +2,7 @@
 
 from bot import Bot
 import logging
-from mastodon import Mastodon
+from mastodon import Mastodon, MastodonServerError
 import re
 from report import Report
 
@@ -25,8 +25,8 @@ class MastodonBot(Bot):
             return mentions
         try:
             notifications = m.notifications()
-        except Exception:
-            logger.error("Unknown Mastodon API Error.", exc_info=True)
+        except MastodonServerError:
+            logger.error("Unknown Mastodon API Error: 502")
             return mentions
         for status in notifications:
             if (status['type'] == 'mention' and

active_bots/telegrambot.py

@@ -32,6 +32,12 @@ class TelegramBot(Bot):
                     logger.error("Unknown Telegram error code: " + str(update))
                 return reports
             user.save_seen_tg(update.update_id)
+            if update.message.photo:
+                tb.send_message(
+                    update.message.sender.id,
+                    "Sending Photos is not supported for privacy reasons. Can "
+                    "you describe it as text instead?")
+                continue
             if update.message.text.lower() == "/start":
                 user.add_telegram_subscribers(update.message.sender.id)
                 tb.send_message(

bots/mail/settings.tpl

@@ -0,0 +1,19 @@
+
+<div>
+    <h2>Edit your mail subscription page</h2>
+    <p>
+        There is also a page where users can subscribe to mail notifications:
+        <a href="/city/mail/{{city}}" target="_blank">Ticketfrei {{city}}</a>.
+        You can change what your users will read there, and adjust it to your
+        needs.
+    </p>
+    <p>
+        So this is the default text we suggest:
+    </p>
+    <form action="/settings/mail_md" method="post">
+        <textarea id="mail_md" rows="20" cols="70" name="mail_md" wrap="physical">{{mail_md}}</textarea>
+        <input name='csrf' value='{{csrf}}' type='hidden' />
+        <input name='confirm' value='Save' type='submit'/>
+    </form>
+</div>
+

bots/mastodon/settings.tpl

@@ -0,0 +1,51 @@
+
+<section>
+    <h2>Log in with Mastodon</h2>
+        <form action="/login/mastodon" method='post'>
+            <label for="email">E-Mail of your Mastodon-Account</label>
+            <input type="text" placeholder="Enter Email" name="email" id="email" required>
+
+            <label for="pass">Mastodon Password</label>
+            <input type="password" placeholder="Enter Password" name="pass" id="pass" required>
+
+            <label>Mastodon instance:
+                <input type='text' name='instance_url' list='instances' placeholder='social.example.net'/>
+            </label>
+            <datalist id='instances'>
+                <option value=''>
+                <option value='anticapitalist.party'>
+                <option value='awoo.space'>
+                <option value='cybre.space'>
+                <option value='mastodon.social'>
+                <option value='glitch.social'>
+                <option value='botsin.space'>
+                <option value='witches.town'>
+                <option value='social.wxcafe.net'>
+                <option value='monsterpit.net'>
+                <option value='mastodon.xyz'>
+                <option value='a.weirder.earth'>
+                <option value='chitter.xyz'>
+                <option value='sins.center'>
+                <option value='dev.glitch.social'>
+                <option value='computerfairi.es'>
+                <option value='niu.moe'>
+                <option value='icosahedron.website'>
+                <option value='hostux.social'>
+                <option value='hyenas.space'>
+                <option value='instance.business'>
+                <option value='mastodon.sdf.org'>
+                <option value='pawoo.net'>
+                <option value='pouet.it'>
+                <option value='scalie.business'>
+                <option value='sleeping.town'>
+                <option value='social.koyu.space'>
+                <option value='sunshinegardens.org'>
+                <option value='vcity.network'>
+                <option value='octodon.social'>
+                <option value='soc.ialis.me'>
+            </datalist>
+            <input name='csrf' value='{{csrf}}' type='hidden' />
+            <input name='confirm' value='Log in' type='submit'/>
+        </form>
+</section>
+

bots/telegram/settings.tpl

@@ -0,0 +1,23 @@
+
+<%
+# todo: hide this part, if there is already a telegram bot connected.
+%>
+<div>
+    <h2>Connect with Telegram</h2>
+    <p>
+        If you have a Telegram account, you can register a bot there. Just
+        write to @botfather. There are detailed instructions on
+        <a href="https://botsfortelegram.com/project/the-bot-father/" target="_blank">
+        Bots for Telegram</a>.
+    </p>
+    <p>
+        The botfather will give you an API key - with the API key, Ticketfrei
+        can use the Telegram bot. Enter it here:
+    </p>
+    <form action="/settings/telegram" method="post">
+        <input type="text" name="apikey" placeholder="Telegram bot API key" id="apikey">
+        <input name='csrf' value='{{csrf}}' type='hidden' />
+        <input name='confirm' value='Login with Telegram' type='submit'/>
+    </form>
+</div>
+

bots/twitter/settings.tpl

@@ -0,0 +1,10 @@
+
+<a class='button' style="padding: 1.5em;" href="/login/twitter">
+    <picture>
+        <source type='image/webp' sizes='20px' srcset="/static-cb/1517673283/twitter-20.webp 20w,/static-cb/1517673283/twitter-40.webp 40w,/static-cb/1517673283/twitter-80.webp 80w,"/>
+        <source type='image/png' sizes='20px' srcset="/static-cb/1517673283/twitter-20.png 20w,/static-cb/1517673283/twitter-40.png 40w,/static-cb/1517673283/twitter-80.png 80w,"/>
+        <img src="https://patriciaannbridewell.files.wordpress.com/2014/04/official-twitter-logo-tile.png" alt="" />
+    </picture>
+    Log in with Twitter
+</a>
+

frontend.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import bottle
+from os import listdir, path
 from bottle import get, post, redirect, request, response, view
 from config import config
 from db import db

sqlalchemy.py

@@ -0,0 +1,160 @@
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.engine import create_engine
+from sqlalchemy.schema import Table, Column, ForeignKey, UniqueConstraint
+from sqlalchemy.types import Integer, String, DateTime, BLOB, REAL
+#from sqlalchemy.orm import relationship, backref
+
+# Get Base class where table objects inherit from
+Base = declarative_base()
+engine = create_engine("sqlite:///:memory:")
+
+
+class User(Base):
+    __tablename__ = 'user'
+    id = Column(Integer, primary_key=True)
+    passhash = Column(String)
+    enabled = Column(Integer, default=1)
+
+    def __repr__(self):
+        return '<User(id=%s, passhash=%s, enabled=%s)>' % (
+            self.id, self.passhash, self.enabled)
+
+
+class Email(Base):
+    __tablename__ = 'email'
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer, ForeignKey('user.id'))
+    email = Column(String)
+
+    # necessary? https://docs.sqlalchemy.org/en/13/orm/tutorial.html#building-a-relationship
+    # user = relationship(User, back_populates='email')
+
+# necessary?
+# User.email = relationship('email', order_by=Email.id, back_populates='user')
+
+class TriggerPatterns(Base):
+    __tablename__ = 'triggerpatterns'
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer, ForeignKey('user.id'))
+    patterns = Column(String)
+
+class BadWords(Base):
+    __tablename__ = 'badwords'
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer, ForeignKey('user.id'))
+    words = Column(String)
+
+class MastodonInstances(Base):
+    __tablename__ = 'mastodon_instances'
+    id = Column(Integer, primary_key=True)
+    instance = Column(String)
+    client_id = Column(String)
+    client_secret = Column(String)
+
+class MastodonAccounts(Base):
+    __tablename__ = 'mastodon_accounts'
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer, ForeignKey('user.id'))
+    access_token = Column(String)
+    instance_id = Column(Integer, ForeignKey('mastodon_instances.id'))
+    active = Column(Integer)  # could be default=1
+
+class SeenToots(Base):
+    __tablename__ = 'seen_toots'
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer, ForeignKey('user.id'))
+    toot_uri = Column(String)
+
+class SeenTelegrams(Base):
+    __tablename__ = 'seen_telegrams'
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer, ForeignKey('user.id'))
+    tg_id = Column(Integer)
+
+class TwitterRequestTokens(Base):
+    __tablename__ = 'twitter_request_tokens'
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer, ForeignKey('user.id'))
+    request_token = Column(String)
+    request_token_secret = Column(String)
+
+class TwitterAccounts(Base):
+    __tablename__ = 'twitter_accounts'
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer, ForeignKey('user.id'))
+    client_id = Column(String)
+    client_secret = Column(String)
+    active = Column(Integer)  # could be default=1
+
+class TelegramAccounts(Base):
+    __tablename__ = 'telegram_accounts'
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer, ForeignKey('user.id'))
+    apikey = Column(String)
+    active = Column(Integer)  # could be default=1
+
+class SeenTweets(Base):
+    __tablename__ = 'seen_tweets'
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer, ForeignKey('user.id'))
+    tweet_id = Column(String)
+
+class SeenDMs(Base):
+    __tablename__ = 'seen_dms'
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer, ForeignKey('user.id'))
+    twitter_accounts = Column(Integer, ForeignKey('twitter_accounts.id'))
+    message_id = Column(String)
+
+class TelegramSubscribers(Base):
+    __tablename__ = 'telegram_subscribers'
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer, ForeignKey('user.id'))
+    subscriber_id = Column(Integer)
+
+    # how to get this to work?
+    # https://docs.sqlalchemy.org/en/13/dialects/sqlite.html#on-conflict-support-for-constraints
+    # UniqueConstraint('id', 'subscriber_id', sqlite_on_conflict='IGNORE')
+
+class Mailinglist(Base):
+    __tablename__ = 'mailinglist'
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer, ForeignKey('user.id'))
+    email = Column(String)
+
+    # It would be good to have a Unique on conflict ignore here, just as in
+    # telegram_subscribers.
+
+class SeenMail(Base):
+    __tablename__ = 'seen_mail'
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer, ForeignKey('user.id'))
+    mail_date = Column(REAL) # could be Datetime, too
+
+class TwitterLastRequest(Base):
+    __tablename__ = 'twitter_last_request'
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer, ForeignKey('user.id'))
+    date = Column(Integer)
+
+class Cities(Base):
+    __tablename__ = 'cities'
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer, ForeignKey('user.id'))
+    city = Column(String)
+    markdown = Column(String)
+    mail_md = Column(String)
+    masto_link = Column(String)
+    twit_link = Column(String)
+
+    # how to get this to work?
+    # https://docs.sqlalchemy.org/en/13/dialects/sqlite.html#on-conflict-support-for-constraints
+    # UniqueConstraint('id', 'city', sqlite_on_conflict='IGNORE')
+
+class Secret(Base):
+    __tablename__ = 'secret'
+    id = Column(Integer, primary_key=True)
+    secret = Column(BLOB)
+
+Base.metadata.create_all(engine)
+

template/settings.tpl

@@ -7,86 +7,15 @@
     <div id="enablebutton" style="float: right; padding: 2em;" color="red">Enable</div>
 % end
 
-<a class='button' style="padding: 1.5em;" href="/login/twitter">
-    <picture>
-        <source type='image/webp' sizes='20px' srcset="/static-cb/1517673283/twitter-20.webp 20w,/static-cb/1517673283/twitter-40.webp 40w,/static-cb/1517673283/twitter-80.webp 80w,"/>
-        <source type='image/png' sizes='20px' srcset="/static-cb/1517673283/twitter-20.png 20w,/static-cb/1517673283/twitter-40.png 40w,/static-cb/1517673283/twitter-80.png 80w,"/>
-        <img src="https://patriciaannbridewell.files.wordpress.com/2014/04/official-twitter-logo-tile.png" alt="" />
-    </picture>
-    Log in with Twitter
-</a>
-
-<section>
-    <h2>Log in with Mastodon</h2>
-        <form action="/login/mastodon" method='post'>
-            <label for="email">E-Mail of your Mastodon-Account</label>
-            <input type="text" placeholder="Enter Email" name="email" id="email" required>
-
-            <label for="pass">Mastodon Password</label>
-            <input type="password" placeholder="Enter Password" name="pass" id="pass" required>
-
-            <label>Mastodon instance:
-                <input type='text' name='instance_url' list='instances' placeholder='social.example.net'/>
-            </label>
-            <datalist id='instances'>
-                <option value=''>
-                <option value='anticapitalist.party'>
-                <option value='awoo.space'>
-                <option value='cybre.space'>
-                <option value='mastodon.social'>
-                <option value='glitch.social'>
-                <option value='botsin.space'>
-                <option value='witches.town'>
-                <option value='social.wxcafe.net'>
-                <option value='monsterpit.net'>
-                <option value='mastodon.xyz'>
-                <option value='a.weirder.earth'>
-                <option value='chitter.xyz'>
-                <option value='sins.center'>
-                <option value='dev.glitch.social'>
-                <option value='computerfairi.es'>
-                <option value='niu.moe'>
-                <option value='icosahedron.website'>
-                <option value='hostux.social'>
-                <option value='hyenas.space'>
-                <option value='instance.business'>
-                <option value='mastodon.sdf.org'>
-                <option value='pawoo.net'>
-                <option value='pouet.it'>
-                <option value='scalie.business'>
-                <option value='sleeping.town'>
-                <option value='social.koyu.space'>
-                <option value='sunshinegardens.org'>
-                <option value='vcity.network'>
-                <option value='octodon.social'>
-                <option value='soc.ialis.me'>
-            </datalist>
-            <input name='csrf' value='{{csrf}}' type='hidden' />
-            <input name='confirm' value='Log in' type='submit'/>
-        </form>
-</section>
-
 <%
-# todo: hide this part, if there is already a telegram bot connected.
+# import all the settings templates from bots/*/settings.tpl
+import os
+bots = os.listdir('bots')
+
+for bot in bots:
+    include('bots/' + bot + '/settings.tpl', csrf=csrf, city=city)
+end
 %>
-<div>
-    <h2>Connect with Telegram</h2>
-    <p>
-        If you have a Telegram account, you can register a bot there. Just
-        write to @botfather. There are detailed instructions on
-        <a href="https://botsfortelegram.com/project/the-bot-father/" target="_blank">
-        Bots for Telegram</a>.
-    </p>
-    <p>
-        The botfather will give you an API key - with the API key, Ticketfrei
-        can use the Telegram bot. Enter it here:
-    </p>
-    <form action="/settings/telegram" method="post">
-        <input type="text" name="apikey" placeholder="Telegram bot API key" id="apikey">
-        <input name='csrf' value='{{csrf}}' type='hidden' />
-        <input name='confirm' value='Login with Telegram' type='submit'/>
-    </form>
-</div>
 
 <div>
     <h2>Edit your city page</h2>
@@ -113,24 +42,6 @@
     </form>
 </div>
 
-<div>
-    <h2>Edit your mail subscription page</h2>
-    <p>
-        There is also a page where users can subscribe to mail notifications:
-        <a href="/city/mail/{{city}}" target="_blank">Ticketfrei {{city}}</a>.
-        You can change what your users will read there, and adjust it to your
-        needs.
-    </p>
-    <p>
-        So this is the default text we suggest:
-    </p>
-    <form action="/settings/mail_md" method="post">
-        <textarea id="mail_md" rows="20" cols="70" name="mail_md" wrap="physical">{{mail_md}}</textarea>
-        <input name='csrf' value='{{csrf}}' type='hidden' />
-        <input name='confirm' value='Save' type='submit'/>
-    </form>
-</div>
-
 <div>
     <h2>Edit your trigger patterns</h2>
     <p>