[email] Fix email bot

kibicara/config.py

@@ -25,13 +25,15 @@ from sys import argv
 config = {
     'database_connection': 'sqlite:////tmp/kibicara.sqlite',
     'frontend_path': None,
-    'root_url': 'http://localhost:8000/',
+    'root_url': 'http://localhost:8000',
 }
 """ Default configuration.
 
 The default configuration gets overwritten by a configuration file if one exists.
 """
 
+args = None
+
 if argv[0].endswith('kibicara'):
     parser = ArgumentParser()
     parser.add_argument(
@@ -46,6 +48,20 @@ if argv[0].endswith('kibicara'):
     )
     args = parser.parse_args()
 
+if argv[0].endswith('kibicara_mda'):
+    parser = ArgumentParser()
+    parser.add_argument(
+        '-f',
+        '--config',
+        dest='configfile',
+        default='/etc/kibicara.conf',
+        help='path to config file',
+    )
+    # the MDA passes the recipient address as command line argument
+    parser.add_argument("recipient")
+    args = parser.parse_args()
+
+if args is not None:
     try:
         with open(args.configfile) as configfile:
             config.update(load(configfile))

kibicara/kibicara_mda.py

@@ -1,119 +0,0 @@
-# Copyright (C) 2020 by Maike <maike@systemli.org>
-#
-# SPDX-License-Identifier: 0BSD
-
-import email.parser
-from email.policy import default
-import email.message
-import sys
-import re
-import requests
-from logging import getLogger
-from kibicara.model import Hood
-from kibicara.platforms.email.model import Email
-import argparse
-from asyncio import run
-from fastapi import status
-from ormantic import NoMatch
-
-
-async def async_main(mail=None, hood_name=None):
-    logger = getLogger(__name__)
-
-    # the MDA passes the recipient address as command line argument
-    parser = argparse.ArgumentParser()
-    if hood_name is None:
-        parser.add_argument("recipient_address")
-        args = parser.parse_args()
-        # extract hood name from the envelope recipient address
-        hood_name = args.recipient_address.split('@')[0].lower()
-        if hood_name.startswith('kibicara.'):
-            hood_name = hood_name[9:]
-        else:
-            logger.error("Recipient address didn't start with 'kibicara.'")
-
-    if mail is None:
-        # read mail from STDIN
-        mail = bytes(sys.stdin.read(), encoding='ascii')
-        # parse plaintext to email.EmailMessage object
-        mail = email.parser.BytesParser(policy=default).parsebytes(mail)
-    else:
-        mail = email.parser.Parser(policy=default).parsestr(mail)
-
-    assert type(mail) == email.message.EmailMessage
-
-    # extract relevant data from mail
-    body = mail.get_body(preferencelist=('plain', 'html'))
-    if body['content-type'].subtype == 'plain':
-        text = str(body.get_content())
-    elif body['content-type'].subtype == 'html':
-        text = re.sub(r'<[^>]*>', '', body.get_content())
-
-    try:
-        text = str(text)
-    except UnboundLocalError:
-        print('No suitable message body')
-        exit(1)
-    try:
-        hood = await Hood.objects.get(name=hood_name)
-    except NoMatch:
-        print('No hood with this name')
-        exit(1)
-    email_row = await Email.objects.get(hood=hood)
-    author = mail.get_unixfrom()
-    if author is None:
-        author = mail['From']
-    body = {
-        'text': text,
-        'author': author,
-        'secret': email_row.secret,
-    }
-    response = requests.post(
-        'http://localhost:8000/api/hoods/%d/email/messages/' % hood.id, json=body
-    )
-    if response.status_code == status.HTTP_201_CREATED:
-        exit(0)
-    elif response.status_code == status.HTTP_451_UNAVAILABLE_FOR_LEGAL_REASONS:
-        print("Message was't accepted: " + text)
-    elif response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY:
-        print("Malformed request: " + str(response.json()))
-    elif response.status_code == status.HTTP_401_UNAUTHORIZED:
-        logger.error('Wrong API secret. kibicara_mda seems to be misconfigured')
-    else:
-        print(str(response.status_code))
-    exit(1)
-
-
-def main():
-    run(async_main())
-
-
-if __name__.endswith('kibicara_mda'):
-    mail = """From test@example.com Tue Jun 16 15:33:19 2020
-Return-path: <test@example.com>
-Envelope-to: hood@localhost
-Delivery-date: Tue, 16 Jun 2020 15:33:19 +0200
-Received: from [23.143.35.123] (helo=example.com)
-        by example.com with smtp (Exim 4.89)
-        (envelope-from <test@example.com>)
-        id 1jlC1e-0005ro-PL
-        for hood@localhost; Tue, 16 Jun 2020 15:33:19 +0200
-Message-ID: <B5F50812.F55DFD8B@example.com>
-Date: Tue, 16 Jun 2020 06:53:19 -0700
-Reply-To: "Test" <test@example.com>
-From: "Test" <test@example.com>
-User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1.17) Gecko/20080914 Thunderbird/2.0.0.17
-MIME-Version: 1.0
-To: <hood@localhost>
-Subject: Chat: test
-Content-Type: multipart/mixed; boundary="AqNPlAX243a8sip3B7kXv8UKD8wuti"
-
-
---AqNPlAX243a8sip3B7kXv8UKD8wuti
-Content-Type: text/plain; charset=utf-8
-
-test
-
---AqNPlAX243a8sip3B7kXv8UKD8wuti--
-    """
-    run(async_main(mail=mail, hood_name='hood'))

kibicara/platforms/email/bot.py

@@ -1,49 +1,50 @@
 # Copyright (C) 2020 by Maike <maike@systemli.org>
+# Copyright (C) 2020 by Cathy Hu <cathy.hu@fau.de>
+# Copyright (C) 2020 by Thomas Lindner <tom@dl6tom.de>
 #
 # SPDX-License-Identifier: 0BSD
 
-from kibicara.platforms.email.model import EmailSubscribers, Email
-from kibicara.platformapi import Censor, Spawner
-from kibicara.email import send_email
 from kibicara.config import config
+from kibicara.email import send_email
+from kibicara.model import Hood
+from kibicara.platformapi import Censor, Spawner
+from kibicara.platforms.email.model import EmailSubscribers
 from kibicara.webapi.admin import to_token
-from smtplib import SMTPException
 from logging import getLogger
+from smtplib import SMTPException
 
 
 logger = getLogger(__name__)
 
 
 class EmailBot(Censor):
-    def __init__(self, email_model):
+    def __init__(self, hood):
-        super().__init__(email_model.hood)
+        super().__init__(hood)
-        self.model = email_model
-        self.messages = []
 
     async def run(self):
         """ Loop which waits for new messages and sends emails to all subscribers. """
         while True:
             message = await self.receive()
-            logger.info("Received Email from %s: %s" % (message.author, message.text))
+            logger.debug(
-            for subscriber in EmailSubscribers.objects.filter(hood=self.hood.id):
+                'Received message from censor (%s): %s' % (self.hood.name, message.text)
+            )
+            logger.debug('a')
+            for subscriber in await EmailSubscribers.objects.filter(
+                hood=self.hood
+            ).all():
                 token = to_token(email=subscriber.email, hood=self.hood.id)
-                unsubscribe_link = (
+                body = (
-                    config['root_url']
+                    '%s\n\n--\n'
-                    + 'api/hoods/%d/email/unsubscribe/' % self.hood.id
+                    'If you want to stop receiving these mails,'
-                    + token
+                    'follow this link: %s/api/hoods/%d/email/unsubscribe/%s'
-                )
+                ) % (message.text, config['root_url'], self.hood.id, token)
-                message.text += (
-                    "\n\n--\nIf you want to stop receiving these mails, "
-                    "follow this link: " + unsubscribe_link
-                )
                 try:
+                    logger.debug('Trying to send: \n%s' % body)
                     send_email(
-                        subscriber.email,
+                        subscriber.email, "Kibicara " + self.hood.name, body=body,
-                        "Kibicara " + self.hood.name,
-                        body=message.text,
                     )
                 except (ConnectionRefusedError, SMTPException):
-                    logger.exception("Sending subscription confirmation email failed.")
+                    logger.exception("Sending email to subscriber failed.")
 
 
-spawner = Spawner(Email, EmailBot)
+spawner = Spawner(Hood, EmailBot)

kibicara/platforms/email/mda.py

@@ -0,0 +1,64 @@
+# Copyright (C) 2020 by Maike <maike@systemli.org>
+# Copyright (C) 2020 by Cathy Hu <cathy.hu@fau.de>
+# Copyright (C) 2020 by Thomas Lindner <tom@dl6tom.de>
+#
+# SPDX-License-Identifier: 0BSD
+
+from argparse import ArgumentParser
+from asyncio import run as asyncio_run
+from email.parser import BytesParser
+from email.policy import default
+from fastapi import status
+from kibicara.config import args, config
+from kibicara.model import Hood
+from kibicara.platforms.email.model import Email
+from logging import getLogger
+from ormantic import NoMatch
+from re import sub
+from requests import post
+from sys import stdin
+
+
+logger = getLogger(__name__)
+
+
+class Main:
+    def __init__(self):
+        asyncio_run(self.__run())
+
+    async def __run(self):
+        # extract email from the recipient
+        email_name = args.recipient.lower()
+        try:
+            email = await Email.objects.get(name=email_name)
+        except NoMatch:
+            logger.error('No recipient with this name')
+            exit(1)
+
+        # read mail from STDIN and parse to EmailMessage object
+        message = BytesParser(policy=default).parsebytes(stdin.buffer.read())
+
+        # extract relevant data from mail
+        text = sub(
+            r'<[^>]*>',
+            '',
+            message.get_body(preferencelist=('plain', 'html')).get_content(),
+        )
+
+        response = post(
+            '%s/api/hoods/%d/email/messages/' % (config['root_url'], email.hood.pk),
+            json={'text': text, 'secret': email.secret},
+        )
+        if response.status_code == status.HTTP_201_CREATED:
+            exit(0)
+        elif response.status_code == status.HTTP_451_UNAVAILABLE_FOR_LEGAL_REASONS:
+            logger.error('Message was\'t accepted: %s' % text)
+        elif response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY:
+            logger.error('Malformed request: %s' % response.json())
+        elif response.status_code == status.HTTP_401_UNAUTHORIZED:
+            logger.error('Wrong API secret. kibicara_mda seems to be misconfigured')
+        else:
+            logger.error(
+                'REST-API failed with response status %d' % response.status_code
+            )
+        exit(1)

kibicara/platforms/email/model.py

@@ -1,4 +1,6 @@
-# Copyright (C) 2020 by Maike <tom@dl6tom.de>
+# Copyright (C) 2020 by Maike <maike@systemli.org>
+# Copyright (C) 2020 by Cathy Hu <cathy.hu@fau.de>
+# Copyright (C) 2020 by Thomas Lindner <tom@dl6tom.de>
 #
 # SPDX-License-Identifier: 0BSD
 
@@ -6,23 +8,24 @@ from kibicara.model import Hood, Mapping
 from ormantic import Integer, ForeignKey, Model, Text
 
 
+class Email(Model):
+    """ This table is used to track the names. It also stores the token secret. """
+
+    id: Integer(primary_key=True) = None
+    hood: ForeignKey(Hood)
+    name: Text(unique=True)
+    secret: Text()
+
+    class Mapping(Mapping):
+        table_name = 'email'
+
+
 class EmailSubscribers(Model):
     """ This table stores all subscribers, who want to receive messages via email. """
 
     id: Integer(primary_key=True) = None
     hood: ForeignKey(Hood)
-    email: Text()
+    email: Text(unique=True)
 
     class Mapping(Mapping):
         table_name = 'email_subscribers'
-
-
-class Email(Model):
-    """ This table is used to track the hood ID. It also stores the token secret. """
-
-    id: Integer(primary_key=True) = None
-    hood: ForeignKey(Hood, unique=True)
-    secret: Text()
-
-    class Mapping(Mapping):
-        table_name = 'email'

kibicara/platforms/email/webapi.py

@@ -1,41 +1,46 @@
 # Copyright (C) 2020 by Maike <maike@systemli.org>
+# Copyright (C) 2020 by Cathy Hu <cathy.hu@fau.de>
+# Copyright (C) 2020 by Thomas Lindner <tom@dl6tom.de>
 #
 # SPDX-License-Identifier: 0BSD
 
-from fastapi import APIRouter, Depends, HTTPException, status
+from fastapi import APIRouter, Depends, HTTPException, Response, status
 from kibicara.platforms.email.bot import spawner
 from kibicara.platforms.email.model import Email, EmailSubscribers
 from kibicara.platformapi import Message
 from kibicara.config import config
 from kibicara.email import send_email
-from kibicara.webapi.hoods import get_hood, get_hood_unauthorized
-from pydantic import BaseModel
-from ormantic.exceptions import NoMatch
-from sqlite3 import IntegrityError
 from kibicara.webapi.admin import from_token, to_token
-from os import urandom
+from kibicara.webapi.hoods import get_hood, get_hood_unauthorized
-from smtplib import SMTPException
 from logging import getLogger
+from ormantic.exceptions import NoMatch
+from os import urandom
+from pydantic import BaseModel
+from smtplib import SMTPException
+from sqlite3 import IntegrityError
 
 
 logger = getLogger(__name__)
 
 
+class BodyEmail(BaseModel):
+    name: str
+
+
 class BodyMessage(BaseModel):
     """ This model shows which values are supplied by the MDA listener script. """
 
     text: str
-    author: str
     secret: str
 
 
-class Subscriber(BaseModel):
+class BodySubscriber(BaseModel):
     """ This model holds the email address of a fresh subscriber. """
 
     email: str
 
 
-async def get_email(hood):
+async def get_email(email_id: int, hood=Depends(get_hood)):
     """ Get Email row by hood.
     You can specify an email_id to nail it down, but it works without as well.
 
@@ -43,7 +48,14 @@ async def get_email(hood):
     :return: Email row of the found email bot.
     """
     try:
-        return await Email.objects.get(hood=hood)
+        return await Email.objects.get(id=email_id, hood=hood)
+    except NoMatch:
+        return HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+
+
+async def get_subscriber(subscriber_id: int, hood=Depends(get_hood)):
+    try:
+        return await EmailSubscriber.objects.get(id=subscriber_id, hood=hood)
     except NoMatch:
         return HTTPException(status_code=status.HTTP_404_NOT_FOUND)
 
@@ -52,53 +64,78 @@ async def get_email(hood):
 router = APIRouter()
 
 
+@router.get('/')
+async def email_read_all(hood=Depends(get_hood)):
+    return await Email.objects.filter(hood=hood).all()
+
+
 @router.post('/', status_code=status.HTTP_201_CREATED)
-async def email_create(hood=Depends(get_hood)):
+async def email_create(values: BodyEmail, response: Response, hood=Depends(get_hood)):
     """ Create an Email bot. Call this when creating a hood.
 
     :param hood: Hood row of the hood the Email bot is supposed to belong to.
     :return: Email row of the new email bot.
     """
+    if not values.name.startswith('kibicara-'):
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail='Recipient address didn\'t start with kibicara-',
+        )
     try:
-        email_row = await Email.objects.create(hood=hood, secret=urandom(32).hex())
+        email = await Email.objects.create(
-        spawner.start(email_row)
+            hood=hood, secret=urandom(32).hex(), **values.__dict__
-        return email_row
+        )
+        spawner.start(email)
+        response.headers['Location'] = '%d' % hood.id
+        return email
     except IntegrityError:
         raise HTTPException(status_code=status.HTTP_409_CONFLICT)
 
 
-@router.delete('/', status_code=status.HTTP_204_NO_CONTENT)
+@router.get('/{email_id}')
-async def email_delete(hood=Depends(get_hood)):
+async def email_read(email=Depends(get_email)):
-    """ Delete an Email bot. Call this when deleting a hood.
+    return email
-    Stops and deletes the Email bot as well as all subscribers.
+
+
+@router.put('/{email_id}', status_code=status.HTTP_204_NO_CONTENT)
+async def email_update(email=Depends(get_email)):
+    await email.update()
+
+
+@router.delete('/{email_id}', status_code=status.HTTP_204_NO_CONTENT)
+async def email_delete(email=Depends(get_email)):
+    """ Delete an Email bot.
+    Stops and deletes the Email bot.
 
     :param hood: Hood the Email bot belongs to.
     """
-    email_row = await get_email(hood)
+    await email.delete()
-    spawner.stop(email_row)
-    await EmailSubscribers.objects.delete_many(hood=hood.id)
-    await email_row.delete()
 
 
 @router.post('/subscribe/', status_code=status.HTTP_202_ACCEPTED)
-async def email_subscribe(subscriber: Subscriber, hood=Depends(get_hood_unauthorized)):
+async def email_subscribe(
+    subscriber: BodySubscriber, hood=Depends(get_hood_unauthorized)
+):
     """ Send a confirmation mail to subscribe to messages via email.
 
     :param subscriber: Subscriber object, holds the email address.
     :param hood: Hood the Email bot belongs to.
     :return: Returns status code 200 after sending confirmation email.
     """
-    token = to_token(email=subscriber.email)
+    token = to_token(hood=hood.id, email=subscriber.email)
-    confirm_link = (
+    confirm_link = '%s/api/%d/email/subscribe/confirm/%s' % (
-        config['root_url'] + "api/" + str(hood.id) + "/email/subscribe/confirm/" + token
+        config['root_url'],
+        hood.id,
+        token,
     )
     try:
         send_email(
             subscriber.email,
             "Subscribe to Kibicara " + hood.name,
             sender=hood.name,
-            body="To confirm your subscription, follow this link: " + confirm_link,
+            body='To confirm your subscription, follow this link: ' + confirm_link,
         )
+        return {}
     except ConnectionRefusedError:
         logger.info(token)
         logger.error("Sending subscription confirmation email failed.", exc_info=True)
@@ -118,8 +155,12 @@ async def email_subscribe_confirm(token, hood=Depends(get_hood_unauthorized)):
     :return: Returns status code 200 after adding the subscriber to the database.
     """
     payload = from_token(token)
+    # If token.hood and url.hood are different, raise an error:
+    if hood.id is not payload['hood']:
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST)
     try:
         await EmailSubscribers.objects.create(hood=hood.id, email=payload['email'])
+        return {}
     except IntegrityError:
         raise HTTPException(status_code=status.HTTP_409_CONFLICT)
 
@@ -136,9 +177,17 @@ async def email_unsubscribe(token, hood=Depends(get_hood_unauthorized)):
     # If token.hood and url.hood are different, raise an error:
     if hood.id is not payload['hood']:
         raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST)
-    await EmailSubscribers.objects.delete_many(
+    await EmailSubscribers.objects.delete(hood=payload['hood'], email=payload['email'])
-        hood=payload['hood'], email=payload['email']
+
-    )
+
+@router.get('/subscribers/')
+async def subscribers_read_all(hood=Depends(get_hood)):
+    return await EmailSubscribers.objects.filter(hood=hood).all()
+
+
+@router.get('/subscribers/{subscriber_id}')
+async def subscribers_read(subscriber=Depends(get_subscriber)):
+    return subscriber
 
 
 @router.post('/messages/', status_code=status.HTTP_201_CREATED)
@@ -151,22 +200,21 @@ async def email_message_create(
     :param hood: Hood the Email bot belongs to.
     :return: returns status code 201 if the message is accepted by the censor.
     """
-    # get bot via "To:" header
+    for email in await Email.objects.filter(hood=hood).all():
-    try:
+        if message.secret == email.secret:
-        email_row = await get_email(hood)
+            # check API secret
-    except HTTPException as exc:
+            logger.warning(str(message))
-        raise exc
+            logger.warning(str(email))
-    # check API secret
+            # pass message.text to bot.py
-    logger.warning(str(message))
+            if await spawner.get(hood).publish(Message(message.text)):
-    logger.warning(str(email_row))
+                logger.warning("Message was accepted: " + message.text)
-    if message.secret != email_row.secret:
+                return {}
-        logger.warning(
+            else:
-            "Someone is trying to submit an email without the correct API secret"
+                logger.warning("Message was't accepted: " + message.text)
-        )
+                raise HTTPException(
-        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED)
+                    status_code=status.HTTP_451_UNAVAILABLE_FOR_LEGAL_REASONS
-    # pass message.text to bot.py
+                )
-    if await spawner.get(email_row).publish(Message(message.text)):
+    logger.warning(
-        logger.warning("Message was accepted: " + message.text)
+        "Someone is trying to submit an email without the correct API secret"
-    else:
+    )
-        logger.warning("Message was't accepted: " + message.text)
+    raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED)
-        raise HTTPException(status_code=status.HTTP_451_UNAVAILABLE_FOR_LEGAL_REASONS)

setup.py

@@ -10,7 +10,7 @@ setup(
     entry_points={
         'console_scripts': [
             'kibicara=kibicara.kibicara:Main',
-            'kibicara_mda=kibicara.kibicara_mda:main',
+            'kibicara_mda=kibicara.platforms.email.mda:Main',
         ]
     },
     install_requires=[