From ea58f4a150ad309dc6f6aad188e94e7b1c9def8d Mon Sep 17 00:00:00 2001 From: missytake Date: Sat, 1 Apr 2023 20:43:02 +0200 Subject: [PATCH] [misc] Wrote migration script ticketfrei 2 -> 3 --- backend/setup.cfg | 1 + backend/src/kibicara/migratefromticketfrei.py | 209 ++++++++++++++++++ 2 files changed, 210 insertions(+) create mode 100644 backend/src/kibicara/migratefromticketfrei.py diff --git a/backend/setup.cfg b/backend/setup.cfg index 9526031..687cf14 100644 --- a/backend/setup.cfg +++ b/backend/setup.cfg @@ -43,6 +43,7 @@ where = src console_scripts = kibicara = kibicara.kibicara:Main kibicara_mda = kibicara.platforms.email.mda:Main + migrate_from_ticketfrei2 = kibicara.migratefromticketfrei:Main [tox:tox] envlist = lint, py310 diff --git a/backend/src/kibicara/migratefromticketfrei.py b/backend/src/kibicara/migratefromticketfrei.py new file mode 100644 index 0000000..db8a81c --- /dev/null +++ b/backend/src/kibicara/migratefromticketfrei.py @@ -0,0 +1,209 @@ +import sqlite3 +import argparse +from time import time, sleep +from os import urandom +from tortoise import Tortoise +import asyncio + +from kibicara.model import Admin, Hood, IncludePattern, ExcludePattern +from kibicara.platforms.mastodon.model import MastodonInstance, MastodonAccount +from kibicara.platforms.telegram.model import Telegram, TelegramSubscriber +from kibicara.platforms.email.model import Email, EmailSubscriber +from kibicara.platforms.twitter.model import Twitter + + +class OldDataBase: + def __init__(self, old_database_path): + self.conn = sqlite3.connect(old_database_path) + self.cur = self.conn.cursor() + + def execute(self, *args, **kwargs): + if len(args) > 1: + print(args[0], args[1][0]) + return self.cur.execute(*args, **kwargs) + + def commit(self): + start_time = time() + while 1: + try: + self.conn.commit() + break + except sqlite3.OperationalError: + # another thread may be writing, give it a chance to finish + sleep(0.1) + if time() - start_time > 5: + # if it takes this long, something is wrong + raise + + def close(self): + self.conn.close() + + +class Main: + def __init__(self): + parser = argparse.ArgumentParser() + parser.add_argument( + "old_database_path", help="path to the ticketfrei2 sqlite3 database" + ) + parser.add_argument( + "new_database_path", + help="path to the ticketfrei3 sqlite3 database", + default="ticketfrei3.sqlite", + ) + args = parser.parse_args() + + # open old database + self.old_db = OldDataBase(args.old_database_path) + + # open new database + asyncio.run(self.new_database(args.new_database_path)) + + async def new_database(self, new_database_path): + await Tortoise.init( + db_url=f"sqlite://{new_database_path}", + modules={ + "models": [ + "kibicara.model", + "kibicara.platforms.email.model", + "kibicara.platforms.mastodon.model", + "kibicara.platforms.telegram.model", + "kibicara.platforms.test.model", + "kibicara.platforms.twitter.model", + ] + }, + ) + await Tortoise.generate_schemas() + + # read table per table and write it to new database. + # mastodon instances + old_mastodon_instances = self.old_db.execute( + "SELECT * FROM mastodon_instances" + ).fetchall() + for instance in old_mastodon_instances: + url = instance[1] + client_id = instance[2] + client_secret = instance[3] + print(url) + print(client_id) + print(client_secret) + await MastodonInstance.create( + name=url, client_id=client_id, client_secret=client_secret + ) + + old_users = self.old_db.execute("SELECT * FROM user;").fetchall() + for user in old_users: + print(user) + user_id = user[0] + user_passhash = user[1] + user_enabled = user[2] + + if user_enabled == 0: + print(f"skipping user {user_id}, inactive") + + email = self.old_db.execute( + "SELECT email FROM email WHERE user_id=?", (user_id,) + ).fetchone()[0] + print("email:", email) + admin = await Admin.create(email=email, passhash=user_passhash) + + city = self.old_db.execute( + "SELECT * FROM cities WHERE user_id=?", (user_id,) + ).fetchone() + city_name = city[2] + city_markdown = city[3] + # print(city_name) + # print(city_markdown) + hood = await Hood.create(name=city_name, landingpage=city_markdown) + await hood.admins.add(admin) + + patterns = self.old_db.execute( + "SELECT patterns FROM triggerpatterns WHERE user_id=?", (user_id,) + ).fetchone()[0] + for pattern in patterns.splitlines(): + print("trigger:", pattern) + await IncludePattern.create(hood=hood, pattern=pattern) + + badwords = self.old_db.execute( + "SELECT words FROM badwords WHERE user_id=?", (user_id,) + ).fetchone()[0] + for badword in badwords.splitlines(): + print("bad word:", badword) + await ExcludePattern.create(hood=hood, pattern=badword) + + mastodon_account = self.old_db.execute( + "SELECT * FROM mastodon_accounts WHERE user_id=?", (user_id,) + ).fetchone() + if mastodon_account: + instance_url = self.old_db.execute( + "SELECT instance FROM mastodon_instances WHERE id=?", + (mastodon_account[3],), + ).fetchone()[0] + new_instance = await MastodonInstance.get(name=instance_url) + access_token = mastodon_account[2] + mastodon_enabled = mastodon_account[4] + await MastodonAccount.create( + hood=hood, + instance=new_instance, + access_token=access_token, + enabled=mastodon_enabled, + ) + # print(instance_url) + # print(access_token) + # print(mastodon_enabled) + + telegram_account = self.old_db.execute( + "SELECT apikey, active FROM telegram_accounts WHERE user_id=?", + (user_id,), + ).fetchone() + if telegram_account[0] != "": + telegram_apikey = telegram_account[0] + telegram_enabled = telegram_account[1] + # print(telegram_apikey) + # print(telegram_enabled) + telegram = await Telegram.create( + hood=hood, + api_token=telegram_apikey, + enabled=telegram_enabled, + welcome_message="", + ) + telegram_subscribers = self.old_db.execute( + "SELECT subscriber_id FROM telegram_subscribers WHERE user_id=?", + (user_id,), + ).fetchall() + for subscriber in telegram_subscribers: + subscriber_id = subscriber[0] + await TelegramSubscriber.create(bot=telegram, user_id=subscriber_id) + + mail_subscribers = self.old_db.execute( + "SELECT email FROM mailinglist WHERE user_id=?", (user_id,) + ).fetchall() + if mail_subscribers is not []: + email_name = f"kibicara-{city_name}" + email = await Email.create( + hood=hood, name=email_name, secret=urandom(32).hex() + ) + for subscriber in mail_subscribers: + subscriber_email = subscriber[0] + await EmailSubscriber.create(email=subscriber_email, hood=hood) + # print("subscriber:", subscriber_email) + + twitter = self.old_db.execute( + "SELECT * FROM twitter_accounts WHERE user_id=?", (user_id,) + ).fetchone() + if twitter: + client_id = twitter[2] + client_secret = twitter[3] + seen_tweet = self.old_db.execute( + "SELECT tweet_id FROM seen_tweets WHERE user_id=?", (user_id,) + ).fetchone()[0] + await Twitter.create( + hood=hood, + dms_since_id=1, # didn't work in ticketfrei2 anyway + mentions_since_id=seen_tweet, + access_token=client_id, + access_token_secret=client_secret, + username=" ", + verified=False, + enabled=True, + ) + await Tortoise.close_connections()