From eb713a3f40ca9be0e532fb64ebbd135a49500859 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Feb 2023 19:44:31 +0000 Subject: [PATCH 1/5] chore(deps): bump sqlalchemy from 1.4.37 to 2.0.4 Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 1.4.37 to 2.0.4. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/main/CHANGES.rst) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) --- updated-dependencies: - dependency-name: sqlalchemy dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e1ba4dfd..bff288df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,7 +25,7 @@ pytest-ordering==0.6 python-dotenv==1.0.0 requests==2.28.2 six==1.16.0 -sqlalchemy==1.4.37 +sqlalchemy==2.0.4 toml==0.10.2 twitchAPI==3.9.0 urllib3==1.26.14 From ff7cd76b4fd9383e5d4a2904035286729f31c9b0 Mon Sep 17 00:00:00 2001 From: JayDwee Date: Tue, 28 Feb 2023 20:34:55 +0000 Subject: [PATCH 2/5] fix: sqlalchely 2 upgrade --- koala/db.py | 24 +++++++----------------- tests/cogs/base/test_core.py | 2 +- tests/cogs/intro_cog/test_db.py | 3 ++- tests/cogs/twitch_alert/test_db.py | 11 +++++------ tests/test_koalabot.py | 4 ++-- 5 files changed, 17 insertions(+), 27 deletions(-) diff --git a/koala/db.py b/koala/db.py index a0f7919e..c3bc169b 100644 --- a/koala/db.py +++ b/koala/db.py @@ -59,7 +59,7 @@ def _get_sql_url(db_path, encrypted: bool, db_key=None): @contextmanager -def session_manager(): +def session_manager() -> Session: """ Provide a transactional scope around a series of operations """ @@ -213,25 +213,15 @@ def get_all_available_guild_extensions(guild_id: int, session: Session): # [extension.extension_id for extension in session.execute(sql_select_all).all()] -def fetch_all_tables(): +def clear_all_tables(): """ - Fetches all table names within the database + Clears all the data from the given tables """ with session_manager() as session: - return [table.name for table in - session.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;").all()] - - -def clear_all_tables(tables): - """ - Clears al the data from the given tables - - :param tables: a list of all tables to be cleared - """ - with session_manager() as session: - for table in tables: - session.execute('DELETE FROM ' + table + ';') - session.commit() + for table in reversed(mapper_registry.metadata.sorted_tables): + print('Clear table %s' % table) + session.execute(table.delete()) + session.commit() setup() diff --git a/tests/cogs/base/test_core.py b/tests/cogs/base/test_core.py index db103d1b..91979766 100644 --- a/tests/cogs/base/test_core.py +++ b/tests/cogs/base/test_core.py @@ -208,7 +208,7 @@ async def test_list_enabled_extensions(bot: commands.Bot): @mock.patch("koalabot.ENABLED_COGS", ["announce"]) @pytest.mark.asyncio async def test_get_extensions(bot: commands.Bot): - koalabot.load_all_cogs(bot) + await koalabot.load_all_cogs(bot) guild: discord.Guild = dpytest.get_config().guilds[0] resp = core.get_all_available_guild_extensions(guild.id) print(resp) diff --git a/tests/cogs/intro_cog/test_db.py b/tests/cogs/intro_cog/test_db.py index 25d9778f..eee9f540 100644 --- a/tests/cogs/intro_cog/test_db.py +++ b/tests/cogs/intro_cog/test_db.py @@ -137,4 +137,5 @@ async def test_on_member_join(): @pytest.fixture(scope='session', autouse=True) def setup_db(): - koala_db.clear_all_tables(koala_db.fetch_all_tables()) + + koala_db.clear_all_tables() diff --git a/tests/cogs/twitch_alert/test_db.py b/tests/cogs/twitch_alert/test_db.py index 6c6522d6..4d56f72a 100644 --- a/tests/cogs/twitch_alert/test_db.py +++ b/tests/cogs/twitch_alert/test_db.py @@ -26,6 +26,7 @@ from koala.cogs.twitch_alert import utils from koala.cogs.twitch_alert.models import TwitchAlerts, TeamInTwitchAlert, UserInTwitchTeam, UserInTwitchAlert from koala.db import session_manager, setup +from koala.models import mapper_registry # Constants DB_PATH = "Koala.db" @@ -66,13 +67,11 @@ def twitch_alert_db_manager_tables(twitch_alert_db_manager): def test_create_tables(): setup() tables = ['TwitchAlerts', 'UserInTwitchAlert', 'TeamInTwitchAlert', 'UserInTwitchTeam'] - sql_check_table_exists = "SELECT name FROM sqlite_master " \ - "WHERE type='table' AND " \ - "name IN ('TwitchAlerts', 'UserInTwitchAlert', 'TeamInTwitchAlert', 'UserInTwitchTeam');" - with session_manager() as session: - tables_found = session.execute(sql_check_table_exists).all() + tables_found = mapper_registry.metadata.tables for table in tables_found: - assert table.name in tables + if table in tables: + tables.remove(table) + assert tables == [] def test_new_ta(twitch_alert_db_manager_tables): diff --git a/tests/test_koalabot.py b/tests/test_koalabot.py index bb92a1cc..7194cfa2 100644 --- a/tests/test_koalabot.py +++ b/tests/test_koalabot.py @@ -20,7 +20,7 @@ # Own modules import koalabot -from koala.db import clear_all_tables, fetch_all_tables +from koala.db import clear_all_tables from tests.tests_utils.utils import FakeAuthor from tests.tests_utils.last_ctx_cog import LastCtxCog @@ -43,7 +43,7 @@ async def test_ctx(bot: commands.Bot): @pytest.fixture(scope='session', autouse=True) def setup_db(): - clear_all_tables(fetch_all_tables()) + clear_all_tables() @pytest_asyncio.fixture(scope='function', autouse=True) From 6b1d2bf755aed8bcb1b9bde326e140346da4c699 Mon Sep 17 00:00:00 2001 From: JayDwee Date: Tue, 28 Feb 2023 20:49:17 +0000 Subject: [PATCH 3/5] fix: provide module --- koala/db.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/koala/db.py b/koala/db.py index c3bc169b..ae15d40e 100644 --- a/koala/db.py +++ b/koala/db.py @@ -13,8 +13,10 @@ from contextlib import contextmanager from functools import wraps from pathlib import Path +from typing import Generator from sqlalchemy import select, delete, and_, create_engine, func as sql_func +from sqlalchemy.dialects.sqlite import pysqlcipher, pysqlite from sqlalchemy.orm import sessionmaker # Own modules @@ -41,25 +43,23 @@ def with_session(*args, **kwargs): return with_session -def _get_sql_url(db_path, encrypted: bool, db_key=None): +def _create_engine(db_path, encrypted: bool, db_key=None): if encrypted: - return "sqlite+pysqlcipher://:x'" + db_key + "'@/" + db_path + return create_engine("sqlite+pysqlcipher://:x'" + db_key + "'@/" + db_path, module=pysqlcipher) else: - return "sqlite:///" + db_path + return create_engine("sqlite:///" + db_path, module=pysqlite) CONFIG_DIR = get_arg_config_path() DATABASE_PATH = format_config_path(CONFIG_DIR, "Koala.db" if ENCRYPTED_DB else "windows_Koala.db") logger.debug("Database Path: "+DATABASE_PATH) -engine = create_engine(_get_sql_url(db_path=DATABASE_PATH, - encrypted=ENCRYPTED_DB, - db_key=DB_KEY), future=True) -Session = sessionmaker(future=True) +engine = _create_engine(db_path=DATABASE_PATH, encrypted=ENCRYPTED_DB, db_key=DB_KEY) +Session = sessionmaker() Session.configure(bind=engine) @contextmanager -def session_manager() -> Session: +def session_manager() -> Generator[Session]: """ Provide a transactional scope around a series of operations """ From be1046fefb5a7842916941160c20ba513de7ab53 Mon Sep 17 00:00:00 2001 From: JayDwee Date: Tue, 28 Feb 2023 21:00:11 +0000 Subject: [PATCH 4/5] Revert "fix: provide module" This reverts commit 6b1d2bf755aed8bcb1b9bde326e140346da4c699. --- koala/db.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/koala/db.py b/koala/db.py index ae15d40e..c3bc169b 100644 --- a/koala/db.py +++ b/koala/db.py @@ -13,10 +13,8 @@ from contextlib import contextmanager from functools import wraps from pathlib import Path -from typing import Generator from sqlalchemy import select, delete, and_, create_engine, func as sql_func -from sqlalchemy.dialects.sqlite import pysqlcipher, pysqlite from sqlalchemy.orm import sessionmaker # Own modules @@ -43,23 +41,25 @@ def with_session(*args, **kwargs): return with_session -def _create_engine(db_path, encrypted: bool, db_key=None): +def _get_sql_url(db_path, encrypted: bool, db_key=None): if encrypted: - return create_engine("sqlite+pysqlcipher://:x'" + db_key + "'@/" + db_path, module=pysqlcipher) + return "sqlite+pysqlcipher://:x'" + db_key + "'@/" + db_path else: - return create_engine("sqlite:///" + db_path, module=pysqlite) + return "sqlite:///" + db_path CONFIG_DIR = get_arg_config_path() DATABASE_PATH = format_config_path(CONFIG_DIR, "Koala.db" if ENCRYPTED_DB else "windows_Koala.db") logger.debug("Database Path: "+DATABASE_PATH) -engine = _create_engine(db_path=DATABASE_PATH, encrypted=ENCRYPTED_DB, db_key=DB_KEY) -Session = sessionmaker() +engine = create_engine(_get_sql_url(db_path=DATABASE_PATH, + encrypted=ENCRYPTED_DB, + db_key=DB_KEY), future=True) +Session = sessionmaker(future=True) Session.configure(bind=engine) @contextmanager -def session_manager() -> Generator[Session]: +def session_manager() -> Session: """ Provide a transactional scope around a series of operations """ From 4402d7b350b97704a73be07c58fc7f5b9ad572aa Mon Sep 17 00:00:00 2001 From: JayDwee Date: Tue, 28 Feb 2023 21:01:00 +0000 Subject: [PATCH 5/5] chore: add module to create engine --- koala/db.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/koala/db.py b/koala/db.py index c3bc169b..fc238d40 100644 --- a/koala/db.py +++ b/koala/db.py @@ -14,6 +14,13 @@ from functools import wraps from pathlib import Path +if os.name == 'nt': + print("Windows Detected: Database Encryption Disabled") + import sqlite3 +else: + print("Linux Detected: Database Encryption Enabled") + from pysqlcipher3 import dbapi2 as sqlite3 + from sqlalchemy import select, delete, and_, create_engine, func as sql_func from sqlalchemy.orm import sessionmaker @@ -53,13 +60,13 @@ def _get_sql_url(db_path, encrypted: bool, db_key=None): logger.debug("Database Path: "+DATABASE_PATH) engine = create_engine(_get_sql_url(db_path=DATABASE_PATH, encrypted=ENCRYPTED_DB, - db_key=DB_KEY), future=True) + db_key=DB_KEY), module=sqlite3) Session = sessionmaker(future=True) Session.configure(bind=engine) @contextmanager -def session_manager() -> Session: +def session_manager(): """ Provide a transactional scope around a series of operations """