From 0b01a406ca1b481df7fc192bff508566881479da Mon Sep 17 00:00:00 2001 From: Elliana May Date: Mon, 16 Dec 2024 06:30:29 +0000 Subject: [PATCH 1/7] chore: add dep --- poetry.lock | 66 +++++++++++++++++++++++++++++++++++++++++--------- pyproject.toml | 3 ++- 2 files changed, 56 insertions(+), 13 deletions(-) diff --git a/poetry.lock b/poetry.lock index 523ab6f03..215444627 100644 --- a/poetry.lock +++ b/poetry.lock @@ -684,6 +684,31 @@ files = [ [package.dependencies] ploomber-core = "*" +[[package]] +name = "markdown-it-py" +version = "3.0.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, + {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, +] + +[package.dependencies] +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +code-style = ["pre-commit (>=3.0,<4.0)"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + [[package]] name = "markupsafe" version = "2.1.2" @@ -760,6 +785,18 @@ files = [ [package.dependencies] traitlets = "*" +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +optional = false +python-versions = ">=3.7" +groups = ["dev"] +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] + [[package]] name = "monotonic" version = "1.6" @@ -1424,6 +1461,22 @@ pytest = ">=4.6" [package.extras] testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"] +[[package]] +name = "pytest-markdown-docs" +version = "0.7.1" +description = "Run markdown code fences through pytest" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "pytest_markdown_docs-0.7.1-py3-none-any.whl", hash = "sha256:916ffa0911892c439773fd363b3bc64afc7e819925bfaec859afdcff2f587cc6"}, + {file = "pytest_markdown_docs-0.7.1.tar.gz", hash = "sha256:20c27d5946601d5b45306685059612396fa737e931c0d28cb28aa0ae97f4e71b"}, +] + +[package.dependencies] +markdown-it-py = ">=2.2.0,<4.0" +pytest = ">=7.0.0" + [[package]] name = "pytest-remotedata" version = "0.4.1" @@ -1480,7 +1533,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1488,16 +1540,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1514,7 +1558,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1522,7 +1565,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1922,4 +1964,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.8,<4" -content-hash = "6f046554bc7605f1a73bc65bc913bdf0f775c41bc754f0cd3b78391784dd999d" +content-hash = "0f45246321fff5543b4078c95e42bdda0ece451dafd615bbdae4d949008d3be7" diff --git a/pyproject.toml b/pyproject.toml index 245351261..d523742e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ pytest-cov = {extras = ["coverage"], version = "^5.0.0"} snapshottest = "^0.6.0" pytest-remotedata = "^0.4.0" toml = "^0.10.2" +pytest-markdown-docs = "^0.7.1" [tool.poetry.group.devtools.dependencies] pdbpp = "^0.10.3" @@ -39,7 +40,7 @@ pre-commit = { version = "^4.0.0", markers = "python_version >= '3.9'" } duckdb = "duckdb_engine:Dialect" [tool.pytest.ini_options] -addopts = "--hypothesis-show-statistics --strict --strict-markers" +addopts = "--hypothesis-show-statistics --strict --strict-markers --markdown-docs" xfail_strict = true [tool.mypy] From 991b16e8b5ad7bf5112a0838e59598771ad7c7c8 Mon Sep 17 00:00:00 2001 From: Samuel Montgomery-Blinn Date: Tue, 3 Dec 2024 20:25:08 -0500 Subject: [PATCH 2/7] Update README.md to correct auto-inc example Also removes the ">>>" and "..." characters so the example can be cut and paste directly into terminal. --- README.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index a8d99fa70..3688acff3 100644 --- a/README.md +++ b/README.md @@ -107,22 +107,22 @@ When defining an Integer column as a primary key, `SQLAlchemy` uses the `SERIAL` The following example demonstrates how to create an auto-incrementing ID column for a simple table: ```python ->>> import sqlalchemy ->>> engine = sqlalchemy.create_engine('duckdb:////path/to/duck.db') ->>> metadata = sqlalchemy.MetaData(engine) ->>> user_id_seq = sqlalchemy.Sequence('user_id_seq') ->>> users_table = sqlalchemy.Table( -... 'users', -... metadata, -... sqlalchemy.Column( -... 'id', -... sqlalchemy.Integer, -... user_id_seq, -... server_default=user_id_seq.next_value(), -... primary_key=True, -... ), -... ) ->>> metadata.create_all(bind=engine) +import sqlalchemy +engine = sqlalchemy.create_engine('duckdb:////path/to/duck.db') +metadata = sqlalchemy.MetaData() +user_id_seq = sqlalchemy.Sequence('user_id_seq') +users_table = sqlalchemy.Table( + 'users', + metadata, + sqlalchemy.Column( + 'id', + sqlalchemy.Integer, + user_id_seq, + server_default=user_id_seq.next_value(), + primary_key=True, + ), + ) +metadata.create_all(bind=engine) ``` ### Pandas `read_sql()` chunksize From 8fd58354e9209fe78799e93923a4469f63ec20d5 Mon Sep 17 00:00:00 2001 From: Elliana May Date: Mon, 16 Dec 2024 06:30:52 +0000 Subject: [PATCH 3/7] chore: ignore noxfile.py --- noxfile.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/noxfile.py b/noxfile.py index 46e90aef7..7a865c04d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,8 +1,13 @@ from contextlib import contextmanager from typing import Generator -import github_action_utils as gha -import nox +try: + import github_action_utils as gha + import nox +except ImportError: + import pytest + + pytest.skip(allow_module_level=True) nox.options.default_venv_backend = "uv" nox.options.error_on_external_run = True From bc167674fac81ea0c51a478d470f1dc27385f204 Mon Sep 17 00:00:00 2001 From: Elliana May Date: Mon, 16 Dec 2024 06:31:45 +0000 Subject: [PATCH 4/7] docs: fix syntax and imports --- README.md | 26 +++++++++++++++++--------- duckdb_engine/datatypes.py | 15 +++++++++------ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 3688acff3..9fda52c17 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,8 @@ Alex Monahan's great demo of this on [his blog](https://alex-monahan.github.io/2 You can configure DuckDB by passing `connect_args` to the create_engine function ```python +from sqlalchemy.engine import create_engine + create_engine( 'duckdb:///:memory:', connect_args={ @@ -87,10 +89,16 @@ The supported configuration parameters are listed in the [DuckDB docs](https://d ## How to register a pandas DataFrame ```python +import pandas as pd +from sqlalchemy import text +from sqlalchemy.engine import create_engine + conn = create_engine("duckdb:///:memory:").connect() +df = pd.DataFrame([{'id': 0}]) + # with SQLAlchemy 1.3 -conn.execute("register", ("dataframe_name", pd.DataFrame(...))) +conn.execute("register", ("dataframe_name", df)) # with SQLAlchemy 1.4+ conn.execute(text("register(:name, :df)"), {"name": "test_df", "df": df}) @@ -121,7 +129,7 @@ users_table = sqlalchemy.Table( server_default=user_id_seq.next_value(), primary_key=True, ), - ) +) metadata.create_all(bind=engine) ``` @@ -131,12 +139,12 @@ metadata.create_all(bind=engine) The `pandas.read_sql()` method can read tables from `duckdb_engine` into DataFrames, but the `sqlalchemy.engine.result.ResultProxy` trips up when `fetchmany()` is called. Therefore, for now `chunksize=None` (default) is necessary when reading duckdb tables into DataFrames. For example: -```python ->>> import pandas as pd ->>> import sqlalchemy ->>> engine = sqlalchemy.create_engine('duckdb:////path/to/duck.db') ->>> df = pd.read_sql('users', engine) ### Works as expected ->>> df = pd.read_sql('users', engine, chunksize=25) ### Throws an exception +```python notest +import pandas as pd +import sqlalchemy +engine = sqlalchemy.create_engine('duckdb:////path/to/duck.db') +df = pd.read_sql('users', engine) ### Works as expected +df = pd.read_sql('users', engine, chunksize=25) ### Throws an exception ``` ### Unsigned integer support @@ -149,7 +157,7 @@ SQLAlchemy's companion library `alembic` can optionally be used to manage databa This support can be enabling by adding an Alembic implementation class for the `duckdb` dialect. -```python +```python notest from alembic.ddl.impl import DefaultImpl class AlembicDuckDBImpl(DefaultImpl): diff --git a/duckdb_engine/datatypes.py b/duckdb_engine/datatypes.py index 363f6e878..ab2455d08 100644 --- a/duckdb_engine/datatypes.py +++ b/duckdb_engine/datatypes.py @@ -115,11 +115,12 @@ class Struct(TypeEngine): ```python from duckdb_engine.datatypes import Struct - from sqlalchemy import Table, Column, String + from sqlalchemy import Table, Column, String, MetaData Table( 'hello', - Column('name', Struct({'first': String, 'last': String}) + MetaData(), + Column('name', Struct({'first': String, 'last': String})) ) ``` @@ -138,11 +139,12 @@ class Map(TypeEngine): ```python from duckdb_engine.datatypes import Map - from sqlalchemy import Table, Column, String + from sqlalchemy import Table, Column, String, MetaData Table( 'hello', - Column('name', Map(String, String) + MetaData(), + Column('name', Map(String, String)) ) ``` """ @@ -179,11 +181,12 @@ class Union(TypeEngine): ```python from duckdb_engine.datatypes import Union - from sqlalchemy import Table, Column, String + from sqlalchemy import Table, Column, String, MetaData Table( 'hello', - Column('name', Union({"name": String, "age": String}) + MetaData(), + Column('name', Union({"name": String, "age": String})) ) ``` """ From 979feb6116f3122be4fd3e682199995c0490f298 Mon Sep 17 00:00:00 2001 From: Elliana May Date: Mon, 16 Dec 2024 08:35:06 +0000 Subject: [PATCH 5/7] docs: fix another code sample --- README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9fda52c17..b5fcb1bc9 100644 --- a/README.md +++ b/README.md @@ -89,21 +89,23 @@ The supported configuration parameters are listed in the [DuckDB docs](https://d ## How to register a pandas DataFrame ```python +breakpoint() import pandas as pd -from sqlalchemy import text +from sqlalchemy import text, __version__ as sqla_version from sqlalchemy.engine import create_engine conn = create_engine("duckdb:///:memory:").connect() df = pd.DataFrame([{'id': 0}]) -# with SQLAlchemy 1.3 -conn.execute("register", ("dataframe_name", df)) +if sqla_version.startswith('1.3.'): + # with SQLAlchemy 1.3 + conn.execute("register", ("dataframe_name", df)) +else: + # with SQLAlchemy 1.4+ + conn.execute(text("register(:name, :df)"), {"name": "dataframe_name", "df": df}) -# with SQLAlchemy 1.4+ -conn.execute(text("register(:name, :df)"), {"name": "test_df", "df": df}) - -conn.execute("select * from dataframe_name") +conn.execute(text("select * from dataframe_name")) ``` ## Things to keep in mind From b180deb93d022ea4bf178bd5c904019a2de672c1 Mon Sep 17 00:00:00 2001 From: Elliana May Date: Mon, 16 Dec 2024 08:37:54 +0000 Subject: [PATCH 6/7] docs: add blacken-docs --- .pre-commit-config.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 55124544c..ac4c831c4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,3 +36,9 @@ repos: - --fix - --exit-non-zero-on-fix - id: ruff-format + - repo: https://github.com/adamchainz/blacken-docs + rev: "1.19.1" + hooks: + - id: blacken-docs + additional_dependencies: + - black==22.12.0 From 7dded512abffcc4d62142614d44c5b8bfd02b459 Mon Sep 17 00:00:00 2001 From: Elliana May Date: Mon, 16 Dec 2024 08:38:32 +0000 Subject: [PATCH 7/7] docs: run blacken docs --- .pre-commit-config.yaml | 3 ++ README.md | 77 ++++++++++++++++++++++---------------- duckdb_engine/datatypes.py | 14 +++---- 3 files changed, 52 insertions(+), 42 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ac4c831c4..923e38e5c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,5 +40,8 @@ repos: rev: "1.19.1" hooks: - id: blacken-docs + args: + - --line-length + - '80' additional_dependencies: - black==22.12.0 diff --git a/README.md b/README.md index b5fcb1bc9..407dcf9c5 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,13 @@ DuckDB Engine also has a conda feedstock available, the instructions for the use Once you've installed this package, you should be able to just use it, as SQLAlchemy does a python path search ```python -from sqlalchemy import Column, Integer, Sequence, String, create_engine +from sqlalchemy import ( + Column, + Integer, + Sequence, + String, + create_engine, +) from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.session import Session @@ -46,7 +52,11 @@ Base = declarative_base() class FakeModel(Base): # type: ignore __tablename__ = "fake" - id = Column(Integer, Sequence("fakemodel_id_sequence"), primary_key=True) + id = Column( + Integer, + Sequence("fakemodel_id_sequence"), + primary_key=True, + ) name = Column(String) @@ -74,13 +84,11 @@ You can configure DuckDB by passing `connect_args` to the create_engine function from sqlalchemy.engine import create_engine create_engine( - 'duckdb:///:memory:', + "duckdb:///:memory:", connect_args={ - 'read_only': False, - 'config': { - 'memory_limit': '500mb' - } - } + "read_only": False, + "config": {"memory_limit": "500mb"}, + }, ) ``` @@ -89,21 +97,23 @@ The supported configuration parameters are listed in the [DuckDB docs](https://d ## How to register a pandas DataFrame ```python -breakpoint() import pandas as pd from sqlalchemy import text, __version__ as sqla_version from sqlalchemy.engine import create_engine conn = create_engine("duckdb:///:memory:").connect() -df = pd.DataFrame([{'id': 0}]) +df = pd.DataFrame([{"id": 0}]) -if sqla_version.startswith('1.3.'): +if sqla_version.startswith("1.3."): # with SQLAlchemy 1.3 conn.execute("register", ("dataframe_name", df)) else: # with SQLAlchemy 1.4+ - conn.execute(text("register(:name, :df)"), {"name": "dataframe_name", "df": df}) + conn.execute( + text("register(:name, :df)"), + {"name": "dataframe_name", "df": df}, + ) conn.execute(text("select * from dataframe_name")) ``` @@ -118,19 +128,20 @@ The following example demonstrates how to create an auto-incrementing ID column ```python import sqlalchemy -engine = sqlalchemy.create_engine('duckdb:////path/to/duck.db') -metadata = sqlalchemy.MetaData() -user_id_seq = sqlalchemy.Sequence('user_id_seq') + +engine = sqlalchemy.create_engine("duckdb:///:memory:") +metadata = sqlalchemy.MetaData(engine) +user_id_seq = sqlalchemy.Sequence("user_id_seq") users_table = sqlalchemy.Table( - 'users', - metadata, - sqlalchemy.Column( - 'id', - sqlalchemy.Integer, - user_id_seq, - server_default=user_id_seq.next_value(), - primary_key=True, - ), + "users", + metadata, + sqlalchemy.Column( + "id", + sqlalchemy.Integer, + user_id_seq, + server_default=user_id_seq.next_value(), + primary_key=True, + ), ) metadata.create_all(bind=engine) ``` @@ -144,9 +155,10 @@ The `pandas.read_sql()` method can read tables from `duckdb_engine` into DataFra ```python notest import pandas as pd import sqlalchemy -engine = sqlalchemy.create_engine('duckdb:////path/to/duck.db') -df = pd.read_sql('users', engine) ### Works as expected -df = pd.read_sql('users', engine, chunksize=25) ### Throws an exception + +engine = sqlalchemy.create_engine("duckdb:////path/to/duck.db") +df = pd.read_sql("users", engine) ### Works as expected +df = pd.read_sql("users", engine, chunksize=25) ### Throws an exception ``` ### Unsigned integer support @@ -162,6 +174,7 @@ This support can be enabling by adding an Alembic implementation class for the ` ```python notest from alembic.ddl.impl import DefaultImpl + class AlembicDuckDBImpl(DefaultImpl): """Alembic implementation for DuckDB.""" @@ -180,13 +193,11 @@ Until the DuckDB python client allows you to natively preload extensions, I've a from sqlalchemy import create_engine create_engine( - 'duckdb:///:memory:', + "duckdb:///:memory:", connect_args={ - 'preload_extensions': ['https'], - 'config': { - 's3_region': 'ap-southeast-1' - } - } + "preload_extensions": ["https"], + "config": {"s3_region": "ap-southeast-1"}, + }, ) ``` diff --git a/duckdb_engine/datatypes.py b/duckdb_engine/datatypes.py index ab2455d08..8d2b960c6 100644 --- a/duckdb_engine/datatypes.py +++ b/duckdb_engine/datatypes.py @@ -118,9 +118,9 @@ class Struct(TypeEngine): from sqlalchemy import Table, Column, String, MetaData Table( - 'hello', + "hello", MetaData(), - Column('name', Struct({'first': String, 'last': String})) + Column("name", Struct({"first": String, "last": String})), ) ``` @@ -141,11 +141,7 @@ class Map(TypeEngine): from duckdb_engine.datatypes import Map from sqlalchemy import Table, Column, String, MetaData - Table( - 'hello', - MetaData(), - Column('name', Map(String, String)) - ) + Table("hello", MetaData(), Column("name", Map(String, String))) ``` """ @@ -184,9 +180,9 @@ class Union(TypeEngine): from sqlalchemy import Table, Column, String, MetaData Table( - 'hello', + "hello", MetaData(), - Column('name', Union({"name": String, "age": String})) + Column("name", Union({"name": String, "age": String})), ) ``` """