Skip to content

Commit 186abe0

Browse files
NightMarcherNightMarcher
committed
feat: add param merge_fields to upsert_on_duplicate
1 parent 468a2a8 commit 186abe0

File tree

7 files changed

+53
-37
lines changed

7 files changed

+53
-37
lines changed

‎README.md

+9-9
Original file line numberDiff line numberDiff line change
@@ -119,24 +119,24 @@ Generate sql and execute
119119
```python
120120
await AccountMgr.upsert_on_duplicate(
121121
[
122-
{"id": 7, "gender": 1, "name": "斉藤 修平", "locale": "ja_JP", "extend": {}},
123-
{"id": 8, "gender": 1, "name": "Ojas Salvi", "locale": "en_IN", "extend": {}},
124-
{"id": 9, "gender": 1, "name": "羊淑兰", "locale": "zh_CN", "extend": {}}
122+
{"id": 10, "gender": 1, "name": "田中 知実", "locale": "ja_JP", "extend": {"rdm": 1}},
123+
{"id": 11, "gender": 2, "name": "Tara Chadha", "locale": "en_IN", "extend": {"rdm": 10}},
124+
{"id": 12, "gender": 2, "name": "吴磊", "locale": "zh_CN", "extend": {"rdm": 9}},
125125
],
126126
insert_fields=["id", "gender", "name", "locale", "extend"],
127-
upsert_fields=["name", "locale"],
128-
using_values=False,
127+
upsert_fields=["gender", "name"],
128+
merge_fields=["extend"],
129129
)
130130
```
131131
Generate sql and execute
132132
```sql
133133
INSERT INTO account
134134
(id, gender, name, locale, extend)
135135
VALUES
136-
(7, 1, '斉藤 修平', 'ja_JP', '{}'),
137-
(8, 1, 'Ojas Salvi', 'en_IN', '{}'),
138-
(9, 1, '羊淑兰', 'zh_CN', '{}')
139-
AS `new_account` ON DUPLICATE KEY UPDATE name=`new_account`.name, locale=`new_account`.locale
136+
(10, 1, '田中 知実', 'ja_JP', '{"rdm": 1}'),
137+
(11, 2, 'Tara Chadha', 'en_IN', '{"rdm": 10}'),
138+
(12, 2, '吴磊', 'zh_CN', '{"rdm": 9}')
139+
AS `new_account` ON DUPLICATE KEY UPDATE gender=`new_account`.gender, name=`new_account`.name, extend=JSON_MERGE_PATCH(COALESCE(account.extend, '{}'), `new_account`.extend)
140140
```
141141

142142
### **insert_into_select**

‎examples/service/routers/account.py

+6-2
Original file line numberDiff line numberDiff line change
@@ -160,12 +160,16 @@ async def bulk_upsert_view():
160160
"gender": faker.random_int(0, 2),
161161
"name": faker.name(),
162162
"locale": locale.value,
163-
"extend": {},
163+
"extend": {"rdm": faker.random_int(0, 10)},
164164
})
165+
from json import dumps
166+
print(dumps(dicts, ensure_ascii=False))
165167
row_cnt = await AccountMgr.upsert_on_duplicate(
166168
dicts,
167169
insert_fields=["id", "gender", "name", "locale", "extend"],
168-
# upsert_fields=["name", "locale"],
170+
upsert_fields=["gender", "name"],
171+
merge_fields=["extend"],
172+
using_values=True,
169173
)
170174
return {"row_cnt": row_cnt}
171175

‎fastapi_esql/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
timing,
2323
)
2424

25-
__version__ = "0.0.10"
25+
__version__ = "0.0.11"
2626

2727
__all__ = [
2828
"QsParsingError",

‎fastapi_esql/orm/base_manager.py

+2
Original file line numberDiff line numberDiff line change
@@ -150,13 +150,15 @@ async def upsert_on_duplicate(
150150
dicts: List[Dict[str, Any]],
151151
insert_fields: List[str],
152152
upsert_fields: Optional[List[str]] = None,
153+
merge_fields: Optional[List[str]] = None,
153154
using_values: bool = False,
154155
):
155156
sql = SQLizer.upsert_on_duplicate(
156157
cls.table,
157158
dicts,
158159
insert_fields,
159160
upsert_fields,
161+
merge_fields,
160162
using_values,
161163
)
162164
return await CursorHandler.sum_row_cnt(sql, cls.rw_conn, logger)

‎fastapi_esql/utils/sqlizer.py

+10-1
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,7 @@ def upsert_on_duplicate(
196196
dicts: List[Dict[str, Any]],
197197
insert_fields: List[str],
198198
upsert_fields: Optional[List[str]] = None,
199+
merge_fields: Optional[List[str]] = None,
199200
using_values: bool = False,
200201
) -> Optional[str]:
201202
if not all([table, dicts, insert_fields]):
@@ -208,13 +209,21 @@ def upsert_on_duplicate(
208209
# NOTE Beginning with MySQL 8.0.19, it is possible to use an alias for the row
209210
# https://dev.mysql.com/doc/refman/8.0/en/insert-on-duplicate.html
210211
on_duplicate = ""
211-
if upsert_fields:
212+
if upsert_fields or merge_fields:
213+
upsert_fields = upsert_fields or []
214+
merge_fields = merge_fields or []
212215
if using_values:
213216
upserts = [f"{field}=VALUES({field})" for field in upsert_fields]
217+
for mf in merge_fields:
218+
dict_obj = f"COALESCE({table}.{mf}, '{{}}')"
219+
upserts.append(f"{mf}=JSON_MERGE_PATCH({dict_obj}, VALUES({mf}))")
214220
on_duplicate = f"ON DUPLICATE KEY UPDATE {', '.join(upserts)}"
215221
else:
216222
new_table = f"`new_{table}`"
217223
upserts = [f"{field}={new_table}.{field}" for field in upsert_fields]
224+
for mf in merge_fields:
225+
dict_obj = f"COALESCE({table}.{mf}, '{{}}')"
226+
upserts.append(f"{mf}=JSON_MERGE_PATCH({dict_obj}, {new_table}.{mf})")
218227
on_duplicate = f"AS {new_table} ON DUPLICATE KEY UPDATE {', '.join(upserts)}"
219228

220229
sql = """

‎pyproject.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[tool.poetry]
22
name = "fastapi-efficient-sql"
3-
version = "0.0.10"
3+
version = "0.0.11"
44
description = "Generate bulk DML SQL and execute them based on Tortoise ORM and mysql8.0+, and integrated with FastAPI."
55
authors = ["BryanLee <[email protected]>"]
66
keywords = ["sql", "fastapi", "tortoise-orm", "mysql8", "bulk-operation"]

‎tests/test_sqlizer.py

+24-23
Original file line numberDiff line numberDiff line change
@@ -212,61 +212,62 @@ def test_upsert_on_duplicate(self):
212212
old_sql = SQLizer.upsert_on_duplicate(
213213
self.table,
214214
[
215-
{"id": 7, "gender": 1, "name": "斉藤 修平", "locale": "ja_JP", "extend": {}},
216-
{"id": 8, "gender": 1, "name": "Ojas Salvi", "locale": "en_IN", "extend": {}},
217-
{"id": 9, "gender": 1, "name": "羊淑兰", "locale": "zh_CN", "extend": {}}
215+
{"id": 10, "gender": 2, "name": "佐々木 美加子", "locale": "ja_JP", "extend": {"rdm": 6}},
216+
{"id": 11, "gender": 0, "name": "Seher Bumb", "locale": "en_IN", "extend": {"rdm": 4}},
217+
{"id": 12, "gender": 0, "name": "谢冬梅", "locale": "zh_CN", "extend": {"rdm": 6}},
218218
],
219219
insert_fields=["id", "gender", "name", "locale", "extend"],
220-
upsert_fields=["name", "locale"],
220+
upsert_fields=["gender", "name"],
221+
merge_fields=["extend"],
221222
using_values=True,
222223
)
223224
assert old_sql == """
224225
INSERT INTO account
225226
(id, gender, name, locale, extend)
226227
VALUES
227-
(7, 1, '斉藤 修平', 'ja_JP', '{}'),
228-
(8, 1, 'Ojas Salvi', 'en_IN', '{}'),
229-
(9, 1, '羊淑兰', 'zh_CN', '{}')
230-
ON DUPLICATE KEY UPDATE name=VALUES(name), locale=VALUES(locale)
228+
(10, 2, '佐々木 美加子', 'ja_JP', '{"rdm": 6}'),
229+
(11, 0, 'Seher Bumb', 'en_IN', '{"rdm": 4}'),
230+
(12, 0, '谢冬梅', 'zh_CN', '{"rdm": 6}')
231+
ON DUPLICATE KEY UPDATE gender=VALUES(gender), name=VALUES(name), extend=JSON_MERGE_PATCH(COALESCE(account.extend, '{}'), VALUES(extend))
231232
"""
232233

233234
new_sql = SQLizer.upsert_on_duplicate(
234235
self.table,
235236
[
236-
{"id": 7, "gender": 1, "name": "斉藤 修平", "locale": "ja_JP", "extend": {}},
237-
{"id": 8, "gender": 1, "name": "Ojas Salvi", "locale": "en_IN", "extend": {}},
238-
{"id": 9, "gender": 1, "name": "羊淑兰", "locale": "zh_CN", "extend": {}}
237+
{"id": 10, "gender": 1, "name": "田中 知実", "locale": "ja_JP", "extend": {"rdm": 1}},
238+
{"id": 11, "gender": 2, "name": "Tara Chadha", "locale": "en_IN", "extend": {"rdm": 10}},
239+
{"id": 12, "gender": 2, "name": "吴磊", "locale": "zh_CN", "extend": {"rdm": 9}},
239240
],
240241
insert_fields=["id", "gender", "name", "locale", "extend"],
241-
upsert_fields=["name", "locale"],
242-
using_values=False,
242+
upsert_fields=["gender", "name"],
243+
merge_fields=["extend"],
243244
)
244245
assert new_sql == """
245246
INSERT INTO account
246247
(id, gender, name, locale, extend)
247248
VALUES
248-
(7, 1, '斉藤 修平', 'ja_JP', '{}'),
249-
(8, 1, 'Ojas Salvi', 'en_IN', '{}'),
250-
(9, 1, '羊淑兰', 'zh_CN', '{}')
251-
AS `new_account` ON DUPLICATE KEY UPDATE name=`new_account`.name, locale=`new_account`.locale
249+
(10, 1, '田中 知実', 'ja_JP', '{"rdm": 1}'),
250+
(11, 2, 'Tara Chadha', 'en_IN', '{"rdm": 10}'),
251+
(12, 2, '吴磊', 'zh_CN', '{"rdm": 9}')
252+
AS `new_account` ON DUPLICATE KEY UPDATE gender=`new_account`.gender, name=`new_account`.name, extend=JSON_MERGE_PATCH(COALESCE(account.extend, '{}'), `new_account`.extend)
252253
"""
253254

254255
only_insert_sql = SQLizer.upsert_on_duplicate(
255256
self.table,
256257
[
257-
{"id": 7, "gender": 1, "name": "斉藤 修平", "locale": "ja_JP", "extend": {}},
258-
{"id": 8, "gender": 1, "name": "Ojas Salvi", "locale": "en_IN", "extend": {}},
259-
{"id": 9, "gender": 1, "name": "羊淑兰", "locale": "zh_CN", "extend": {}}
258+
{"id": 10, "gender": 2, "name": "池田 幹", "locale": "ja_JP", "extend": {"rdm": 9}},
259+
{"id": 11, "gender": 0, "name": "Sana Mani", "locale": "en_IN", "extend": {"rdm": 0}},
260+
{"id": 12, "gender": 0, "name": "刘柳", "locale": "zh_CN", "extend": {"rdm": 9}},
260261
],
261262
insert_fields=["id", "gender", "name", "locale", "extend"],
262263
)
263264
assert only_insert_sql == """
264265
INSERT INTO account
265266
(id, gender, name, locale, extend)
266267
VALUES
267-
(7, 1, '斉藤 修平', 'ja_JP', '{}'),
268-
(8, 1, 'Ojas Salvi', 'en_IN', '{}'),
269-
(9, 1, '羊淑兰', 'zh_CN', '{}')
268+
(10, 2, '池田 幹', 'ja_JP', '{"rdm": 9}'),
269+
(11, 0, 'Sana Mani', 'en_IN', '{"rdm": 0}'),
270+
(12, 0, '刘柳', 'zh_CN', '{"rdm": 9}')
270271
"""
271272

272273
def test_insert_into_select(self):

0 commit comments

Comments
 (0)