From ccd0c9d97c5e2ff328265271dc9926892d968d64 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 27 Apr 2022 07:52:31 -0700 Subject: [PATCH] draft of ilike filter --- .../blueprints/client/api.py | 8 ++++++++ .../blueprints/client/query.py | 19 ++++++++++++++++++- .../blueprints/client/schemas.py | 10 ++-------- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/materializationengine/blueprints/client/api.py b/materializationengine/blueprints/client/api.py index 06444f53..960f09df 100644 --- a/materializationengine/blueprints/client/api.py +++ b/materializationengine/blueprints/client/api.py @@ -475,6 +475,11 @@ def post(self, datastack_name: str, table_name: str): "tablename": { "column_name": [[min_x, min_y, min_z], [max_x, max_y, max_z]] } + "filter_like_dict": { + "tablename":{ + "column_name": "%phrase_searching%" + } + } } Returns: pyarrow.buffer: a series of bytes that can be deserialized using pyarrow.deserialize @@ -549,6 +554,7 @@ def _format_filter(filter, table_in, seg_table): data.get("filter_equal_dict", None), table_name, seg_table ), filter_spatial=data.get("filter_spatial_dict", None), + filter_like=data.get("filter_like_dict", None), select_columns=data.get("select_columns", None), consolidate_positions=not args["split_positions"], offset=data.get("offset", None), @@ -677,6 +683,7 @@ def post(self, datastack_name: str, version: int, table_name: str): filter_notin_dict=data.get("filter_notin_dict", None), filter_equal_dict=data.get("filter_equal_dict", None), filter_spatial=data.get("filter_spatial_dict", None), + filter_like=data.get("filter_like_dict", None), select_columns=data.get("select_columns", None), consolidate_positions=not args["split_positions"], offset=data.get("offset", None), @@ -786,6 +793,7 @@ def post(self, datastack_name: str, version: int): filter_notin_dict=data.get("filter_notin_dict", None), filter_equal_dict=data.get("filter_equal_dict", None), filter_spatial=data.get("filter_spatial_dict", None), + filter_like=data.get("filter_like_dict", None), select_columns=data.get("select_columns", None), consolidate_positions=not args["split_positions"], offset=data.get("offset", None), diff --git a/materializationengine/blueprints/client/query.py b/materializationengine/blueprints/client/query.py index 72c1ed81..df4fef8c 100644 --- a/materializationengine/blueprints/client/query.py +++ b/materializationengine/blueprints/client/query.py @@ -15,6 +15,7 @@ from geoalchemy2.types import Geometry from multiwrapper import multiprocessing_utils as mu from sqlalchemy import func, not_ +from sqlalchemy.sql.operators import ilike_op from sqlalchemy.orm import Query, Session from sqlalchemy.sql.sqltypes import Boolean, Integer from sqlalchemy.ext.declarative import DeclarativeMeta @@ -238,6 +239,7 @@ def specific_query( filter_notin_dict=None, filter_equal_dict=None, filter_spatial=None, + filter_like=None, select_columns=None, consolidate_positions=True, return_wkb=False, @@ -268,6 +270,10 @@ def specific_query( outer layer: keys are table_namess inner layer: keys are column names, values are [min,max] as list of lists e.g. [[0,0,0], [1,1,1]] + filter_like: dict of dicts + outer layer: keys are table_namess + inner layer: keys are column names, values are a string to pass to ilike, + columns must be string types select_columns: list of str consolidate_positions: whether to make the position columns arrays of x,y,z offset: int @@ -378,7 +384,18 @@ def specific_query( bounding_box = filter_table_dict[column_name] filter = make_spatial_filter(model, column_name, bounding_box) filter_args.append((filter,)) - + + if filter_like is not None: + for filter_table, filter_table_dict in filter_like.items(): + for column_name in filter_table_dict.keys(): + filter_value = filter_table_dict[column_name] + filter_args.append( + ( + ilike_op( + model_dict[filter_table].__dict__[column_name], filter_value + ), + ) + ) df = _query( sqlalchemy_session, engine, diff --git a/materializationengine/blueprints/client/schemas.py b/materializationengine/blueprints/client/schemas.py index 0953109d..3f5e23d9 100644 --- a/materializationengine/blueprints/client/schemas.py +++ b/materializationengine/blueprints/client/schemas.py @@ -40,20 +40,14 @@ class SimpleQuerySchema(Schema): filter_notin_dict = fields.Dict() filter_equal_dict = fields.Dict() filter_spatial_dict = fields.Dict() + filter_like_dict = fields.Dict() select_columns = fields.List(fields.Str) offset = fields.Integer() limit = fields.Integer() -class ComplexQuerySchema(Schema): +class ComplexQuerySchema(SimpleQuerySchema): tables = fields.List( fields.List(fields.Str, validate=Length(equal=2)), required=True ) - filter_in_dict = fields.Dict() - filter_notin_dict = fields.Dict() - filter_equal_dict = fields.Dict() - filter_spatial_dict = fields.Dict() - select_columns = fields.List(fields.Str) - offset = fields.Integer() - limit = fields.Integer() suffixes = fields.List(fields.Str)