Merge branch '4-vector-types' into 'main'

Support vector types #4

See merge request objectbox/objectbox-python!5

Author: greenrobot

.gitlab-ci.yml

@@ -30,6 +30,7 @@ build:
   stage: test
   script:
     - pip3 install --user pytest
+    - pip3 install --user numpy
     - rm -r objectbox
     - pip3 install --user --force-reinstall dist/*.whl
     - ${PYTHON} -m pytest
@@ -59,9 +60,9 @@ test:linux:x64:3.10:
   extends: .test:linux:x64
   image: python:3.10
 
-test:linux:armv7hf:
-  extends: .test-python3
-  tags: [armv7hf, shell, linux, python3]
+# test:linux:armv7hf:
+#   extends: .test-python3
+#   tags: [armv7hf, shell, linux, python3]
 
 # aarch64 version not published
 #test:linux:aarch64:
@@ -71,6 +72,9 @@ test:linux:armv7hf:
 test:mac:x64:
   extends: .test-python3
   tags: [mac, x64, shell, python3]
+  script:
+    - make depend
+    - make test
 
 test:windows:x64:
   extends: .test

download-c-lib.py

@@ -6,7 +6,7 @@
 # Script used to download objectbox-c shared libraries for all supported platforms. Execute by running `make get-lib`
 # on first checkout of this repo and any time after changing the objectbox-c lib version.
 
-version = "v0.14.0"  # see objectbox/c.py required_version
+version = "v0.18.1"  # see objectbox/c.py required_version
 variant = 'objectbox'  # or 'objectbox-sync'
 
 base_url = "https://github.com/objectbox/objectbox-c/releases/download/"

objectbox/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2021 ObjectBox Ltd. All rights reserved.
+# Copyright 2019-2023 ObjectBox Ltd. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -31,7 +31,7 @@
 ]
 
 # Python binding version
-version = Version(0, 4, 0)
+version = Version(0, 5, 0)
 
 
 def version_info():

objectbox/c.py

@@ -24,7 +24,7 @@
 
 # Version of the library used by the binding. This version is checked at runtime to ensure binary compatibility.
 # Don't forget to update download-c-lib.py when upgrading to a newer version.
-required_version = "0.14.0"
+required_version = "0.18.1"
 
 
 def shlib_name(library: str) -> str:
@@ -395,6 +395,10 @@ def c_voidp_as_bytes(voidp, size):
 OBXPropertyType_Date = 10
 OBXPropertyType_Relation = 11
 OBXPropertyType_ByteVector = 23
+OBXPropertyType_IntVector = 26
+OBXPropertyType_LongVector = 27
+OBXPropertyType_FloatVector = 28
+OBXPropertyType_DoubleVector = 29
 OBXPropertyType_StringVector = 30
 
 OBXPropertyFlags_ID = 1

objectbox/model/entity.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2021 ObjectBox Ltd. All rights reserved.
+# Copyright 2019-2023 ObjectBox Ltd. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,6 +14,7 @@
 
 
 import flatbuffers
+import numpy as np
 from objectbox.c import *
 from objectbox.model.properties import Property
 
@@ -64,8 +65,14 @@ def fill_properties(self):
                 self.id_property = prop
 
             if prop._fb_type == flatbuffers.number_types.UOffsetTFlags:
-                assert prop._ob_type in [OBXPropertyType_String, OBXPropertyType_ByteVector], \
-                    "programming error - invalid type OB & FB type combination"
+                assert prop._ob_type in [
+                    OBXPropertyType_String,
+                    OBXPropertyType_ByteVector,
+                    OBXPropertyType_IntVector,
+                    OBXPropertyType_LongVector,
+                    OBXPropertyType_FloatVector,
+                    OBXPropertyType_DoubleVector,
+                ], "programming error - invalid type OB & FB type combination"
                 self.offset_properties.append(prop)
 
             # print('Property {}.{}: {} (ob:{} fb:{})'.format(self.name, prop._name, prop._py_type, prop._ob_type, prop._fb_type))
@@ -78,8 +85,11 @@ def fill_properties(self):
     def get_value(self, object, prop: Property):
         # in case value is not overwritten on the object, it's the Property object itself (= as defined in the Class)
         val = getattr(object, prop._name)
-        if val == prop:
-            return prop._py_type()  # default (empty) value for the given type
+        if prop._py_type == np.ndarray:
+            if (val == np.array(prop)).all():
+                return np.array([])
+        elif val == prop:
+                return prop._py_type()  # default (empty) value for the given type
         return val
 
     def get_object_id(self, object) -> int:
@@ -99,6 +109,14 @@ def marshal(self, object, id: int) -> bytearray:
                 offsets[prop._id] = builder.CreateString(val.encode('utf-8'))
             elif prop._ob_type == OBXPropertyType_ByteVector:
                 offsets[prop._id] = builder.CreateByteVector(val)
+            elif prop._ob_type == OBXPropertyType_IntVector:
+                offsets[prop._id] = builder.CreateNumpyVector(np.array(val, dtype=np.int32))
+            elif prop._ob_type == OBXPropertyType_LongVector:
+                offsets[prop._id] = builder.CreateNumpyVector(np.array(val, dtype=np.int64))
+            elif prop._ob_type == OBXPropertyType_FloatVector:
+                offsets[prop._id] = builder.CreateNumpyVector(np.array(val, dtype=np.float32))
+            elif prop._ob_type == OBXPropertyType_DoubleVector:
+                offsets[prop._id] = builder.CreateNumpyVector(np.array(val, dtype=np.float64))
             else:
                 assert False, "programming error - invalid type OB & FB type combination"
 
@@ -140,12 +158,20 @@ def unmarshal(self, data: bytes):
                 # access the FB byte vector information
                 start = table.Vector(o)
                 size = table.VectorLen(o)
-
                 # slice the vector as a requested type
                 val = prop._py_type(table.Bytes[start:start+size])
+            elif prop._ob_type == OBXPropertyType_IntVector:
+                val = table.GetVectorAsNumpy(flatbuffers.number_types.Int32Flags, o)
+            elif prop._ob_type == OBXPropertyType_LongVector:
+                val = table.GetVectorAsNumpy(flatbuffers.number_types.Int64Flags, o)
+            elif prop._ob_type == OBXPropertyType_FloatVector:
+                val = table.GetVectorAsNumpy(flatbuffers.number_types.Float32Flags, o)
+            elif prop._ob_type == OBXPropertyType_DoubleVector:
+                val = table.GetVectorAsNumpy(flatbuffers.number_types.Float64Flags, o)
             else:
                 val = table.Get(prop._fb_type, o + table.Pos)
-
+            if prop._py_type == list:
+                val = val.tolist()
             setattr(obj, prop._name, val)
         return obj
 

objectbox/model/properties.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2021 ObjectBox Ltd. All rights reserved.
+# Copyright 2019-2023 ObjectBox Ltd. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@
 
 from objectbox.c import *
 import flatbuffers.number_types
+import numpy as np
 
 
 class PropertyType(IntEnum):
@@ -31,6 +32,10 @@ class PropertyType(IntEnum):
     # date = OBXPropertyType_Date
     # relation = OBXPropertyType_Relation
     byteVector = OBXPropertyType_ByteVector
+    intVector = OBXPropertyType_IntVector
+    longVector = OBXPropertyType_LongVector
+    floatVector = OBXPropertyType_FloatVector
+    doubleVector = OBXPropertyType_DoubleVector
     # stringVector = OBXPropertyType_StringVector
 
 
@@ -47,6 +52,10 @@ class PropertyType(IntEnum):
     # PropertyType.date: flatbuffers.number_types.Int64Flags,
     # PropertyType.relation: flatbuffers.number_types.Int64Flags,
     PropertyType.byteVector: flatbuffers.number_types.UOffsetTFlags,
+    PropertyType.intVector: flatbuffers.number_types.UOffsetTFlags,
+    PropertyType.longVector: flatbuffers.number_types.UOffsetTFlags,
+    PropertyType.floatVector: flatbuffers.number_types.UOffsetTFlags,
+    PropertyType.doubleVector: flatbuffers.number_types.UOffsetTFlags,
     # PropertyType.stringVector: flatbuffers.number_types.UOffsetTFlags,
 }
 
@@ -77,6 +86,8 @@ def __determine_ob_type(self) -> OBXPropertyType:
             return OBXPropertyType_Long
         elif ts == bytes:  # or ts == bytearray: might require further tests on read objects due to mutability
             return OBXPropertyType_ByteVector
+        elif ts == list or ts == np.ndarray:
+            return OBXPropertyType_DoubleVector
         elif ts == float:
             return OBXPropertyType_Double
         elif ts == bool:

requirements.txt

@@ -1,5 +1,6 @@
 pip
 wheel
-flatbuffers==2.0
+flatbuffers==23.5.26
 pytest>=4.4.1
-twine
+twine
+numpy

setup.py

@@ -46,7 +46,7 @@
     ],
 
     install_requires=[
-       'flatbuffers==2.0',
+       'flatbuffers==23.5.26',
     ],
 
     packages=setuptools.find_packages(),

tests/common.py

@@ -3,6 +3,7 @@
 import shutil
 import pytest
 from tests.model import TestEntity
+import numpy as np
 
 test_dir = 'testdata'
 
@@ -23,7 +24,7 @@ def autocleanup():
 def load_empty_test_objectbox(name: str = "") -> objectbox.ObjectBox:
     model = objectbox.Model()
     from objectbox.model import IdUid
-    model.entity(TestEntity, last_property_id=IdUid(10, 1010))
+    model.entity(TestEntity, last_property_id=IdUid(18, 1018))
     model.last_entity_id = IdUid(1, 1)
 
     db_name = test_dir if len(name) == 0 else test_dir + "/" + name
@@ -35,6 +36,14 @@ def assert_equal_prop(actual, expected, default):
     assert actual == expected or (isinstance(
         expected, objectbox.model.Property) and actual == default)
 
+def assert_equal_prop_vector(actual, expected, default):
+    assert (actual == np.array(expected)).all() or (isinstance(
+        expected, objectbox.model.Property) and actual == default)
+
+# compare approx values for list
+def assert_equal_prop_list(actual, expected, default):
+    assert pytest.approx(actual) == expected or (isinstance(
+        expected, objectbox.model.Property) and actual == default)
 
 def assert_equal(actual: TestEntity, expected: TestEntity):
     """Check that two TestEntity objects have the same property data"""
@@ -46,3 +55,11 @@ def assert_equal(actual: TestEntity, expected: TestEntity):
     assert_equal_prop(actual.float64, expected.float64, 0)
     assert_equal_prop(actual.float32, expected.float32, 0)
     assert_equal_prop(actual.bytes, expected.bytes, b'')
+    assert_equal_prop_vector(actual.ints, expected.ints, np.array([]))
+    assert_equal_prop_vector(actual.longs, expected.longs, np.array([]))
+    assert_equal_prop_vector(actual.floats, expected.floats, np.array([]))
+    assert_equal_prop_vector(actual.doubles, expected.doubles, np.array([]))
+    assert_equal_prop_list(actual.ints_list, expected.ints_list, [])
+    assert_equal_prop_list(actual.longs_list, expected.longs_list, [])
+    assert_equal_prop_list(actual.floats_list, expected.floats_list, [])
+    assert_equal_prop_list(actual.doubles_list, expected.doubles_list, [])

tests/model.py

@@ -1,4 +1,5 @@
 from objectbox.model import *
+import numpy as np
 
 
 @Entity(id=1, uid=1)
@@ -13,6 +14,14 @@ class TestEntity:
     float64 = Property(float, type=PropertyType.double, id=8, uid=1008)
     float32 = Property(float, type=PropertyType.float, id=9, uid=1009)
     bytes = Property(bytes, id=10, uid=1010)
+    ints = Property(np.ndarray, type=PropertyType.intVector, id=11, uid=1011)
+    longs = Property(np.ndarray, type=PropertyType.longVector, id=12, uid=1012)
+    floats = Property(np.ndarray, type=PropertyType.floatVector, id=13, uid=1013)
+    doubles = Property(np.ndarray, type=PropertyType.doubleVector, id=14, uid=1014)
+    ints_list = Property(list, type=PropertyType.intVector, id=15, uid=1015)
+    longs_list = Property(list, type=PropertyType.longVector, id=16, uid=1016)
+    floats_list = Property(list, type=PropertyType.floatVector, id=17, uid=1017)
+    doubles_list = Property(list, type=PropertyType.doubleVector, id=18, uid=1018)
     transient = ""  # not "Property" so it's not stored
 
     def __init__(self, string: str = ""):

tests/test_box.py

@@ -1,5 +1,6 @@
 import pytest
 import objectbox
+import numpy as np
 from tests.model import TestEntity
 from tests.common import autocleanup, load_empty_test_objectbox, assert_equal
 
@@ -29,12 +30,19 @@ def test_box_basics():
     object.float64 = 4.2
     object.float32 = 1.5
     object.bytes = bytes([1, 1, 2, 3, 5])
+    object.ints = np.array([1, 2, 3, 555, 120, 222], dtype=np.int32)
+    object.longs = np.array([9182, 8273, 7364, 6455, 55462547], dtype=np.int64)
+    object.floats = np.array([0.1, 1.2, 2.3, 3.4, 4.5], dtype=np.float32)
+    object.doubles = np.array([99.99, 88.88, 77.77, 66.66, 55.595425], dtype=np.float64)
+    object.ints_list = [91, 82, 73, 64, 55]
+    object.longs_list = [4568, 8714, 1234, 5678, 9012240941]
+    object.floats_list = [0.11, 1.22, 2.33, 3.44, 4.5595]
+    object.doubles_list = [99.1999, 88.2888, 77.3777, 66.4666, 55.6597555]
     object.transient = "abcd"
 
     id = box.put(object)
     assert id == 5
     assert id == object.id
-
     # check the count
     assert not box.is_empty()
     assert box.count() == 2