tweaks to NeoAccess.import_json_dump()

New optional argument extended_validation.   Additional pytests.
Updated pyproject.toml

Author: BrainAnnex

README.md

@@ -4,4 +4,4 @@ https://brainannex.org/guide.php
 
 
 This library used to be distributed together with the web app "Brain Annex";
-but, as of version 4.0.3, it's being independently released.
+but, starting with version 4.0.3, it's being independently released.

pyproject.toml

@@ -35,5 +35,5 @@ keywords = [
 Homepage = "https://brainannex.org/guide.php"
 "Home-page" = "https://brainannex.org/guide.php"
 "Bug Tracker" = "https://github.com/BrainAnnex/brain-annex/issues"
-Source = "https://github.com/BrainAnnex/brain-annex/tree/main/BrainAnnex/modules/neo_access"
+Source = "https://github.com/BrainAnnex/neoaccess"
 Documentation = "https://brainannex.org/guide.php"

src/neoaccess/neoaccess.py

@@ -630,7 +630,7 @@ def get_nodes(self, match: Union[int, dict],
         :param match:           EITHER an integer with a Neo4j node id,
                                 OR a dictionary of data to identify a node, or set of nodes, as returned by match()
 
-        :param return_internal_id:   Flag indicating whether to also include the Neo4j internal node ID in the returned data
+        :param return_internal_id:  Flag indicating whether to also include the Neo4j internal node ID in the returned data
                                     (using "internal_id" as its key in the returned dictionary)
         :param return_labels:   Flag indicating whether to also include the Neo4j label names in the returned data
                                     (using "neo4j_labels" as its key in the returned dictionary)
@@ -2319,7 +2319,7 @@ def drop_all_indexes(self, including_constraints=True) -> None:
             if self.apoc:
                 self.query("call apoc.schema.assert({},{})")
             else:
-                self.drop_all_constraints()    # TODO: it doesn't work in version 5.5 of the database
+                self.drop_all_constraints()    # TODO: it doesn't work in version 5.5 of the Neo4j database
 
         indexes = self.get_indexes()
         for name in indexes['name']:
@@ -2815,85 +2815,122 @@ def list_importer(self, l: list, labels, level) -> [int]:
 
 
 
-
-    def import_json_dump(self, json_str: str) -> str:
+    def import_json_dump(self, json_str: str, extended_validation = True) -> str:
         """
-        Used to import data from a database dump done with export_dbase_json() or export_nodes_rels_json()
-        Import nodes and/or relationships into the database, as directed by the given data dump in JSON form.
-        Note: the id's of the nodes need to be shifted,
+        Used to import data from a database dump that was done with export_dbase_json() or export_nodes_rels_json().
+
+        Import nodes and relationships into the database, as specified in the JSON code
+        that was created by the earlier data dump.
+
+        IMPORTANT: the internal id's of the nodes need to be shifted,
               because one cannot force the Neo4j internal id's to be any particular value...
               and, besides (if one is importing into an existing database), particular id's may already be taken.
-        :param json_str:    A JSON string with the format specified under export_dbase_json()
-        :return:            A status message with import details if successful, or raise an Exception if not
+
+        :param json_str:            A JSON string with the format specified under export_dbase_json()
+        :param extended_validation: If True, an attempt is made to try to avoid partial imports,
+                                        by running extended validations prior to importing
+                                        (it will make a first pass thru the data, and hence take longer)
+
+        :return:                    A status message with import details if successful;
+                                        or raise an Exception if not.
+                                        If an error does occur during import then the import is aborted -
+                                        and the number of imported nodes & relationships is returned in the Exception raised.
         """
 
         try:
-            json_list = json.loads(json_str)    # Turn the string (representing a JSON list) into a list
+            json_list = json.loads(json_str)    # Turn the string (which represents a JSON list) into a list
         except Exception as ex:
-            raise Exception(f"Incorrectly-formatted JSON string. {ex}")
+            raise Exception(f"import_json_dump(): incorrectly-formatted JSON string. {ex}")
 
         if self.debug:
             print("json_list: ", json_list)
 
-        assert type(json_list) == list, "The JSON string does not represent the expected list"
+        assert type(json_list) == list, \
+            "import_json_dump(): the JSON string does not represent a list"
+
 
         id_shifting = {}    # To map the Neo4j internal ID's specified in the JSON data dump
                             #       into the ID's of newly-created nodes
 
-        # Do an initial pass for correctness, to try to avoid partial imports
-        for i, item in enumerate(json_list):
-            # We use item.get(key_name) to handle without error situation where the key is missing
-            if (item.get("type") != "node") and (item.get("type") != "relationship"):
-                raise Exception(f"Item in list index {i} must have a 'type' of either 'node' or 'relationship'.  Nothing imported.  Item: {item}")
-
-            if item["type"] == "node":
-                if "id" not in item:
-                    raise Exception(f"Item in list index {i} is marked as 'node' but it lacks an 'id'.  Nothing imported.  Item: {item}")
-
-            elif item["type"] == "relationship":
-                if "label" not in item:
-                    raise Exception(f"Item in list index {i} is marked as 'relationship' but lacks a 'label'.  Nothing imported.  Item: {item}")
-                if "start" not in item:
-                    raise Exception(f"Item in list index {i} is marked as 'relationship' but lacks a 'start' value.  Nothing imported.  Item: {item}")
-                if "end" not in item:
-                    raise Exception(f"Item in list index {i} is marked as 'relationship' but lacks a 'end' value.  Nothing imported.  Item: {item}")
-                if "id" not in item["start"]:
-                    raise Exception(f"Item in list index {i} is marked as 'relationship' but its 'start' value lacks an 'id'.  Nothing imported.  Item: {item}")
-                if "id" not in item["end"]:
-                    raise Exception(f"Item in list index {i} is marked as 'relationship' but its 'end' value lacks an 'id'.  Nothing imported.  Item: {item}")
-
-
-        # First, process all the nodes, and in the process create the id_shifting map
+        if extended_validation:
+            # Do an initial pass for correctness, to help avoid partial imports.
+            # TODO: maybe also check the validity of the start and end nodes of relationships
+            for i, item in enumerate(json_list):
+                assert type(item) == dict, \
+                    f"import_json_dump(): Item in list index {i} should be a dict, but instead it's of type {type(item)}.  Nothing imported.  Item: {item}"
+                # We use item.get(key_name) to handle without error situation where the key is missing
+                if (item.get("type") != "node") and (item.get("type") != "relationship"):
+                    raise Exception(f"import_json_dump(): Item in list index {i} must be a dict with a 'type' key, "
+                                    f"whose value is either 'node' or 'relationship'.  Nothing imported.  Item: {item}")
+
+                if item["type"] == "node":
+                    if "id" not in item:
+                        raise Exception(f"import_json_dump(): Item in list index {i} is marked as 'node' but it lacks an 'id'.  Nothing imported.  Item: {item}")
+                    try:
+                        int(item["id"])
+                    except ValueError:
+                        raise Exception(f"import_json_dump(): Item in list index {i} has an 'id' key whose value ({item['id']}) doesn't correspond to an integer.  "
+                                        f"Nothing imported.  Item: {item}")
+
+                elif item["type"] == "relationship":
+                    if "label" not in item:
+                        raise Exception(f"import_json_dump(): Item in list index {i} is marked as 'relationship' but lacks a 'label'.  Nothing imported.  Item: {item}")
+                    if "start" not in item:
+                        raise Exception(f"import_json_dump(): Item in list index {i} is marked as 'relationship' but lacks a 'start' value.  Nothing imported.  Item: {item}")
+                    if "end" not in item:
+                        raise Exception(f"import_json_dump(): Item in list index {i} is marked as 'relationship' but lacks a 'end' value.  Nothing imported.  Item: {item}")
+                    if "id" not in item["start"]:
+                        raise Exception(f"import_json_dump(): Item in list index {i} is marked as 'relationship' but its 'start' value lacks an 'id'.  Nothing imported.  Item: {item}")
+                    if "id" not in item["end"]:
+                        raise Exception(f"import_json_dump(): Item in list index {i} is marked as 'relationship' but its 'end' value lacks an 'id'.  Nothing imported.  Item: {item}")
+
+
+        # First, process all the node data, and create the nodes; while doing that, generate the id_shifting map
         num_nodes_imported = 0
-        for item in json_list:
-            if item["type"] == "node":
-                #print("ADDING NODE: ", item)
-                #print(f'     Creating node with label `{item["labels"][0]}` and properties {item["properties"]}')
-                old_id = int(item["id"])
-                new_id = self.create_node(item["labels"][0], item["properties"])  # TODO: Only the 1st label is used for now
-                id_shifting[old_id] = new_id
-                num_nodes_imported += 1
+        try:
+            for item in json_list:
+                if item["type"] == "node":
+                    #print("ADDING NODE: ", item)
+                    #print(f'     Creating node with labels `{item["labels"]}` and properties {item["properties"]}')
+                    old_id = int(item["id"])
+                    new_id = self.create_node(item["labels"], item["properties"])  # Note: any number of labels can be imported
+                    id_shifting[old_id] = new_id
+                    num_nodes_imported += 1
+        except Exception as ex:
+            raise Exception(f"import_json_dump(): the import process was INTERRUPTED "
+                            f"after importing {num_nodes_imported} node(s) and 0 relationship(s). Reason: " + str(ex))
+
 
         #print("id_shifting map:", id_shifting)
 
         # Then process all the relationships, linking to the correct (newly-created) nodes by using the id_shifting map
+        # (node: item types that aren't either "node" nor "relationship" are currently being ignored during the import)
         num_rels_imported = 0
-        for item in json_list:
-            if item["type"] == "relationship":
-                #print("ADDING RELATIONSHIP: ", item)
-                rel_name = item["label"]
-                #rel_props = item["properties"]
-                rel_props = item.get("properties")      # Also works if no "properties" is present (relationships may lack it)
-
-                start_id_original = int(item["start"]["id"])
-                end_id_original = int(item["end"]["id"])
-
-                start_id_shifted = id_shifting[start_id_original]
-                end_id_shifted = id_shifting[end_id_original]
-                #print(f'     Creating relationship named `{rel_name}` from node {start_id_shifted} to node {end_id_shifted},  with properties {rel_props}')
-
-                self.link_nodes_by_ids(start_id_shifted, end_id_shifted, rel_name, rel_props)
-                num_rels_imported += 1
+        try:
+            for item in json_list:
+                if item["type"] == "relationship":
+                    #print("ADDING RELATIONSHIP: ", item)
+                    rel_name = item["label"]
+                    #rel_props = item["properties"]
+                    rel_props = item.get("properties")      # Also works if no "properties" is present (relationships may lack it)
+
+                    start_id_original = int(item["start"]["id"])
+                    end_id_original = int(item["end"]["id"])
+
+                    if start_id_original not in id_shifting:
+                        raise Exception(f"cannot add a relationship `{rel_name}` starting at node with id {start_id_original}, because no node with that id was imported")
+                    if end_id_original not in id_shifting:
+                        raise Exception(f"cannot add a relationship `{rel_name}` ending at node with id {start_id_original}, because no node with that id was imported")
+
+                    start_id_shifted = id_shifting[start_id_original]
+                    end_id_shifted = id_shifting[end_id_original]
+
+                    #print(f'     Creating relationship named `{rel_name}` from node {start_id_shifted} to node {end_id_shifted},  with properties {rel_props}')
+                    self.link_nodes_by_ids(start_id_shifted, end_id_shifted, rel_name, rel_props)
+                    num_rels_imported += 1
+        except Exception as ex:
+            raise Exception(f"import_json_dump(): the import process was INTERRUPTED "
+                            f"after importing {num_nodes_imported} node(s) and {num_rels_imported} relationship(s). Reason: " + str(ex))
 
 
         return f"Successful import of {num_nodes_imported} node(s) and {num_rels_imported} relationship(s)"
@@ -3002,4 +3039,9 @@ def indent_chooser(self, level: int) -> str:
 
 
     def _debug_local(self) -> str:
-        return "local"
+        """
+        Use to test the switch from a local to remote repository, for debugging
+
+        :return:
+        """
+        return "remote"

tests/test_neoaccess_import_export.py

@@ -286,19 +286,99 @@ def test_import_json_data(db):
                                                                 # Incorrectly-formatted JSON string. Expecting value: line 1 column 1 (char 0)
     with pytest.raises(Exception):
         assert db.import_json_dump('{"a": "this is good JSON, but not a list!"}')   # This ought to raise an Exception:
-                                                                                    # "The JSON string does not represent the expected list"
-    # TODO: extend
+                                                                                    # "The JSON string does not represent a list"
+    with pytest.raises(Exception):                      # This ought to raise an Exception:
+        assert db.import_json_dump('[1, 2, 3]')         # Item in list index 0 should be a dict, but instead it's of type <class 'int'>.  Item: 1
+
+    with pytest.raises(Exception):
+        assert db.import_json_dump('[{"bad_type":"node"}]')     # The dict in 0-th element lacks a "type" key
+
+    with pytest.raises(Exception):
+        assert db.import_json_dump('[{"type":"bad_value"}]')    # The dict in 0-th element doesn't have an acceptable value
+                                                                # for the "type" key
+
+    with pytest.raises(Exception):
+        assert db.import_json_dump('[{"type":"node"}]')         # Missing "id" key
+
+    with pytest.raises(Exception):
+        assert db.import_json_dump('[{"type":"node", "id": "NOT_INTEGER"}]')    # Non-integer id
+
+    with pytest.raises(Exception):
+        json = '[{"id":"4","type":"relationship","label":"is_friends_with","start":{"id":"123","labels":["User"]},"end":{"id":"456","labels":["Person", "Client"]}}]'
+        db.import_json_dump(json)                               # Trying to add a relationship between non-existing nodes
+
 
     # Now, test actual imports
 
-    # Completely clear the database
-    db.empty_dbase()
+    db.empty_dbase()    # Completely clear the database
 
+    # Import a 1st node
     json = '[{"type":"node","id":"123","labels":["User"],"properties":{"name":"Eve"}}]'
     details = db.import_json_dump(json)
     assert details == "Successful import of 1 node(s) and 0 relationship(s)"
     match = db.match(labels="User", properties={"name": "Eve"})
     retrieved_records = db.get_nodes(match)
     assert len(retrieved_records) == 1
+    match_all = db.match()
+    retrieved_records = db.get_nodes(match_all)
+    assert len(retrieved_records) == 1
+
+    # Import a 2nd node
+    json = '[{"type":"node","id":"456","labels":["Person", "Client"],"properties":{"name":"Adam"}}]'
+    details = db.import_json_dump(json)
+    assert details == "Successful import of 1 node(s) and 0 relationship(s)"
+    match = db.match(labels=["Person", "Client"], properties={"name": "Adam"})
+    retrieved_records = db.get_nodes(match)
+    assert len(retrieved_records) == 1
+    retrieved_records = db.get_nodes(match_all)
+    assert len(retrieved_records) == 2
+
+
+    # Starting with an empty database, re-import the 2 earlier nodes, but this time both at once
+    db.empty_dbase()    # Completely clear the database
+
+    json = '[{"type":"node","id":"123","labels":["User"],"properties":{"name":"Eve"}},\n' \
+           '{"type":"node","id":"456","labels":["Person", "Client"],"properties":{"name":"Adam"}}]'
+    details = db.import_json_dump(json)
+    assert details == "Successful import of 2 node(s) and 0 relationship(s)"
+
+    match = db.match(labels="User", properties={"name": "Eve"})
+    retrieved_records = db.get_nodes(match)
+    assert len(retrieved_records) == 1
+
+    match = db.match(labels=["Person", "Client"], properties={"name": "Adam"})
+    retrieved_records = db.get_nodes(match)
+    assert len(retrieved_records) == 1
+
+    retrieved_records = db.get_nodes(match_all)
+    assert len(retrieved_records) == 2
+
+
+    # Starting with an empty database, re-import the 2 earlier nodes, but this time both at once - and with a relationship between them
+    db.empty_dbase()    # Completely clear the database
+
+    json = '[{"type":"node","id":"123","labels":["User"],"properties":{"name":"Eve"}},\n' \
+           '{"type":"node","id":"456","labels":["Person", "Client"],"properties":{"name":"Adam"}},\n' \
+           '{"id":"7","type":"relationship","label":"is_friends_with","start":{"id":"123","labels":["User"]},"end":{"id":"456","labels":["Person", "Client"]}}]'
+    details = db.import_json_dump(json)
+    assert details == "Successful import of 2 node(s) and 1 relationship(s)"
+
+    match = db.match(labels="User", properties={"name": "Eve"})
+    retrieved_records = db.get_nodes(match, return_internal_id=True)
+    assert len(retrieved_records) == 1
+    id_eve = retrieved_records[0]["internal_id"]
+
+    match = db.match(labels=["Person", "Client"], properties={"name": "Adam"})
+    retrieved_records = db.get_nodes(match, return_internal_id=True)
+    assert len(retrieved_records) == 1
+    id_adam = retrieved_records[0]["internal_id"]
+
+    retrieved_records = db.get_nodes(match_all)
+    assert len(retrieved_records) == 2
 
-    # TODO: extend
+    q = '''
+        MATCH (eve :User {name: "Eve"}) - [:is_friends_with] -> (adam :Person:Client {name: "Adam"})
+        RETURN count(eve) AS num_eve, count(adam) AS num_adam, id(eve) AS id_eve, id(adam) AS id_adam
+        '''
+    result = db.query(q, single_row=True)
+    assert result == {'num_eve': 1, 'num_adam': 1, 'id_eve': id_eve, 'id_adam': id_adam}