Merge pull request #188 from buildingSMART/IVS-516-Use-spf-only-in-parse-info-task

IVS-516 - Use simple_spf only in parse_info task

Author: Ghesselink

.github/workflows/ci_cd.yml

@@ -86,7 +86,7 @@ jobs:
           source venv/bin/activate
           # use version of ifcopenshell with desired schema parsing
           # TODO: revert to pyPI when schema parsing is published in the future
-          wget -O /tmp/ifcopenshell_python.zip "https://s3.amazonaws.com/ifcopenshell-builds/ifcopenshell-python-311-v0.8.1-c49ca69-linux64.zip"
+          wget -O /tmp/ifcopenshell_python.zip "https://s3.amazonaws.com/ifcopenshell-builds/ifcopenshell-python-311-v0.8.3-ec13294-linux64.zip"
           mkdir -p venv/lib/python3.11/site-packages
           unzip -d venv/lib/python3.11/site-packages /tmp/ifcopenshell_python.zip
 

backend/Makefile

@@ -14,23 +14,23 @@ venv:
 install: venv
 	$(PIP) install --upgrade pip
 	$(PIP) install -r requirements.txt
-	wget -O /tmp/ifcopenshell_python.zip "https://s3.amazonaws.com/ifcopenshell-builds/ifcopenshell-python-311-v0.8.1-c49ca69-linux64.zip"
+	wget -O /tmp/ifcopenshell_python.zip "https://s3.amazonaws.com/ifcopenshell-builds/ifcopenshell-python-311-v0.8.3-ec13294-linux64.zip"
 	mkdir -p $(VIRTUAL_ENV)/lib/python3.11/site-packages
 	unzip -f -d $(VIRTUAL_ENV)/lib/python3.11/site-packages /tmp/ifcopenshell_python.zip
 	rm /tmp/ifcopenshell_python.zip
 
 install-macos: venv
 	$(PIP) install --upgrade pip
 	$(PIP) install -r requirements.txt
-	wget -O /tmp/ifcopenshell_python.zip "https://s3.amazonaws.com/ifcopenshell-builds/ifcopenshell-python-311-v0.8.1-c49ca69-macos64.zip"
+	wget -O /tmp/ifcopenshell_python.zip "https://s3.amazonaws.com/ifcopenshell-builds/ifcopenshell-python-311-v0.8.3-ec13294-macos64.zip"
 	mkdir -p $(VIRTUAL_ENV)/lib/python3.11/site-packages
 	unzip /tmp/ifcopenshell_python.zip -d $(VIRTUAL_ENV)/lib/python3.11/site-packages
 	rm /tmp/ifcopenshell_python.zip
 
 install-macos-m1: venv
 	$(PIP) install --upgrade pip
 	$(PIP) install -r requirements.txt
-	wget -O /tmp/ifcopenshell_python.zip "https://s3.amazonaws.com/ifcopenshell-builds/ifcopenshell-python-311-v0.8.1-c49ca69-macosm164.zip"
+	wget -O /tmp/ifcopenshell_python.zip "https://s3.amazonaws.com/ifcopenshell-builds/ifcopenshell-python-311-v0.8.3-ec13294-macosm164.zip"
 	mkdir -p $(VIRTUAL_ENV)/lib/python3.11/site-packages
 	unzip /tmp/ifcopenshell_python.zip -d $(VIRTUAL_ENV)/lib/python3.11/site-packages
 	rm /tmp/ifcopenshell_python.zip

backend/apps/ifc_validation/checks/header_policy/mvd_parser.py

@@ -16,26 +16,17 @@ def collect_test_files():
 @pytest.mark.parametrize("f", collect_test_files())
 def test_invocation(f):
     filename, outcome, field = f[0], f[1], f[2]
-    try:
-        file = ifcopenshell.open(filename)
-        header = HeaderStructure(file=file, purepythonparser=False)
-    except ifcopenshell.SchemaError:
-        file = ifcopenshell.simple_spf.open(filename)
-        header = HeaderStructure(file=file, purepythonparser=True)
+    file = ifcopenshell.simple_spf.open(filename, only_header=True)
+    header = HeaderStructure(file=file)
 
     assert (field not in header.validation_errors) if outcome == 'pass' else (field in header.validation_errors)
 
 
 def run_single_file(filename=''):
     if filename:
-        try:
-            file = ifcopenshell.open(filename)
-            header = HeaderStructure(file=file, purepythonparser=False)
-            print(header.validation_errors)
-        except ifcopenshell.SchemaError:
-            file = ifcopenshell.simple_spf.open(filename)
-            header = HeaderStructure(file=file, purepythonparser=True)
-            print(header.validation_errors)
+        file = ifcopenshell.simple_spf.open(filename, only_header=True)
+        print(header.validation_errors)
+        header = HeaderStructure(file=file)
 
 if __name__ == "__main__":
     if len(sys.argv) == 2:

backend/apps/ifc_validation/checks/header_policy/test_header_validation.py

@@ -1,6 +1,6 @@
 import sys
 from pydantic import Field, field_validator, model_validator
-from typing import Tuple, Union, Optional
+from typing import Tuple, Any, Optional
 import ifcopenshell
 from ifcopenshell import validate, SchemaError, simple_spf
 import re
@@ -12,7 +12,6 @@
 import yaml
 import os
 from config import ConfiguredBaseModel
-from mvd_parser import parse_mvd
 
 
 def ifcopenshell_pre_validation(file):
@@ -80,8 +79,7 @@ def validate_and_split_originating_system(attributes):
 
 
 class HeaderStructure(ConfiguredBaseModel):
-    file: Union[ifcopenshell.file, simple_spf.file]
-    purepythonparser : bool = False
+    file: Optional[simple_spf.file] = None
     validation_errors : list = Field(default_factory=list)  
 
     description: Optional[Tuple[str, ...]] = Field(default=None)
@@ -98,20 +96,19 @@ class HeaderStructure(ConfiguredBaseModel):
     application_name: Optional[str] = Field(default=None)
     schema_identifier: Optional[str] = Field(default=None)
     version: Optional[str] = Field(default=None)
-    mvd: Optional[str] = Field(default=None)
+    mvd: Optional[Any] = Field(default=None)  # Expected: ifcopenshell.util.mvd_info.AutoCommitList
+    comments: Optional[Any] = Field(default=None)  # Expected: ifcopenshell.util.mvd_info.AutoCommitList
 
 
     @model_validator(mode='before')
     def populate_header(cls, values):
         if (file := values.get('file')):
 
-            purepythonparser = values.get('purepythonparser')
             header = file.header
             file_description = header.file_description
             file_name = header.file_name
 
             fields = [
-                (file_description, 'description', 0),
                 (file_description, 'implementation_level', 1),
                 (file_name, 'name', 0),
                 (file_name, 'time_stamp', 1),
@@ -122,41 +119,35 @@ def populate_header(cls, values):
                 (file_name, 'authorization', 6)
             ]
 
-            attributes = {field: getattr(obj, field) if not purepythonparser else obj[index]
-                for obj, field, index in fields
-            }
+            attributes = {field: getattr(obj, field) for obj, field, index in fields}
 
             attributes['validation_errors'] = []
-            attributes['mvd'] = ''
+            attributes['mvd'] = file.mvd.view_definitions
+            attributes['comments'] = file.mvd.comments
             attributes['schema_identifier'] = ''
 
             attributes = validate_and_split_originating_system(attributes)
 
-            errors_from_pre_validation = ifcopenshell_pre_validation(file) if not purepythonparser else []
-            for error in errors_from_pre_validation:
-                attributes['validation_errors'].append(error)
-                attributes[error] = None
+            if file.mvd.description:
+                errors_from_pre_validation = ifcopenshell_pre_validation(file)
+                for error in errors_from_pre_validation:
+                    attributes['validation_errors'].append(error)
+                    attributes[error] = None
 
-            values.update(attributes)            
+            values.update(attributes)         
 
         return values
 
-
-    @field_validator('description')
+    
+    @field_validator('comments', mode='after')
     def validate_description(cls, v, values):
         """
         https://github.com/buildingSMART/IFC4.x-IF/tree/header-policy/docs/IFC-file-header#description
         For grammar refer to https://standards.buildingsmart.org/documents/Implementation/ImplementationGuide_IFCHeaderData_Version_1.0.2.pdf
         """
-        if v:
-            header_description_text = ' '.join(v)
-            parsed_description = parse_mvd(header_description_text)
-            view_definitions = parsed_description.mvd
-            values.data['mvd'] = view_definitions
-
-            
+        if v:            
             # comments is a free textfield, but constrainted to 256 characters
-            if len(parsed_description.comments) > 256:
+            if len(v) > 256:
                 values.data.get('validation_errors').append(values.field_name)
                 return v if type(v) == tuple else v
 
@@ -167,30 +158,30 @@ def validate_description(cls, v, values):
     @field_validator('mvd', mode='after')
     def validate_and_set_mvd(cls, v, values):
         """
+        https://github.com/buildingSMART/IFC4.x-IF/tree/header-policy/docs/IFC-file-header#description
+        For grammar refer to https://standards.buildingsmart.org/documents/Implementation/ImplementationGuide_IFCHeaderData_Version_1.0.2.pdf
         This function runs after the other fields. It validates the mvd based on the grammar done in  
         the 'description' field. The function checks the constraints on the mvds and returns 
         the comma separated list into a single string.
         """
-        view_definitions = values.data.get('mvd')
-        if not view_definitions:
+        if not v:
             values.data.get('validation_errors').append('description')
             return v
 
         allowed_descriptions = yaml.safe_load(open(os.path.join(os.path.dirname(__file__), 'valid_descriptions.yaml')))
         schema_identifier = values.data.get('file').schema_identifier
 
-        view_definitions_set = {view for view in view_definitions} 
+        view_definitions_set = {view for view in v} 
 
         if any("AddOnView" in view for view in view_definitions_set):
             if not any("CoordinationView" in view for view in view_definitions_set):
                 values.data.get('validation_errors').append('description')  # AddOnView MVD without CoordinationView        
 
-        for mvd in view_definitions:
+        for mvd in v:
             if mvd not in allowed_descriptions.get(schema_identifier, [False]) and not 'AddOnView' in mvd:
                 values.data.get('validation_errors').append('description')
 
-        return ', '.join(view_definitions)
-        
+        return ', '.join(v)
 
 
     @field_validator('time_stamp')
@@ -238,11 +229,10 @@ def main():
 
     filename = sys.argv[1] 
     try:
-        file = ifcopenshell.open(filename)
-        header = HeaderStructure(file=file, purepythonparser=False)
-    except SchemaError as e:
-        file = ifcopenshell.simple_spf.open(filename)
-        header = HeaderStructure(file=file, purepythonparser=True)
+        file = ifcopenshell.simple_spf.open(filename, only_header=True)
+        header = HeaderStructure(file=file)
+    except ifcopenshell.simple_spf.SyntaxError:
+        header = HeaderStructure(file=None, validation_errors=["syntax_error"])
     except Exception as e:
         print(f"Error opening file '{filename}': {e}")
         sys.exit(1)

backend/apps/ifc_validation/checks/header_policy/tests/originating_system/fail/dot_after_specifier.ifc renamed to backend/apps/ifc_validation/checks/header_policy/tests/syntax_error/dot_after_specifier.ifc

@@ -104,7 +104,6 @@ def test_schema_validation_task_creates_error_validation_outcomes(self):
         for outcome in outcomes:
             self.assertEqual(outcome.severity, ValidationOutcome.OutcomeSeverity.ERROR)
             self.assertEqual(outcome.outcome_code, ValidationOutcome.ValidationOutcomeCode.SCHEMA_ERROR)
-            self.assertIsNotNone(outcome.instance)
 
     def test_schema_validation_task_creates_error_validation_outcomes_2(self):
 
@@ -124,11 +123,10 @@ def test_schema_validation_task_creates_error_validation_outcomes_2(self):
 
         outcomes = ValidationOutcome.objects.filter(validation_task__request_id=request.id)
         self.assertIsNotNone(outcomes)
-        self.assertEqual(len(outcomes), 3)
+        self.assertEqual(len(outcomes), 5)
         for outcome in outcomes:
             self.assertEqual(outcome.severity, ValidationOutcome.OutcomeSeverity.ERROR)
             self.assertEqual(outcome.outcome_code, ValidationOutcome.ValidationOutcomeCode.SCHEMA_ERROR)
-            self.assertIsNotNone(outcome.instance)
 
     def test_schema_validation_task_creates_error_validation_outcomes_3(self):
 
@@ -148,7 +146,7 @@ def test_schema_validation_task_creates_error_validation_outcomes_3(self):
 
         outcomes = ValidationOutcome.objects.filter(validation_task__request_id=request.id)
         self.assertIsNotNone(outcomes)
-        self.assertEqual(len(outcomes), 8)
+        self.assertEqual(len(outcomes), 11)
         for outcome in outcomes:
             self.assertEqual(outcome.severity, ValidationOutcome.OutcomeSeverity.ERROR)
             self.assertEqual(outcome.outcome_code, ValidationOutcome.ValidationOutcomeCode.SCHEMA_ERROR)
@@ -171,7 +169,7 @@ def test_schema_validation_task_creates_error_validation_outcomes_4(self):
 
         outcomes = ValidationOutcome.objects.filter(validation_task__request_id=request.id)
         self.assertIsNotNone(outcomes)
-        self.assertEqual(len(outcomes), 8)
+        self.assertEqual(len(outcomes), 11)
         for outcome in outcomes:
             self.assertEqual(outcome.severity, ValidationOutcome.OutcomeSeverity.ERROR)
             self.assertEqual(outcome.outcome_code, ValidationOutcome.ValidationOutcomeCode.SCHEMA_ERROR)

backend/apps/ifc_validation/checks/header_policy/tests/description/fail/empty_mvd.ifc renamed to backend/apps/ifc_validation/checks/header_policy/tests/syntax_error/empty_mvd.ifc

@@ -37,7 +37,7 @@ RUN --mount=type=cache,target=/root/.cache \
     pip install --no-cache-dir -r /app/backend/requirements.txt && \    
     # use version of ifcopenshell with desired schema parsing
     # TODO: revert to pyPI when schema parsing is published in the future
-    wget -O /tmp/ifcopenshell_python.zip "https://s3.amazonaws.com/ifcopenshell-builds/ifcopenshell-python-311-v0.8.1-c49ca69-linux64.zip" && \
+    wget -O /tmp/ifcopenshell_python.zip "https://s3.amazonaws.com/ifcopenshell-builds/ifcopenshell-python-311-v0.8.3-ec13294-linux64.zip" && \
     mkdir -p /opt/venv/lib/python3.11/site-packages  && \
     unzip -d /opt/venv/lib/python3.11/site-packages /tmp/ifcopenshell_python.zip && \
     # some cleanup