implement lanchain to replace nextjs tools for text extraction

.env.example

@@ -0,0 +1,19 @@
+#URNEX RD DATABASE SETTINGS
+SQL_NAME = "local"
+SQL_USER="SQL_USER"
+SQL_PASSWORD="SQL_PASSWORD"
+SQL_LOCAL_SERVER="SQL_LOCAL_SERVER"
+SQL_LOCAL_DRIVER ="SQL Server"
+SQL_DATABASE="DATABASE"
+SQL_ENCRYPT=false
+enableArithAbort=true
+
+
+#other
+MEDIA_ROOT=path
+DJANGO_DEBUG=true
+
+#Django settings
+SECRET_KEY = 'django-insecure-8$j=u)jo!srkxm=6&(dp95gg20x#*bwfe@r5w)uf!r387v9ser'
+JWT_SIGNING_KEY = 'JWT_SIGNING_KEY'
+OPEN_AI_KEY=DFDFDFDF

.gitignore

@@ -1,3 +1,165 @@
+/node_modules
+/temp_media
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+venv/
+ENV/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Django stuff:
+*.log
+*.pot
+*.pyc
+*.pyo
+*.pyd
+db.sqlite3
+media
+*.sqlite3
+
+# If your database settings are using postgres
+*.pgsql
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+# Pipenv
+pipenv
+pipenv.*
+pipenv/pipfile
+pipenv/pyvenv.cfg
+pipenv/pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# celery beat schedule file
+celerybeat-schedule
+celerybeat.pid
+
+# dotenv
+.env
+
+
+# virtualenv
+.venv
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# venv & ENV folders
+env/
+venv/
+ENV/
+.env
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# cProfile Dumps
+*.pstat
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+coverage.xml
+*.cover
+.hypothesis/
+
+# Pytest
+.cache
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# IDEs and editors
+.idea/
+.vscode/
+*.swp
+*.swo
+*.sublime-project
+*.sublime-workspace
+.vscode/
+.vscode-test/
+.vscode-recommendations
+.vscode-server
+
+# system specific files
+.DS_Store
+Thumbs.db
 
 logs/
 sandbox/

.prettierrc

@@ -0,0 +1,11 @@
+{
+  "plugins": ["prettier-plugin-django"],
+  "overrides": [
+    {
+      "files": "*.html",
+      "options": {
+        "parser": "django-html"
+      }
+    }
+  ]
+}

ai_file_reader/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

ai_file_reader/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class AiFileReaderConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'ai_file_reader'

ai_file_reader/event_handler.py

@@ -0,0 +1,36 @@
+from django.conf import settings
+from typing_extensions import override
+import openai
+
+# open_ai_key = settings.OPENAI_API_KEY
+# client = openai.OpenAI(api_key=open_ai_key)
+
+class EventHandler:
+    def __init__(self):
+        self.output_json = None
+
+    def get_output_json(self):
+        return self.output_json
+
+    def process_messages(self, messages, client):
+        try:
+            if not messages:
+                return {'error': 'No messages returned'}
+
+            message_content = messages[0].content[0].text  # Get the first message text
+            annotations = message_content.annotations
+            citations = []
+
+            for index, annotation in enumerate(annotations):
+                message_content.value = message_content.value.replace(annotation.text, f"[{index}]")
+                if file_citation := getattr(annotation, "file_citation", None):
+                    cited_file = client.files.retrieve(file_citation.file_id)
+                    citations.append(f"[{index}] {cited_file.filename}")
+
+            return {
+                'message': message_content.value,
+                'citations': citations
+            }
+
+        except Exception as e:
+            return {'error': str(e)}

ai_file_reader/form_data_type.py

@@ -0,0 +1,115 @@
+from dataclasses import dataclass
+from typing import Optional, List, Union
+from datetime import date
+
+@dataclass
+class ChemicalComposition:
+    casNo: Optional[str]
+    chemicalName: Optional[str]
+    percentage: float
+
+@dataclass
+class PPE:
+    health: Optional[int]
+    flammability: Optional[int]
+    physical: Optional[int]
+    protection: str
+
+@dataclass
+class FormDataType:
+    # General Information
+    WPSID: Optional[str]
+    chemNo: Optional[str]
+    chemDesc: Optional[str]
+    commodity: Optional[str]
+    grade: Optional[str]
+    testing: Optional[str]
+    testProps: Optional[str]
+    brandName: Optional[str]
+    companyName: Optional[str]
+    manuId: Optional[int]
+    offsetsRuledOut_brand_manu: Optional[str]
+    outsourced_only: Optional[str]
+    composition: List[ChemicalComposition]
+    functions: Optional[List[str]]
+    contain_cl: str
+    water_reaction: str
+    dioxane: str
+
+    # SDS Validation (specs)
+    sdsSaved: Optional[str]
+    coaSaved: Optional[str]
+    sds_date: Optional[date]
+    hazardous: Optional[str]
+    oxidizer: Optional[str]
+    corrosive: Optional[str]
+    color: Optional[str]
+    odor: Optional[str]
+    appearance: Optional[str]
+    form: Optional[str]
+    flashPoint: Optional[str]
+    freezePoint: Optional[str]
+    spg: Optional[float]
+    pH_neat: Optional[float]
+    vaporPressure: Optional[str]
+    meltingPoint: Optional[str]
+    blkDens: Optional[float]
+    pH_1percent: Optional[float]
+    pictograms: Optional[List[str]]
+    solubility: Optional[str]
+    viscosity: Optional[str]
+    molarMass: Optional[str]
+
+    # PPE Validations
+    ppe: Optional[PPE]
+    respiratorRequired: Optional[str]
+    respiratorType: Optional[str]
+    respiratorNo: Optional[str]
+    maskType: Optional[str]
+    catridgeType: Optional[str]
+    storageLocation: str
+    container: List[str]
+
+    # Regulatory Validation
+    tsca: str
+    sara313Reportable: str
+    sara313CasNo: Optional[str]
+    prop65: str
+    propConc: Optional[List[ChemicalComposition]]
+
+    # Environment Validation
+    readilyBiodegradable: str
+    bioMethod: Optional[str]
+    bioAmount: Optional[float]
+    marinePollutant: str
+    canadian_DSL_NDSL: str
+    voc: str
+    percentVoc: Optional[float]
+    percentLVPVOC: Optional[float]
+    derivedFromNaturalSrc: str
+    percentNaturallyDerived: Optional[float]
+    bioBased: str
+    percentBiobased: Optional[float]
+
+    # Tier II Reportable
+    tier_II_reportable: str
+    tier_II_amount: Optional[float]
+    tier_II_EHS: Optional[str]
+    tier_II_hazard_composition: Optional[str]
+    tier_II_hazard_form: Optional[str]
+    tier_II_hazard_class: Optional[List[str]]
+
+    # Allergens validation
+    containsBacteria: str
+    containsPalm_PalmKernelOil: str
+    palmOil_Other_certified: str
+    otherOils: str
+
+    # Used in formulations
+    certifications: List[str]
+    notes: Optional[str]
+
+    # Files
+    sdsFile: Optional[Union[str, bytes]]
+    coaFile: Optional[Union[str, bytes]]
+    otherFiles: Optional[List[Union[str, bytes]]]

ai_file_reader/models.py

@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.