Chore: put tesseract multiple languages splitter "+" in constant (#2226)

^^^

Author: yuming-long

CHANGELOG.md

@@ -6,6 +6,7 @@
 * **Refactor pdfminer code.** The pdfminer code is moved from `unstructured-inference` to `unstructured`.
 * **Improve handling of auth data for fsspec connectors.** Leverage an extension of the dataclass paradigm to support a `sensitive` annotation for fields related to auth (i.e. passwords, tokens). Refactor all fsspec connectors to use explicit access configs rather than a generic dictionary.
 * **Add glob support for fsspec connectors** Similar to the glob support in the ingest local source connector, similar filters are now enabled on all fsspec based source connectors to limit files being partitioned.
+* Define a constant for the splitter "+" used in tesseract ocr languages.
 
 ### Features
 

unstructured/partition/lang.py

@@ -6,6 +6,7 @@
 
 from unstructured.documents.elements import Element
 from unstructured.logger import logger
+from unstructured.partition.utils.constants import TESSERACT_LANGUAGES_SPLITTER
 
 # pytesseract.get_languages(config="") only shows user installed language packs,
 # so manually include the list of all currently supported Tesseract languages
@@ -160,7 +161,7 @@ def prepare_languages_for_tesseract(languages: Optional[List[str]] = ["eng"]):
         )
         return "eng"
 
-    return "+".join(converted_languages)
+    return TESSERACT_LANGUAGES_SPLITTER.join(converted_languages)
 
 
 def check_languages(languages: Optional[List[str]], ocr_languages: Optional[str]):
@@ -196,7 +197,7 @@ def convert_old_ocr_languages_to_languages(ocr_languages: str):
     Assumption: ocr_languages is in tesseract plus sign format
     """
 
-    return ocr_languages.split("+")
+    return ocr_languages.split(TESSERACT_LANGUAGES_SPLITTER)
 
 
 def convert_language_to_tesseract(lang: str) -> str:
@@ -223,17 +224,17 @@ def convert_language_to_tesseract(lang: str) -> str:
     # try to match ISO 639-3 code
     if lang_iso639.part3 in pytesseract_langs_3:
         matched_langcodes = _get_all_tesseract_langcodes_with_prefix(lang_iso639.part3)
-        return "+".join(matched_langcodes)
+        return TESSERACT_LANGUAGES_SPLITTER.join(matched_langcodes)
 
     # try to match ISO 639-2b
     elif lang_iso639.part2b in pytesseract_langs_3:
         matched_langcodes = _get_all_tesseract_langcodes_with_prefix(lang_iso639.part2b)
-        return "+".join(matched_langcodes)
+        return TESSERACT_LANGUAGES_SPLITTER.join(matched_langcodes)
 
     # try to match ISO 639-2t
     elif lang_iso639.part2t in pytesseract_langs_3:
         matched_langcodes = _get_all_tesseract_langcodes_with_prefix(lang_iso639.part2t)
-        return "+".join(matched_langcodes)
+        return TESSERACT_LANGUAGES_SPLITTER.join(matched_langcodes)
 
     else:
         logger.warning(f"{lang} is not a language supported by Tesseract.")

unstructured/partition/utils/constants.py

@@ -36,3 +36,5 @@ class PartitionStrategy:
 
 # this field is defined by pytesseract/unstructured.pytesseract
 TESSERACT_TEXT_HEIGHT = "height"
+
+TESSERACT_LANGUAGES_SPLITTER = "+"