feat/add file data annotation

requirements/cli.in

@@ -1,3 +1,4 @@
 uvicorn
 fastapi
 click
+unstructured

requirements/cli.txt

@@ -11,25 +11,48 @@ anyio==4.4.0
     #   httpx
     #   starlette
     #   watchfiles
+backoff==2.2.1
+    # via unstructured
+beautifulsoup4==4.12.3
+    # via unstructured
 certifi==2024.6.2
     # via
     #   httpcore
     #   httpx
+    #   requests
+    #   unstructured-client
+chardet==5.2.0
+    # via unstructured
+charset-normalizer==3.3.2
+    # via
+    #   requests
+    #   unstructured-client
 click==8.1.7
     # via
     #   -r requirements//cli.in
+    #   nltk
     #   typer
     #   uvicorn
+dataclasses-json==0.6.7
+    # via
+    #   unstructured
+    #   unstructured-client
+deepdiff==7.0.1
+    # via unstructured-client
 dnspython==2.6.1
     # via email-validator
 email-validator==2.2.0
     # via fastapi
+emoji==2.12.1
+    # via unstructured
 exceptiongroup==1.2.1
     # via anyio
 fastapi==0.111.0
     # via -r requirements//cli.in
 fastapi-cli==0.0.4
     # via fastapi
+filetype==1.2.0
+    # via unstructured
 h11==0.14.0
     # via
     #   httpcore
@@ -39,56 +62,137 @@ httpcore==1.0.5
 httptools==0.6.1
     # via uvicorn
 httpx==0.27.0
-    # via fastapi
+    # via
+    #   fastapi
+    #   unstructured-client
 idna==3.7
     # via
     #   anyio
     #   email-validator
     #   httpx
+    #   requests
+    #   unstructured-client
 jinja2==3.1.4
     # via fastapi
+joblib==1.4.2
+    # via nltk
+jsonpath-python==1.0.6
+    # via unstructured-client
+langdetect==1.0.9
+    # via unstructured
+lxml==5.2.2
+    # via unstructured
 markdown-it-py==3.0.0
     # via rich
 markupsafe==2.1.5
     # via jinja2
+marshmallow==3.21.3
+    # via
+    #   dataclasses-json
+    #   unstructured-client
 mdurl==0.1.2
     # via markdown-it-py
-orjson==3.10.5
+mypy-extensions==1.0.0
+    # via
+    #   typing-inspect
+    #   unstructured-client
+nest-asyncio==1.6.0
+    # via unstructured-client
+nltk==3.8.1
+    # via unstructured
+numpy==1.26.4
+    # via unstructured
+ordered-set==4.1.0
+    # via deepdiff
+orjson==3.10.6
     # via fastapi
+packaging==24.1
+    # via
+    #   marshmallow
+    #   unstructured-client
 pydantic==2.8.0
     # via fastapi
 pydantic-core==2.20.0
     # via pydantic
 pygments==2.18.0
     # via rich
+pypdf==4.2.0
+    # via unstructured-client
+python-dateutil==2.9.0.post0
+    # via unstructured-client
 python-dotenv==1.0.1
     # via uvicorn
+python-iso639==2024.4.27
+    # via unstructured
+python-magic==0.4.27
+    # via unstructured
 python-multipart==0.0.9
     # via fastapi
 pyyaml==6.0.1
     # via uvicorn
+rapidfuzz==3.9.4
+    # via unstructured
+regex==2024.5.15
+    # via nltk
+requests==2.32.3
+    # via
+    #   requests-toolbelt
+    #   unstructured
+    #   unstructured-client
+requests-toolbelt==1.0.0
+    # via unstructured-client
 rich==13.7.1
     # via typer
 shellingham==1.5.4
     # via typer
+six==1.16.0
+    # via
+    #   langdetect
+    #   python-dateutil
+    #   unstructured-client
 sniffio==1.3.1
     # via
     #   anyio
     #   httpx
+soupsieve==2.5
+    # via beautifulsoup4
 starlette==0.37.2
     # via fastapi
+tabulate==0.9.0
+    # via unstructured
+tqdm==4.66.4
+    # via
+    #   nltk
+    #   unstructured
 typer==0.12.3
     # via fastapi-cli
 typing-extensions==4.12.2
     # via
     #   anyio
+    #   emoji
     #   fastapi
     #   pydantic
     #   pydantic-core
+    #   pypdf
     #   typer
+    #   typing-inspect
+    #   unstructured
+    #   unstructured-client
     #   uvicorn
+typing-inspect==0.9.0
+    # via
+    #   dataclasses-json
+    #   unstructured-client
 ujson==5.10.0
     # via fastapi
+unstructured==0.14.9
+    # via -r requirements//cli.in
+unstructured-client==0.23.8
+    # via unstructured
+urllib3==2.2.2
+    # via
+    #   requests
+    #   unstructured-client
 uvicorn[standard]==0.30.1
     # via
     #   -r requirements//cli.in
@@ -99,3 +203,5 @@ watchfiles==0.22.0
     # via uvicorn
 websockets==12.0
     # via uvicorn
+wrapt==1.16.0
+    # via unstructured

scripts/pip-compile.sh

@@ -1,6 +1,18 @@
 #!/usr/bin/env bash
 
+# python version must match lowest supported (3.9)
+major=3
+minor=10
+if ! python -c "import sys; assert sys.version_info.major == $major and sys.version_info.minor == $minor"; then
+	echo "python version not equal to expected $major.$minor: $(python --version)"
+	exit 1
+fi
+
 while IFS= read -r -d '' file; do
+	filename=${file%.in}
+	txtfilename="${filename}.txt"
+	echo "Removing $txtfilename"
+	rm -f $txtfilename
 	echo "Running 'pip-compile --upgrade $file'"
 	pip-compile --upgrade --verbose "$file"
 done < <(find requirements/ -type f -name "*.in" -maxdepth 1 -print0)