diff --git a/.github/workflows/schoolai-build-publish.yml b/.github/workflows/schoolai-build-publish.yml new file mode 100644 index 00000000..4601136c --- /dev/null +++ b/.github/workflows/schoolai-build-publish.yml @@ -0,0 +1,34 @@ +name: SchoolAI Build and Publish + +# Builds the SchoolAI fork image and publishes it to our Artifact Registry so +# the webapp (apps/web RAG pipeline) can run a build. +# +# Mirrors the webapp build pattern (see web-app: +# .github/workflows/webapp-staging-deploy.yml -> build_webapp) by reusing the +# shared SchoolAI/github-actions-workflows sai-docker-build.yaml workflow. + +on: + workflow_call: + # Keep workflow_dispatch so operators can trigger a manual staging deploy + # even though `staging-deploy.yml` is now the primary entrypoint on main. + workflow_dispatch: + +jobs: + build_unstructured_api: + uses: SchoolAI/github-actions-workflows/.github/workflows/sai-docker-build.yaml@main + with: + # amd64 only — matches the cluster nodeSelector (kubernetes.io/arch: amd64) + # and how the other SchoolAI services build. + build_amd_enabled: true + build_arm_enabled: false + docker_build_amd_runs_on: sai-7cpu-26gb-amd-perf + # Produces images at: + # us-central1-docker.pkg.dev/schoolai-global/docker/unstructured-api: + docker_repo: us-central1-docker.pkg.dev/schoolai-global/docker + docker_image_name: unstructured-api + dockerfile: ./Dockerfile + docker_build_args: | + PIPELINE_PACKAGE=general + push_image: true + secrets: + GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY_STAGING }} diff --git a/prepline_general/api/general.py b/prepline_general/api/general.py index 61d996d2..e9e3ab4c 100644 --- a/prepline_general/api/general.py +++ b/prepline_general/api/general.py @@ -223,6 +223,7 @@ def pipeline_api( new_after_n_chars: Optional[int], overlap: int, overlap_all: bool, + include_orig_elements: bool = True, # ---------------------- filename: str = "", file_content_type: Optional[str] = None, @@ -283,6 +284,7 @@ def pipeline_api( "new_after_n_chars": new_after_n_chars, "overlap": overlap, "overlap_all": overlap_all, + "include_orig_elements": include_orig_elements, "starting_page_number": starting_page_number, "include_slide_notes": include_slide_notes, }, @@ -338,6 +340,7 @@ def pipeline_api( "max_characters": max_characters, "overlap": overlap, "overlap_all": overlap_all, + "include_orig_elements": include_orig_elements, "extract_image_block_types": extract_image_block_types, "extract_image_block_to_payload": extract_image_block_to_payload, "unique_element_ids": unique_element_ids, @@ -368,6 +371,7 @@ def pipeline_api( "max_characters": max_characters, "overlap": overlap, "overlap_all": overlap_all, + "include_orig_elements": include_orig_elements, "extract_image_block_types": extract_image_block_types, "extract_image_block_to_payload": extract_image_block_to_payload, "unique_element_ids": unique_element_ids, @@ -716,6 +720,7 @@ def response_generator(is_multipart: bool): new_after_n_chars=form_params.new_after_n_chars, overlap=form_params.overlap, overlap_all=form_params.overlap_all, + include_orig_elements=form_params.include_orig_elements, starting_page_number=form_params.starting_page_number, include_slide_notes=form_params.include_slide_notes, ) diff --git a/prepline_general/api/models/form_params.py b/prepline_general/api/models/form_params.py index 8b96bfda..081175dd 100644 --- a/prepline_general/api/models/form_params.py +++ b/prepline_general/api/models/form_params.py @@ -35,6 +35,7 @@ class GeneralFormParams(BaseModel): new_after_n_chars: Optional[int] overlap: int overlap_all: bool + include_orig_elements: bool starting_page_number: Optional[int] = None include_slide_notes: bool @@ -236,6 +237,18 @@ def as_form( examples=[True], ), ] = False, + include_orig_elements: Annotated[ + bool, + Form( + title="Include Orig Elements", + description="""When `True` (the default), the elements used to form each chunk are +added to that chunk's `.metadata.orig_elements` as a gzipped+base64 blob. Set to `False` to omit +them and produce a much smaller payload — useful for large tables, where this blob is duplicated +into every chunk and can balloon the response size dramatically.""", + examples=[False], + ), + BeforeValidator(SmartValueParser[bool]().value_or_first_element), + ] = True, starting_page_number: Annotated[ Optional[int], Form( @@ -283,6 +296,7 @@ def as_form( new_after_n_chars=new_after_n_chars, overlap=overlap, overlap_all=overlap_all, + include_orig_elements=include_orig_elements, unique_element_ids=unique_element_ids, starting_page_number=starting_page_number, include_slide_notes=include_slide_notes,