Merge branch 'main' into async

nicpottier · web-flow · commit c3e9c22cc86b · 2025-07-22T07:36:15.000-07:00
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -0,0 +1,11 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile
+{
+  "name": "ADT Press",
+  "build": {
+    // Sets the run context to one level up instead of the .devcontainer folder.
+    "context": "..",
+    // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename.
+    "dockerfile": "../Dockerfile"
+  }
+}
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,12 @@
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+*.db
+*.sqlite3
+.env
+.venv
+.git
+.gitignore
+tests/
+# Removed exclusion of README.md
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -30,6 +30,9 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v6
 
+      - name: Update lockfile
+        run: uv lock
+
       - name: Install the project
         run: uv sync --locked --all-extras --dev
 
diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml
@@ -0,0 +1,33 @@
+name: MyPy
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Graphviz
+        uses: ts-graphviz/setup-graphviz@v2
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+
+      - name: Update lockfile
+        run: uv lock
+
+      - name: Install the project
+        run: uv sync --locked --all-extras --dev
+
+      - name: Run mypy
+        run: uv run mypy adt-press.py
diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml
@@ -36,6 +36,9 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v6
 
+      - name: Update lockfile
+        run: uv lock
+
       - name: Install the project
         run: uv sync --locked --all-extras --dev
 
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -23,6 +23,9 @@ jobs:
     - name: Install uv
       uses: astral-sh/setup-uv@v6
 
+    - name: Update lockfile
+      run: uv lock
+
     - name: Install the project
       run: uv sync --locked --all-extras --dev
 
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,25 @@
+FROM python:3.13-slim
+
+# Set work directory
+WORKDIR /app
+
+# Install system dependencies for graphviz and opencv
+RUN apt-get update && apt-get install -y \
+    graphviz \
+    libgl1 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install uv
+RUN pip install --upgrade pip && pip install uv
+
+# Copy project files
+COPY . .
+
+# Sync and install dependencies using uv
+RUN uv sync
+
+# Pre-create the .venv by running a dummy command
+RUN uv run -- python -c "import sys; print('venv ready:', sys.prefix)"
+
+# Set default command (adjust as needed)
+CMD ["python"]
diff --git a/README.md b/README.md
@@ -23,6 +23,7 @@ The [sample report](https://unicef.github.io/adt-press/) can help in better unde
 
 - Python 3.13 or higher
 - UV package manager (recommended)
+- **You must set the environment variable `OPENAI_API_KEY` with your OpenAI API key for the application to work.**
 
 ## Installation
 
@@ -111,4 +112,67 @@ uv run pytest
 - `config/`: Configuration files
 - `prompts/`: LLM prompt templates
 - `templates/`: HTML templates
-- `tests/`: Test files
+- `tests/`: Test files
+
+## Docker
+
+Build the image:
+
+```bash
+docker build -t adt-press .
+```
+
+Run the container:
+
+```bash
+docker run --rm adt-press
+```
+
+To run a specific command inside the container (for example, to execute `uv run adt-press.py` with a PDF file):
+
+```bash
+docker run --rm adt-press uv run adt-press.py pdf_path=/data/yourfile.pdf
+```
+
+Replace `/data/yourfile.pdf` with the path to your PDF file inside the container.
+
+---
+
+#### VS Code: "Reopen in Container"
+
+If you use Visual Studio Code, you can take advantage of the **"Reopen in Container"** feature for a full-featured development environment inside Docker.  
+This allows you to edit, run, and debug your code directly within the container.
+
+To use this, add a `.devcontainer` configuration to your project and select "Reopen in Container" from the VS Code command palette.  
+You will need to have the **Dev Containers** extension installed in VS Code to use this feature.
+
+---
+
+**Note:**  
+The folder `.devcontainer` needs to be in the root of your project, containing a `devcontainer.json` file with the following content:
+
+```json
+{
+  "name": "ADT Press",
+  "build": {
+    // Sets the run context to one level up instead of the .devcontainer folder.
+    "context": "..",
+    // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename.
+    "dockerfile": "../Dockerfile"
+  }
+}
+```
+
+---
+
+**Environment Variable Required**
+
+> **Note:**  
+> You must set the environment variable `OPENAI_API_KEY` with your OpenAI API key for the application to work.
+>
+> - When running the Dockerized version, you need to set the `OPENAI_API_KEY` variable every time you run the container.  
+>   For example:
+>   ```bash
+>   docker run --rm -e OPENAI_API_KEY=your-key-here adt-press
+>   ```
+> - When using VS Code "Reopen in Container", you can add the variable to your `.env` file or set it in the container terminal before running your scripts.
diff --git a/adt-press.py b/adt-press.py
@@ -4,7 +4,7 @@
 import structlog
 from hamilton import driver, registry, telemetry
 from hamilton.lifecycle import NodeExecutionHook
-from omegaconf import OmegaConf
+from omegaconf import DictConfig, OmegaConf
 
 from adt_press.nodes import config_nodes, pdf_nodes, report_nodes
 
@@ -55,7 +55,7 @@ def run_after_node_execution(
 def main() -> None:
     cli_config = OmegaConf.from_cli()
     file_config = OmegaConf.load("config/config.yaml")
-    config = OmegaConf.merge(file_config, cli_config)
+    config = DictConfig(OmegaConf.merge(file_config, cli_config))
     print(OmegaConf.to_yaml(config))
 
     cache_args = {"recompute": True} if config.get("clear_cache", False) else {}
diff --git a/adt_press/llm/image_caption.py b/adt_press/llm/image_caption.py
@@ -10,7 +10,7 @@
 from .prompt import PromptConfig
 
 
-class ImageCaptionResponse(BaseModel):
+class CaptionResponse(BaseModel):
     caption: ImageCaption
 
 
@@ -31,9 +31,9 @@ async def get_image_caption(config: PromptConfig, page: Page, image: Image, lang
 
     prompt = Prompt(template_content)
     client = instructor.from_litellm(acompletion)
-    response = await client.chat.completions.create(
+    response: CaptionResponse = await client.chat.completions.create(
         model=config.model,
-        response_model=ImageCaptionResponse,
+        response_model=CaptionResponse,
         messages=[m.model_dump(exclude_none=True) for m in prompt.chat_messages(context)],
         max_retries=3,
     )
diff --git a/adt_press/llm/image_crop.py b/adt_press/llm/image_crop.py
@@ -27,7 +27,7 @@ async def get_image_crop_coordinates(config: PromptConfig, page: Page, image: Im
 
     prompt = Prompt(template_content)
     client = instructor.from_litellm(acompletion)
-    response = await client.chat.completions.create(
+    response: CropResponse = await client.chat.completions.create(
         model=config.model,
         response_model=CropResponse,
         messages=[m.model_dump(exclude_none=True) for m in prompt.chat_messages(context)],
diff --git a/adt_press/llm/image_meaningfulness.py b/adt_press/llm/image_meaningfulness.py
@@ -29,7 +29,7 @@ async def get_image_meaningfulness(config: PromptConfig, page: Page, image: Imag
 
     prompt = Prompt(template_content)
     client = instructor.from_litellm(acompletion)
-    response = await client.chat.completions.create(
+    response: MeaningfulnessResponse = await client.chat.completions.create(
         model=config.model,
         response_model=MeaningfulnessResponse,
         messages=[m.model_dump(exclude_none=True) for m in prompt.chat_messages(context)],
diff --git a/adt_press/llm/prompt.py b/adt_press/llm/prompt.py
@@ -8,7 +8,7 @@
 class PromptConfig(BaseModel):
     model: str
     template_path: str
-    template_hash: str = Field(default=None, exclude=True)
+    template_hash: str | None = Field(default=None, exclude=True)
     examples: list[dict] = []
     rate_limit: int = 300
 
diff --git a/adt_press/nodes/config_nodes.py b/adt_press/nodes/config_nodes.py
@@ -1,34 +1,34 @@
 from hamilton.function_modifiers import cache
-from omegaconf import OmegaConf
+from omegaconf import DictConfig, OmegaConf
 from pydantic import BaseModel
 
 from adt_press.llm.prompt import PromptConfig
 from adt_press.utils.file import calculate_file_hash
 from adt_press.utils.web import TemplateConfig
 
 
-def config() -> OmegaConf:
+def config() -> DictConfig:
     return OmegaConf.create({})
 
 
 def template_config(output_dir_config: str, template_dir_config: str) -> TemplateConfig:
     return TemplateConfig(output_dir=output_dir_config, template_dir=template_dir_config)
 
 
-def pdf_path_config(config: OmegaConf) -> str:
-    return config["pdf_path"]
+def pdf_path_config(config: DictConfig) -> str:
+    return str(config["pdf_path"])
 
 
-def output_language_config(config: OmegaConf) -> str:
-    return config.get("output_language", "en")
+def output_language_config(config: DictConfig) -> str:
+    return str(config.get("output_language", "en"))
 
 
-def output_dir_config(config: OmegaConf) -> str:
-    return config["output_dir"]
+def output_dir_config(config: DictConfig) -> str:
+    return str(config["output_dir"])
 
 
-def template_dir_config(config: OmegaConf) -> str:
-    return config["template_dir"]
+def template_dir_config(config: DictConfig) -> str:
+    return str(config["template_dir"])
 
 
 @cache(behavior="recompute")
@@ -41,41 +41,41 @@ class PageRangeConfig(BaseModel):
     end: int = 0
 
 
-def page_range_config(config: OmegaConf) -> PageRangeConfig:
+def page_range_config(config: DictConfig) -> PageRangeConfig:
     return PageRangeConfig.model_validate(config.get("page_range", {}))
 
 
 @cache(behavior="recompute")
-def caption_prompt_config(config: OmegaConf) -> PromptConfig:
+def caption_prompt_config(config: DictConfig) -> PromptConfig:
     return PromptConfig.model_validate(config["prompts"]["caption"])
 
 
 @cache(behavior="recompute")
-def crop_prompt_config(config: OmegaConf) -> PromptConfig:
+def crop_prompt_config(config: DictConfig) -> PromptConfig:
     return PromptConfig.model_validate(config["prompts"]["crop"])
 
 
 @cache(behavior="recompute")
-def meaningfulness_prompt_config(config: OmegaConf) -> PromptConfig:
+def meaningfulness_prompt_config(config: DictConfig) -> PromptConfig:
     return PromptConfig.model_validate(config["prompts"]["meaningfulness"])
 
 
-def image_config(config: OmegaConf) -> dict:
-    return config.get("image_filters", {})
+def image_config(config: DictConfig) -> DictConfig:
+    return DictConfig(config.get("image_filters", {}))
 
 
 class ImageSizeFilterConfig(BaseModel):
     max_side: int = 500
     min_side: int = 50
 
 
-def image_size_filter_config(image_config: dict) -> ImageSizeFilterConfig:
+def image_size_filter_config(image_config: DictConfig) -> ImageSizeFilterConfig:
     return ImageSizeFilterConfig.model_validate(image_config.get("size", {}))
 
 
 class BlankImageFilterConfig(BaseModel):
     threshold: int = 2
 
 
-def blank_image_filter_config(image_config: dict) -> BlankImageFilterConfig:
+def blank_image_filter_config(image_config: DictConfig) -> BlankImageFilterConfig:
     return BlankImageFilterConfig.model_validate(image_config.get("blank", {}))
diff --git a/adt_press/nodes/pdf_nodes.py b/adt_press/nodes/pdf_nodes.py
@@ -113,7 +113,7 @@ def pdf_processed_images(
     pdf_image_captions: list[ImageCaption],
     pdf_image_crops: list[ImageCrop],
     pdf_image_meaningfulness: list[ImageMeaningfulness],
-) -> list[Image]:
+) -> list[ProcessedImage]:
     processed_images = []
     for img, size_failures, blank_failures, caption, crop, meaningfulness in zip(
         pdf_raster_images,
diff --git a/adt_press/nodes/report_nodes.py b/adt_press/nodes/report_nodes.py
@@ -1,14 +1,14 @@
 from hamilton.function_modifiers import cache
-from omegaconf import OmegaConf
+from omegaconf import DictConfig, OmegaConf
 
 from adt_press.nodes.config_nodes import TemplateConfig
-from adt_press.utils.image import Image
+from adt_press.utils.image import ProcessedImage
 from adt_press.utils.pdf import Page
 from adt_press.utils.web import render_template
 
 
 @cache(behavior="recompute")
-def report_images(template_config: TemplateConfig, pdf_processed_images: list[Image]) -> str:
+def report_images(template_config: TemplateConfig, pdf_processed_images: list[ProcessedImage]) -> str:
     return render_template(template_config, "image_report.html", dict(images=pdf_processed_images))
 
 
@@ -18,7 +18,7 @@ def report_pages(template_config: TemplateConfig, pdf_pages: list[Page]) -> str:
 
 
 @cache(behavior="recompute")
-def report_config(template_config: TemplateConfig, config: OmegaConf) -> str:
+def report_config(template_config: TemplateConfig, config: DictConfig) -> str:
     return render_template(template_config, "config.html", dict(config=OmegaConf.to_yaml(config)))
 
 
diff --git a/adt_press/utils/image.py b/adt_press/utils/image.py
diff --git a/pyproject.toml b/pyproject.toml
diff --git a/uv.lock b/uv.lock
