API reference

data_models

Data models for the data extraction evaluation toolkit.

base

Core data models regarding annotations.

AnnotationType

Bases: StrEnum

Enumeration of annotation types.

Source code in deet/data_models/base.py

class AnnotationType(StrEnum):
    """Enumeration of annotation types."""

    HUMAN = auto()
    LLM = auto()

Attribute pydantic-model

Bases: BaseModel

Core attribute definition for data extraction tasks.

Represents a single piece of information to be extracted from documents.

Fields:

• prompt (str | None)
• output_data_type (AttributeType)
• attribute_id (int)
• attribute_label (str)

Source code in deet/data_models/base.py

class Attribute(BaseModel):
    """
    Core attribute definition for data extraction tasks.

    Represents a single piece of information to be extracted from documents.
    """

    model_config = ConfigDict()

    prompt: str | None = None  # an optional prompt.
    output_data_type: AttributeType  # One of the defined output data types
    attribute_id: int  # unique identifier for the attribute
    attribute_label: str  # human-readable way of identifying the attribute

    def write_to_csv(self, filepath: Path, mode: Literal["a", "w"] = "a") -> None:
        """
        Write an attribute as a line to a csv file - fields represent columns.

        Args:
            filepath (Path): outfile destination.
            mode (Literal["a", "w"], optional): _w_rite or _a_ppend.
            Defaults to "a" (append).

        """
        dictified = self.model_dump()

        filepath.parent.mkdir(parents=True, exist_ok=True)
        file_exists = filepath.exists() and filepath.stat().st_size > 0
        write_header = not file_exists or mode == "w"

        with filepath.open(mode=mode, newline="", encoding="utf-8") as f:
            writer = csv.DictWriter(f, fieldnames=dictified.keys())

            if write_header:
                writer.writeheader()

            writer.writerow(dictified)

        logger.debug(f"Wrote attribute to {filepath}")

    def populate_prompt_from_dict(
        self, input_dict: dict[str, Any], *, overwrite: bool = True
    ) -> None:
        """
        Populate the `prompt` field in an Attribute instance from a dict.

        The dict must contain following fields:
            - attribute_id
            - prompt
        and attribute_id(dict) must match self.attribute_id.

        NOTE: this would typically be used in a loop to populate
        prompts for a list of attributes from a csv file where every
        row represents an attribute.

        Args:
            input_dict (dict[str, Any]): An input dict, typically a line in a csv file.
            overwrite (bool, optional): Overwrite existing val in `self.prompt`.
            Defaults to True.

        """
        for field in ["attribute_id", "prompt"]:
            if field not in input_dict:
                bad_dict = (
                    "input dict must contain at least `attribute_id` and `prompt`"
                    " fields. currently, it only "
                    f"contains: {', '.join(input_dict.keys())}"
                )
                raise ValueError(bad_dict)

        if int(input_dict["attribute_id"]) != self.attribute_id:
            bad_att_id = (
                f"attribute_id mismatch: input: {input_dict['attribute_id']}. "
                f" self: {self.attribute_id}"
            )
            raise ValueError(bad_att_id)

        if overwrite or (not overwrite and self.prompt is None):
            self.prompt = input_dict["prompt"]
            logger.debug("added prompt  [...] to Attribute instance.")
        else:
            logger.info("overwrite is set to False, no overwrite prompts.")

    def print_tabulated(self) -> None:
        """Print tabulated version of the contents of this attribute."""
        dictified = self.model_dump()
        data = [[k, v] for k, v in dictified.items()]

        print(tabulate(data, headers=["Field", "Value"], tablefmt="simple"))

    def enter_custom_prompt(self, max_tries: int = 5) -> None:
        """Use CLI to add a prompt."""
        self.print_tabulated()
        print("")
        print("Do you want to add a new prompt? y/n. Use CTRL+C to cancel.")
        tries = 0
        while True:
            user_input = input().strip().lower()

            if user_input == "n":
                logger.debug("user chose not to write a prompt...")
                return

            if user_input == "y":
                break

            print("Please answer either `y` or `n`...")
            tries += 1
            if tries >= max_tries:
                return

        def sanitize_prompt(prompt: str) -> str:
            # Remove non-printable/control characters
            return "".join(c for c in prompt if c.isprintable())

        while True:
            print(f"Please enter your prompt (max {MAX_PROMPT_LENGTH} characters): ")
            user_prompt = input().strip()
            user_prompt = sanitize_prompt(user_prompt)
            if len(user_prompt) == 0:
                print("Prompt cannot be empty. Please try again.")
                continue
            if len(user_prompt) > MAX_PROMPT_LENGTH:
                print(f"Prompt exceeds max {MAX_PROMPT_LENGTH} chars. Shorten!.")
                continue
            print("\nYour prompt will be stored as:\n")
            print(f'"{user_prompt}"')
            print("Confirm? y/n")
            confirm = input().strip().lower()
            if confirm == "y":
                self.prompt = user_prompt
                logger.debug(f"wrote prompt {self.prompt[:30]} [...] to prompt field.")
                return
            if confirm == "n":
                print("Prompt entry cancelled. Please enter again or CTRL+C to exit.")
                continue

enter_custom_prompt(max_tries=5)

Use CLI to add a prompt.

Source code in deet/data_models/base.py

def enter_custom_prompt(self, max_tries: int = 5) -> None:
    """Use CLI to add a prompt."""
    self.print_tabulated()
    print("")
    print("Do you want to add a new prompt? y/n. Use CTRL+C to cancel.")
    tries = 0
    while True:
        user_input = input().strip().lower()

        if user_input == "n":
            logger.debug("user chose not to write a prompt...")
            return

        if user_input == "y":
            break

        print("Please answer either `y` or `n`...")
        tries += 1
        if tries >= max_tries:
            return

    def sanitize_prompt(prompt: str) -> str:
        # Remove non-printable/control characters
        return "".join(c for c in prompt if c.isprintable())

    while True:
        print(f"Please enter your prompt (max {MAX_PROMPT_LENGTH} characters): ")
        user_prompt = input().strip()
        user_prompt = sanitize_prompt(user_prompt)
        if len(user_prompt) == 0:
            print("Prompt cannot be empty. Please try again.")
            continue
        if len(user_prompt) > MAX_PROMPT_LENGTH:
            print(f"Prompt exceeds max {MAX_PROMPT_LENGTH} chars. Shorten!.")
            continue
        print("\nYour prompt will be stored as:\n")
        print(f'"{user_prompt}"')
        print("Confirm? y/n")
        confirm = input().strip().lower()
        if confirm == "y":
            self.prompt = user_prompt
            logger.debug(f"wrote prompt {self.prompt[:30]} [...] to prompt field.")
            return
        if confirm == "n":
            print("Prompt entry cancelled. Please enter again or CTRL+C to exit.")
            continue

populate_prompt_from_dict(input_dict, *, overwrite=True)

Populate the prompt field in an Attribute instance from a dict.

The dict must contain following fields

• attribute_id
• prompt

and attribute_id(dict) must match self.attribute_id.

NOTE: this would typically be used in a loop to populate prompts for a list of attributes from a csv file where every row represents an attribute.

Parameters:

Name	Type	Description	Default
input_dict	dict[str, Any]	An input dict, typically a line in a csv file.	required
overwrite	bool	Overwrite existing val in self.prompt.	True

Source code in deet/data_models/base.py

def populate_prompt_from_dict(
    self, input_dict: dict[str, Any], *, overwrite: bool = True
) -> None:
    """
    Populate the `prompt` field in an Attribute instance from a dict.

    The dict must contain following fields:
        - attribute_id
        - prompt
    and attribute_id(dict) must match self.attribute_id.

    NOTE: this would typically be used in a loop to populate
    prompts for a list of attributes from a csv file where every
    row represents an attribute.

    Args:
        input_dict (dict[str, Any]): An input dict, typically a line in a csv file.
        overwrite (bool, optional): Overwrite existing val in `self.prompt`.
        Defaults to True.

    """
    for field in ["attribute_id", "prompt"]:
        if field not in input_dict:
            bad_dict = (
                "input dict must contain at least `attribute_id` and `prompt`"
                " fields. currently, it only "
                f"contains: {', '.join(input_dict.keys())}"
            )
            raise ValueError(bad_dict)

    if int(input_dict["attribute_id"]) != self.attribute_id:
        bad_att_id = (
            f"attribute_id mismatch: input: {input_dict['attribute_id']}. "
            f" self: {self.attribute_id}"
        )
        raise ValueError(bad_att_id)

    if overwrite or (not overwrite and self.prompt is None):
        self.prompt = input_dict["prompt"]
        logger.debug("added prompt  [...] to Attribute instance.")
    else:
        logger.info("overwrite is set to False, no overwrite prompts.")

print_tabulated()

Print tabulated version of the contents of this attribute.

Source code in deet/data_models/base.py

def print_tabulated(self) -> None:
    """Print tabulated version of the contents of this attribute."""
    dictified = self.model_dump()
    data = [[k, v] for k, v in dictified.items()]

    print(tabulate(data, headers=["Field", "Value"], tablefmt="simple"))

write_to_csv(filepath, mode='a')

Write an attribute as a line to a csv file - fields represent columns.

Parameters:

Name	Type	Description	Default
filepath	Path	outfile destination.	required
mode	Literal['a', 'w']	_w_rite or _a_ppend.	'a'

Source code in deet/data_models/base.py

def write_to_csv(self, filepath: Path, mode: Literal["a", "w"] = "a") -> None:
    """
    Write an attribute as a line to a csv file - fields represent columns.

    Args:
        filepath (Path): outfile destination.
        mode (Literal["a", "w"], optional): _w_rite or _a_ppend.
        Defaults to "a" (append).

    """
    dictified = self.model_dump()

    filepath.parent.mkdir(parents=True, exist_ok=True)
    file_exists = filepath.exists() and filepath.stat().st_size > 0
    write_header = not file_exists or mode == "w"

    with filepath.open(mode=mode, newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=dictified.keys())

        if write_header:
            writer.writeheader()

        writer.writerow(dictified)

    logger.debug(f"Wrote attribute to {filepath}")

AttributeType

Bases: StrEnum

Enum of permitted attribute data types.

Source code in deet/data_models/base.py

class AttributeType(StrEnum):
    """Enum of permitted attribute data types."""

    STRING = auto()
    INTEGER = auto()
    FLOAT = auto()
    BOOL = auto()
    LIST = auto()
    DICT = auto()

    def missing_annotation_default(
        self,
    ) -> bool | str | int | float | list[Never] | dict[str, Never]:
        """
        Return default ``output_data`` when no gold-standard annotation exists.

        Used when synthesizing a placeholder annotation (e.g. comparing LLM output
        to gold standard where a value was never annotated).

        Returns a fresh ``list`` or ``dict`` for mutable types so callers do not share
        state.

        Raises:
            ValueError: If this member has no defined default.

        Note:
            This is not ``Enum._missing_``; that hook resolves *unrecognised raw
            values* when constructing enum members, not per-type defaults.

        """
        match self:
            case AttributeType.BOOL:
                return False
            case AttributeType.LIST:
                return []
            case AttributeType.STRING:
                return ""
            case AttributeType.INTEGER:
                return 0
            case AttributeType.FLOAT:
                return 0.0
            case AttributeType.DICT:
                return {}
            case _:
                unsupported = (
                    f"No default for missing annotation when attribute type is {self!s}"
                )
                raise ValueError(unsupported)

    def __str__(self) -> str:
        """Return the string value for JSON serialization."""
        return self.value

    def to_python_type(self) -> type:
        """Map AttributeType to actual Python types."""
        mapping = {
            AttributeType.STRING: str,
            AttributeType.INTEGER: int,
            AttributeType.FLOAT: float,
            AttributeType.BOOL: bool,
            AttributeType.LIST: list,
            AttributeType.DICT: dict,
        }
        return mapping[self]

    def llm_annotation_response_model(self) -> type[BaseModel]:
        """
        Return the shared Pydantic sub-model for LLM responses of this type.

        One model is built and cached per :class:`AttributeType`; attributes
        with the same type reuse it in :func:`build_llm_response_model`.

        Returns
            A Pydantic model class with typed ``output_data`` for this type.

        """
        return _llm_annotation_response_model_for_type(self)

    def to_json_type(self) -> JsonValue:
        """Map AttributeType to JS types for the JSON schema."""
        mapping: JsonDict = {
            AttributeType.STRING: {"type": "string"},
            AttributeType.INTEGER: {"type": "integer"},
            AttributeType.FLOAT: {"type": "number"},
            AttributeType.BOOL: {"type": "boolean"},
            AttributeType.LIST: {
                "type": "array",
                "items": {"type": "object", "additionalProperties": False},
            },
            AttributeType.DICT: {"type": "object", "additionalProperties": False},
        }
        return mapping[self]

__str__()

Return the string value for JSON serialization.

Source code in deet/data_models/base.py

def __str__(self) -> str:
    """Return the string value for JSON serialization."""
    return self.value

llm_annotation_response_model()

Return the shared Pydantic sub-model for LLM responses of this type.

One model is built and cached per :class:AttributeType; attributes with the same type reuse it in :func:build_llm_response_model.

Returns A Pydantic model class with typed output_data for this type.

Source code in deet/data_models/base.py

def llm_annotation_response_model(self) -> type[BaseModel]:
    """
    Return the shared Pydantic sub-model for LLM responses of this type.

    One model is built and cached per :class:`AttributeType`; attributes
    with the same type reuse it in :func:`build_llm_response_model`.

    Returns
        A Pydantic model class with typed ``output_data`` for this type.

    """
    return _llm_annotation_response_model_for_type(self)

missing_annotation_default()

Return default output_data when no gold-standard annotation exists.

Used when synthesizing a placeholder annotation (e.g. comparing LLM output to gold standard where a value was never annotated).

Returns a fresh list or dict for mutable types so callers do not share state.

Raises:

Type	Description
ValueError	If this member has no defined default.

Note

This is not Enum._missing_; that hook resolves unrecognised raw values when constructing enum members, not per-type defaults.

Source code in deet/data_models/base.py

def missing_annotation_default(
    self,
) -> bool | str | int | float | list[Never] | dict[str, Never]:
    """
    Return default ``output_data`` when no gold-standard annotation exists.

    Used when synthesizing a placeholder annotation (e.g. comparing LLM output
    to gold standard where a value was never annotated).

    Returns a fresh ``list`` or ``dict`` for mutable types so callers do not share
    state.

    Raises:
        ValueError: If this member has no defined default.

    Note:
        This is not ``Enum._missing_``; that hook resolves *unrecognised raw
        values* when constructing enum members, not per-type defaults.

    """
    match self:
        case AttributeType.BOOL:
            return False
        case AttributeType.LIST:
            return []
        case AttributeType.STRING:
            return ""
        case AttributeType.INTEGER:
            return 0
        case AttributeType.FLOAT:
            return 0.0
        case AttributeType.DICT:
            return {}
        case _:
            unsupported = (
                f"No default for missing annotation when attribute type is {self!s}"
            )
            raise ValueError(unsupported)

to_json_type()

Map AttributeType to JS types for the JSON schema.

Source code in deet/data_models/base.py

def to_json_type(self) -> JsonValue:
    """Map AttributeType to JS types for the JSON schema."""
    mapping: JsonDict = {
        AttributeType.STRING: {"type": "string"},
        AttributeType.INTEGER: {"type": "integer"},
        AttributeType.FLOAT: {"type": "number"},
        AttributeType.BOOL: {"type": "boolean"},
        AttributeType.LIST: {
            "type": "array",
            "items": {"type": "object", "additionalProperties": False},
        },
        AttributeType.DICT: {"type": "object", "additionalProperties": False},
    }
    return mapping[self]

to_python_type()

Map AttributeType to actual Python types.

Source code in deet/data_models/base.py

def to_python_type(self) -> type:
    """Map AttributeType to actual Python types."""
    mapping = {
        AttributeType.STRING: str,
        AttributeType.INTEGER: int,
        AttributeType.FLOAT: float,
        AttributeType.BOOL: bool,
        AttributeType.LIST: list,
        AttributeType.DICT: dict,
    }
    return mapping[self]

BaseLLMResponse pydantic-model

Bases: BaseModel

Base for all LLM response models.

Config:

• extra: forbid

Source code in deet/data_models/base.py

class BaseLLMResponse(BaseModel):
    """Base for all LLM response models."""

    model_config = ConfigDict(extra="forbid")

DynamicLLMResponseBase pydantic-model

Bases: BaseLLMResponse

The base for dynamically generated schemas.

We can expect that each field is typed as a subclass of LLMAnnotationResponse.

Source code in deet/data_models/base.py

class DynamicLLMResponseBase(BaseLLMResponse):
    """
    The base for dynamically generated schemas.

    We can expect that each field is typed as a subclass of LLMAnnotationResponse.
    """

    def iter_attribute_responses(self) -> Iterator[tuple[str, LLMAnnotationResponse]]:
        """Yield field names and safely typed LLMAnnotationResponses."""
        for field_name in self.__class__.model_fields:
            value = getattr(self, field_name)
            if isinstance(value, LLMAnnotationResponse):
                yield field_name, value

iter_attribute_responses()

Yield field names and safely typed LLMAnnotationResponses.

Source code in deet/data_models/base.py

def iter_attribute_responses(self) -> Iterator[tuple[str, LLMAnnotationResponse]]:
    """Yield field names and safely typed LLMAnnotationResponses."""
    for field_name in self.__class__.model_fields:
        value = getattr(self, field_name)
        if isinstance(value, LLMAnnotationResponse):
            yield field_name, value

GoldStandardAnnotation pydantic-model

Bases: BaseModel

A single gold standard annotation for an attribute.

raw_data stores the data as it comes from source, output_data is computed and coerces raw_data into the correct type. This can change if the AttributeType of the attribute changes.

Fields:

• attribute (Attribute)
• raw_data (Any)
• annotation_type (AnnotationType)
• additional_text (str | None)
• reasoning (str | None)

Source code in deet/data_models/base.py

class GoldStandardAnnotation(BaseModel):
    """
    A single gold standard annotation for an attribute.

    `raw_data` stores the data as it comes from source,
    `output_data` is computed and coerces raw_data into the correct type.
    This can change if the `AttributeType` of the `attribute` changes.
    """

    attribute: Attribute
    raw_data: Any = Field(
        description=(
            "The output data exactly as it was first seen"
            " without any coercion to the correct type"
        )
    )
    annotation_type: AnnotationType
    additional_text: str | None = Field(
        description="Notes provided by the annotator - usually the citation "
        " from the paper containing the context window where the attribute is found",
        default=None,
    )
    reasoning: str | None = Field(
        description="Reasoning, taken from LLM response", default=None
    )

    @model_validator(mode="before")
    @classmethod
    def handle_output_data_input(cls, data: SUPPORTED_TYPES) -> SUPPORTED_TYPES:
        """Catch instantations with output_data and send this to raw_data."""
        if isinstance(data, dict) and "output_data" in data and "raw_data" not in data:
            data["raw_data"] = data.pop("output_data")
        return data

    @computed_field  # type: ignore[prop-decorator]
    @property
    def output_data(self) -> SUPPORTED_TYPES | None:
        """Coerce raw data to correct type based on attribute."""
        strategy = ANNOTATION_COERCION_STRATEGIES.get(self.attribute.output_data_type)

        if strategy:
            return strategy(self.raw_data)

        return self.raw_data

additional_text = None pydantic-field

Notes provided by the annotator - usually the citation from the paper containing the context window where the attribute is found

output_data property

Coerce raw data to correct type based on attribute.

raw_data pydantic-field

The output data exactly as it was first seen without any coercion to the correct type

reasoning = None pydantic-field

Reasoning, taken from LLM response

handle_output_data_input(data) classmethod

Catch instantations with output_data and send this to raw_data.

Source code in deet/data_models/base.py

@model_validator(mode="before")
@classmethod
def handle_output_data_input(cls, data: SUPPORTED_TYPES) -> SUPPORTED_TYPES:
    """Catch instantations with output_data and send this to raw_data."""
    if isinstance(data, dict) and "output_data" in data and "raw_data" not in data:
        data["raw_data"] = data.pop("output_data")
    return data

LLMAnnotationResponse pydantic-model

Bases: BaseModel

LLM response model for a single attribute's annotation.

Used as the base for the per-type sub-models produced by :meth:AttributeType.llm_annotation_response_model. The attribute identity is not stored on this model; it is encoded in the parent field name (attribute_<id>) so the LLM is never asked to repeat (and potentially mismatch) the id.

output_data is typed Any here and is always overridden with the attribute's concrete Python type when the per-type sub-model is built.

Config:

• extra: forbid

Fields:

• output_data (Any)
• additional_text (str | None)
• reasoning (str | None)

Source code in deet/data_models/base.py

class LLMAnnotationResponse(BaseModel):
    """
    LLM response model for a single attribute's annotation.

    Used as the base for the per-type sub-models produced by
    :meth:`AttributeType.llm_annotation_response_model`. The attribute identity
    is *not* stored on this model; it is encoded in the parent field name
    (``attribute_<id>``) so the LLM is never asked to repeat (and potentially
    mismatch) the id.

    ``output_data`` is typed ``Any`` here and is always overridden with the
    attribute's concrete Python type when the per-type sub-model is built.
    """

    output_data: Any = Field(
        ...,
        description="The LLM's annotation for this attribute.",
    )
    additional_text: str | None = Field(
        ...,
        description=(
            "Supporting text from document containing the context window "
            "where the attribute is found"
        ),
    )
    reasoning: str | None = Field(
        ...,
        description="Reasoning or explanation for the annotation decision",
    )

    # Note: arm_id, arm_title, arm_description, item_attribute_full_text_details
    # are not included as they're EPPI-specific metadata the LLM cannot provide

    model_config = ConfigDict(extra="forbid")

additional_text pydantic-field

Supporting text from document containing the context window where the attribute is found

output_data pydantic-field

The LLM's annotation for this attribute.

reasoning pydantic-field

Reasoning or explanation for the annotation decision

LLMInputSchema pydantic-model

Bases: BaseModel

Schema for data going into the LLM.

Config:

• extra: ignore

Fields:

• prompt (str)
• attribute_id (int)
• output_data_type (AttributeType)

Validators:

• fill_prompt

Source code in deet/data_models/base.py

class LLMInputSchema(BaseModel):
    """Schema for data going into the LLM."""

    prompt: str
    attribute_id: int
    output_data_type: AttributeType

    model_config = ConfigDict(extra="ignore")

    @model_validator(mode="before")
    @classmethod
    def fill_prompt(cls, data: dict, fill_from_field: str = "attribute_label") -> dict:
        """
        Fill `prompt` field if empty.

        Args:
            data (dict): the incoming data
            fill_from_field (str, optional): field to use to fill prompt if empty.
                Defaults to "attribute_label".

        Returns:
            dict: the populated data.

        """
        if data["prompt"] is not None:
            return data
        logger.debug(data)
        if fill_from_field not in data:
            no_fill_field = f" '{fill_from_field}' is missing from data"
            raise ValueError(no_fill_field)
        data["prompt"] = data[fill_from_field]
        logger.debug(f"filled `prompt` with {data['prompt']}.")
        return data

fill_prompt(data, fill_from_field='attribute_label') pydantic-validator

Fill prompt field if empty.

Parameters:

Name	Type	Description	Default
data	dict	the incoming data	required
fill_from_field	str	field to use to fill prompt if empty. Defaults to "attribute_label".	'attribute_label'

Returns:

Name	Type	Description
dict	dict	the populated data.

Source code in deet/data_models/base.py

@model_validator(mode="before")
@classmethod
def fill_prompt(cls, data: dict, fill_from_field: str = "attribute_label") -> dict:
    """
    Fill `prompt` field if empty.

    Args:
        data (dict): the incoming data
        fill_from_field (str, optional): field to use to fill prompt if empty.
            Defaults to "attribute_label".

    Returns:
        dict: the populated data.

    """
    if data["prompt"] is not None:
        return data
    logger.debug(data)
    if fill_from_field not in data:
        no_fill_field = f" '{fill_from_field}' is missing from data"
        raise ValueError(no_fill_field)
    data["prompt"] = data[fill_from_field]
    logger.debug(f"filled `prompt` with {data['prompt']}.")
    return data

LLMResponseSchema pydantic-model

Bases: BaseLLMResponse

Static response schema containing a list of StaticLLMAnnotationResponses.

This structure contains a list of StaticLLMAnnotationResponses, where each response has an attribute_id, and output_data_type is untyped.

Responses of this type are cheaper to request, since the json schema passed to the llm is shorter. However such a schema does not require llms to produce exactly one annotation per attribute, or to make sure that output_data_type matches that defined at the attribute level.

Fields:

• annotations (list[StaticLLMAnnotationResponse])

Source code in deet/data_models/base.py

class LLMResponseSchema(BaseLLMResponse):
    """
    Static response schema containing a list of StaticLLMAnnotationResponses.

    This structure contains a list of StaticLLMAnnotationResponses, where
    each response has an attribute_id, and output_data_type is untyped.

    Responses of this type are cheaper to request, since the json schema passed to
    the llm is shorter. However such a schema does not require llms to produce
    exactly one annotation per attribute, or to make sure that output_data_type
    matches that defined at the attribute level.
    """

    annotations: list[StaticLLMAnnotationResponse] = Field(
        ..., description="List of annotations extracted from the document"
    )

annotations pydantic-field

List of annotations extracted from the document

StaticLLMAnnotationResponse pydantic-model

Bases: LLMAnnotationResponse

Untyped LLM annotation response model where attribute_id is defined per annotation.

Fields:

• output_data (Any)
• additional_text (str | None)
• reasoning (str | None)
• attribute_id (int)

Source code in deet/data_models/base.py

class StaticLLMAnnotationResponse(LLMAnnotationResponse):
    """
    Untyped LLM annotation response model where attribute_id is defined per
    annotation.
    """

    attribute_id: int = Field(
        ..., description="The ID of the attribute being annotated."
    )

attribute_id pydantic-field

The ID of the attribute being annotated.

attribute_id_from_response_key(key)

Recover the attribute id from a dynamic-model field name.

Parameters:

Name	Type	Description	Default
key	str	A field name such as "attribute_1234".	required

Returns:

Type	Description
int	The integer attribute id encoded in the key.

Raises:

Type	Description
ValueError	If key does not encode a valid integer attribute id.

Source code in deet/data_models/base.py

def attribute_id_from_response_key(key: str) -> int:
    """
    Recover the attribute id from a dynamic-model field name.

    Args:
        key: A field name such as ``"attribute_1234"``.

    Returns:
        The integer attribute id encoded in the key.

    Raises:
        ValueError: If ``key`` does not encode a valid integer attribute id.

    """
    raw_id = key.removeprefix(ATTRIBUTE_RESPONSE_KEY_PREFIX)
    try:
        return int(raw_id)
    except ValueError as exc:
        msg = f"Cannot recover attribute id from response key {key!r}"
        raise ValueError(msg) from exc

attribute_response_key(attribute_id)

Build the dynamic-model field name used for a given attribute.

Parameters:

Name	Type	Description	Default
attribute_id	int	The attribute's unique identifier.	required

Returns:

Type	Description
str	The field name, e.g. "attribute_1234".

Source code in deet/data_models/base.py

def attribute_response_key(attribute_id: int) -> str:
    """
    Build the dynamic-model field name used for a given attribute.

    Args:
        attribute_id: The attribute's unique identifier.

    Returns:
        The field name, e.g. ``"attribute_1234"``.

    """
    return f"{ATTRIBUTE_RESPONSE_KEY_PREFIX}{attribute_id}"

build_llm_response_model(attributes)

Build a dynamic LLM response model from the selected attributes.

Each attribute becomes a required, correctly typed sub-model keyed by attribute_<id> on the returned root model. Because every key is required and extra="forbid" is set at every level, the JSON schema forces the LLM to return exactly one response per attribute - no missing attributes, no extra/hallucinated attributes, and output_data constrained to the attribute's concrete type. This also yields a schema that providers such as Ollama accept as valid (unlike an Any-typed field).

Parameters:

Name	Type	Description	Default
attributes	list[Attribute]	The attributes to extract; must be non-empty.	required

Returns:

Type	Description
type[DynamicLLMResponseBase]	A dynamically created Pydantic model class suitable for use as a
type[DynamicLLMResponseBase]	structured-output schema and for validating the LLM response.

Raises:

Type	Description
ValueError	If attributes is empty.

Source code in deet/data_models/base.py

def build_llm_response_model(
    attributes: list[Attribute],
) -> type[DynamicLLMResponseBase]:
    """
    Build a dynamic LLM response model from the selected attributes.

    Each attribute becomes a required, correctly typed sub-model keyed by
    ``attribute_<id>`` on the returned root model. Because every key is required
    and ``extra="forbid"`` is set at every level, the JSON schema forces the LLM
    to return exactly one response per attribute - no missing attributes, no
    extra/hallucinated attributes, and ``output_data`` constrained to the
    attribute's concrete type. This also yields a schema that providers such as
    Ollama accept as valid (unlike an ``Any``-typed field).

    Args:
        attributes: The attributes to extract; must be non-empty.

    Returns:
        A dynamically created Pydantic model class suitable for use as a
        structured-output schema and for validating the LLM response.

    Raises:
        ValueError: If ``attributes`` is empty.

    """
    if not attributes:
        msg = "Cannot build an LLM response model from an empty attribute list"
        raise ValueError(msg)

    response_fields: dict[str, tuple[type[BaseModel], Any]] = {}
    for attribute in attributes:
        response_fields[attribute_response_key(attribute.attribute_id)] = (
            attribute.output_data_type.llm_annotation_response_model(),
            Field(...),
        )

    return create_model(
        "DynamicLLMResponse",
        __base__=DynamicLLMResponseBase,
        **cast(Any, response_fields),
    )

coerce_annotation_to_bool(val)

Coerce an annotation to a bool.

Source code in deet/data_models/base.py

def coerce_annotation_to_bool(val: SUPPORTED_TYPES) -> bool:
    """Coerce an annotation to a bool."""
    if isinstance(val, bool):
        return val
    if isinstance(val, str) and val.lower() in ("false", "0"):
        return False

    if isinstance(val, int | float):
        return bool(val)

    return True

coerce_annotation_to_float(val)

Coerce an annotation to a float.

Source code in deet/data_models/base.py

def coerce_annotation_to_float(val: SUPPORTED_TYPES) -> float | None:
    """Coerce an annotation to a float."""
    if isinstance(val, float):
        return val
    if isinstance(val, str | bool | int):
        try:
            return float(val)
        except ValueError:
            logger.warning(f"Could not convert {val} to float")

    logger.warning(f"Unsupported type for float conversion: {type(val).__name__}")
    return None

coerce_annotation_to_int(val)

Coerce an annotation to a int.

Source code in deet/data_models/base.py

def coerce_annotation_to_int(val: SUPPORTED_TYPES) -> int | None:
    """Coerce an annotation to a int."""
    if isinstance(val, int):
        return val
    if isinstance(val, str | float | bool):
        try:
            return int(val)
        except ValueError:
            logger.warning("Could not convert {val} to int")

    logger.warning(f"Unsupported type for int conversion: {type(val).__name__}")
    return None

coerce_annotation_to_list(val)

Coerce an annotation to list.

Source code in deet/data_models/base.py

def coerce_annotation_to_list(val: SUPPORTED_TYPES) -> list:
    """Coerce an annotation to list."""
    if val is None:
        return []
    if isinstance(val, list):
        return val
    if isinstance(val, str):
        list_of_strings = [item.strip() for item in val.split(";;;")]
        try:
            return [int(item) for item in list_of_strings]
        except (ValueError, TypeError):
            logger.debug("Could not convert items to int.")

        try:
            return [float(item) for item in list_of_strings]
        except (ValueError, TypeError):
            logger.debug("Could not convert items to float")

        return list_of_strings
    return [val]

coerce_annotation_to_str(val)

Coerce an annotation to a string.

Source code in deet/data_models/base.py

def coerce_annotation_to_str(val: SUPPORTED_TYPES) -> str:
    """Coerce an annotation to a string."""
    return str(val) if val else ""

documents

Data models concerning documents and how to represent them in deet.

ContextType

Bases: StrEnum

Types of context that can be provided to the LLM.

Source code in deet/data_models/documents.py

class ContextType(StrEnum):
    """Types of context that can be provided to the LLM."""

    # TODO: implement EMPTY = auto()
    FULL_DOCUMENT = auto()
    ABSTRACT_ONLY = auto()

Document pydantic-model

Bases: BaseModel

Represents a document.

This can be used both for references itemised in a document listing gold standard annotations (e.g. eppi.json) AND for a document coming from a file (e.g. pdf) without linking to a gold standard annotations document with references.

Config:

• extra: allow
• validate_assignment: True

Fields:

• name (str)
• citation (ReferenceFileInput)
• context (str | None)
• context_type (ContextType | None)
• document_id (int | str | None)
• document_identity (DocumentIdentity | None)
• parsed_document (ParsedOutput | None)
• original_doc_filepath (Path | None)
• is_final (bool)
• is_linked (bool)

Source code in deet/data_models/documents.py

class Document(BaseModel):
    """
    Represents a document.

    This can be used both for references itemised
    in a document listing gold standard annotations (e.g. eppi.json)
    AND
    for a document coming from a file (e.g. pdf) without
    linking to a gold standard annotations document with references.
    """

    # `extra` allows extra fields, e.g. for EppiDocument.
    model_config = ConfigDict(extra="allow", validate_assignment=True)
    # `validate_assignment` runs model/field validators
    # not only on instantiation, but also when we change values,
    # e.g. when we set is_linked=True, thereby preventing
    # us from saying something is linked if it isn't.

    name: str
    citation: ReferenceFileInput
    context: str | None = None  # new defaults, empty
    context_type: ContextType | None = None
    document_id: int | str | None = None  # add support for str ids, e.g. uuid
    document_identity: DocumentIdentity | None = None

    parsed_document: ParsedOutput | None = None
    original_doc_filepath: Path | None = (
        None  # NOTE -- add S3/blob support when required.
    )

    is_final: bool = False
    is_linked: bool = False

    @property
    def safe_identity(self) -> DocumentIdentity:
        """
        Definitely Return an identity.
        Initialise identity if not set already,
        or raise an error if this is not possible.
        """
        if self.document_identity is None:
            self.init_document_identity()
        if self.document_identity is None:
            no_id = "Failed to initialise document identity"
            raise RuntimeError(no_id)
        return self.document_identity

    @property
    def safe_parsed_document(self) -> ParsedOutput:
        """Return the parsed_document, or raise an error if document is not linked."""
        if self.parsed_document is None:
            unlinked = "Document is not linked, cannot access parsed_document"
            raise RuntimeError(unlinked)
        return self.parsed_document

    @model_validator(mode="after")
    def validate_linking_complete(self) -> Self:
        """Validate linking is completed if `is_linked=True`."""
        base_err_msg = "requirements not met for linking: "
        if self.is_linked:
            if self.context is None:
                no_context_err = base_err_msg + "`context` is empty."
                raise ValueError(no_context_err)
            if (
                self.context_type is None
                or self.context_type != ContextType.FULL_DOCUMENT
            ):
                no_context_type_err = (
                    base_err_msg + "`context_type` is not FULL_DOCUMENT."
                )
                raise ValueError(no_context_type_err)
            if self.document_identity is None:
                no_doc_id_err = (
                    base_err_msg + "`document_identity` is empty.  "
                    "run `init_document_identity() to populate."
                )
                raise ValueError(no_doc_id_err)
            if self.parsed_document is None:
                no_parsed_doc_err = base_err_msg + "`parsed_document` is empty. "
                raise ValueError(no_parsed_doc_err)

        return self

    @model_validator(mode="after")
    def validate_final(self) -> Self:
        """Validate Document is permitted to be `is_final`."""
        base_err_msg = "requirements not met for `Document().is_final`: "
        if self.is_final:
            if self.context is None:
                no_context_err = base_err_msg + "`context` is empty."
                raise ValueError(no_context_err)
            if self.context_type is None:
                bad_context_type_err = base_err_msg + "`context_type` musnt be None."
                raise ValueError(bad_context_type_err)
            if self.document_identity is None:
                no_doc_id_err = (
                    base_err_msg + "`document_identity` is empty.  "
                    "run `init_document_identity() to populate."
                )
                raise ValueError(no_doc_id_err)

        return self

    def init_document_identity(
        self,
        existing_ids: set[int] | None = None,
        *,
        return_id: bool = True,
    ) -> int | None:
        """Initialise document_identity field using available metadata."""
        labs_ref = LabsReference(reference=self.citation)  # convert for easy access
        self.document_identity = DocumentIdentity(
            document_id=None,
            external_id=self.document_id,
            doi=labs_ref.doi,
            first_author=labs_ref.first_author,
            year=str(labs_ref.publication_year),
        )

        logger.info("populating id & id source...")
        self.document_identity.populate_id(existing_ids=existing_ids)
        if self.document_id is None:
            logger.info(
                "populating Document-level `document_id` field with "
                f"newly populated id {self.document_identity.document_id}... "
            )
            self.document_id = self.document_identity.document_id

        if return_id:
            return self.document_identity.document_id
        return None

    def author_year_from_document_identity(
        self, substring_strategy: Literal["longest", "last"]
    ) -> str:
        """
        Create lower-case `author_year` guess from a Document's
        DocumentIdentity field.
        The idea is to take the last name of the first author.

        NOTE: this can probably improved with more knowledge
        of how destiny encodes the first_author field.

        Returns:
            author_year (str): `author_year`

        """
        if self.document_identity is None:
            logger.debug("document identity is None, initialising...")
            self.init_document_identity()

        if (
            self.document_identity is None
            or self.document_identity.first_author is None
            or self.document_identity.year is None
        ):
            whats_what = f"self.document_identity: {self.document_identity}; "
            if self.document_identity is not None:
                whats_what += (
                    "self.document_identity.first_author: "
                    f"{self.document_identity.first_author}; "
                    f"self.document_identity.year: {self.document_identity.year}."
                )
            logger.warning(whats_what)
            raise ValueError
        author_name = self.document_identity.first_author
        year = self.document_identity.year

        name_components = author_name.split(" ")
        if substring_strategy == "longest":
            name_guess = max(name_components, key=len)
        elif substring_strategy == "last":
            name_guess = name_components[-1]
        else:
            missing_name_guess_method_err = (
                f"method {substring_strategy} is not implemented."
            )
            raise NotImplementedError(missing_name_guess_method_err)

        return f"{name_guess.lower()}_{year}"

    def set_abstract_context(self) -> None:
        """Set the abstract, contained in `citation` field, as context."""
        abstract = LabsReference(reference=self.citation).abstract
        if abstract is not None:
            self.context_type = ContextType.ABSTRACT_ONLY
            self.context = abstract
            logger.info(
                "set context type to ABSTRACT_ONLY; set context to abstract."
                f" snippet: {abstract[:20]}"
            )
            return
        no_abstract = "No abstract found"
        raise NoAbstractError(no_abstract)

    def link_parsed_document(
        self,
        parsed_document: ParsedOutput,
        original_doc_filepath: Path | None = None,
    ) -> None:
        """
        Link parsed document and document metadata/`reference`.

        Args:
            parsed_document (ParsedOutput): the output from the parser
            original_doc_filepath (Path): full filepath to the original doc.

        """
        self.parsed_document = parsed_document
        if original_doc_filepath and (
            not original_doc_filepath.is_file() or not original_doc_filepath.exists()
        ):
            logger.warning(
                "supplied `original_doc_filepath` does not resolve. writing None."
            )
            original_doc_filepath = None
        self.original_doc_filepath = original_doc_filepath

    def set_context_from_parsed(self) -> None:
        """Symlink context to parsed_document.text."""
        if self.parsed_document and self.parsed_document.text:
            self.context = self.parsed_document.text
            self.context_type = ContextType.FULL_DOCUMENT
        else:
            logger.warning("no text in parsed_document!")

    def save(self, path: Path) -> None:
        """Save linked document to .json."""
        data = self.model_dump(by_alias=False)

        # convert images to base64 for json serialization
        # NOTE @all -- we leave ourselves open to
        # malicious stuff being injected here. open to
        # suggestions as to how to validate/circumvent.
        # however, we do control what goes in and out,
        # so the danger here might be overstated.
        if self.parsed_document and self.parsed_document.images:
            images_b64 = {}
            for key, img in self.parsed_document.images.items():
                buffer = BytesIO()
                img.save(buffer, format="PNG")
                images_b64[key] = base64.b64encode(buffer.getvalue()).decode()
            data["parsed_document"]["images"] = images_b64

        path.parent.mkdir(parents=True, exist_ok=True)
        with path.open("w", encoding="utf-8") as f:
            json.dump(data, f, indent=2, default=str)
        logger.info(f"Saved Document fulltext link to {path}")

    @classmethod
    def load(cls, path: Path) -> Self:
        """Load linked document from .json."""
        data = json.loads(path.read_text(encoding="utf-8"))
        logger.debug(data)

        # convert base64 back to PIL img
        if data.get("parsed_document") is not None and data.get(
            "parsed_document", {}
        ).get("images"):
            images = {}
            for key, img_b64 in data["parsed_document"]["images"].items():
                img_bytes = base64.b64decode(img_b64)
                images[key] = Image.open(BytesIO(img_bytes))
            data["parsed_document"]["images"] = images

        return cls(**data)

safe_identity property

Definitely Return an identity. Initialise identity if not set already, or raise an error if this is not possible.

safe_parsed_document property

Return the parsed_document, or raise an error if document is not linked.

author_year_from_document_identity(substring_strategy)

Create lower-case author_year guess from a Document's DocumentIdentity field. The idea is to take the last name of the first author.

NOTE: this can probably improved with more knowledge of how destiny encodes the first_author field.

Returns:

Name	Type	Description
author_year	str	author_year

Source code in deet/data_models/documents.py

def author_year_from_document_identity(
    self, substring_strategy: Literal["longest", "last"]
) -> str:
    """
    Create lower-case `author_year` guess from a Document's
    DocumentIdentity field.
    The idea is to take the last name of the first author.

    NOTE: this can probably improved with more knowledge
    of how destiny encodes the first_author field.

    Returns:
        author_year (str): `author_year`

    """
    if self.document_identity is None:
        logger.debug("document identity is None, initialising...")
        self.init_document_identity()

    if (
        self.document_identity is None
        or self.document_identity.first_author is None
        or self.document_identity.year is None
    ):
        whats_what = f"self.document_identity: {self.document_identity}; "
        if self.document_identity is not None:
            whats_what += (
                "self.document_identity.first_author: "
                f"{self.document_identity.first_author}; "
                f"self.document_identity.year: {self.document_identity.year}."
            )
        logger.warning(whats_what)
        raise ValueError
    author_name = self.document_identity.first_author
    year = self.document_identity.year

    name_components = author_name.split(" ")
    if substring_strategy == "longest":
        name_guess = max(name_components, key=len)
    elif substring_strategy == "last":
        name_guess = name_components[-1]
    else:
        missing_name_guess_method_err = (
            f"method {substring_strategy} is not implemented."
        )
        raise NotImplementedError(missing_name_guess_method_err)

    return f"{name_guess.lower()}_{year}"

init_document_identity(existing_ids=None, *, return_id=True)

Initialise document_identity field using available metadata.

Source code in deet/data_models/documents.py

def init_document_identity(
    self,
    existing_ids: set[int] | None = None,
    *,
    return_id: bool = True,
) -> int | None:
    """Initialise document_identity field using available metadata."""
    labs_ref = LabsReference(reference=self.citation)  # convert for easy access
    self.document_identity = DocumentIdentity(
        document_id=None,
        external_id=self.document_id,
        doi=labs_ref.doi,
        first_author=labs_ref.first_author,
        year=str(labs_ref.publication_year),
    )

    logger.info("populating id & id source...")
    self.document_identity.populate_id(existing_ids=existing_ids)
    if self.document_id is None:
        logger.info(
            "populating Document-level `document_id` field with "
            f"newly populated id {self.document_identity.document_id}... "
        )
        self.document_id = self.document_identity.document_id

    if return_id:
        return self.document_identity.document_id
    return None

link_parsed_document(parsed_document, original_doc_filepath=None)

Link parsed document and document metadata/reference.

Parameters:

Name	Type	Description	Default
parsed_document	ParsedOutput	the output from the parser	required
original_doc_filepath	Path	full filepath to the original doc.	None

Source code in deet/data_models/documents.py

def link_parsed_document(
    self,
    parsed_document: ParsedOutput,
    original_doc_filepath: Path | None = None,
) -> None:
    """
    Link parsed document and document metadata/`reference`.

    Args:
        parsed_document (ParsedOutput): the output from the parser
        original_doc_filepath (Path): full filepath to the original doc.

    """
    self.parsed_document = parsed_document
    if original_doc_filepath and (
        not original_doc_filepath.is_file() or not original_doc_filepath.exists()
    ):
        logger.warning(
            "supplied `original_doc_filepath` does not resolve. writing None."
        )
        original_doc_filepath = None
    self.original_doc_filepath = original_doc_filepath

load(path) classmethod

Load linked document from .json.

Source code in deet/data_models/documents.py

@classmethod
def load(cls, path: Path) -> Self:
    """Load linked document from .json."""
    data = json.loads(path.read_text(encoding="utf-8"))
    logger.debug(data)

    # convert base64 back to PIL img
    if data.get("parsed_document") is not None and data.get(
        "parsed_document", {}
    ).get("images"):
        images = {}
        for key, img_b64 in data["parsed_document"]["images"].items():
            img_bytes = base64.b64decode(img_b64)
            images[key] = Image.open(BytesIO(img_bytes))
        data["parsed_document"]["images"] = images

    return cls(**data)

save(path)

Save linked document to .json.

Source code in deet/data_models/documents.py

def save(self, path: Path) -> None:
    """Save linked document to .json."""
    data = self.model_dump(by_alias=False)

    # convert images to base64 for json serialization
    # NOTE @all -- we leave ourselves open to
    # malicious stuff being injected here. open to
    # suggestions as to how to validate/circumvent.
    # however, we do control what goes in and out,
    # so the danger here might be overstated.
    if self.parsed_document and self.parsed_document.images:
        images_b64 = {}
        for key, img in self.parsed_document.images.items():
            buffer = BytesIO()
            img.save(buffer, format="PNG")
            images_b64[key] = base64.b64encode(buffer.getvalue()).decode()
        data["parsed_document"]["images"] = images_b64

    path.parent.mkdir(parents=True, exist_ok=True)
    with path.open("w", encoding="utf-8") as f:
        json.dump(data, f, indent=2, default=str)
    logger.info(f"Saved Document fulltext link to {path}")

set_abstract_context()

Set the abstract, contained in citation field, as context.

Source code in deet/data_models/documents.py

def set_abstract_context(self) -> None:
    """Set the abstract, contained in `citation` field, as context."""
    abstract = LabsReference(reference=self.citation).abstract
    if abstract is not None:
        self.context_type = ContextType.ABSTRACT_ONLY
        self.context = abstract
        logger.info(
            "set context type to ABSTRACT_ONLY; set context to abstract."
            f" snippet: {abstract[:20]}"
        )
        return
    no_abstract = "No abstract found"
    raise NoAbstractError(no_abstract)

set_context_from_parsed()

Symlink context to parsed_document.text.

Source code in deet/data_models/documents.py

def set_context_from_parsed(self) -> None:
    """Symlink context to parsed_document.text."""
    if self.parsed_document and self.parsed_document.text:
        self.context = self.parsed_document.text
        self.context_type = ContextType.FULL_DOCUMENT
    else:
        logger.warning("no text in parsed_document!")

validate_final()

Validate Document is permitted to be is_final.

Source code in deet/data_models/documents.py

@model_validator(mode="after")
def validate_final(self) -> Self:
    """Validate Document is permitted to be `is_final`."""
    base_err_msg = "requirements not met for `Document().is_final`: "
    if self.is_final:
        if self.context is None:
            no_context_err = base_err_msg + "`context` is empty."
            raise ValueError(no_context_err)
        if self.context_type is None:
            bad_context_type_err = base_err_msg + "`context_type` musnt be None."
            raise ValueError(bad_context_type_err)
        if self.document_identity is None:
            no_doc_id_err = (
                base_err_msg + "`document_identity` is empty.  "
                "run `init_document_identity() to populate."
            )
            raise ValueError(no_doc_id_err)

    return self

validate_linking_complete()

Validate linking is completed if is_linked=True.

Source code in deet/data_models/documents.py

@model_validator(mode="after")
def validate_linking_complete(self) -> Self:
    """Validate linking is completed if `is_linked=True`."""
    base_err_msg = "requirements not met for linking: "
    if self.is_linked:
        if self.context is None:
            no_context_err = base_err_msg + "`context` is empty."
            raise ValueError(no_context_err)
        if (
            self.context_type is None
            or self.context_type != ContextType.FULL_DOCUMENT
        ):
            no_context_type_err = (
                base_err_msg + "`context_type` is not FULL_DOCUMENT."
            )
            raise ValueError(no_context_type_err)
        if self.document_identity is None:
            no_doc_id_err = (
                base_err_msg + "`document_identity` is empty.  "
                "run `init_document_identity() to populate."
            )
            raise ValueError(no_doc_id_err)
        if self.parsed_document is None:
            no_parsed_doc_err = base_err_msg + "`parsed_document` is empty. "
            raise ValueError(no_parsed_doc_err)

    return self

DocumentIDSource

Bases: StrEnum

Sources for a given document_id.

Priority is given first to EPPI IDs, or IDs that confirm to the EPPI ID format.

When these do not exist we try strategies that hash information from the document, starting from the external id, and then attempting with other bibliographic information.

Source code in deet/data_models/documents.py

class DocumentIDSource(StrEnum):
    """
    Sources for a given document_id.

    Priority is given first to EPPI IDs, or IDs that confirm to the EPPI ID format.

    When these do not exist we try strategies that hash information from the document,
    starting from the external id, and then
    attempting with other bibliographic information.
    """

    EPPI_ITEM_ID = auto()
    EXTERNAL_ID = auto()
    DOI_AUTHOR_YEAR = auto()
    DOI_ID = auto()
    AUTHOR_YEAR_ID = auto()
    RANDINT = auto()

DocumentIdentity pydantic-model

Bases: BaseModel

A unified identity for a document, deriveable from multiple sources.

document_id: always int, the canonical internal id assigned by deet. in current implementation, mirrors eppi item ids. internal_id: a symlink to document_id. external_id: ID inherited verbatim from source citation/gold standard data. this can be string or int, no validation is performed on it.

Fields:

• document_id (int | None)
• document_id_source (DocumentIDSource | None)
• external_id (str | int | None)
• doi (str | None)
• first_author (str | None)
• year (str | None)

Source code in deet/data_models/documents.py

class DocumentIdentity(BaseModel):
    """
    A unified identity for a document, deriveable from multiple sources.

    `document_id`:
        always int, the canonical internal id assigned by deet.
        in current implementation, mirrors eppi item ids.
    `internal_id`:
        a symlink to `document_id`.
    `external_id`:
        ID inherited verbatim from source citation/gold standard
        data. this can be string or int, no validation is performed on
        it.

    """

    document_id: int | None = None
    document_id_source: DocumentIDSource | None = None
    external_id: str | int | None = None

    # parsed citation info
    doi: str | None
    first_author: str | None
    year: str | None

    @property
    def internal_id(self) -> int | None:
        """Return the internal ID (alias for document_id for backward compatibility)."""
        return self.document_id

    def populate_id(
        self,
        existing_ids: set[int] | None = None,
        hierarchy: list[DocumentIDSource] | None = None,
    ) -> None:
        """
        Populate document_id using a hierarchical list of ID creation methods.

        Tries each method in order until a unique ID is generated. If an ID
        conflicts with existing_ids, tries the next method. RANDINT always
        succeeds as fallback.

        NOTE: we will have to implement some sort of matching thing, if we are
        concerned that an id-collision might be becuase we have already
        parsed&linked a document.

        Args:
            existing_ids: List of existing IDs to check for conflicts.
            hierarchy: Ordered list of DocumentIDSource methods to try.
                Defaults to [EPPI_ITEM_ID, DOI_AUTHOR_YEAR, DOI_ID,
                AUTHOR_YEAR_ID, RANDINT].

        Raises:
            BadDocumentIdError: If unable to generate unique ID (should never
                happen as RANDINT is always in hierarchy).

        """
        if existing_ids is None:
            existing_ids = set()

        if hierarchy is None:
            hierarchy = list(DocumentIDSource)  # retain the order from the enum

        if DocumentIDSource.RANDINT not in hierarchy:
            hierarchy.append(
                DocumentIDSource.RANDINT
            )  # always keep random as a fallback

        attempted_sources = []

        for id_source in hierarchy:
            try:
                id_factory = self._create_id_factory(id_source)
                logger.debug(f"created id_factory: {id_factory.__name__}")
                potential_id = id_factory()
                logger.debug(
                    f"created potential id: {potential_id} "
                    f"using factory {id_factory.__name__}"
                )

                # id collisions?
                if potential_id not in existing_ids:
                    self.document_id = potential_id
                    self.document_id_source = id_source
                    logger.debug(
                        f"successfully created document_id {potential_id} "
                        f"using {id_source}"
                    )
                    return

                logger.debug(
                    f"id {potential_id} from {id_source} conflicts with existing IDs"
                )
                attempted_sources.append(id_source)

            except (BadDocumentIdError, MissingCitationElementError) as e:
                logger.debug(f"Failed to create ID using {id_source}: {e}")
                attempted_sources.append(id_source)
                continue

        failed_sources = ", ".join(str(s) for s in attempted_sources)
        err_msg = (
            f"Failed to generate unique document_id after trying: {failed_sources}"
        )

        if len(attempted_sources) == len(hierarchy):
            max_attempts = 10
            attempts = 0
            for _ in range(max_attempts):
                potential_id = self._random_int_id()
                if potential_id not in existing_ids:
                    self.document_id = potential_id
                    self.document_id_source = DocumentIDSource.RANDINT
                    logger.debug(
                        f"successfully created document_id {potential_id} "
                        f"using {id_source}"
                    )
                    return

        err_msg += f" plus {attempts} randint attempts."
        raise BadDocumentIdError(err_msg)

    def _create_id_factory(self, id_source: DocumentIDSource) -> Callable:
        """
        Return an id-creating method given specific value of DocumentIDSource.

        Returns:
            int: the id.

        """
        id_creation_map = {
            DocumentIDSource.EPPI_ITEM_ID: self._eppi_item_id,
            DocumentIDSource.DOI_ID: self._doi_id,
            DocumentIDSource.AUTHOR_YEAR_ID: self._author_year_id,
            DocumentIDSource.DOI_AUTHOR_YEAR: self._doi_author_year_id,
            DocumentIDSource.EXTERNAL_ID: self._external_id,
            DocumentIDSource.RANDINT: self._random_int_id,
        }

        return id_creation_map[id_source]

    def _eppi_item_id(self) -> int:
        """
        Map an existing item_id (parsed as external_id
        in DocumentIdentity instance).
        """
        # we're going to assume that our `document_id`, received
        # from parsing eppi-json to EppiDocument is always going
        # to be eppi, otherwise this method should be extended to
        # reflect it coming from somewhere else.
        # Either way, it must be a positive integer with a number of digits
        # between MIN_DOCUMENT_ID_DIGITS and MAX_DOCUMENT_ID_DIGITS (inclusive).
        if (
            self.external_id is not None
            and isinstance(self.external_id, int)
            and self.external_id > 0
        ):
            digit_count = len(str(abs(self.external_id)))
            if MIN_DOCUMENT_ID_DIGITS <= digit_count <= MAX_DOCUMENT_ID_DIGITS:
                # Set external_id to the original document_id for EPPi item IDs
                if self.document_id is None:
                    self.document_id = self.external_id
                return self.document_id
        bad_doc_id = f"id {self.external_id} is not a valid eppi item_id."
        raise BadDocumentIdError(bad_doc_id)

    def _external_id(self) -> int:
        """Create an id by hashing the external ID."""
        if self.external_id is not None:
            payload = [str(self.external_id).strip()]
            return hash_n_strings_to_document_id(payload)

        bad_doc_id = (
            "Cannot generate ID from EXTERNAL_ID " "source because external_id is None."
        )
        raise BadDocumentIdError(bad_doc_id)

    def _citation_id_hasher(self, target_fields: list[str]) -> int:
        """Create an id from _n_ citation fields."""
        if not all(field in self.model_dump() for field in target_fields):
            missing_citation = (
                f"required fields are missing in citation. "
                f"required: {', '.join(target_fields)}"
                f"actual: {','.join(self.model_dump())}"
            )
            raise MissingCitationElementError(missing_citation)
        payload = [self.model_dump()[field] for field in target_fields]

        if "" in payload or None in payload:
            none_or_empty = (
                "some or all of target fields are "
                f"None or empty strings: {','.join(target_fields)} "
            )
            raise MissingCitationElementError(none_or_empty)
        return hash_n_strings_to_document_id(payload)

    def _doi_id(self) -> int:
        """Create an integer id as a function of doi."""
        return self._citation_id_hasher(["doi"])

    def _doi_author_year_id(self) -> int:
        """Create an integer id as a function of doi, author and year."""
        return self._citation_id_hasher(["doi", "first_author", "year"])

    def _author_year_id(self) -> int:
        """Create an 8-digit integer id as a function of author and year."""
        return self._citation_id_hasher(["first_author", "year"])

    @staticmethod
    def _random_int_id() -> int:
        """Create a random integer id with 8 digits."""
        return randint(MIN_DOCUMENT_ID, MAX_DOCUMENT_ID)  # noqa: S311

internal_id property

Return the internal ID (alias for document_id for backward compatibility).

populate_id(existing_ids=None, hierarchy=None)

Populate document_id using a hierarchical list of ID creation methods.

Tries each method in order until a unique ID is generated. If an ID conflicts with existing_ids, tries the next method. RANDINT always succeeds as fallback.

NOTE: we will have to implement some sort of matching thing, if we are concerned that an id-collision might be becuase we have already parsed&linked a document.

Parameters:

Name	Type	Description	Default
existing_ids	set[int] | None	List of existing IDs to check for conflicts.	None
hierarchy	list[DocumentIDSource] | None	Ordered list of DocumentIDSource methods to try. Defaults to [EPPI_ITEM_ID, DOI_AUTHOR_YEAR, DOI_ID, AUTHOR_YEAR_ID, RANDINT].	None

Raises:

Type	Description
BadDocumentIdError	If unable to generate unique ID (should never happen as RANDINT is always in hierarchy).

Source code in deet/data_models/documents.py

def populate_id(
    self,
    existing_ids: set[int] | None = None,
    hierarchy: list[DocumentIDSource] | None = None,
) -> None:
    """
    Populate document_id using a hierarchical list of ID creation methods.

    Tries each method in order until a unique ID is generated. If an ID
    conflicts with existing_ids, tries the next method. RANDINT always
    succeeds as fallback.

    NOTE: we will have to implement some sort of matching thing, if we are
    concerned that an id-collision might be becuase we have already
    parsed&linked a document.

    Args:
        existing_ids: List of existing IDs to check for conflicts.
        hierarchy: Ordered list of DocumentIDSource methods to try.
            Defaults to [EPPI_ITEM_ID, DOI_AUTHOR_YEAR, DOI_ID,
            AUTHOR_YEAR_ID, RANDINT].

    Raises:
        BadDocumentIdError: If unable to generate unique ID (should never
            happen as RANDINT is always in hierarchy).

    """
    if existing_ids is None:
        existing_ids = set()

    if hierarchy is None:
        hierarchy = list(DocumentIDSource)  # retain the order from the enum

    if DocumentIDSource.RANDINT not in hierarchy:
        hierarchy.append(
            DocumentIDSource.RANDINT
        )  # always keep random as a fallback

    attempted_sources = []

    for id_source in hierarchy:
        try:
            id_factory = self._create_id_factory(id_source)
            logger.debug(f"created id_factory: {id_factory.__name__}")
            potential_id = id_factory()
            logger.debug(
                f"created potential id: {potential_id} "
                f"using factory {id_factory.__name__}"
            )

            # id collisions?
            if potential_id not in existing_ids:
                self.document_id = potential_id
                self.document_id_source = id_source
                logger.debug(
                    f"successfully created document_id {potential_id} "
                    f"using {id_source}"
                )
                return

            logger.debug(
                f"id {potential_id} from {id_source} conflicts with existing IDs"
            )
            attempted_sources.append(id_source)

        except (BadDocumentIdError, MissingCitationElementError) as e:
            logger.debug(f"Failed to create ID using {id_source}: {e}")
            attempted_sources.append(id_source)
            continue

    failed_sources = ", ".join(str(s) for s in attempted_sources)
    err_msg = (
        f"Failed to generate unique document_id after trying: {failed_sources}"
    )

    if len(attempted_sources) == len(hierarchy):
        max_attempts = 10
        attempts = 0
        for _ in range(max_attempts):
            potential_id = self._random_int_id()
            if potential_id not in existing_ids:
                self.document_id = potential_id
                self.document_id_source = DocumentIDSource.RANDINT
                logger.debug(
                    f"successfully created document_id {potential_id} "
                    f"using {id_source}"
                )
                return

    err_msg += f" plus {attempts} randint attempts."
    raise BadDocumentIdError(err_msg)

GoldStandardAnnotatedDocument pydantic-model

Bases: BaseModel

A document with its gold standard annotations.

Fields:

• document (DocumentType)
• annotations (list[GoldStandardAnnotationType])

Source code in deet/data_models/documents.py

class GoldStandardAnnotatedDocument[
    DocumentType: Document,
    GoldStandardAnnotationType: GoldStandardAnnotation,
](BaseModel):
    """A document with its gold standard annotations."""

    document: DocumentType
    annotations: list[GoldStandardAnnotationType]

    def get_attribute_annotation(self, attribute: Attribute) -> GoldStandardAnnotation:
        """Get the value of the annotation of the corresponding attribute."""
        result = None
        output_data: Any
        for annotation in self.annotations:
            if annotation.attribute.attribute_id == attribute.attribute_id:
                if result is not None:
                    multiple_matches = (
                        "More than one annotation found for "
                        f"attribute: {attribute.attribute_label}. We don't know how to"
                        "interpret which is the canonical version."
                    )
                    raise DuplicateAnnotationError(multiple_matches)
                result = annotation

        if result is None:
            try:
                output_data = attribute.output_data_type.missing_annotation_default()
                reasoning = (
                    "No annotation with this attribute found: reverting to default"
                )
            except ValueError:
                output_data = ""
                reasoning = (
                    "No annotation with this attribute found: "
                    "No default for this attribute type"
                )
            return GoldStandardAnnotation(
                attribute=attribute,
                raw_data=output_data,
                reasoning=reasoning,
                annotation_type=AnnotationType.HUMAN,
            )

        return result

get_attribute_annotation(attribute)

Get the value of the annotation of the corresponding attribute.

Source code in deet/data_models/documents.py

def get_attribute_annotation(self, attribute: Attribute) -> GoldStandardAnnotation:
    """Get the value of the annotation of the corresponding attribute."""
    result = None
    output_data: Any
    for annotation in self.annotations:
        if annotation.attribute.attribute_id == attribute.attribute_id:
            if result is not None:
                multiple_matches = (
                    "More than one annotation found for "
                    f"attribute: {attribute.attribute_label}. We don't know how to"
                    "interpret which is the canonical version."
                )
                raise DuplicateAnnotationError(multiple_matches)
            result = annotation

    if result is None:
        try:
            output_data = attribute.output_data_type.missing_annotation_default()
            reasoning = (
                "No annotation with this attribute found: reverting to default"
            )
        except ValueError:
            output_data = ""
            reasoning = (
                "No annotation with this attribute found: "
                "No default for this attribute type"
            )
        return GoldStandardAnnotation(
            attribute=attribute,
            raw_data=output_data,
            reasoning=reasoning,
            annotation_type=AnnotationType.HUMAN,
        )

    return result

GoldStandardAnnotatedDocumentList pydantic-model

Bases: BaseModel

A list of GoldStandardAnnotatedDocuments (or any subclasses thereof). This list is indexed to enable easy retrieval by document_id.

Fields:

• gold_standard_annotations (Sequence[GoldStandardAnnotatedDocumentType])

Source code in deet/data_models/documents.py

class GoldStandardAnnotatedDocumentList[
    GoldStandardAnnotatedDocumentType: GoldStandardAnnotatedDocument[Any, Any]
](BaseModel):
    """
    A list of GoldStandardAnnotatedDocuments (or any subclasses thereof).
    This list is indexed to enable easy retrieval by document_id.
    """

    gold_standard_annotations: Sequence[GoldStandardAnnotatedDocumentType]

    @cached_property
    def annotation_index(self) -> dict[int, GoldStandardAnnotatedDocumentType]:
        """Cached index to enable retrieving annotated documents by id."""
        return {
            doc.document.safe_identity.document_id: doc
            for doc in self.gold_standard_annotations
        }

    def get_by_id(self, document_id: int) -> GoldStandardAnnotatedDocumentType:
        """
        Get GoldStandardAnnotatedDocuments where document.document_identity
        matches document_identity.
        """
        try:
            return self.annotation_index[document_id]
        except KeyError as err:
            not_found = f"Document with ID {document_id} not found in annotated"
            " doc list"
            raise MissingDocumentError(not_found) from err

annotation_index cached property

Cached index to enable retrieving annotated documents by id.

get_by_id(document_id)

Get GoldStandardAnnotatedDocuments where document.document_identity matches document_identity.

Source code in deet/data_models/documents.py

def get_by_id(self, document_id: int) -> GoldStandardAnnotatedDocumentType:
    """
    Get GoldStandardAnnotatedDocuments where document.document_identity
    matches document_identity.
    """
    try:
        return self.annotation_index[document_id]
    except KeyError as err:
        not_found = f"Document with ID {document_id} not found in annotated"
        " doc list"
        raise MissingDocumentError(not_found) from err

enums

A store for plain enums, so they can quickly be imported in the CLI. We can use these to set argument types and defaults, without needing large imports, that would slow the CLI down during autocomplete, or when asking for --help.

CustomPromptPopulationMethod

Bases: StrEnum

Methods of populating prompts.

Source code in deet/data_models/enums.py

class CustomPromptPopulationMethod(StrEnum):
    """Methods of populating prompts."""

    FILE = auto()
    CLI = auto()

EvaluationStrategyName

Bases: StrEnum

A list of allowable names for evaluation strategies.

Note: make sure that each of these is in the strategy registry in deet.data_models.evaluation_strategies.init.py.

Source code in deet/data_models/enums.py

class EvaluationStrategyName(StrEnum):
    """
    A list of allowable names for evaluation strategies.

    Note: make sure that each of these is in the strategy registry in
    deet.data_models.evaluation_strategies.__init__.py.
    """

    NONE = auto()
    DEV_VAL_TEST = auto()

eppi

EPPI-specific data models extending the core models.

AttributeAnswerCoT pydantic-model

Bases: BaseModel

Detailed answer format for a single attribute with reasoning.

Fields:

• attribute_name (str)
• answer (str)
• reasoning (str)
• citation (str | None)

Source code in deet/data_models/eppi.py

class AttributeAnswerCoT(BaseModel):
    """Detailed answer format for a single attribute with reasoning."""

    attribute_name: str = Field(
        description="The name of the attribute being asked about"
    )
    answer: str = Field(description="The answer to the question, 'True' or 'False'")
    reasoning: str = Field(description="The reasoning behind the answer")
    citation: str | None = Field(
        description="The citation from the Research Information to support the answer"
    )

answer pydantic-field

The answer to the question, 'True' or 'False'

attribute_name pydantic-field

The name of the attribute being asked about

citation pydantic-field

The citation from the Research Information to support the answer

reasoning pydantic-field

The reasoning behind the answer

BatchAnswerFormatCoT pydantic-model

Bases: BaseModel

Batch answers for all attributes with reasoning.

Fields:

• answers (list[AttributeAnswerCoT])

Source code in deet/data_models/eppi.py

class BatchAnswerFormatCoT(BaseModel):
    """Batch answers for all attributes with reasoning."""

    answers: list[AttributeAnswerCoT] = Field(
        description="List of answers for each attribute"
    )

answers pydantic-field

List of answers for each attribute

EppiAttribute pydantic-model

Bases: Attribute

EPPI-specific attribute with additional fields.

Extends the core Attribute class with EPPI-specific metadata and hierarchy information.

Uses alias generators to automatically map camelCase EPPI JSON fields to snake_case Python fields.

Config:

• validate_by_name: True
• validate_by_alias: True

Fields:

• prompt (str | None)
• attribute_id (int)
• attribute_selection_type (EppiAttributeSelectionType)
• output_data_type (AttributeType)
• attribute_label (str)
• attribute_set_description (str | None)
• hierarchy_path (str | None)
• hierarchy_level (int)
• is_leaf (bool)
• parent_attribute_id (int | None)
• attribute_description (str | None)

Source code in deet/data_models/eppi.py

class EppiAttribute(Attribute):
    """
    EPPI-specific attribute with additional fields.

    Extends the core Attribute class with EPPI-specific
    metadata and hierarchy information.

    Uses alias generators to automatically map
    camelCase EPPI JSON fields to snake_case Python fields.
    """

    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)  # type: ignore[typeddict-unknown-key]

    attribute_id: int = Field(
        validation_alias=AliasChoices("AttributeId", "attribute_id")
    )
    attribute_selection_type: EppiAttributeSelectionType = Field(
        default=EppiAttributeSelectionType.UNSPECIFIED,
        validation_alias=AliasChoices(
            "AttributeType", "attribute_type", "attribute_selection_type"
        ),
    )
    output_data_type: AttributeType = DEFAULT_ATTRIBUTE_TYPE
    attribute_label: str = Field(alias="AttributeName")

    # EPPI-specific fields - these map automatically from camelCase JSON
    attribute_set_description: str | None = Field(
        description="Description of the attribute set this attribute belongs to",
        default=None,
    )
    hierarchy_path: str | None = Field(
        description="Dot-separated path showing the hierarchical "
        " position of this attribute",
        default=None,
    )
    hierarchy_level: int = Field(
        description="Numeric level indicating depth in "
        " the attribute hierarchy (0 = root level)",
        default=0,
    )
    is_leaf: bool = Field(
        description="Whether this attribute is a leaf node  (has no child attributes)",
        default=True,
    )
    parent_attribute_id: int | None = Field(
        description="ID of the parent attribute in the hierarchy", default=None
    )
    attribute_description: str | None = Field(
        description="Detailed description explaining what this attribute represents",
        default=None,
    )

attribute_description = None pydantic-field

Detailed description explaining what this attribute represents

attribute_set_description = None pydantic-field

Description of the attribute set this attribute belongs to

hierarchy_level = 0 pydantic-field

Numeric level indicating depth in the attribute hierarchy (0 = root level)

hierarchy_path = None pydantic-field

Dot-separated path showing the hierarchical position of this attribute

is_leaf = True pydantic-field

Whether this attribute is a leaf node (has no child attributes)

parent_attribute_id = None pydantic-field

ID of the parent attribute in the hierarchy

EppiAttributeSelectionType

Bases: StrEnum

AttributeType as it appears in eppi json.

Source code in deet/data_models/eppi.py

class EppiAttributeSelectionType(StrEnum):
    """`AttributeType` as it appears in eppi json."""

    SELECTABLE = "Selectable (show checkbox)"
    OUTCOME = "Outcome"
    INTERVENTION = "Intervention"
    NOT_SELECTABLE = "Not Selectable (no checkbox)"
    UNSPECIFIED = "Unspecified"

    @classmethod
    def _missing_(cls, value: object) -> "EppiAttributeSelectionType | None":
        """Handle case-insensitive assignment & lookup."""
        if isinstance(value, str):
            for member in cls:
                if member.value.lower() == value.lower():
                    return member
        return None

EppiCodeSet pydantic-model

Bases: BaseModel

Represents a single CodeSet from EPPI JSON.

CodeSets contain hierarchical attribute definitions used in EPPI-Reviewer.

Fields:

• attributes (dict[str, Any] | None)

Source code in deet/data_models/eppi.py

class EppiCodeSet(BaseModel):
    """
    Represents a single CodeSet from EPPI JSON.

    CodeSets contain hierarchical attribute definitions used in EPPI-Reviewer.
    """

    attributes: dict[str, Any] | None = Field(alias="Attributes", default=None)

    def get_attributes_list(self) -> list[dict[str, Any]]:
        """Extract AttributesList from the CodeSet."""
        if self.attributes and "AttributesList" in self.attributes:
            return self.attributes["AttributesList"]
        return []

get_attributes_list()

Extract AttributesList from the CodeSet.

Source code in deet/data_models/eppi.py

def get_attributes_list(self) -> list[dict[str, Any]]:
    """Extract AttributesList from the CodeSet."""
    if self.attributes and "AttributesList" in self.attributes:
        return self.attributes["AttributesList"]
    return []

EppiDocument pydantic-model

Bases: Document

EPPI-specific document.

Uses alias generators to automatically map camelCase EPPI JSON fields to snake_case Python fields.

Config:

• validate_by_name: True
• validate_by_alias: True

Fields:

• citation (ReferenceFileInput)
• context (str | None)
• context_type (ContextType | None)
• document_identity (DocumentIdentity | None)
• parsed_document (ParsedOutput | None)
• original_doc_filepath (Path | None)
• is_final (bool)
• is_linked (bool)
• name (str)
• document_id (int | str)
• parent_title (str | None)
• short_title (str | None)
• date_created (datetime | None)
• created_by (str | None)
• edited_by (str | None)
• year (int | None)
• month (str | None)
• abstract (str | None)
• authors (str | None)
• keywords (str | None)
• doi (str | None)

Validators:

• empty_year_string_to_none → year
• parse_date_string → date_created
• populate_citation_field

Source code in deet/data_models/eppi.py

class EppiDocument(Document):
    """
    EPPI-specific document.

    Uses alias generators to automatically map
    camelCase EPPI JSON fields to snake_case Python fields.
    """

    name: str = Field(default="", validation_alias=AliasChoices("Title", "name"))
    document_id: int | str = Field(
        validation_alias=AliasChoices("ItemId", "document_id")
    )

    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)  # type: ignore[typeddict-unknown-key]

    parent_title: str | None = Field(
        default=None, validation_alias=AliasChoices("ParentTitle", "parent_title")
    )
    short_title: str | None = Field(
        default=None, validation_alias=AliasChoices("ShortTitle", "short_title")
    )
    date_created: datetime | None = Field(
        default=None, validation_alias=AliasChoices("DateCreated", "date_created")
    )
    created_by: str | None = Field(
        default=None, validation_alias=AliasChoices("CreatedBy", "created_by")
    )
    edited_by: str | None = Field(
        default=None, validation_alias=AliasChoices("EditedBy", "edited_by")
    )
    year: int | None = Field(
        default=None, validation_alias=AliasChoices("Year", "year")
    )
    month: str | None = Field(
        default=None, validation_alias=AliasChoices("Month", "month")
    )
    abstract: str | None = Field(
        default=None, validation_alias=AliasChoices("Abstract", "abstract")
    )
    authors: str | None = Field(
        default=None, validation_alias=AliasChoices("Authors", "authors")
    )
    keywords: str | None = Field(
        default=None, validation_alias=AliasChoices("Keywords", "keywords")
    )
    doi: str | None = Field(default=None, validation_alias=AliasChoices("DOI", "doi"))

    @field_validator("year", mode="before")
    @classmethod
    def empty_year_string_to_none(cls, value: Any) -> int | None:  # noqa:ANN401
        """Parse an empty string year to None or return as is."""
        if value == "" or value is None:
            return None
        if isinstance(value, int):
            return value
        if isinstance(value, str):
            stripped_value = value.strip()
            if stripped_value == "":
                return None
            if stripped_value.isdigit():
                return int(stripped_value)
        bad_year = f"unable to parse year: {value!r}"
        raise ValueError(bad_year)

    @field_validator("date_created", mode="before")
    @classmethod
    def parse_date_string(cls, value: str) -> datetime | None:
        """Parse a string datetime to native datetime."""
        if value is None or value == "":
            return None
        if isinstance(value, datetime):
            return value
        if isinstance(value, str):
            # add as we encounter other formats, if ever relevant
            formats = [
                "%d/%m/%Y",  # OG EPPI
                "%Y-%m-%d %H:%M:%S%z",  # ISO format with timezone,
                # result of dumping is_final EppiDocument to json
                "%Y-%m-%d",  # simple ISO date
            ]

            for fmt in formats:
                try:
                    return datetime.strptime(value, fmt).replace(tzinfo=UTC)
                except ValueError:
                    continue
            no_parsage = "unable to parse date_created."
            raise ValueError(no_parsage)

        return None

    @model_validator(mode="before")
    @classmethod
    def populate_citation_field(cls, data: dict[str, Any]) -> dict:
        """
        Populate the `citation` field with a Destiny
        reference derived from the EPPI data.
        """
        # if not isinstance(data, dict):
        #     return data
        if "citation" in data:
            # we have already created citation,
            # no need to do it again
            return data

        citation = parse_citation_to_destiny(reference=data)
        data["citation"] = citation

        return data

empty_year_string_to_none(value) pydantic-validator

Parse an empty string year to None or return as is.

Source code in deet/data_models/eppi.py

@field_validator("year", mode="before")
@classmethod
def empty_year_string_to_none(cls, value: Any) -> int | None:  # noqa:ANN401
    """Parse an empty string year to None or return as is."""
    if value == "" or value is None:
        return None
    if isinstance(value, int):
        return value
    if isinstance(value, str):
        stripped_value = value.strip()
        if stripped_value == "":
            return None
        if stripped_value.isdigit():
            return int(stripped_value)
    bad_year = f"unable to parse year: {value!r}"
    raise ValueError(bad_year)

parse_date_string(value) pydantic-validator

Parse a string datetime to native datetime.

Source code in deet/data_models/eppi.py

@field_validator("date_created", mode="before")
@classmethod
def parse_date_string(cls, value: str) -> datetime | None:
    """Parse a string datetime to native datetime."""
    if value is None or value == "":
        return None
    if isinstance(value, datetime):
        return value
    if isinstance(value, str):
        # add as we encounter other formats, if ever relevant
        formats = [
            "%d/%m/%Y",  # OG EPPI
            "%Y-%m-%d %H:%M:%S%z",  # ISO format with timezone,
            # result of dumping is_final EppiDocument to json
            "%Y-%m-%d",  # simple ISO date
        ]

        for fmt in formats:
            try:
                return datetime.strptime(value, fmt).replace(tzinfo=UTC)
            except ValueError:
                continue
        no_parsage = "unable to parse date_created."
        raise ValueError(no_parsage)

    return None

populate_citation_field(data) pydantic-validator

Populate the citation field with a Destiny reference derived from the EPPI data.

Source code in deet/data_models/eppi.py

@model_validator(mode="before")
@classmethod
def populate_citation_field(cls, data: dict[str, Any]) -> dict:
    """
    Populate the `citation` field with a Destiny
    reference derived from the EPPI data.
    """
    # if not isinstance(data, dict):
    #     return data
    if "citation" in data:
        # we have already created citation,
        # no need to do it again
        return data

    citation = parse_citation_to_destiny(reference=data)
    data["citation"] = citation

    return data

EppiGoldStandardAnnotatedDocument pydantic-model

Bases: GoldStandardAnnotatedDocument[EppiDocument, EppiGoldStandardAnnotation]

EPPI-specific gold standard annotated document.

Fields:

• document (DocumentType)
• annotations (list[GoldStandardAnnotationType])

Source code in deet/data_models/eppi.py

class EppiGoldStandardAnnotatedDocument(
    GoldStandardAnnotatedDocument[EppiDocument, EppiGoldStandardAnnotation]
):
    """EPPI-specific gold standard annotated document."""

EppiGoldStandardAnnotation pydantic-model

Bases: GoldStandardAnnotation

EPPI-specific gold standard annotation.

In EPPI-Reviewer context, an "arm" refers to a study group or intervention group within a research study (e.g., "Treatment Group", "Control Group", "Placebo Group"). Each annotation is associated with a specific arm to indicate which study group the extracted information relates to.

Fields:

• attribute (Attribute)
• raw_data (Any)
• annotation_type (AnnotationType)
• additional_text (str | None)
• reasoning (str | None)
• arm_id (int | None)
• arm_title (str | None)
• arm_description (str | None)
• item_attribute_full_text_details (list[EppiItemAttributeFullTextDetails] | None)

Source code in deet/data_models/eppi.py

class EppiGoldStandardAnnotation(GoldStandardAnnotation):
    """
    EPPI-specific gold standard annotation.

    In EPPI-Reviewer context, an "arm" refers to a study group or intervention group
    within a research study (e.g., "Treatment Group", "Control Group", "Placebo Group").
    Each annotation is associated with a specific arm to indicate which study group
    the extracted information relates to.
    """

    arm_id: int | None = Field(
        description="ID of the study arm this annotation relates to", default=None
    )
    arm_title: str | None = Field(
        description="Title or name of the study arm", default=None
    )
    arm_description: str | None = Field(
        description="Detailed description of the study arm", default=None
    )
    item_attribute_full_text_details: list[EppiItemAttributeFullTextDetails] | None = (
        Field(
            description="List of detailed text extracts and "
            " arm-specific information for this annotation",
            default=None,
        )
    )

arm_description = None pydantic-field

Detailed description of the study arm

arm_id = None pydantic-field

ID of the study arm this annotation relates to

arm_title = None pydantic-field

Title or name of the study arm

item_attribute_full_text_details = None pydantic-field

List of detailed text extracts and arm-specific information for this annotation

EppiItemAttributeFullTextDetails pydantic-model

Bases: BaseModel

EPPI-specific item attribute full text details.

Arm specific information, exact text keywords for the attribute.

Fields:

• item_document_id (int | None)
• text (str | None)
• item_arm (str | None)

Validators:

• validate_at_least_one_field

Source code in deet/data_models/eppi.py

class EppiItemAttributeFullTextDetails(BaseModel):
    """
    EPPI-specific item attribute full text details.

    Arm specific information, exact text keywords for the attribute.
    """

    item_document_id: int | None = None
    text: str | None = None
    item_arm: str | None = None

    @model_validator(mode="before")
    @classmethod
    def validate_at_least_one_field(cls, data: dict) -> dict:
        """Ensure at least one field is not None."""
        if all(v is None for k, v in data.items()):
            msg = (
                "At least one field must be provided "
                "(item_document_id, text, or item_arm)"
            )
            raise ValueError(msg)
        return data

validate_at_least_one_field(data) pydantic-validator

Ensure at least one field is not None.

Source code in deet/data_models/eppi.py

@model_validator(mode="before")
@classmethod
def validate_at_least_one_field(cls, data: dict) -> dict:
    """Ensure at least one field is not None."""
    if all(v is None for k, v in data.items()):
        msg = (
            "At least one field must be provided "
            "(item_document_id, text, or item_arm)"
        )
        raise ValueError(msg)
    return data

EppiRawData pydantic-model

Bases: BaseModel

Represents the complete EPPI JSON structure.

This model validates and structures the raw EPPI JSON data, making it easier to work with and validate.

Fields:

• code_sets (list[EppiCodeSet])
• references (list[dict[str, Any]])

Source code in deet/data_models/eppi.py

class EppiRawData(BaseModel):
    """
    Represents the complete EPPI JSON structure.

    This model validates and structures the raw EPPI JSON data,
    making it easier to work with and validate.
    """

    code_sets: list[EppiCodeSet] = Field(alias="CodeSets", default=[])
    references: list[dict[str, Any]] = Field(alias="References", default=[])

    def extract_all_attributes(
        self, flatten_hierarchy_func: Callable[[list], list]
    ) -> list[dict[str, Any]]:
        """
        Extract and flatten attributes from all CodeSets.

        Args:
            flatten_hierarchy_func: Function to flatten attribute hierarchy

        Returns:
            List of flattened attribute dictionaries

        """
        all_attributes = []
        for codeset in self.code_sets:
            attributes_list = codeset.get_attributes_list()
            if attributes_list:
                flattened = flatten_hierarchy_func(attributes_list)
                all_attributes.extend(flattened)
        return all_attributes

extract_all_attributes(flatten_hierarchy_func)

Extract and flatten attributes from all CodeSets.

Parameters:

Name	Type	Description	Default
flatten_hierarchy_func	Callable[[list], list]	Function to flatten attribute hierarchy	required

Returns:

Type	Description
list[dict[str, Any]]	List of flattened attribute dictionaries

Source code in deet/data_models/eppi.py

def extract_all_attributes(
    self, flatten_hierarchy_func: Callable[[list], list]
) -> list[dict[str, Any]]:
    """
    Extract and flatten attributes from all CodeSets.

    Args:
        flatten_hierarchy_func: Function to flatten attribute hierarchy

    Returns:
        List of flattened attribute dictionaries

    """
    all_attributes = []
    for codeset in self.code_sets:
        attributes_list = codeset.get_attributes_list()
        if attributes_list:
            flattened = flatten_hierarchy_func(attributes_list)
            all_attributes.extend(flattened)
    return all_attributes

parse_citation_to_destiny(reference)

Create a ReferenceFileInput object from document data.

NOTE: we are not using the wrapping parser method in repository as it is for the whole document, and if it fails, we wouldn't be able to map a destiny reference.

See https://github.com/destiny-evidence/destiny-repository/issues/458

Parameters:

Name	Type	Description	Default
reference	dict[str, Any]	one reference from the eppi json.	required

Source code in deet/data_models/eppi.py

def parse_citation_to_destiny(reference: dict[str, Any]) -> ReferenceFileInput:
    """
    Create a ReferenceFileInput object from document data.

    NOTE: we are not using the wrapping parser method in
    repository as it is for the whole document, and
    if it fails, we wouldn't be able to map a destiny reference.

    See https://github.com/destiny-evidence/destiny-repository/issues/458

    Args:
        reference: one reference from the eppi json.

    """
    if "DOI" in reference:
        reference["DOI"] = sanitise_doi(reference["DOI"])
    raw_enhancement_content = [
        c
        for c in [
            (
                eppi_destiny_parser._parse_abstract_enhancement(reference),  # noqa: SLF001
                EnhancementType.ABSTRACT,
            ),
            (
                eppi_destiny_parser._parse_bibliographic_enhancement(reference),  # noqa: SLF001
                EnhancementType.BIBLIOGRAPHIC,
            ),
            (
                eppi_destiny_parser._create_annotation_enhancement(),  # noqa: SLF001
                EnhancementType.ANNOTATION,
            ),
        ]
        if c[0] is not None
    ]

    enhancements = [
        EnhancementFileInput(
            source=eppi_destiny_parser.parser_source,
            visibility=Visibility.PUBLIC,
            content=content[0],  # type:ignore[arg-type]
            enhancement_type=content[1],  # type:ignore[call-arg]
        )
        for content in raw_enhancement_content
    ]

    destiny_ids = None
    try:
        destiny_ids = eppi_destiny_parser._parse_identifiers(  # noqa: SLF001
            ref_to_import=reference
        )
    except ExternalIdentifierNotFoundError as e:
        logger.warning(f"no identifier for reference. storing `None`. error: {e}")

    return ReferenceFileInput(
        visibility=Visibility.PUBLIC,
        identifiers=destiny_ids,
        enhancements=enhancements,
    )

sanitise_doi(doi_candidate, *, raise_on_fail=False)

Clean DOI strings in EPPI jsons.

Source code in deet/data_models/eppi.py

def sanitise_doi(doi_candidate: str, *, raise_on_fail: bool = False) -> str:
    """Clean DOI strings in EPPI jsons."""
    doi = DOI_REGEX.search(doi_candidate)
    if doi and isinstance(doi, re.Match):
        return doi[0]
    if raise_on_fail:
        bad_doi = f"doi {doi} is bad."
        raise ValueError(bad_doi)
    logger.debug(
        "not found a valid DOI, returning empty string."
        " to modify this behaviour, set raise_on_fail=True"
    )
    return ""

evaluation

Data models to help with evaluation.

AttributeCountMetric dataclass

Bases: AttributeMetric

An integer count metric (e.g. n_gold_instances).

Source code in deet/data_models/evaluation.py

@dataclass(slots=True)
class AttributeCountMetric(AttributeMetric):
    """An integer count metric (e.g. ``n_gold_instances``)."""

    value: int

AttributeMetric dataclass

Base row for one metric on one attribute in an extraction run.

Source code in deet/data_models/evaluation.py

@dataclass(slots=True)
class AttributeMetric:
    """Base row for one metric on one attribute in an extraction run."""

    attribute: Attribute
    metric_name: str
    extraction_run_id: str

AttributeMetricsReport pydantic-model

Bases: BaseModel

Per-attribute metrics block for metrics.json / wide metrics.csv.

Reuses :class:~deet.data_models.base.AttributeType for attribute_type. Count keys are integers; score keys are floats (None means not computable — omitted from JSON, blank in CSV).

Fields:

• attribute_id (int)
• attribute_label (str)
• attribute_type (AttributeType)
• counts (dict[str, int])
• metrics (dict[str, float | None])

Source code in deet/data_models/evaluation.py

class AttributeMetricsReport(BaseModel):
    """
    Per-attribute metrics block for ``metrics.json`` / wide ``metrics.csv``.

    Reuses :class:`~deet.data_models.base.AttributeType` for ``attribute_type``.
    Count keys are integers; score keys are floats (``None`` means not
    computable — omitted from JSON, blank in CSV).
    """

    attribute_id: int
    attribute_label: str
    attribute_type: AttributeType
    counts: dict[str, int] = Field(default_factory=dict)
    metrics: dict[str, float | None] = Field(default_factory=dict)

AttributeScoreMetric dataclass

Bases: AttributeMetric

A float score metric (may be None when not computable).

Source code in deet/data_models/evaluation.py

@dataclass(slots=True)
class AttributeScoreMetric(AttributeMetric):
    """A float score metric (may be ``None`` when not computable)."""

    value: float | None

RunMetricsReport pydantic-model

Bases: BaseModel

Machine-readable evaluation report for one extraction run.

Serialises to metrics.json and wide metrics.csv (one entry / row per attribute, metric names as keys).

Fields:

• extraction_run_id (str)
• format_version (int)
• attributes (list[AttributeMetricsReport])

Source code in deet/data_models/evaluation.py

class RunMetricsReport(BaseModel):
    """
    Machine-readable evaluation report for one extraction run.

    Serialises to ``metrics.json`` and wide ``metrics.csv`` (one entry / row per
    attribute, metric names as keys).
    """

    extraction_run_id: str
    format_version: int = 1
    attributes: list[AttributeMetricsReport] = Field(default_factory=list)

    @classmethod
    def from_attribute_metrics(
        cls,
        *,
        extraction_run_id: str,
        calculated_metrics: list[AttributeMetric],
        format_version: int = 1,
    ) -> "RunMetricsReport":
        """
        Group row-level metric values into a run report.

        :class:`AttributeCountMetric` values go under ``counts`` as integers;
        :class:`AttributeScoreMetric` values go under ``metrics`` (including
        ``None`` for blank CSV cells). :meth:`to_json` omits ``None`` scores.

        Args:
            extraction_run_id: Run folder / run id.
            calculated_metrics: Flat metric rows from the evaluator.
            format_version: Schema version written to JSON.

        Returns:
            A :class:`RunMetricsReport` ready to serialise.

        """
        by_attribute: dict[tuple[int, str], AttributeMetricsReport] = {}
        for metric in calculated_metrics:
            attr_key = (
                metric.attribute.attribute_id,
                metric.attribute.attribute_label,
            )
            if attr_key not in by_attribute:
                by_attribute[attr_key] = AttributeMetricsReport(
                    attribute_id=metric.attribute.attribute_id,
                    attribute_label=metric.attribute.attribute_label,
                    attribute_type=metric.attribute.output_data_type,
                )
            if isinstance(metric, AttributeCountMetric):
                by_attribute[attr_key].counts[metric.metric_name] = metric.value
            elif isinstance(metric, AttributeScoreMetric):
                by_attribute[attr_key].metrics[metric.metric_name] = metric.value

        attributes = sorted(
            by_attribute.values(),
            key=lambda report: report.attribute_label,
        )
        return cls(
            extraction_run_id=extraction_run_id,
            format_version=format_version,
            attributes=attributes,
        )

    @classmethod
    def from_json(cls, filepath: Path) -> "RunMetricsReport":
        """
        Load a report from a ``metrics.json`` file.

        Args:
            filepath: Path to a JSON file written by :meth:`to_json`.

        Returns:
            Validated :class:`RunMetricsReport`.

        """
        return cls.model_validate_json(filepath.read_text(encoding="utf-8"))

    def to_json(self, filepath: Path) -> None:
        """
        Write this report to ``metrics.json``.

        ``None`` score values are omitted so inapplicable keys are absent.

        Args:
            filepath: Destination path (must end in ``.json``).

        """
        filepath.parent.mkdir(parents=True, exist_ok=True)
        payload = self.model_dump(mode="json")
        for attr_payload in payload["attributes"]:
            attr_payload["metrics"] = {
                key: value
                for key, value in attr_payload["metrics"].items()
                if value is not None
            }
        filepath.write_text(
            json.dumps(payload, indent=2) + "\n",
            encoding="utf-8",
        )

    def to_csv(self, filepath: Path) -> None:
        """
        Write this report as a wide ``metrics.csv`` (one row per attribute).

        ``None`` score values become empty cells.

        Args:
            filepath: Destination path (must end in ``.csv``).

        Raises:
            ValueError: If ``filepath`` does not end in ``.csv``.

        """
        if filepath.suffix != ".csv":
            bad_filetype = "file ending must be .csv"
            raise ValueError(bad_filetype)

        base_columns = [
            "extraction_run_id",
            "attribute_id",
            "attribute_label",
            "attribute_type",
        ]
        preferred = preferred_metric_column_names()
        metric_names: set[str] = set()
        for attr_report in self.attributes:
            metric_names.update(attr_report.counts)
            metric_names.update(attr_report.metrics)

        ordered_metric_columns = [name for name in preferred if name in metric_names]
        ordered_metric_columns.extend(
            sorted(name for name in metric_names if name not in ordered_metric_columns)
        )
        fieldnames = base_columns + ordered_metric_columns

        filepath.parent.mkdir(parents=True, exist_ok=True)
        with filepath.open("w", newline="", encoding="utf-8") as file_handle:
            writer = csv.DictWriter(file_handle, fieldnames=fieldnames)
            writer.writeheader()
            for attr_report in self.attributes:
                row: dict[str, object] = {
                    "extraction_run_id": self.extraction_run_id,
                    "attribute_id": attr_report.attribute_id,
                    "attribute_label": attr_report.attribute_label,
                    "attribute_type": attr_report.attribute_type.value,
                }
                for name in ordered_metric_columns:
                    if name in attr_report.counts:
                        row[name] = attr_report.counts[name]
                    elif name in attr_report.metrics:
                        value = attr_report.metrics[name]
                        row[name] = "" if value is None else value
                    else:
                        row[name] = ""
                writer.writerow(row)

from_attribute_metrics(*, extraction_run_id, calculated_metrics, format_version=1) classmethod

Group row-level metric values into a run report.

:class:AttributeCountMetric values go under counts as integers; :class:AttributeScoreMetric values go under metrics (including None for blank CSV cells). :meth:to_json omits None scores.

Parameters:

Name	Type	Description	Default
extraction_run_id	str	Run folder / run id.	required
calculated_metrics	list[AttributeMetric]	Flat metric rows from the evaluator.	required
format_version	int	Schema version written to JSON.	1

Returns:

Name	Type	Description
A	RunMetricsReport	class:RunMetricsReport ready to serialise.

Source code in deet/data_models/evaluation.py

@classmethod
def from_attribute_metrics(
    cls,
    *,
    extraction_run_id: str,
    calculated_metrics: list[AttributeMetric],
    format_version: int = 1,
) -> "RunMetricsReport":
    """
    Group row-level metric values into a run report.

    :class:`AttributeCountMetric` values go under ``counts`` as integers;
    :class:`AttributeScoreMetric` values go under ``metrics`` (including
    ``None`` for blank CSV cells). :meth:`to_json` omits ``None`` scores.

    Args:
        extraction_run_id: Run folder / run id.
        calculated_metrics: Flat metric rows from the evaluator.
        format_version: Schema version written to JSON.

    Returns:
        A :class:`RunMetricsReport` ready to serialise.

    """
    by_attribute: dict[tuple[int, str], AttributeMetricsReport] = {}
    for metric in calculated_metrics:
        attr_key = (
            metric.attribute.attribute_id,
            metric.attribute.attribute_label,
        )
        if attr_key not in by_attribute:
            by_attribute[attr_key] = AttributeMetricsReport(
                attribute_id=metric.attribute.attribute_id,
                attribute_label=metric.attribute.attribute_label,
                attribute_type=metric.attribute.output_data_type,
            )
        if isinstance(metric, AttributeCountMetric):
            by_attribute[attr_key].counts[metric.metric_name] = metric.value
        elif isinstance(metric, AttributeScoreMetric):
            by_attribute[attr_key].metrics[metric.metric_name] = metric.value

    attributes = sorted(
        by_attribute.values(),
        key=lambda report: report.attribute_label,
    )
    return cls(
        extraction_run_id=extraction_run_id,
        format_version=format_version,
        attributes=attributes,
    )

from_json(filepath) classmethod

Load a report from a metrics.json file.

Parameters:

Name	Type	Description	Default
filepath	Path	Path to a JSON file written by :meth:to_json.	required

Returns:

Name	Type	Description
Validated	RunMetricsReport	class:RunMetricsReport.

Source code in deet/data_models/evaluation.py

@classmethod
def from_json(cls, filepath: Path) -> "RunMetricsReport":
    """
    Load a report from a ``metrics.json`` file.

    Args:
        filepath: Path to a JSON file written by :meth:`to_json`.

    Returns:
        Validated :class:`RunMetricsReport`.

    """
    return cls.model_validate_json(filepath.read_text(encoding="utf-8"))

to_csv(filepath)

Write this report as a wide metrics.csv (one row per attribute).

None score values become empty cells.

Parameters:

Name	Type	Description	Default
filepath	Path	Destination path (must end in .csv).	required

Raises:

Type	Description
ValueError	If filepath does not end in .csv.

Source code in deet/data_models/evaluation.py

def to_csv(self, filepath: Path) -> None:
    """
    Write this report as a wide ``metrics.csv`` (one row per attribute).

    ``None`` score values become empty cells.

    Args:
        filepath: Destination path (must end in ``.csv``).

    Raises:
        ValueError: If ``filepath`` does not end in ``.csv``.

    """
    if filepath.suffix != ".csv":
        bad_filetype = "file ending must be .csv"
        raise ValueError(bad_filetype)

    base_columns = [
        "extraction_run_id",
        "attribute_id",
        "attribute_label",
        "attribute_type",
    ]
    preferred = preferred_metric_column_names()
    metric_names: set[str] = set()
    for attr_report in self.attributes:
        metric_names.update(attr_report.counts)
        metric_names.update(attr_report.metrics)

    ordered_metric_columns = [name for name in preferred if name in metric_names]
    ordered_metric_columns.extend(
        sorted(name for name in metric_names if name not in ordered_metric_columns)
    )
    fieldnames = base_columns + ordered_metric_columns

    filepath.parent.mkdir(parents=True, exist_ok=True)
    with filepath.open("w", newline="", encoding="utf-8") as file_handle:
        writer = csv.DictWriter(file_handle, fieldnames=fieldnames)
        writer.writeheader()
        for attr_report in self.attributes:
            row: dict[str, object] = {
                "extraction_run_id": self.extraction_run_id,
                "attribute_id": attr_report.attribute_id,
                "attribute_label": attr_report.attribute_label,
                "attribute_type": attr_report.attribute_type.value,
            }
            for name in ordered_metric_columns:
                if name in attr_report.counts:
                    row[name] = attr_report.counts[name]
                elif name in attr_report.metrics:
                    value = attr_report.metrics[name]
                    row[name] = "" if value is None else value
                else:
                    row[name] = ""
            writer.writerow(row)

to_json(filepath)

Write this report to metrics.json.

None score values are omitted so inapplicable keys are absent.

Parameters:

Name	Type	Description	Default
filepath	Path	Destination path (must end in .json).	required

Source code in deet/data_models/evaluation.py

def to_json(self, filepath: Path) -> None:
    """
    Write this report to ``metrics.json``.

    ``None`` score values are omitted so inapplicable keys are absent.

    Args:
        filepath: Destination path (must end in ``.json``).

    """
    filepath.parent.mkdir(parents=True, exist_ok=True)
    payload = self.model_dump(mode="json")
    for attr_payload in payload["attributes"]:
        attr_payload["metrics"] = {
            key: value
            for key, value in attr_payload["metrics"].items()
            if value is not None
        }
    filepath.write_text(
        json.dumps(payload, indent=2) + "\n",
        encoding="utf-8",
    )

preferred_metric_column_names()

Preferred wide-CSV metric column order.

Score bases and stratification suffixes are derived from :data:~deet.evaluators.metrics.METRICS_BY_ATTRIBUTE_TYPE and :data:~deet.evaluators.source_fidelity.SOURCE_FIDELITY_ATTRIBUTE_TYPES. Custom metrics passed at evaluate time are not listed here; they are appended alphabetically when writing CSV (see :meth:RunMetricsReport.to_csv).

Source code in deet/data_models/evaluation.py

def preferred_metric_column_names() -> list[str]:
    """
    Preferred wide-CSV metric column order.

    Score bases and stratification suffixes are derived from
    :data:`~deet.evaluators.metrics.METRICS_BY_ATTRIBUTE_TYPE` and
    :data:`~deet.evaluators.source_fidelity.SOURCE_FIDELITY_ATTRIBUTE_TYPES`.
    Custom metrics passed at evaluate time are not listed here; they are
    appended alphabetically when writing CSV (see :meth:`RunMetricsReport.to_csv`).
    """
    columns: list[str] = list(COUNT_METRIC_COLUMN_ORDER)
    for base in _SCORE_METRIC_BASE_ORDER:
        suffixes = (
            _STRATIFICATION_SUFFIXES if base in _METRICS_WITH_STRATIFICATION else ("",)
        )
        columns.extend(f"{base}{suffix}" for suffix in suffixes)
    return columns

evaluation_strategies

Public interfact for evaluation strategies.

Exports the base class and concrete strategy implementations, maintains the strategy registry that maps strategy names to their constructors.

base

Evaluation strategy framework: dividing documents between stages for evaluation.

This module defines base classes for managing how documents are partitioned across evaluation stages. This module provides three abstractions:

• BaseEvaluationStage: An enum of the concrete stages for a strategy.
• BaseSplits: A Pydantic model persisting the partition of documents across stages and the current stage of an experiment.
• BaseEvaluationStrategy: The orchestrator, managing splits and providing interactive and programmatic interfaces for moving documents between stages.

Concrete strategies implement these abstractions and are registered in the package __init__.py.

BaseEvaluationStage

Bases: StrEnum

Defines the stages of an evaluation strategy.

Each of these stages is mapped to a list of IDs in the splits model below.

Source code in deet/data_models/evaluation_strategies/base.py

class BaseEvaluationStage(StrEnum):
    """
    Defines the stages of an evaluation strategy.

    Each of these stages is mapped to a list of IDs in the splits model below.
    """

BaseEvaluationStrategy

Bases: ABC

Orchestrate how a project's documents are divided into evaluation stages.

An evaluation strategy defines stages (via a BaseEvaluationStage enum) and how to move documents through them.

When deet.extractors.cli_helpers.run_extraction_pipeline is called, it filters documents down to those that are assigned to the current stage.

Each subclass implements its own methods for managing splits, but each must implement a run_splits_wizard method, which is called when a user runs deet experiments splits.

Source code in deet/data_models/evaluation_strategies/base.py

class BaseEvaluationStrategy[SplitsT: BaseSplits](ABC):
    """
    Orchestrate how a project's documents are divided into evaluation stages.

    An evaluation strategy defines stages (via a `BaseEvaluationStage` enum)
    and how to move documents through them.

    When `deet.extractors.cli_helpers.run_extraction_pipeline` is called, it
    filters documents down to those that are assigned to the current stage.

    Each subclass implements its own methods for managing splits, but each
    must implement a `run_splits_wizard` method, which is called when a user
    runs `deet experiments splits`.
    """

    name: EvaluationStrategyName

    def __init__(self, project: DeetProject) -> None:
        """Initialise and set splits."""
        self._project = project
        self.splits = self._load_splits(project)

    @abstractmethod
    def _load_splits(self, project: DeetProject) -> SplitsT:
        """Load or init the splits model for this strategy and project."""

    def get_active_ids(self, project: DeetProject) -> list[int]:
        """Return the document IDs in the current stage to run the pipeline on."""
        return self.splits.active_ids

    def snapshot(self, artefacts: ExperimentArtefacts) -> None:
        """Persist stategy state with an experiment run's artefacts."""
        self.splits.dump_to_json(artefacts.evaluation_splits_snapshot)

    @abstractmethod
    def run_splits_wizard(
        self,
        project: DeetProject,
        *,
        action: str | None = None,
        size: int | None = None,
        experiment: str | None = None,
    ) -> None:
        """
        Run the interactive workflow for moving documents between stages.

        By default (``action=None``), presents an interactive prompt listing the
        actions available in the current stage and strategy, and the user selects
        one. Optionally, ``--action`` can be passed to skip the prompt (for
        scripted use). Unknown or stage-inappropriate actions raise an error rather
        than silently failing.

        Further details (e.g. how many documents to move) are prompted for
        interactively if not provided (``size``, ``experiment``). Subclasses
        implement the concrete stage transitions and validation logic.

        Note:
            The valid actions depend on both the strategy and the current stage,
            which are only known at runtime. Static CLI subcommands would
            misleadingly list all actions in ``--help`` regardless of whether they
            apply to the project's configured strategy and current stage. Instead,
            the action is discovered interactively (the prompt shows what's valid
            *now*) or specified as an unconstrained string for scripted use.

        """

__init__(project)

Initialise and set splits.

Source code in deet/data_models/evaluation_strategies/base.py

def __init__(self, project: DeetProject) -> None:
    """Initialise and set splits."""
    self._project = project
    self.splits = self._load_splits(project)

get_active_ids(project)

Return the document IDs in the current stage to run the pipeline on.

Source code in deet/data_models/evaluation_strategies/base.py

def get_active_ids(self, project: DeetProject) -> list[int]:
    """Return the document IDs in the current stage to run the pipeline on."""
    return self.splits.active_ids

run_splits_wizard(project, *, action=None, size=None, experiment=None) abstractmethod

Run the interactive workflow for moving documents between stages.

By default (action=None), presents an interactive prompt listing the actions available in the current stage and strategy, and the user selects one. Optionally, --action can be passed to skip the prompt (for scripted use). Unknown or stage-inappropriate actions raise an error rather than silently failing.

Further details (e.g. how many documents to move) are prompted for interactively if not provided (size, experiment). Subclasses implement the concrete stage transitions and validation logic.

Note

The valid actions depend on both the strategy and the current stage, which are only known at runtime. Static CLI subcommands would misleadingly list all actions in --help regardless of whether they apply to the project's configured strategy and current stage. Instead, the action is discovered interactively (the prompt shows what's valid now) or specified as an unconstrained string for scripted use.

Source code in deet/data_models/evaluation_strategies/base.py

@abstractmethod
def run_splits_wizard(
    self,
    project: DeetProject,
    *,
    action: str | None = None,
    size: int | None = None,
    experiment: str | None = None,
) -> None:
    """
    Run the interactive workflow for moving documents between stages.

    By default (``action=None``), presents an interactive prompt listing the
    actions available in the current stage and strategy, and the user selects
    one. Optionally, ``--action`` can be passed to skip the prompt (for
    scripted use). Unknown or stage-inappropriate actions raise an error rather
    than silently failing.

    Further details (e.g. how many documents to move) are prompted for
    interactively if not provided (``size``, ``experiment``). Subclasses
    implement the concrete stage transitions and validation logic.

    Note:
        The valid actions depend on both the strategy and the current stage,
        which are only known at runtime. Static CLI subcommands would
        misleadingly list all actions in ``--help`` regardless of whether they
        apply to the project's configured strategy and current stage. Instead,
        the action is discovered interactively (the prompt shows what's valid
        *now*) or specified as an unconstrained string for scripted use.

    """

snapshot(artefacts)

Persist stategy state with an experiment run's artefacts.

Source code in deet/data_models/evaluation_strategies/base.py

def snapshot(self, artefacts: ExperimentArtefacts) -> None:
    """Persist stategy state with an experiment run's artefacts."""
    self.splits.dump_to_json(artefacts.evaluation_splits_snapshot)

BaseSplits pydantic-model

Bases: BaseModel

Base object for persisting how document IDs are used for an experiment.

An evaluation strategy divides a project's documents into stages (e.g. dev, validation, and test). This model stores that partition, alongside the stage the experiment is currently in, and serialises to/from JSON.

Subclasses define the concrete shape and must: - declare one model field per stage - bind StageT to the strategy's stage enum - set the _STAGE_FIELD_NAMES class var mapping each stage in the enum to its model field

This contract is enforced on definition, and ensures that generic methods can be defined here for DRYness.

Fields:

• current_stage (StageT)

Source code in deet/data_models/evaluation_strategies/base.py

class BaseSplits[StageT: BaseEvaluationStage](BaseModel):
    """
    Base object for persisting how document IDs are used for an experiment.

    An evaluation strategy divides a project's documents into stages
    (e.g. dev, validation, and test). This model stores that partition,
    alongside the stage the experiment is currently in, and serialises to/from
    JSON.

    Subclasses define the concrete shape and must:
    - declare one model field per stage
    - bind `StageT` to the strategy's stage enum
    - set the `_STAGE_FIELD_NAMES` class var mapping each stage in the enum
        to its model field

    This contract is enforced on definition, and ensures that generic methods
    can be defined here for DRYness.
    """

    _STAGE_FIELD_NAMES: ClassVar[dict[BaseEvaluationStage, str]]

    @classmethod
    def __pydantic_init_subclass__(cls, **kwargs) -> None:
        """Validate whether a subclass honours contract defined by this base class."""
        super().__pydantic_init_subclass__(**kwargs)

        if not hasattr(cls, "_STAGE_FIELD_NAMES"):
            undeclared = f"{cls.__name__} must define _STAGE_FIELD_NAMES"
            raise TypeError(undeclared)

        undefined = [
            field
            for field in cls._STAGE_FIELD_NAMES.values()
            if field not in cls.model_fields
        ]
        if undefined:
            bad_fields = (
                f"{cls.__name__}._STAGE_FIELD_NAMES maps to undefined"
                f" model fields(s): {undefined}"
            )
            raise TypeError(bad_fields)

        stage_enum = cls.model_fields["current_stage"].annotation
        if isinstance(stage_enum, type) and issubclass(stage_enum, BaseEvaluationStage):
            mapped = set(cls._STAGE_FIELD_NAMES)
            expected: set[BaseEvaluationStage] = set(stage_enum)
            if mapped != expected:
                mismatch = (
                    f"{cls.__name__}._STAGE_FIELD_NAMES keys must match "
                    f"{stage_enum.__name__} exactly; "
                    f"missing={expected - mapped}, unexpected={mapped - expected}"
                )
                raise TypeError(mismatch)

    current_stage: StageT

    def dump_to_json(self, path: Path) -> None:
        """Persist split state to json."""
        path.write_text(self.model_dump_json(indent=2), encoding="utf-8")

    def _get_list_for_stage(self, stage: StageT) -> list[int]:
        """Get the the ids in the field corresponding the stage."""
        return getattr(self, self._STAGE_FIELD_NAMES[stage])

    def get_unassigned_ids(self, project_doc_ids: Collection[int]) -> list[int]:
        """Filter a collection of document IDs to those which have not been assigned."""
        assigned = set()
        for field_name in self._STAGE_FIELD_NAMES.values():
            assigned.update(getattr(self, field_name))

        return [doc_id for doc_id in project_doc_ids if doc_id not in assigned]

    def add_to_stage(
        self,
        stage: StageT,
        project_doc_ids: Collection[int],
        size: int,
    ) -> int:
        """Sample from unassigned and add to a stage."""
        unassigned = self.get_unassigned_ids(project_doc_ids)
        target_list = self._get_list_for_stage(stage)

        if size <= 0:
            too_small = "Sample size must be greater than 0."
            raise SplitsValidationError(too_small)
        if len(unassigned) < size:
            incompatible_size = (
                f"Tried to assign {size} docs to the development set"
                f" but only {len(unassigned)} are unassigned"
            )
            raise SplitsValidationError(incompatible_size)

        target_ids = random.sample(unassigned, size)
        target_list.extend(target_ids)
        return len(target_ids)

    @property
    def active_ids(self) -> list[int]:
        """Return the current active ids, based on the current stage."""
        return self._get_list_for_stage(self.current_stage)

active_ids property

Return the current active ids, based on the current stage.

__pydantic_init_subclass__(**kwargs) classmethod

Validate whether a subclass honours contract defined by this base class.

Source code in deet/data_models/evaluation_strategies/base.py

@classmethod
def __pydantic_init_subclass__(cls, **kwargs) -> None:
    """Validate whether a subclass honours contract defined by this base class."""
    super().__pydantic_init_subclass__(**kwargs)

    if not hasattr(cls, "_STAGE_FIELD_NAMES"):
        undeclared = f"{cls.__name__} must define _STAGE_FIELD_NAMES"
        raise TypeError(undeclared)

    undefined = [
        field
        for field in cls._STAGE_FIELD_NAMES.values()
        if field not in cls.model_fields
    ]
    if undefined:
        bad_fields = (
            f"{cls.__name__}._STAGE_FIELD_NAMES maps to undefined"
            f" model fields(s): {undefined}"
        )
        raise TypeError(bad_fields)

    stage_enum = cls.model_fields["current_stage"].annotation
    if isinstance(stage_enum, type) and issubclass(stage_enum, BaseEvaluationStage):
        mapped = set(cls._STAGE_FIELD_NAMES)
        expected: set[BaseEvaluationStage] = set(stage_enum)
        if mapped != expected:
            mismatch = (
                f"{cls.__name__}._STAGE_FIELD_NAMES keys must match "
                f"{stage_enum.__name__} exactly; "
                f"missing={expected - mapped}, unexpected={mapped - expected}"
            )
            raise TypeError(mismatch)

add_to_stage(stage, project_doc_ids, size)

Sample from unassigned and add to a stage.

Source code in deet/data_models/evaluation_strategies/base.py

def add_to_stage(
    self,
    stage: StageT,
    project_doc_ids: Collection[int],
    size: int,
) -> int:
    """Sample from unassigned and add to a stage."""
    unassigned = self.get_unassigned_ids(project_doc_ids)
    target_list = self._get_list_for_stage(stage)

    if size <= 0:
        too_small = "Sample size must be greater than 0."
        raise SplitsValidationError(too_small)
    if len(unassigned) < size:
        incompatible_size = (
            f"Tried to assign {size} docs to the development set"
            f" but only {len(unassigned)} are unassigned"
        )
        raise SplitsValidationError(incompatible_size)

    target_ids = random.sample(unassigned, size)
    target_list.extend(target_ids)
    return len(target_ids)

dump_to_json(path)

Persist split state to json.

Source code in deet/data_models/evaluation_strategies/base.py

def dump_to_json(self, path: Path) -> None:
    """Persist split state to json."""
    path.write_text(self.model_dump_json(indent=2), encoding="utf-8")

get_unassigned_ids(project_doc_ids)

Filter a collection of document IDs to those which have not been assigned.

Source code in deet/data_models/evaluation_strategies/base.py

def get_unassigned_ids(self, project_doc_ids: Collection[int]) -> list[int]:
    """Filter a collection of document IDs to those which have not been assigned."""
    assigned = set()
    for field_name in self._STAGE_FIELD_NAMES.values():
        assigned.update(getattr(self, field_name))

    return [doc_id for doc_id in project_doc_ids if doc_id not in assigned]

dev_val_test

The dynamic dev-val-test splitting strategy.

This is described in detail at https://destiny-evidence.github.io/evaluation-book/index-1/#chunked-evaluation-data

DevValTestEvaluationStage

Bases: BaseEvaluationStage

Describes the possible evaluation stages.

Inherits from BaseEvaluationStage, and defines each of the following stages for the dev-val-test strategy.

• DEVELOPMENT is used to iterate and improve prompts/configuration.
• VALIDATION is used to validate prompts on data they have not been tuned for.
• TEST is used for a final assessment.

Source code in deet/data_models/evaluation_strategies/dev_val_test.py

class DevValTestEvaluationStage(BaseEvaluationStage):
    """
    Describes the possible evaluation stages.

    Inherits from BaseEvaluationStage, and defines each of the following stages
    for the dev-val-test strategy.

    - DEVELOPMENT is used to iterate and improve prompts/configuration.
    - VALIDATION is used to validate prompts on data they have not been tuned for.
    - TEST is used for a final assessment.
    """

    DEVELOPMENT = auto()
    VALIDATION = auto()
    TEST = auto()

DevValTestEvaluationStrategy

Bases: BaseEvaluationStrategy[DevValTestSplits]

Strategy to manage dynamic splitting into dev-val-test.

Source code in deet/data_models/evaluation_strategies/dev_val_test.py

class DevValTestEvaluationStrategy(BaseEvaluationStrategy[DevValTestSplits]):
    """Strategy to manage dynamic splitting into dev-val-test."""

    name = EvaluationStrategyName.DEV_VAL_TEST

    def _load_splits(self, project: DeetProject) -> DevValTestSplits:
        """Make a new splits object with all project IDs."""
        return DevValTestSplits.load_or_init(project.evaluation_splits_path)

    def add_to_development(
        self,
        project_doc_ids: list[int],
        size: int | None = None,
    ) -> None:
        """
        Add unassigned documents to the development pool.

        Randomly samples from the unassigns documents and adds them
        to the development set.

        Persists the updated splits to dist and notifies the user of actions taken.

        Args:
            project_doc_ids: All document IDs in the project
                (to determine which are unassigned)
            size: Number of documents to randomly sample and add (prompted for if None).

        Raises:
            SplitsValidationError: If size exceeds the number of unassigned documents

        """
        from InquirerPy import inquirer

        if not size:
            size = int(
                inquirer.number(
                    message="How many documents would you like to add?"
                ).execute()
            )
        try:
            n_added = self.splits.add_to_stage(
                DevValTestEvaluationStage.DEVELOPMENT, project_doc_ids, size
            )
            self.splits.dump_to_json(self._project.evaluation_splits_path)

        except SplitsValidationError as e:
            fail_with_message(str(e))

        notify(
            f"Added {n_added} documents to development set."
            f" This now contains {len(self.splits.development_ids)} documents."
            f" {len(self.splits.get_unassigned_ids(project_doc_ids))}"
            " are still unassigned."
        )

    def validate_run(
        self, size: int, project_doc_ids: list[int], experiment: ExperimentArtefacts
    ) -> None:
        """
        Evaluate a past experiment config and eval against a fresh validation set.

        Randomly samples previously unsassigned documents into the validation set,
        evaluates given experiment config and prompts against them,
        and updates the splits state with the validation run ID for later reference

        Persists the updated splits and a snapshot of strategy state with the
        experiment artefacts.

        Args:
            size: Number of documents to randomly sample
            project_doc_ids: All document IDs in the project (to determine unassigned)
            experiment: The ExperimentArtefacts to evaluate (from a prior run)

        Raises:
            SplitsValidationError: If size exceeds the number of unassigned documents

        Note:
            The CLI calls `choose_experiment()` interactively, then passes it here.
            For programmatic use, provide the experiment directly.

        """
        from deet.extractors.cli_helpers import (
            evaluate_extraction_pipeline,
            run_extraction_pipeline,
        )

        try:
            n_added = self.splits.add_to_stage(
                DevValTestEvaluationStage.VALIDATION, project_doc_ids, size
            )
            self.splits.current_stage = DevValTestEvaluationStage.VALIDATION
        except SplitsValidationError as e:
            fail_with_message(str(e))

        notify(
            f"Added {n_added} documents to validation set"
            f" ({len(self.splits.get_unassigned_ids(project_doc_ids))}"
            " are still unassigned)."
            f"\nEvaluating experiment: {experiment.run_id}"
            " using these documents"
        )

        run_output, processed_annotation_data, experiment_artefacts, _config = (
            run_extraction_pipeline(
                deet_project=self._project,
                prompt_csv_path=experiment.prompts_snapshot,
                config_path=experiment.config_snapshot,
                run_name="VALIDATION",
            )
        )
        evaluate_extraction_pipeline(
            processed_annotation_data=processed_annotation_data,
            run_output=run_output,
            experiment_artefacts=experiment_artefacts,
        )
        self.splits.validation_run_id = experiment_artefacts.run_id
        self.splits.dump_to_json(self._project.evaluation_splits_path)
        self.snapshot(experiment_artefacts)

    def run_validation_interactive(
        self, project_doc_ids: list[int], size: int | None, experiment: str | None
    ) -> None:
        """Orchestrate validation (prompt for args and dispatch validation)."""
        from InquirerPy import inquirer

        from deet.ui.terminal.prompts import select_experiment

        if size is None:
            size = int(
                inquirer.number(
                    message="How many documents would you like to add?"
                ).execute()
            )
        if experiment is None:
            selected_experiment = select_experiment(self._project)
        else:
            selected_experiment = ExperimentArtefacts(
                base_dir=self._project.experiments_dir / experiment
            )
        self.validate_run(
            size=size, project_doc_ids=project_doc_ids, experiment=selected_experiment
        )

    def accept_validation(self, project_doc_ids: list[int]) -> None:
        """Accept the results of validation run and do final evaluation of test set."""
        from deet.extractors.cli_helpers import (
            evaluate_extraction_pipeline,
            run_extraction_pipeline,
        )
        from deet.ui import fail_with_message

        try:
            self.finalise_test(project_doc_ids)
        except SplitsValidationError as e:
            fail_with_message(str(e))

        if self.splits.validation_run_id is None:
            fail_with_message("No validation run id")
        selected_experiment = ExperimentArtefacts(
            base_dir=self._project.experiments_dir / self.splits.validation_run_id
        )

        run_output, processed_annotation_data, experiment_artefacts, _config = (
            run_extraction_pipeline(
                deet_project=self._project,
                prompt_csv_path=selected_experiment.prompts_snapshot,
                config_path=selected_experiment.config_snapshot,
                run_name="FINAL_TEST",
            )
        )
        self.snapshot(experiment_artefacts)
        evaluate_extraction_pipeline(
            processed_annotation_data=processed_annotation_data,
            run_output=run_output,
            experiment_artefacts=experiment_artefacts,
        )

    def finalise_test(self, project_doc_ids: Collection[int]) -> None:
        """Add all remaining docs to test."""
        unassigned = self.splits.get_unassigned_ids(project_doc_ids)

        if len(unassigned) == 0:
            none_remaining = (
                "No unassigned documents left for testing."
                " Add more documents to the project to continue."
            )
            raise SplitsValidationError(none_remaining)

        self.splits.test_ids = unassigned
        self.splits.current_stage = DevValTestEvaluationStage.TEST
        self.splits.dump_to_json(self._project.evaluation_splits_path)

    def reject_validation(self) -> None:
        """Merge validation IDs into development and continue developing."""
        self.splits.development_ids.extend(self.splits.validation_ids)
        self.splits.validation_ids = []
        self.splits.current_stage = DevValTestEvaluationStage.DEVELOPMENT
        self.splits.dump_to_json(self._project.evaluation_splits_path)

    def _get_action_for_stage(
        self,
        choices: list[EvaluationDecisionSpec],
        stage: DevValTestEvaluationStage,
        action: str | None = None,
    ) -> EvaluationDecisionSpec:
        """Get an action spec, validating it exists for this stage."""
        from deet.ui.terminal.prompts import select_from_list

        try:
            return select_from_list(choices, item_key="name", selected_value=action)
        except KeyError:
            valid = [c["name"] for c in choices]
            fail_with_message(f"Action '{action}' not valid. Choose from: {valid}")

    def run_splits_wizard(
        self,
        project: DeetProject,
        *,
        action: str | None = None,
        size: int | None = None,
        experiment: str | None = None,
    ) -> None:
        """Run the splits wizard."""
        project_doc_ids = project.get_all_doc_ids()
        unassigned = self.splits.get_unassigned_ids(project_doc_ids)

        notify(
            f"Strategy:    {self.name}\n"
            f"Stage:       {self.splits.current_stage}\n"
            f"Development: {len(self.splits.development_ids)} documents\n"
            f"Validation:  {len(self.splits.validation_ids)} documents\n"
            f"Test:        {len(self.splits.test_ids)} documents\n"
            f"Unassigned:  {len(unassigned)} documents"
        )

        if self.splits.current_stage == DevValTestEvaluationStage.DEVELOPMENT:
            choices = list(STAGE_ACTIONS[DevValTestEvaluationStage.DEVELOPMENT])
            if self.splits.development_ids:
                choices.append(ADD_TO_DEVELOPMENT)

            selected_action = self._get_action_for_stage(
                choices, self.splits.current_stage, action
            )
            selected_action["execute"](self, project_doc_ids, size, experiment)

        if self.splits.current_stage == DevValTestEvaluationStage.VALIDATION:
            choices = STAGE_ACTIONS[DevValTestEvaluationStage.VALIDATION]
            selected_action = self._get_action_for_stage(
                choices, self.splits.current_stage, action
            )
            selected_action["execute"](self, project_doc_ids, size, experiment)

        elif self.splits.current_stage == DevValTestEvaluationStage.TEST:
            notify("Test is complete. No further action available")

accept_validation(project_doc_ids)

Accept the results of validation run and do final evaluation of test set.

Source code in deet/data_models/evaluation_strategies/dev_val_test.py

def accept_validation(self, project_doc_ids: list[int]) -> None:
    """Accept the results of validation run and do final evaluation of test set."""
    from deet.extractors.cli_helpers import (
        evaluate_extraction_pipeline,
        run_extraction_pipeline,
    )
    from deet.ui import fail_with_message

    try:
        self.finalise_test(project_doc_ids)
    except SplitsValidationError as e:
        fail_with_message(str(e))

    if self.splits.validation_run_id is None:
        fail_with_message("No validation run id")
    selected_experiment = ExperimentArtefacts(
        base_dir=self._project.experiments_dir / self.splits.validation_run_id
    )

    run_output, processed_annotation_data, experiment_artefacts, _config = (
        run_extraction_pipeline(
            deet_project=self._project,
            prompt_csv_path=selected_experiment.prompts_snapshot,
            config_path=selected_experiment.config_snapshot,
            run_name="FINAL_TEST",
        )
    )
    self.snapshot(experiment_artefacts)
    evaluate_extraction_pipeline(
        processed_annotation_data=processed_annotation_data,
        run_output=run_output,
        experiment_artefacts=experiment_artefacts,
    )

add_to_development(project_doc_ids, size=None)

Add unassigned documents to the development pool.

Randomly samples from the unassigns documents and adds them to the development set.

Persists the updated splits to dist and notifies the user of actions taken.

Parameters:

Name	Type	Description	Default
project_doc_ids	list[int]	All document IDs in the project (to determine which are unassigned)	required
size	int | None	Number of documents to randomly sample and add (prompted for if None).	None

Raises:

Type	Description
SplitsValidationError	If size exceeds the number of unassigned documents

Source code in deet/data_models/evaluation_strategies/dev_val_test.py

def add_to_development(
    self,
    project_doc_ids: list[int],
    size: int | None = None,
) -> None:
    """
    Add unassigned documents to the development pool.

    Randomly samples from the unassigns documents and adds them
    to the development set.

    Persists the updated splits to dist and notifies the user of actions taken.

    Args:
        project_doc_ids: All document IDs in the project
            (to determine which are unassigned)
        size: Number of documents to randomly sample and add (prompted for if None).

    Raises:
        SplitsValidationError: If size exceeds the number of unassigned documents

    """
    from InquirerPy import inquirer

    if not size:
        size = int(
            inquirer.number(
                message="How many documents would you like to add?"
            ).execute()
        )
    try:
        n_added = self.splits.add_to_stage(
            DevValTestEvaluationStage.DEVELOPMENT, project_doc_ids, size
        )
        self.splits.dump_to_json(self._project.evaluation_splits_path)

    except SplitsValidationError as e:
        fail_with_message(str(e))

    notify(
        f"Added {n_added} documents to development set."
        f" This now contains {len(self.splits.development_ids)} documents."
        f" {len(self.splits.get_unassigned_ids(project_doc_ids))}"
        " are still unassigned."
    )

finalise_test(project_doc_ids)

Add all remaining docs to test.

Source code in deet/data_models/evaluation_strategies/dev_val_test.py

def finalise_test(self, project_doc_ids: Collection[int]) -> None:
    """Add all remaining docs to test."""
    unassigned = self.splits.get_unassigned_ids(project_doc_ids)

    if len(unassigned) == 0:
        none_remaining = (
            "No unassigned documents left for testing."
            " Add more documents to the project to continue."
        )
        raise SplitsValidationError(none_remaining)

    self.splits.test_ids = unassigned
    self.splits.current_stage = DevValTestEvaluationStage.TEST
    self.splits.dump_to_json(self._project.evaluation_splits_path)

reject_validation()

Merge validation IDs into development and continue developing.

Source code in deet/data_models/evaluation_strategies/dev_val_test.py

def reject_validation(self) -> None:
    """Merge validation IDs into development and continue developing."""
    self.splits.development_ids.extend(self.splits.validation_ids)
    self.splits.validation_ids = []
    self.splits.current_stage = DevValTestEvaluationStage.DEVELOPMENT
    self.splits.dump_to_json(self._project.evaluation_splits_path)

run_splits_wizard(project, *, action=None, size=None, experiment=None)

Run the splits wizard.

Source code in deet/data_models/evaluation_strategies/dev_val_test.py

def run_splits_wizard(
    self,
    project: DeetProject,
    *,
    action: str | None = None,
    size: int | None = None,
    experiment: str | None = None,
) -> None:
    """Run the splits wizard."""
    project_doc_ids = project.get_all_doc_ids()
    unassigned = self.splits.get_unassigned_ids(project_doc_ids)

    notify(
        f"Strategy:    {self.name}\n"
        f"Stage:       {self.splits.current_stage}\n"
        f"Development: {len(self.splits.development_ids)} documents\n"
        f"Validation:  {len(self.splits.validation_ids)} documents\n"
        f"Test:        {len(self.splits.test_ids)} documents\n"
        f"Unassigned:  {len(unassigned)} documents"
    )

    if self.splits.current_stage == DevValTestEvaluationStage.DEVELOPMENT:
        choices = list(STAGE_ACTIONS[DevValTestEvaluationStage.DEVELOPMENT])
        if self.splits.development_ids:
            choices.append(ADD_TO_DEVELOPMENT)

        selected_action = self._get_action_for_stage(
            choices, self.splits.current_stage, action
        )
        selected_action["execute"](self, project_doc_ids, size, experiment)

    if self.splits.current_stage == DevValTestEvaluationStage.VALIDATION:
        choices = STAGE_ACTIONS[DevValTestEvaluationStage.VALIDATION]
        selected_action = self._get_action_for_stage(
            choices, self.splits.current_stage, action
        )
        selected_action["execute"](self, project_doc_ids, size, experiment)

    elif self.splits.current_stage == DevValTestEvaluationStage.TEST:
        notify("Test is complete. No further action available")

run_validation_interactive(project_doc_ids, size, experiment)

Orchestrate validation (prompt for args and dispatch validation).

Source code in deet/data_models/evaluation_strategies/dev_val_test.py

def run_validation_interactive(
    self, project_doc_ids: list[int], size: int | None, experiment: str | None
) -> None:
    """Orchestrate validation (prompt for args and dispatch validation)."""
    from InquirerPy import inquirer

    from deet.ui.terminal.prompts import select_experiment

    if size is None:
        size = int(
            inquirer.number(
                message="How many documents would you like to add?"
            ).execute()
        )
    if experiment is None:
        selected_experiment = select_experiment(self._project)
    else:
        selected_experiment = ExperimentArtefacts(
            base_dir=self._project.experiments_dir / experiment
        )
    self.validate_run(
        size=size, project_doc_ids=project_doc_ids, experiment=selected_experiment
    )

validate_run(size, project_doc_ids, experiment)

Evaluate a past experiment config and eval against a fresh validation set.

Randomly samples previously unsassigned documents into the validation set, evaluates given experiment config and prompts against them, and updates the splits state with the validation run ID for later reference

Persists the updated splits and a snapshot of strategy state with the experiment artefacts.

Parameters:

Name	Type	Description	Default
size	int	Number of documents to randomly sample	required
project_doc_ids	list[int]	All document IDs in the project (to determine unassigned)	required
experiment	ExperimentArtefacts	The ExperimentArtefacts to evaluate (from a prior run)	required

Raises:

Type	Description
SplitsValidationError	If size exceeds the number of unassigned documents

Note

The CLI calls choose_experiment() interactively, then passes it here. For programmatic use, provide the experiment directly.

Source code in deet/data_models/evaluation_strategies/dev_val_test.py

def validate_run(
    self, size: int, project_doc_ids: list[int], experiment: ExperimentArtefacts
) -> None:
    """
    Evaluate a past experiment config and eval against a fresh validation set.

    Randomly samples previously unsassigned documents into the validation set,
    evaluates given experiment config and prompts against them,
    and updates the splits state with the validation run ID for later reference

    Persists the updated splits and a snapshot of strategy state with the
    experiment artefacts.

    Args:
        size: Number of documents to randomly sample
        project_doc_ids: All document IDs in the project (to determine unassigned)
        experiment: The ExperimentArtefacts to evaluate (from a prior run)

    Raises:
        SplitsValidationError: If size exceeds the number of unassigned documents

    Note:
        The CLI calls `choose_experiment()` interactively, then passes it here.
        For programmatic use, provide the experiment directly.

    """
    from deet.extractors.cli_helpers import (
        evaluate_extraction_pipeline,
        run_extraction_pipeline,
    )

    try:
        n_added = self.splits.add_to_stage(
            DevValTestEvaluationStage.VALIDATION, project_doc_ids, size
        )
        self.splits.current_stage = DevValTestEvaluationStage.VALIDATION
    except SplitsValidationError as e:
        fail_with_message(str(e))

    notify(
        f"Added {n_added} documents to validation set"
        f" ({len(self.splits.get_unassigned_ids(project_doc_ids))}"
        " are still unassigned)."
        f"\nEvaluating experiment: {experiment.run_id}"
        " using these documents"
    )

    run_output, processed_annotation_data, experiment_artefacts, _config = (
        run_extraction_pipeline(
            deet_project=self._project,
            prompt_csv_path=experiment.prompts_snapshot,
            config_path=experiment.config_snapshot,
            run_name="VALIDATION",
        )
    )
    evaluate_extraction_pipeline(
        processed_annotation_data=processed_annotation_data,
        run_output=run_output,
        experiment_artefacts=experiment_artefacts,
    )
    self.splits.validation_run_id = experiment_artefacts.run_id
    self.splits.dump_to_json(self._project.evaluation_splits_path)
    self.snapshot(experiment_artefacts)

DevValTestSplits pydantic-model

Bases: BaseSplits

Model to record how documents are allocated across dev-val-test splits.

Fields:

• current_stage (DevValTestEvaluationStage)
• development_ids (list[int])
• validation_ids (list[int])
• test_ids (list[int])
• validation_run_id (str | None)

Source code in deet/data_models/evaluation_strategies/dev_val_test.py

class DevValTestSplits(BaseSplits):
    """Model to record how documents are allocated across dev-val-test splits."""

    _STAGE_FIELD_NAMES: ClassVar[dict[BaseEvaluationStage, str]] = {
        DevValTestEvaluationStage.DEVELOPMENT: "development_ids",
        DevValTestEvaluationStage.VALIDATION: "validation_ids",
        DevValTestEvaluationStage.TEST: "test_ids",
    }

    current_stage: DevValTestEvaluationStage = DevValTestEvaluationStage.DEVELOPMENT
    development_ids: list[int] = Field(default_factory=list)
    validation_ids: list[int] = Field(default_factory=list)
    test_ids: list[int] = Field(default_factory=list)

    validation_run_id: str | None = None

    @classmethod
    def load_or_init(cls, file_path: Path) -> DevValTestSplits:
        """
        Load splits from file, or initialise if empty or invalid.

        Note:
            It may be invalid if the evaluation strategy was switched.
            In this case, it would make sense to start a fresh instantiation

        """
        if not file_path.exists():
            logger.warning(
                "No splits file exists at {file_path}. Instantiating fresh instance"
            )
            return cls()
        try:
            return cls.model_validate_json(file_path.read_text(encoding="utf-8"))
        except ValidationError:
            logger.warning(
                "Existing splits file does not match this strategy's schema"
                " Instantiating fresh splits."
            )
            return cls()

load_or_init(file_path) classmethod

Load splits from file, or initialise if empty or invalid.

Note

It may be invalid if the evaluation strategy was switched. In this case, it would make sense to start a fresh instantiation

Source code in deet/data_models/evaluation_strategies/dev_val_test.py

@classmethod
def load_or_init(cls, file_path: Path) -> DevValTestSplits:
    """
    Load splits from file, or initialise if empty or invalid.

    Note:
        It may be invalid if the evaluation strategy was switched.
        In this case, it would make sense to start a fresh instantiation

    """
    if not file_path.exists():
        logger.warning(
            "No splits file exists at {file_path}. Instantiating fresh instance"
        )
        return cls()
    try:
        return cls.model_validate_json(file_path.read_text(encoding="utf-8"))
    except ValidationError:
        logger.warning(
            "Existing splits file does not match this strategy's schema"
            " Instantiating fresh splits."
        )
        return cls()

EvaluationDecisionSpec

Bases: TypedDict

Definition of the shape of choices presented to user, and how to act on them.

Source code in deet/data_models/evaluation_strategies/dev_val_test.py

class EvaluationDecisionSpec(TypedDict):
    """Definition of the shape of choices presented to user, and how to act on them."""

    name: str
    description: str
    execute: Callable[
        [DevValTestEvaluationStrategy, list[int], int | None, str | None], None
    ]

null

Null Evaluation strategy (== state prior to introduction of strategies).

NullEvaluationStrategy

Bases: BaseEvaluationStrategy[NullSplits]

The default evaluation strategy (use all documents).

Source code in deet/data_models/evaluation_strategies/null.py

class NullEvaluationStrategy(BaseEvaluationStrategy[NullSplits]):
    """The default evaluation strategy (use all documents)."""

    name = EvaluationStrategyName.NONE

    def _load_splits(self, project: "DeetProject") -> NullSplits:
        """Make a new splits object with all project IDs."""
        return NullSplits(all_ids=project.get_all_doc_ids())

    def run_splits_wizard(
        self,
        project: DeetProject,
        *,
        action: str | None = None,
        size: int | None = None,
        experiment: str | None = None,
    ) -> None:
        """Run the splits wizard."""
        notify(
            "No evaluation strategy is selected."
            " Evaluation will be run against all project documents"
        )

run_splits_wizard(project, *, action=None, size=None, experiment=None)

Run the splits wizard.

Source code in deet/data_models/evaluation_strategies/null.py

def run_splits_wizard(
    self,
    project: DeetProject,
    *,
    action: str | None = None,
    size: int | None = None,
    experiment: str | None = None,
) -> None:
    """Run the splits wizard."""
    notify(
        "No evaluation strategy is selected."
        " Evaluation will be run against all project documents"
    )

NullSplits pydantic-model

Bases: BaseSplits

Records document IDs used in the null strategy.

Fields:

• current_stage (NullStage)
• all_ids (list[int])

Source code in deet/data_models/evaluation_strategies/null.py

class NullSplits(BaseSplits):
    """Records document IDs used in the null strategy."""

    _STAGE_FIELD_NAMES: ClassVar[dict[BaseEvaluationStage, str]] = {
        NullStage.ALL: "all_ids"
    }

    current_stage: NullStage = NullStage.ALL

    all_ids: list[int]

NullStage

Bases: BaseEvaluationStage

Stages of the null strategy.

In this case, the only stage is ALL -> evaluate all documents.

Source code in deet/data_models/evaluation_strategies/null.py

class NullStage(BaseEvaluationStage):
    """
    Stages of the null strategy.

    In this case, the only stage is ALL -> evaluate all documents.
    """

    ALL = auto()

extraction

Data models for LLM extraction outputs.

DocumentExtractionResult pydantic-model

Bases: BaseModel

Result of extracting data from a single document via an LLM.

Fields:

• annotations (list[GoldStandardAnnotation])
• messages (list[dict[str, Any]])
• input_tokens (int)
• output_tokens (int)
• model (str | None)
• total_cost_usd (float | None)
• llm_call_seconds (float)
• timestamp (datetime)

Validators:

• compute_total_cost_usd

Source code in deet/data_models/extraction.py

class DocumentExtractionResult(BaseModel):
    """Result of extracting data from a single document via an LLM."""

    annotations: list[GoldStandardAnnotation]
    messages: list[dict[str, Any]]
    input_tokens: int = 0
    output_tokens: int = 0
    model: str | None = None
    total_cost_usd: float | None = None
    llm_call_seconds: float = 0.0
    timestamp: datetime = Field(default_factory=lambda: datetime.now(tz=UTC))

    @model_validator(mode="after")
    def compute_total_cost_usd(self) -> Self:
        """Populate ``total_cost_usd`` from tokens and model (``estimate_cost_usd``)."""
        if self.model is None:
            self.total_cost_usd = None
            return self
        prompt_c, completion_c = estimate_cost_usd(
            self.model,
            prompt_tokens=self.input_tokens,
            completion_tokens=self.output_tokens,
        )
        merged = merge_prompt_completion_cost_usd(prompt_c, completion_c)
        self.total_cost_usd = round(merged, 6) if merged is not None else None
        return self

compute_total_cost_usd() pydantic-validator

Populate total_cost_usd from tokens and model (estimate_cost_usd).

Source code in deet/data_models/extraction.py

@model_validator(mode="after")
def compute_total_cost_usd(self) -> Self:
    """Populate ``total_cost_usd`` from tokens and model (``estimate_cost_usd``)."""
    if self.model is None:
        self.total_cost_usd = None
        return self
    prompt_c, completion_c = estimate_cost_usd(
        self.model,
        prompt_tokens=self.input_tokens,
        completion_tokens=self.output_tokens,
    )
    merged = merge_prompt_completion_cost_usd(prompt_c, completion_c)
    self.total_cost_usd = round(merged, 6) if merged is not None else None
    return self

DocumentParsingStats pydantic-model

Bases: BaseModel

Parsing timing for a single document during document preparation.

Fields:

• parsing_seconds (float | None)
• parsing_skipped (bool)

Source code in deet/data_models/extraction.py

class DocumentParsingStats(BaseModel):
    """Parsing timing for a single document during document preparation."""

    parsing_seconds: float | None = None
    parsing_skipped: bool = True

ExtractionPipelineStage

Bases: StrEnum

Named stages timed during an extraction pipeline run.

Source code in deet/data_models/extraction.py

class ExtractionPipelineStage(StrEnum):
    """Named stages timed during an extraction pipeline run."""

    annotation_conversion = "annotation_conversion"
    prompt_population = "prompt_population"
    document_preparation = "document_preparation"
    llm_extraction = "llm_extraction"
    artifact_export = "artifact_export"

ExtractionRunMetadata pydantic-model

Bases: BaseModel

Aggregate metadata for a batch extraction run.

Fields:

• model (str | None)
• total_input_tokens (int)
• total_output_tokens (int)
• total_cost_usd (float | None)
• per_document (dict[str, PerDocumentExtractionStats])
• total_pipeline_duration_seconds (float | None)
• stage_durations_seconds (dict[str, float])
• notes (RunMetadataNotes)
• timestamp (datetime)

Source code in deet/data_models/extraction.py

class ExtractionRunMetadata(BaseModel):
    """Aggregate metadata for a batch extraction run."""

    model: str | None = None
    total_input_tokens: int = 0
    total_output_tokens: int = 0
    total_cost_usd: float | None = None
    per_document: dict[str, PerDocumentExtractionStats] = Field(default_factory=dict)
    total_pipeline_duration_seconds: float | None = None
    stage_durations_seconds: dict[str, float] = Field(default_factory=dict)
    notes: RunMetadataNotes = Field(default_factory=RunMetadataNotes)
    timestamp: datetime = Field(default_factory=lambda: datetime.now(tz=UTC))

ExtractionRunOutput pydantic-model

Bases: BaseModel

Top-level output from a batch extraction run.

Fields:

• annotated_documents (list[GoldStandardAnnotatedDocument])
• metadata (ExtractionRunMetadata)

Source code in deet/data_models/extraction.py

class ExtractionRunOutput(BaseModel):
    """Top-level output from a batch extraction run."""

    annotated_documents: list[GoldStandardAnnotatedDocument]
    metadata: ExtractionRunMetadata

PerDocumentExtractionStats pydantic-model

Bases: BaseModel

Per-document tokens and timing for an extraction run.

Fields:

• input_tokens (int)
• output_tokens (int)
• parsing_seconds (float | None)
• parsing_skipped (bool)
• llm_call_seconds (float)

Source code in deet/data_models/extraction.py

class PerDocumentExtractionStats(BaseModel):
    """Per-document tokens and timing for an extraction run."""

    input_tokens: int = 0
    output_tokens: int = 0
    parsing_seconds: float | None = None
    parsing_skipped: bool = True
    llm_call_seconds: float = 0.0

RunMetadataNotes pydantic-model

Bases: BaseModel

Human-readable notes for timing fields in run metadata.

Fields:

• total_pipeline_duration_seconds (str)
• stage_durations_seconds (dict[str, str])
• per_document (dict[str, str])

Source code in deet/data_models/extraction.py

class RunMetadataNotes(BaseModel):
    """Human-readable notes for timing fields in run metadata."""

    total_pipeline_duration_seconds: str = (
        "Wall-clock time for the extraction run, from loading project data "
        "through export of experiment snapshots. Includes time spent answering "
        "the interactive config wizard when no config file is provided."
    )
    stage_durations_seconds: dict[str, str] = Field(
        default_factory=pipeline_stage_notes,
    )
    per_document: dict[str, str] = Field(
        default_factory=lambda: {
            "parsing_seconds": (
                "Parse/read time during document preparation; null when "
                "parsing_skipped is true."
            ),
            "parsing_skipped": (
                "True when full text came from cache and was not parsed this run."
            ),
            "llm_call_seconds": (
                "Wall-clock time for the LLM request, including local prompt setup."
            ),
        },
    )

pipeline_stage_notes()

Return human-readable notes keyed by pipeline stage name.

Source code in deet/data_models/extraction.py

def pipeline_stage_notes() -> dict[str, str]:
    """Return human-readable notes keyed by pipeline stage name."""
    return {
        stage.value: PIPELINE_STAGE_NOTES[stage] for stage in ExtractionPipelineStage
    }

pipeline

Models to employ for implementing DEET jobs in sequential, harmonised pipelines.

BaseExecutor

Bases: ABC

Abstract base class for all executors.

Source code in deet/data_models/pipeline.py

class BaseExecutor(ABC):
    """Abstract base class for all executors."""

    @abstractmethod
    def _execute(
        self,
        job: "Job",
        args: list[Any] | None = None,
        kwargs: dict[str, Any] | None = None,
    ) -> Any:  # noqa: ANN401
        pass

CodeExecutor

Bases: BaseExecutor

Executor for Python callable.

Source code in deet/data_models/pipeline.py

class CodeExecutor(BaseExecutor):
    """Executor for Python callable."""

    def _execute(
        self,
        job: "Job",
        args: list[Any] | None = None,
        kwargs: dict[str, Any] | None = None,
    ) -> Any:  # noqa: ANN401
        args = args or []
        kwargs = kwargs or {}
        if isinstance(job.job, Path):
            malspecified_job = "job has to be a callable, not a path to a script."
            raise JobExecutionError(malspecified_job)
        result = job.job(*args, **kwargs)
        if job.capture_output:
            return result
        return None

EgressMethod

Bases: StrEnum

An enum of egree methods for a PipelineStage.

Source code in deet/data_models/pipeline.py

class EgressMethod(StrEnum):
    """An enum of egree methods for a PipelineStage."""

    FILE = auto()
    MEMORY = auto()

Executor

A wrapper for all kinds of executors.

Source code in deet/data_models/pipeline.py

class Executor:
    """A wrapper for all kinds of executors."""

    def __init__(self, executor: BaseExecutor) -> None:
        """Init new executor instance."""
        self.executor = executor

    def execute(
        self,
        job: "Job",
        args: list[Any] | None = None,
        kwargs: dict[str, Any] | None = None,
    ) -> Any:  # noqa: ANN401
        """Execute a job."""
        return self.executor._execute(job, args=args, kwargs=kwargs)  # noqa: SLF001

__init__(executor)

Init new executor instance.

Source code in deet/data_models/pipeline.py

def __init__(self, executor: BaseExecutor) -> None:
    """Init new executor instance."""
    self.executor = executor

execute(job, args=None, kwargs=None)

Execute a job.

Source code in deet/data_models/pipeline.py

def execute(
    self,
    job: "Job",
    args: list[Any] | None = None,
    kwargs: dict[str, Any] | None = None,
) -> Any:  # noqa: ANN401
    """Execute a job."""
    return self.executor._execute(job, args=args, kwargs=kwargs)  # noqa: SLF001

IngressMethod

Bases: StrEnum

An enum of ingress methods for a PipelineStage.

Source code in deet/data_models/pipeline.py

class IngressMethod(StrEnum):
    """An enum of ingress methods for a PipelineStage."""

    FILE = auto()
    MEMORY = auto()
    HTTP = auto()  # we may need to download data
    RANDOM = auto()  # there might be jobs & pipeline stages where we start with a seed

Job pydantic-model

Bases: BaseModel

The attributes describing a specific job.

Config:

• arbitrary_types_allowed: True

Fields:

• name (str)
• job_format (JobFormat)
• job_type (JobType | list[JobType])
• language (Language)
• ingress_method (IngressMethod | None)
• egress_method (EgressMethod)
• job (Callable | Path)
• script_args (list[str] | None)
• func_args (list[Any] | None)
• func_kwargs (dict[str, Any] | None)
• capture_output (bool)
• executor (Executor)

Source code in deet/data_models/pipeline.py

class Job(BaseModel):
    """The attributes describing a specific job."""

    name: str
    job_format: JobFormat
    job_type: JobType | list[JobType]
    language: Language
    ingress_method: IngressMethod | None = (
        None  # we may have a job that starts with no data
    )
    egress_method: EgressMethod
    job: Callable | Path
    script_args: list[str] | None
    func_args: list[Any] | None = None
    func_kwargs: dict[str, Any] | None = None
    capture_output: bool = True
    executor: Executor

    model_config = ConfigDict(arbitrary_types_allowed=True)

    def run_job(self) -> None | str:
        """Run the job defined in this model instance."""
        logger.debug(f"Running job {self.name}")
        logger.debug(f"func args: {self.func_args}")
        logger.debug(f"func kwargs: {self.func_kwargs}")

        # Use func_args/func_kwargs for CODE jobs, script_args for SCRIPT jobs
        if self.job_format == JobFormat.CODE:
            output = self.executor.execute(
                job=self, args=self.func_args, kwargs=self.func_kwargs
            )
        elif self.job_format == JobFormat.SCRIPT:
            output = self.executor.execute(job=self, args=self.script_args)

        if self.capture_output:
            logger.debug(output)
        return output

run_job()

Run the job defined in this model instance.

Source code in deet/data_models/pipeline.py

def run_job(self) -> None | str:
    """Run the job defined in this model instance."""
    logger.debug(f"Running job {self.name}")
    logger.debug(f"func args: {self.func_args}")
    logger.debug(f"func kwargs: {self.func_kwargs}")

    # Use func_args/func_kwargs for CODE jobs, script_args for SCRIPT jobs
    if self.job_format == JobFormat.CODE:
        output = self.executor.execute(
            job=self, args=self.func_args, kwargs=self.func_kwargs
        )
    elif self.job_format == JobFormat.SCRIPT:
        output = self.executor.execute(job=self, args=self.script_args)

    if self.capture_output:
        logger.debug(output)
    return output

JobExecutionError

Bases: Exception

To raise when a job hits a generic error.

Source code in deet/data_models/pipeline.py

class JobExecutionError(Exception):
    """To raise when a job hits a generic error."""

JobFormat

Bases: StrEnum

An enum of job formats.

Jobs are the building blocks of pipeline stages. The job format describes the 'medium' in which the job is provided to the job object.

Source code in deet/data_models/pipeline.py

class JobFormat(StrEnum):
    """
    An enum of job formats.

    Jobs are the building blocks of pipeline stages.
    The job format describes the 'medium' in which the job
    is provided to the job object.
    """

    SCRIPT = auto()  # when job is supplied in a file.
    CODE = auto()  # when job is supplied within the pipeline.

JobType

Bases: StrEnum

An enum of job types.

This is a descriptive label of the broad category of what the job is doing.

Source code in deet/data_models/pipeline.py

class JobType(StrEnum):
    """
    An enum of job types.

    This is a descriptive label of the broad category
    of what the job is doing.
    """

    DATA_PROCESSING = auto()
    DATA_COLLECTION = auto()
    CLASSIFICATION = auto()
    EXTRACTION = auto()

Language

Bases: StrEnum

An enum of permitted languages a job can be specified in.

Source code in deet/data_models/pipeline.py

class Language(StrEnum):
    """An enum of permitted languages a job can be specified in."""

    PYTHON = auto()
    R = auto()
    SHELL = auto()
    SQL = auto()
    LLM_PROMPT = auto()

MissingBinaryError

Bases: Exception

To raise when we're missing a binary required to run a script.

Source code in deet/data_models/pipeline.py

class MissingBinaryError(Exception):
    """To raise when we're missing a binary required to run a script."""

Pipeline pydantic-model

Bases: BaseModel

A complete pipeline consisting of several PipelineStage objects.

Fields:

• name (str)
• stages (list[PipelineStage])

Source code in deet/data_models/pipeline.py

class Pipeline(BaseModel):
    """A complete pipeline consisting of several `PipelineStage` objects."""

    model_config = ConfigDict()

    name: str
    stages: list[PipelineStage]

    def run(self) -> None:
        """Run all pipeline stages."""
        logger.info(f"Pipeline {self.name} has {len(self.stages)} stages.")
        for stage in self.stages:
            stage.run_jobs()

run()

Run all pipeline stages.

Source code in deet/data_models/pipeline.py

def run(self) -> None:
    """Run all pipeline stages."""
    logger.info(f"Pipeline {self.name} has {len(self.stages)} stages.")
    for stage in self.stages:
        stage.run_jobs()

PipelineStage pydantic-model

Bases: BaseModel

A stage in a DEET pipeline.

Fields:

• name (str)
• skip_jobs_if_failed (bool)
• input_file (Path | None)
• data (Any | None)
• jobs (Job | list[Job])
• logfile (Path | None)
• default_func_args (list[Any] | None)
• default_func_kwargs (dict[str, Any] | None)

Validators:

• convert_jobs_to_list → jobs

Source code in deet/data_models/pipeline.py

class PipelineStage(BaseModel):
    """A stage in a DEET pipeline."""

    model_config = ConfigDict()

    name: str
    skip_jobs_if_failed: bool = False
    input_file: Path | None = None  # handled in job
    data: Any | None = None  # handled in job
    jobs: Job | list[Job]
    logfile: Path | None = None
    default_func_args: list[Any] | None = None
    default_func_kwargs: dict[str, Any] | None = None

    @field_validator("jobs", mode="before")
    @classmethod
    def convert_jobs_to_list(cls, v: Job | list[Job]) -> list[Job]:
        """Convert jobs to list of jobs if just one job supplied."""
        if isinstance(v, Job):
            v = [v]
        return v

    @staticmethod
    def write_stage_logfile(payload: str, filepath: Path) -> None:
        """Write logfile for a specific stage."""
        filepath.write_text(payload, encoding="utf-8")

    def run_jobs(
        self,
        func_args: list[Any] | None = None,
        func_kwargs: dict[str, Any] | None = None,
    ) -> None:
        """Run all jobs in a pipeline stage."""
        if isinstance(
            self.jobs, Job
        ):  # for mypy -- can we remove this given field_validator?
            self.jobs = [self.jobs]

        args_to_use = func_args or self.default_func_args
        kwargs_to_use = func_kwargs or self.default_func_kwargs

        logger.info(f"Pipeline stage {self.name} has {len(self.jobs)} stages.")
        for i, job in enumerate(self.jobs):
            logger.info(
                f"Running job {i + 1} out of {len(self.jobs)}, name: {job.name}."
            )
            try:
                # Use job's own args if they exist, otherwise fall
                # back to stage/method args
                if job.job_format == JobFormat.CODE:
                    args_to_use = job.func_args or func_args or self.default_func_args
                    kwargs_to_use = (
                        job.func_kwargs or func_kwargs or self.default_func_kwargs
                    )

                    # Only override if the job doesn't have its own args
                    if job.func_args is None:
                        job.func_args = args_to_use
                    if job.func_kwargs is None:
                        job.func_kwargs = kwargs_to_use

                job_output = job.run_job()
                if (
                    job.capture_output
                    and job_output is not None
                    and self.logfile is not None
                ):
                    self.write_stage_logfile(payload=job_output, filepath=self.logfile)
                # should we `yield` the job_output here?
            except Exception as e:
                if not self.skip_jobs_if_failed:
                    raise
                logger.error(
                    f"Encountered error {e} on job {i}, {job.name}"
                    f" in pipeline stage {self.name}, moving to next job..."
                )
                logger.error(f"Error type: {type(e).__name__}")
                logger.error(f"Error message: {e}")
                logger.error("Stack trace:")
                logger.error(traceback.format_exc())
                logger.info("Moving to next job...")

                continue

convert_jobs_to_list(v) pydantic-validator

Convert jobs to list of jobs if just one job supplied.

Source code in deet/data_models/pipeline.py

@field_validator("jobs", mode="before")
@classmethod
def convert_jobs_to_list(cls, v: Job | list[Job]) -> list[Job]:
    """Convert jobs to list of jobs if just one job supplied."""
    if isinstance(v, Job):
        v = [v]
    return v

run_jobs(func_args=None, func_kwargs=None)

Run all jobs in a pipeline stage.

Source code in deet/data_models/pipeline.py

def run_jobs(
    self,
    func_args: list[Any] | None = None,
    func_kwargs: dict[str, Any] | None = None,
) -> None:
    """Run all jobs in a pipeline stage."""
    if isinstance(
        self.jobs, Job
    ):  # for mypy -- can we remove this given field_validator?
        self.jobs = [self.jobs]

    args_to_use = func_args or self.default_func_args
    kwargs_to_use = func_kwargs or self.default_func_kwargs

    logger.info(f"Pipeline stage {self.name} has {len(self.jobs)} stages.")
    for i, job in enumerate(self.jobs):
        logger.info(
            f"Running job {i + 1} out of {len(self.jobs)}, name: {job.name}."
        )
        try:
            # Use job's own args if they exist, otherwise fall
            # back to stage/method args
            if job.job_format == JobFormat.CODE:
                args_to_use = job.func_args or func_args or self.default_func_args
                kwargs_to_use = (
                    job.func_kwargs or func_kwargs or self.default_func_kwargs
                )

                # Only override if the job doesn't have its own args
                if job.func_args is None:
                    job.func_args = args_to_use
                if job.func_kwargs is None:
                    job.func_kwargs = kwargs_to_use

            job_output = job.run_job()
            if (
                job.capture_output
                and job_output is not None
                and self.logfile is not None
            ):
                self.write_stage_logfile(payload=job_output, filepath=self.logfile)
            # should we `yield` the job_output here?
        except Exception as e:
            if not self.skip_jobs_if_failed:
                raise
            logger.error(
                f"Encountered error {e} on job {i}, {job.name}"
                f" in pipeline stage {self.name}, moving to next job..."
            )
            logger.error(f"Error type: {type(e).__name__}")
            logger.error(f"Error message: {e}")
            logger.error("Stack trace:")
            logger.error(traceback.format_exc())
            logger.info("Moving to next job...")

            continue

write_stage_logfile(payload, filepath) staticmethod

Write logfile for a specific stage.

Source code in deet/data_models/pipeline.py

@staticmethod
def write_stage_logfile(payload: str, filepath: Path) -> None:
    """Write logfile for a specific stage."""
    filepath.write_text(payload, encoding="utf-8")

ScriptExecutor

Bases: BaseExecutor

An executor class for different kinds of scripts.

Source code in deet/data_models/pipeline.py

class ScriptExecutor(BaseExecutor):
    """An executor class for different kinds of scripts."""

    def __init__(
        self,
        python_path: Path | None = None,
        r_path: Path | None = None,
        bash_path: Path = Path("/bin/bash"),
    ) -> None:
        """Create ScriptExecutor instance."""
        python_which = shutil.which("python")
        r_which = shutil.which("R")

        self.python_path = (
            python_path
            if python_path
            else (Path(python_which) if python_which else None)
        )
        self.r_path = r_path if r_path else (Path(r_which) if r_which else None)
        self.bash_path = bash_path

        logger.debug(f"python path: {self.python_path}")
        logger.debug(f"r path: {self.r_path}")
        logger.debug(f"bash path: {self.bash_path}")

    def _execute(
        self,
        job: "Job",
        args: list[str] | None = None,
        kwargs: dict[str, Any] | None = None,
    ) -> Any:  # noqa: ANN401
        if not isinstance(job.job, Path):
            malspecified_job = (
                "ScriptExecutor requires job.job to be a Path, not a Callable."
            )
            raise JobExecutionError(malspecified_job)

        if job.language == Language.PYTHON:
            return self.python_executor(
                job.job, args=args, capture_output=job.capture_output
            )
        if job.language == Language.R:
            return self.r_executor(
                job.job, args=args, capture_output=job.capture_output
            )
        if job.language == Language.SHELL:
            return self.bash_executor(
                job.job, args=args, capture_output=job.capture_output
            )
        missing_language = (
            f"Script execution not implemented for language: {job.language}"
        )
        raise NotImplementedError(missing_language)

    @staticmethod
    def verify_filetype(filename: str, filetype: Literal[".py", ".R", ".sh"]) -> bool:
        """
        Verify a given file is of a given filetype via checking the ending.

        Args:
            filename (str): Name of file.
            filetype (Literal[".py", ".R", ".sh"]):
                    file ending.

        Raises:
            WrongFiletypeError: When ending doesnt match the input.

        Returns:
            bool: True if OK.

        """
        if not filename[-len(filetype) :] == filetype:
            raise WrongFiletypeError
        return True

    def python_executor(
        self, script_path: Path, args: list[str] | None, *, capture_output: bool = True
    ) -> None | str:
        """
        Execute a python script.

        Args:
            script_path (Path): file path to script.
            args (list[str]): args to run with script.
            capture_output (bool, optional): Defaults to True.

        Returns:
            None | str: output from stdout or None.

        """
        self.verify_filetype(script_path.name, ".py")
        if self.python_path is None:
            python_missing = "can't find python binary. please find it/install."
            raise MissingBinaryError(python_missing)

        cmd: list[str] = [str(self.python_path), str(script_path)]
        if args:
            cmd.extend(args)

        env = restricted_env.copy()
        env.update({"PYTHONPATH": ""})

        output = subprocess.run(  # noqa: S603
            cmd,
            check=True,
            capture_output=capture_output,
            text=True,
            env=env,
        )
        if capture_output:
            return (
                output.stderr
            )  # loguru writes to stderr; we're using loguru for messages.
        # if you want to capture `print`-ed messages, capture stdout.
        return None

    def r_executor(
        self, script_path: Path, args: list[str] | None, *, capture_output: bool = True
    ) -> None | str:
        """Execute an R script."""
        self.verify_filetype(script_path.name, ".R")
        if self.r_path is None:
            r_missing = "can't find r binary. please find it/install."
            raise MissingBinaryError(r_missing)

        cmd: list[str] = [str(self.r_path), str(script_path)]
        if args:
            cmd.extend(args)

        env = restricted_env.copy()
        env.update(
            {
                "R_LIBS_USER": "",  # prevent loading from user library paths
                "R_PROFILE_USER": "",  # disable user profile scripts
                "R_ENVIRON_USER": "",  # disable user environment files
                "R_HISTFILE": "",  # disable history file
            }
        )

        output = subprocess.run(  # noqa: S603
            cmd,
            check=True,
            capture_output=capture_output,
            text=True,
            env=env,
        )
        if capture_output:
            return output.stdout
        return None

    def bash_executor(
        self, script_path: Path, args: list[str] | None, *, capture_output: bool = True
    ) -> None | str:
        """Execute a bash script."""
        self.verify_filetype(script_path.name, ".sh")
        if self.r_path is None:
            r_missing = "can't find bash binary. please find it/install."
            raise MissingBinaryError(r_missing)

        cmd: list[str] = [str(self.bash_path), str(script_path)]
        if args:
            cmd.extend(args)

        env = restricted_env.copy()
        env.update(
            {
                "SHELL": str(self.bash_path),
                "IFS": " \t\n",  # safe input field separator
                "ENV": "",  # disable shell startup file
                "BASH_ENV": "",  # disable bash startup file
            }
        )

        output = subprocess.run(  # noqa: S603
            cmd,
            check=True,
            capture_output=capture_output,
            text=True,
            env=env,
        )
        if capture_output:
            return output.stdout
        return None

__init__(python_path=None, r_path=None, bash_path=Path('/bin/bash'))

Create ScriptExecutor instance.

Source code in deet/data_models/pipeline.py

def __init__(
    self,
    python_path: Path | None = None,
    r_path: Path | None = None,
    bash_path: Path = Path("/bin/bash"),
) -> None:
    """Create ScriptExecutor instance."""
    python_which = shutil.which("python")
    r_which = shutil.which("R")

    self.python_path = (
        python_path
        if python_path
        else (Path(python_which) if python_which else None)
    )
    self.r_path = r_path if r_path else (Path(r_which) if r_which else None)
    self.bash_path = bash_path

    logger.debug(f"python path: {self.python_path}")
    logger.debug(f"r path: {self.r_path}")
    logger.debug(f"bash path: {self.bash_path}")

bash_executor(script_path, args, *, capture_output=True)

Execute a bash script.

Source code in deet/data_models/pipeline.py

def bash_executor(
    self, script_path: Path, args: list[str] | None, *, capture_output: bool = True
) -> None | str:
    """Execute a bash script."""
    self.verify_filetype(script_path.name, ".sh")
    if self.r_path is None:
        r_missing = "can't find bash binary. please find it/install."
        raise MissingBinaryError(r_missing)

    cmd: list[str] = [str(self.bash_path), str(script_path)]
    if args:
        cmd.extend(args)

    env = restricted_env.copy()
    env.update(
        {
            "SHELL": str(self.bash_path),
            "IFS": " \t\n",  # safe input field separator
            "ENV": "",  # disable shell startup file
            "BASH_ENV": "",  # disable bash startup file
        }
    )

    output = subprocess.run(  # noqa: S603
        cmd,
        check=True,
        capture_output=capture_output,
        text=True,
        env=env,
    )
    if capture_output:
        return output.stdout
    return None

python_executor(script_path, args, *, capture_output=True)

Execute a python script.

Parameters:

Name	Type	Description	Default
script_path	Path	file path to script.	required
args	list[str]	args to run with script.	required
capture_output	bool	Defaults to True.	True

Returns:

Type	Description
None | str	None | str: output from stdout or None.

Source code in deet/data_models/pipeline.py

def python_executor(
    self, script_path: Path, args: list[str] | None, *, capture_output: bool = True
) -> None | str:
    """
    Execute a python script.

    Args:
        script_path (Path): file path to script.
        args (list[str]): args to run with script.
        capture_output (bool, optional): Defaults to True.

    Returns:
        None | str: output from stdout or None.

    """
    self.verify_filetype(script_path.name, ".py")
    if self.python_path is None:
        python_missing = "can't find python binary. please find it/install."
        raise MissingBinaryError(python_missing)

    cmd: list[str] = [str(self.python_path), str(script_path)]
    if args:
        cmd.extend(args)

    env = restricted_env.copy()
    env.update({"PYTHONPATH": ""})

    output = subprocess.run(  # noqa: S603
        cmd,
        check=True,
        capture_output=capture_output,
        text=True,
        env=env,
    )
    if capture_output:
        return (
            output.stderr
        )  # loguru writes to stderr; we're using loguru for messages.
    # if you want to capture `print`-ed messages, capture stdout.
    return None

r_executor(script_path, args, *, capture_output=True)

Execute an R script.

Source code in deet/data_models/pipeline.py

def r_executor(
    self, script_path: Path, args: list[str] | None, *, capture_output: bool = True
) -> None | str:
    """Execute an R script."""
    self.verify_filetype(script_path.name, ".R")
    if self.r_path is None:
        r_missing = "can't find r binary. please find it/install."
        raise MissingBinaryError(r_missing)

    cmd: list[str] = [str(self.r_path), str(script_path)]
    if args:
        cmd.extend(args)

    env = restricted_env.copy()
    env.update(
        {
            "R_LIBS_USER": "",  # prevent loading from user library paths
            "R_PROFILE_USER": "",  # disable user profile scripts
            "R_ENVIRON_USER": "",  # disable user environment files
            "R_HISTFILE": "",  # disable history file
        }
    )

    output = subprocess.run(  # noqa: S603
        cmd,
        check=True,
        capture_output=capture_output,
        text=True,
        env=env,
    )
    if capture_output:
        return output.stdout
    return None

verify_filetype(filename, filetype) staticmethod

Verify a given file is of a given filetype via checking the ending.

Parameters:

Name	Type	Description	Default
filename	str	Name of file.	required
filetype	Literal[".py", ".R", ".sh"]	file ending.	required

Raises:

Type	Description
WrongFiletypeError	When ending doesnt match the input.

Returns:

Name	Type	Description
bool	bool	True if OK.

Source code in deet/data_models/pipeline.py

@staticmethod
def verify_filetype(filename: str, filetype: Literal[".py", ".R", ".sh"]) -> bool:
    """
    Verify a given file is of a given filetype via checking the ending.

    Args:
        filename (str): Name of file.
        filetype (Literal[".py", ".R", ".sh"]):
                file ending.

    Raises:
        WrongFiletypeError: When ending doesnt match the input.

    Returns:
        bool: True if OK.

    """
    if not filename[-len(filetype) :] == filetype:
        raise WrongFiletypeError
    return True

WrongFiletypeError

Bases: Exception

Raise for wrong filetype.

Source code in deet/data_models/pipeline.py

class WrongFiletypeError(Exception):
    """Raise for wrong filetype."""

    def __init__(
        self,
        msg: str = "Supplied filetype is not correct.",
        *args,  # noqa: ANN002
        **kwargs,
    ) -> None:
        """Init the exception with default message."""
        super().__init__(msg, *args, **kwargs)

__init__(msg='Supplied filetype is not correct.', *args, **kwargs)

Init the exception with default message.

Source code in deet/data_models/pipeline.py

def __init__(
    self,
    msg: str = "Supplied filetype is not correct.",
    *args,  # noqa: ANN002
    **kwargs,
) -> None:
    """Init the exception with default message."""
    super().__init__(msg, *args, **kwargs)

jobify(name, job_type=JobType.DATA_PROCESSING, func_args=None, func_kwargs=None, *, capture_output=True)

Decorate to wrap a function as a Job instance.

Source code in deet/data_models/pipeline.py

def jobify(
    name: str,
    job_type: JobType | list[JobType] = JobType.DATA_PROCESSING,
    func_args: list[Any] | None = None,
    func_kwargs: dict[str, Any] | None = None,
    *,
    capture_output: bool = True,
) -> Callable[[F], Job]:
    """Decorate to wrap a function as a Job instance."""

    def decorator(func: F) -> Job:
        """Wrap around target callable."""
        return Job(
            name=name,
            job_format=JobFormat.CODE,
            job_type=job_type,
            language=Language.PYTHON,
            ingress_method=None,
            egress_method=EgressMethod.MEMORY,
            job=func,
            script_args=None,
            func_args=func_args,
            func_kwargs=func_kwargs,
            capture_output=capture_output,
            executor=Executor(executor=CodeExecutor()),
        )

    return decorator

stage_from_job(job=None, stage_name=None, input_file=None, logfile=None, *, skip_jobs_if_failed=False)

stage_from_job(
    job: Job,
    stage_name: str | None = None,
    input_file: Path | None = None,
    logfile: Path | None = None,
    *,
    skip_jobs_if_failed: bool = False,
) -> PipelineStage

stage_from_job(
    job: None = None,
    stage_name: str | None = None,
    input_file: Path | None = None,
    logfile: Path | None = None,
    *,
    skip_jobs_if_failed: bool = False,
) -> Callable[[Job], PipelineStage]

Create a PipelineStage from a single Job.

Can be used as a function or as a decorator.

Parameters:

Name	Type	Description	Default
job	Job | None	The Job to wrap in a PipelineStage. If None, returns a decorator.	None
stage_name	str | None	Name for the stage. Defaults to job name if not provided.	None
input_file	Path | None	Optional input file for the stage.	None
logfile	Path | None	Optional logfile for the stage.	None
skip_jobs_if_failed	bool	Whether to skip remaining jobs if one fails.	False

Returns:

Type	Description
PipelineStage | Callable[[Job], PipelineStage]	PipelineStage or decorator function.

Examples:

As a function:

>>> stage = stage_from_job(my_job, stage_name="my_stage")

As a decorator:

>>> @stage_from_job(stage_name="my_stage")
... @jobify(name="my_job")
... def my_function():
...     pass

Source code in deet/data_models/pipeline.py

def stage_from_job(
    job: Job | None = None,
    stage_name: str | None = None,
    input_file: Path | None = None,
    logfile: Path | None = None,
    *,
    skip_jobs_if_failed: bool = False,
) -> PipelineStage | Callable[[Job], PipelineStage]:
    """
    Create a PipelineStage from a single Job.

    Can be used as a function or as a decorator.

    Args:
        job: The Job to wrap in a PipelineStage. If None, returns a decorator.
        stage_name: Name for the stage. Defaults to job name if not provided.
        input_file: Optional input file for the stage.
        logfile: Optional logfile for the stage.
        skip_jobs_if_failed: Whether to skip remaining jobs if one fails.

    Returns:
        PipelineStage or decorator function.

    Examples:
        As a function:
        >>> stage = stage_from_job(my_job, stage_name="my_stage")

        As a decorator:
        >>> @stage_from_job(stage_name="my_stage")
        ... @jobify(name="my_job")
        ... def my_function():
        ...     pass

    """

    def _create_stage(j: Job) -> PipelineStage:
        return PipelineStage(
            name=stage_name or j.name,
            skip_jobs_if_failed=skip_jobs_if_failed,
            input_file=input_file,
            data=None,
            jobs=j,
            logfile=logfile,
        )

    # ff job is provided, return the stage directly
    if job is not None:
        return _create_stage(job)

    # otherwise, return a decorator
    return _create_stage

processed_gold_standard_annotations

Data models for procesed annotation data.

ProcessedAnnotationData pydantic-model

Bases: ProcessedAttributeData[AttributeT]

Structured result from annotation processing.

This model provides a clean, validated structure for all processed annotation data with useful properties and methods.

Fields:

• attributes (list[AttributeT])
• documents (list[DocumentType])
• annotations (list[GoldStandardAnnotationType])
• annotated_documents (list[GoldStandardAnnotatedDocumentType])
• attribute_id_to_label (dict[int, str])

Source code in deet/data_models/processed_gold_standard_annotations.py

class ProcessedAnnotationData[
    AttributeT: Attribute,
    DocumentType: Document,
    GoldStandardAnnotationType: GoldStandardAnnotation,
    GoldStandardAnnotatedDocumentType: GoldStandardAnnotatedDocument,
](ProcessedAttributeData[AttributeT]):
    """
    Structured result from annotation processing.

    This model provides a clean, validated structure for all processed
    annotation data with useful properties and methods.
    """

    documents: list[DocumentType]
    annotations: list[GoldStandardAnnotationType]
    annotated_documents: list[GoldStandardAnnotatedDocumentType]
    attribute_id_to_label: dict[int, str]

    @property
    def total_documents(self) -> int:
        """Total number of documents processed."""
        return len(self.documents)

    @property
    def total_annotations(self) -> int:
        """Total number of annotations processed."""
        return len(self.annotations)

    @property
    def total_annotated_documents(self) -> int:
        """Total number of documents with annotations."""
        return len(self.annotated_documents)

    @property
    def all_doc_ids(self) -> list[int]:
        """Return a list of document IDs present in the dataset."""
        return [
            doc.safe_identity.document_id
            for doc in self.documents
            if doc.safe_identity.document_id is not None
        ]

    def get_attributes_by_attribute_type(
        self, attribute_type: AttributeType
    ) -> list[AttributeT]:
        """Get all attributes of a specific type."""
        return [
            attr for attr in self.attributes if attr.output_data_type == attribute_type
        ]

    def get_documents_with_annotations(self) -> list[DocumentType]:
        """Get only documents that have annotations."""
        annotated_doc_ids = {
            doc.document.document_id for doc in self.annotated_documents
        }
        return [doc for doc in self.documents if doc.document_id in annotated_doc_ids]

    def get_annotations_by_annotation_type(
        self, annotation_type: AnnotationType
    ) -> list[GoldStandardAnnotationType]:
        """Get all annotations of a specific type (human/llm)."""
        return [
            ann for ann in self.annotations if ann.annotation_type == annotation_type
        ]

    def get_attribute_by_id(self, attribute_id: int) -> Attribute | None:
        """Get an attribute by its ID."""
        for attr in self.attributes:
            if attr.attribute_id == attribute_id:
                return attr
        return None

    def export_linkage_mapper_csv(
        self,
        file_path: Path,
        document_base_dir: Path | None = None,
        path_type: Literal["full", "relative", "file"] = "file",
    ) -> None:
        """
        Export a csv mapper to link document IDs and filenames.

        If document_base_dir is not None, then attempt to pre-populate this.
        """
        existing_ids: set[int] = set()
        for d in self.documents:
            if d.document_identity is None or d.document_identity.document_id is None:
                d.init_document_identity(existing_ids=existing_ids)
            if d.document_identity and d.document_identity.document_id is not None:
                existing_ids.add(d.document_identity.document_id)

        pre_fill: dict[int, Path] = {}
        if document_base_dir is not None:
            linker = DocumentReferenceLinker(
                references=self.documents,
                document_base_dir=document_base_dir,
            )
            # Use all available strategies including FILENAME_EXTERNAL_ID
            # for better coverage
            pre_fill = linker.guess_file_paths()
            logger.info(
                f"pre-filled {len(pre_fill)} file paths from {document_base_dir}"
            )

        with file_path.open("w", newline="", encoding="utf-8") as f:
            writer = csv.DictWriter(
                f, fieldnames=["document_id", "external_id", "name", "file_path"]
            )
            writer.writeheader()
            for d in self.documents:
                if (
                    d.document_identity is None
                    or d.document_identity.document_id is None
                ):
                    d.init_document_identity()
                doc_id = d.document_identity.document_id  # type: ignore[union-attr]
                external_id = d.document_identity.external_id  # type: ignore[union-attr]
                if doc_id is None:
                    no_doc_identity_err = (
                        f"document_identity was not set for document {d}"
                    )
                    raise ValueError(no_doc_identity_err)

                file_path_result = pre_fill.get(doc_id)  # type:ignore[assignment]
                if isinstance(file_path_result, Path):
                    if path_type == "full":
                        file_path_result = file_path_result.absolute()
                    elif path_type == "relative":
                        pass
                    elif path_type == "file":
                        file_path_result = Path(file_path_result.name)
                    else:
                        bad_filepath_formatting_err = (
                            f"path_type {path_type} is "
                            "not permitted. use `full`, `relative`, `file`."
                        )
                        raise NotImplementedError(bad_filepath_formatting_err)

                writer.writerow(
                    {
                        "document_id": doc_id,
                        "external_id": external_id,
                        "name": d.name,
                        "file_path": file_path_result,
                    }
                )

    def filter_documents_by_ids(self, target_ids: Collection[int]) -> None:
        """Filter documents and annotated documents in-place to match target IDs."""
        initial_doc_count = len(self.documents)
        target_set = set(target_ids)
        self.documents = [
            doc for doc in self.documents if doc.safe_identity.document_id in target_set
        ]
        self.annotated_documents = [
            adoc
            for adoc in self.annotated_documents
            if adoc.document.safe_identity.document_id in target_set
        ]
        doc_count = len(self.documents)
        logger.info(f"Filtered from {initial_doc_count}" f" to {doc_count}")

all_doc_ids property

Return a list of document IDs present in the dataset.

total_annotated_documents property

Total number of documents with annotations.

total_annotations property

Total number of annotations processed.

total_documents property

Total number of documents processed.

export_linkage_mapper_csv(file_path, document_base_dir=None, path_type='file')

Export a csv mapper to link document IDs and filenames.

If document_base_dir is not None, then attempt to pre-populate this.

Source code in deet/data_models/processed_gold_standard_annotations.py

def export_linkage_mapper_csv(
    self,
    file_path: Path,
    document_base_dir: Path | None = None,
    path_type: Literal["full", "relative", "file"] = "file",
) -> None:
    """
    Export a csv mapper to link document IDs and filenames.

    If document_base_dir is not None, then attempt to pre-populate this.
    """
    existing_ids: set[int] = set()
    for d in self.documents:
        if d.document_identity is None or d.document_identity.document_id is None:
            d.init_document_identity(existing_ids=existing_ids)
        if d.document_identity and d.document_identity.document_id is not None:
            existing_ids.add(d.document_identity.document_id)

    pre_fill: dict[int, Path] = {}
    if document_base_dir is not None:
        linker = DocumentReferenceLinker(
            references=self.documents,
            document_base_dir=document_base_dir,
        )
        # Use all available strategies including FILENAME_EXTERNAL_ID
        # for better coverage
        pre_fill = linker.guess_file_paths()
        logger.info(
            f"pre-filled {len(pre_fill)} file paths from {document_base_dir}"
        )

    with file_path.open("w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(
            f, fieldnames=["document_id", "external_id", "name", "file_path"]
        )
        writer.writeheader()
        for d in self.documents:
            if (
                d.document_identity is None
                or d.document_identity.document_id is None
            ):
                d.init_document_identity()
            doc_id = d.document_identity.document_id  # type: ignore[union-attr]
            external_id = d.document_identity.external_id  # type: ignore[union-attr]
            if doc_id is None:
                no_doc_identity_err = (
                    f"document_identity was not set for document {d}"
                )
                raise ValueError(no_doc_identity_err)

            file_path_result = pre_fill.get(doc_id)  # type:ignore[assignment]
            if isinstance(file_path_result, Path):
                if path_type == "full":
                    file_path_result = file_path_result.absolute()
                elif path_type == "relative":
                    pass
                elif path_type == "file":
                    file_path_result = Path(file_path_result.name)
                else:
                    bad_filepath_formatting_err = (
                        f"path_type {path_type} is "
                        "not permitted. use `full`, `relative`, `file`."
                    )
                    raise NotImplementedError(bad_filepath_formatting_err)

            writer.writerow(
                {
                    "document_id": doc_id,
                    "external_id": external_id,
                    "name": d.name,
                    "file_path": file_path_result,
                }
            )

filter_documents_by_ids(target_ids)

Filter documents and annotated documents in-place to match target IDs.

Source code in deet/data_models/processed_gold_standard_annotations.py

def filter_documents_by_ids(self, target_ids: Collection[int]) -> None:
    """Filter documents and annotated documents in-place to match target IDs."""
    initial_doc_count = len(self.documents)
    target_set = set(target_ids)
    self.documents = [
        doc for doc in self.documents if doc.safe_identity.document_id in target_set
    ]
    self.annotated_documents = [
        adoc
        for adoc in self.annotated_documents
        if adoc.document.safe_identity.document_id in target_set
    ]
    doc_count = len(self.documents)
    logger.info(f"Filtered from {initial_doc_count}" f" to {doc_count}")

get_annotations_by_annotation_type(annotation_type)

Get all annotations of a specific type (human/llm).

Source code in deet/data_models/processed_gold_standard_annotations.py

def get_annotations_by_annotation_type(
    self, annotation_type: AnnotationType
) -> list[GoldStandardAnnotationType]:
    """Get all annotations of a specific type (human/llm)."""
    return [
        ann for ann in self.annotations if ann.annotation_type == annotation_type
    ]

get_attribute_by_id(attribute_id)

Get an attribute by its ID.

Source code in deet/data_models/processed_gold_standard_annotations.py

def get_attribute_by_id(self, attribute_id: int) -> Attribute | None:
    """Get an attribute by its ID."""
    for attr in self.attributes:
        if attr.attribute_id == attribute_id:
            return attr
    return None

get_attributes_by_attribute_type(attribute_type)

Get all attributes of a specific type.

Source code in deet/data_models/processed_gold_standard_annotations.py

def get_attributes_by_attribute_type(
    self, attribute_type: AttributeType
) -> list[AttributeT]:
    """Get all attributes of a specific type."""
    return [
        attr for attr in self.attributes if attr.output_data_type == attribute_type
    ]

get_documents_with_annotations()

Get only documents that have annotations.

Source code in deet/data_models/processed_gold_standard_annotations.py

def get_documents_with_annotations(self) -> list[DocumentType]:
    """Get only documents that have annotations."""
    annotated_doc_ids = {
        doc.document.document_id for doc in self.annotated_documents
    }
    return [doc for doc in self.documents if doc.document_id in annotated_doc_ids]

ProcessedAttributeData pydantic-model

Bases: BaseModel

Structured result from annotation processing.

Contains only attributes, so the ProcessedAnnotationData class can subclass this

Fields:

• attributes (list[AttributeT])

Source code in deet/data_models/processed_gold_standard_annotations.py

class ProcessedAttributeData[AttributeT: Attribute](BaseModel):
    """
    Structured result from annotation processing.

    Contains only attributes, so the ProcessedAnnotationData class can
    subclass this
    """

    attributes: list[AttributeT]

    def _custom_prompts_cli(self) -> None:
        """
        Use an interactive CLI to have the user enter custom prompts.

        Args:
            attribute (Attribute): a single (Eppi)Attribute

        """
        for attribute in self.attributes:
            attribute.enter_custom_prompt()

    def export_attributes_csv_file(self, filepath: Path) -> None:
        """
        Write a csv file containing all attributes for prompt population.

        Args:
            filepath (Path): outfile path.


        """
        if filepath.suffix != ".csv":
            bad_filetype = "file ending must be .csv"
            raise ValueError(bad_filetype)
        filepath.unlink(missing_ok=True)
        for attribute in self.attributes:
            attribute.write_to_csv(filepath=filepath)

        logger.info(f"wrote attributes to file {filepath}.")

    @staticmethod
    def _validate_csv_file(filepath: Path) -> None:
        """Validate csv file exists and has correct extension."""
        if not filepath.exists():
            no_file = f"CSV file not found: {filepath}"
            raise FileNotFoundError(no_file)

        if filepath.suffix != ".csv":
            bad_suffix = "File must have .csv extension"
            raise ValueError(bad_suffix)

    @staticmethod
    def _validate_csv_headers(fieldnames: Sequence[str] | None) -> None:
        """Validate csv has required headers."""
        if fieldnames is None:
            empty_csv = "csv file is empty or has no headers"
            raise ValueError(empty_csv)

        required_fields = ["attribute_id", "prompt"]
        for field in required_fields:
            if field not in fieldnames:
                csv_missing_fields = (
                    f"csv must contain '{field}' column. "
                    f"Found columns: {', '.join(fieldnames)}"
                )
                raise ValueError(csv_missing_fields)

    def _process_csv_row(
        self,
        row: dict[str, Any],
        csv_attribute_ids_with_prompts: set[int],
        *,
        overwrite: bool = True,
    ) -> bool:
        """
        Process a single csv row and update the matching attribute.

        Returns
            bool: True if row was processed successfully, False otherwise

        """
        try:
            attribute_id = int(row.get("attribute_id"))  # type:ignore[arg-type]
        except ValueError as e:
            logger.warning(e)
            return False

        if (row.get("prompt") == "") or (row.get("prompt") is None):
            logger.debug(
                "prompt field is empty, "
                f"so we don't want this attribute {attribute_id}."
            )
            return False

        # Track attribute IDs that have non-empty prompts
        csv_attribute_ids_with_prompts.add(attribute_id)

        matching_attribute = None
        for attribute in self.attributes:
            if attribute.attribute_id == attribute_id:
                matching_attribute = attribute
                break

        if matching_attribute is None:
            logger.warning(f"No attribute found with ID {attribute_id}, skipping row")
            return False

        # Update attribute with prompt and data type
        try:
            matching_attribute.populate_prompt_from_dict(row, overwrite=overwrite)
            csv_attr_type = AttributeType(row.get("output_data_type"))  # type:ignore[arg-type]
            matching_attribute.output_data_type = csv_attr_type

        except ValueError as e:
            logger.error(
                f"Error processing row for attribute {attribute_id}: {e}. "
                "Setting attribute type to bool."
            )
            matching_attribute.output_data_type = DEFAULT_ATTRIBUTE_TYPE
            return False
        else:
            return True

    def _filter_attributes_by_csv(
        self,
        csv_attribute_ids_with_prompts: set[int],
        *,
        retain_only_csv_attributes: bool = True,
    ) -> None:
        """Filter attributes based on CSV content and retention policy."""
        if retain_only_csv_attributes:
            original_count = len(self.attributes)
            # Only keep attributes that are in CSV AND have non-empty prompts
            self.attributes = [
                attr
                for attr in self.attributes
                if attr.attribute_id in csv_attribute_ids_with_prompts
            ]
            logger.info(
                f"filtered attributes from {original_count} to {len(self.attributes)} "
                f"(retained only those in CSV with non-empty prompts)"
            )

    def _import_prompts_csv_file(
        self,
        filepath: Path,
        *,
        retain_only_csv_attributes: bool = True,
        overwrite: bool = True,
    ) -> None:
        """
        Import prompts from a csv file.

        Args:
            filepath (Path): attribute/prompt input file.
            retain_only_csv_attributes (bool, optional): if True, filter self.attributes
                to only include attributes with ids & a non-null prompt found in csv.
                Defaults to True.
            overwrite (bool, optional): Overwrite existing prompts. Defaults to True.

        """
        self._validate_csv_file(filepath)

        csv_attribute_ids_with_prompts: set[int] = set()
        rows_processed = 0

        with filepath.open(mode="r", newline="", encoding="utf-8") as f:
            reader = csv.DictReader(f)
            if reader.fieldnames is not None:
                reader.fieldnames = [
                    name.lstrip("\ufeff").strip() if name else name
                    for name in reader.fieldnames
                ]

            self._validate_csv_headers(reader.fieldnames)

            for row in reader:
                if self._process_csv_row(
                    row=row,
                    csv_attribute_ids_with_prompts=csv_attribute_ids_with_prompts,
                    overwrite=overwrite,
                ):
                    rows_processed += 1

            logger.info(f"Processed {rows_processed} prompts from {filepath}")

        self._filter_attributes_by_csv(
            csv_attribute_ids_with_prompts=csv_attribute_ids_with_prompts,
            retain_only_csv_attributes=retain_only_csv_attributes,
        )

    def populate_custom_prompts(
        self,
        method: CustomPromptPopulationMethod,
        filepath: Path | None = None,
        **kwargs,
    ) -> None:
        """
        Populate custom prompts.

        Args:
            method (CustomPromptPopulationMethod)
            filepath (Path | None): infile path.

        Raises:
            FileNotFoundError: if method is file and there's no filepath.

        """
        if method == "cli":
            self._custom_prompts_cli()
        elif method == "file":
            if filepath is None:
                missing_filepath = "please specify a filepath!"
                raise FileNotFoundError(missing_filepath)
            self._import_prompts_csv_file(filepath=filepath, **kwargs)
        else:
            not_impl = f"method {method} is not implemented. use cli or file."
            raise NotImplementedError(not_impl)

    @property
    def total_attributes(self) -> int:
        """Total number of attributes processed."""
        return len(self.attributes)

total_attributes property

Total number of attributes processed.

export_attributes_csv_file(filepath)

Write a csv file containing all attributes for prompt population.

Parameters:

Name	Type	Description	Default
filepath	Path	outfile path.	required

Source code in deet/data_models/processed_gold_standard_annotations.py

def export_attributes_csv_file(self, filepath: Path) -> None:
    """
    Write a csv file containing all attributes for prompt population.

    Args:
        filepath (Path): outfile path.


    """
    if filepath.suffix != ".csv":
        bad_filetype = "file ending must be .csv"
        raise ValueError(bad_filetype)
    filepath.unlink(missing_ok=True)
    for attribute in self.attributes:
        attribute.write_to_csv(filepath=filepath)

    logger.info(f"wrote attributes to file {filepath}.")

populate_custom_prompts(method, filepath=None, **kwargs)

Populate custom prompts.

Parameters:

Name	Type	Description	Default
filepath	Path | None	infile path.	None

Raises:

Type	Description
FileNotFoundError	if method is file and there's no filepath.

Source code in deet/data_models/processed_gold_standard_annotations.py

def populate_custom_prompts(
    self,
    method: CustomPromptPopulationMethod,
    filepath: Path | None = None,
    **kwargs,
) -> None:
    """
    Populate custom prompts.

    Args:
        method (CustomPromptPopulationMethod)
        filepath (Path | None): infile path.

    Raises:
        FileNotFoundError: if method is file and there's no filepath.

    """
    if method == "cli":
        self._custom_prompts_cli()
    elif method == "file":
        if filepath is None:
            missing_filepath = "please specify a filepath!"
            raise FileNotFoundError(missing_filepath)
        self._import_prompts_csv_file(filepath=filepath, **kwargs)
    else:
        not_impl = f"method {method} is not implemented. use cli or file."
        raise NotImplementedError(not_impl)

ProcessedEppiAnnotationData pydantic-model

Bases: ProcessedAnnotationData[EppiAttribute, EppiDocument, EppiGoldStandardAnnotation, EppiGoldStandardAnnotatedDocument]

Structured result from EPPI annotation processing.

This differs from Base ProcessedAnnotationData by specifying raw_data as an EppiRawData object

Fields:

• attributes (list[AttributeT])
• documents (list[DocumentType])
• annotations (list[GoldStandardAnnotationType])
• annotated_documents (list[GoldStandardAnnotatedDocumentType])
• attribute_id_to_label (dict[int, str])
• raw_data (EppiRawData | None)

Source code in deet/data_models/processed_gold_standard_annotations.py

class ProcessedEppiAnnotationData(
    ProcessedAnnotationData[
        EppiAttribute,
        EppiDocument,
        EppiGoldStandardAnnotation,
        EppiGoldStandardAnnotatedDocument,
    ]
):
    """
    Structured result from EPPI annotation processing.

    This differs from Base ProcessedAnnotationData by specifying raw_data as an
    EppiRawData object
    """

    raw_data: EppiRawData | None = None

    def populate_custom_prompts(
        self,
        method: CustomPromptPopulationMethod,
        filepath: Path | None = None,
        **kwargs: Any,  # noqa: ANN401
    ) -> None:
        """
        Populate custom prompts, then recompute each annotation's ``raw_data``.

        An EPPI ``Report.json`` (or similar) only carries EPPI-Reviewer field names;
        it does not set each attribute's ``output_data_type`` in DEET. Ingested
        attributes default to ``bool`` from the codeset. The custom prompt
        definition file (e.g. a CSV with ``output_data_type`` such as
        ``string`` or ``bool``) is the source of those types, not the JSON. After
        the base method updates prompts and types, we re-apply
        ``eppi_output_data_from_eppi_fields`` so ``raw_data`` matches
        ``AdditionalText`` and the updated type (e.g. ``"32"`` for a string count,
        not boolean ``True`` from initial ingest only).
        """
        super().populate_custom_prompts(method, filepath=filepath, **kwargs)
        self._recompute_eppi_raw_data_from_additional_text()

    def _recompute_eppi_raw_data_from_additional_text(self) -> None:
        """Re-derive each gold ``raw_data`` from types and ``AdditionalText``."""
        # Import inside the method: ``eppi_annotation_converter`` already imports
        # ``ProcessedEppiAnnotationData`` from this module, so a top-level import
        # would risk a circular import.
        from deet.processors.eppi_annotation_converter import (
            eppi_output_data_from_eppi_fields,
        )

        for ann in self.annotations:
            ann.raw_data = eppi_output_data_from_eppi_fields(
                ann.attribute.output_data_type,
                additional_text=ann.additional_text or "",
            )

populate_custom_prompts(method, filepath=None, **kwargs)

Populate custom prompts, then recompute each annotation's raw_data.

An EPPI Report.json (or similar) only carries EPPI-Reviewer field names; it does not set each attribute's output_data_type in DEET. Ingested attributes default to bool from the codeset. The custom prompt definition file (e.g. a CSV with output_data_type such as string or bool) is the source of those types, not the JSON. After the base method updates prompts and types, we re-apply eppi_output_data_from_eppi_fields so raw_data matches AdditionalText and the updated type (e.g. "32" for a string count, not boolean True from initial ingest only).

Source code in deet/data_models/processed_gold_standard_annotations.py

def populate_custom_prompts(
    self,
    method: CustomPromptPopulationMethod,
    filepath: Path | None = None,
    **kwargs: Any,  # noqa: ANN401
) -> None:
    """
    Populate custom prompts, then recompute each annotation's ``raw_data``.

    An EPPI ``Report.json`` (or similar) only carries EPPI-Reviewer field names;
    it does not set each attribute's ``output_data_type`` in DEET. Ingested
    attributes default to ``bool`` from the codeset. The custom prompt
    definition file (e.g. a CSV with ``output_data_type`` such as
    ``string`` or ``bool``) is the source of those types, not the JSON. After
    the base method updates prompts and types, we re-apply
    ``eppi_output_data_from_eppi_fields`` so ``raw_data`` matches
    ``AdditionalText`` and the updated type (e.g. ``"32"`` for a string count,
    not boolean ``True`` from initial ingest only).
    """
    super().populate_custom_prompts(method, filepath=filepath, **kwargs)
    self._recompute_eppi_raw_data_from_additional_text()

project

Data models for DeetProject.

DeetProjects handle the one-time definition of configuration options, and create standardised directory structures to store resources like prompt csvs, link maps, experiment results.

DeetProject pydantic-model

Bases: BaseModel

A deet "project" that lives in a directory. Configuration options are defined here once, and elicited through an interactive wizard.

Config:

• json_encoders: {Path: str}
• extra: ignore

Fields:

• name (str)
• gold_standard_data_format (SupportedImportFormat)
• gold_standard_data_path (Path)
• pdf_dir (Path | None)
• evaluation_strategy (EvaluationStrategyName)
• created_at (datetime)

Validators:

• check_suffix → gold_standard_data_path
• _process_pdf_dir → pdf_dir

Source code in deet/data_models/project.py

class DeetProject(BaseModel):
    """
    A deet "project" that lives in a directory.
    Configuration options are defined here once, and elicited through an
        interactive wizard.
    """

    # Wizard fields, configurable by users
    name: Annotated[
        str,
        UI(
            help="Give your project a name. This will help you to identify it later",
            valid="Must be at least 1 character",
        ),
    ] = Field(..., description="The name of a deet project", min_length=1)

    gold_standard_data_format: Annotated[
        SupportedImportFormat,
        UI(
            help=(
                "The format of a file describing documents,"
                " attributes, and annotations."
            )
        ),
    ] = Field(..., description="Format of gold standard annotations")

    gold_standard_data_path: Annotated[
        Path,
        UI(
            help=(
                "A file containing a list of documents from which you wish to"
                " extract data"
                ", and (optionally) a set of human annotations to be used"
                " to evaluate "
                "automatic extraction."
            ),
            instructions="press Tab to autocomplete, '/' to go to next directory",
            valid="Must be a valid .csv or .json path",
        ),
    ] = Field(..., description="Path to gold standard annotated data")

    pdf_dir: Annotated[
        Path | None,
        UI(
            help=(
                "If you want to extract data from full texts, "
                "choose a directory that contains your pdfs."
                " You will have an opportunity to link this later"
                " using a 'link map' created here"
            )
        ),
    ] = Field(None, description="Path to folder containing PDFs")

    evaluation_strategy: EvaluationStrategyName = EvaluationStrategyName.NONE

    # Project metadata
    created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))

    # Root defaults to cwd
    _root: Path = PrivateAttr(default_factory=Path.cwd)

    @property
    def root(self) -> Path:
        """Return project root."""
        return self._root

    # Computed paths - file and folder structure within project dir
    @property
    def experiments_dir(self) -> Path:
        """
        Return path to experiments directory.

        Each time we run data extraction in this project, the results
        of the experiment will be stored here.
        """
        return self.root / "data-extraction-experiments"

    @property
    def completed_experiments(self) -> list[ExperimentArtefacts]:
        """Return all completed experiments."""
        return sorted(
            (
                exp
                for exp in (
                    ExperimentArtefacts(base_dir=path)
                    for path in self.experiments_dir.iterdir()
                    if path.is_dir()
                )
                if exp.is_complete
            ),
            key=lambda exp: exp.run_id,
            reverse=True,
        )

    @property
    def prompt_csv_path(self) -> Path:
        """Return path to prompt definition file."""
        return self.root / "prompts" / "prompt_definitions.csv"

    @property
    def link_map_path(self) -> Path:
        """Return path to link map."""
        return self.root / "link_map.csv"

    @property
    def linked_documents_path(self) -> Path:
        """Return path to linked documents folder."""
        return self.root / "linked_documents"

    @property
    def config_path(self) -> Path:
        """Return path to config file."""
        return self.root / "default_extraction_config.yaml"

    @property
    def evaluation_splits_path(self) -> Path:
        """Return path to file recording evaluation split state."""
        return self.root / "evaluation_splits.json"

    @property
    def gold_standard_data_abspath(self) -> Path:
        """
        Return a usable path to the gold-standard data.

        ``gold_standard_data_path`` is stored relative to the project root (and
        never persisted as an absolute path, so ``project.yaml`` stays portable).
        This joins it with the root only for I/O.
        """
        return self.root / self.gold_standard_data_path

    @property
    def pdf_dir_abspath(self) -> Path | None:
        """
        Return a usable path to the pdf directory, or None if unset.

        ``pdf_dir`` is stored relative to the project root and joined with it here
        only for I/O; it is never persisted as an absolute path.
        """
        if self.pdf_dir is None:
            return None
        return self.root / self.pdf_dir

    # Configuration and validation
    model_config = ConfigDict(
        json_encoders={Path: str},
        extra="ignore",
    )

    @field_validator("gold_standard_data_path", mode="after")
    @classmethod
    def check_suffix(cls, value: Path) -> Path:
        """Check if extension is supported."""
        if value.suffix not in SUPPORTED_EXTENSIONS:
            unsupported_ext = f"Unsupported extension, allowed: {SUPPORTED_EXTENSIONS}"
            raise ValueError(unsupported_ext)
        return value

    @field_validator("pdf_dir", mode="before")
    @classmethod
    def _process_pdf_dir(cls, value: object) -> object | None:
        """Parse empty string to None (not cwd) before Path coercion."""
        if isinstance(value, str) and value.strip() == "":
            return None
        return value

    def anchor_to(self, root: Path, source_dir: Path | None = None) -> None:
        """
        Anchor the project to ``root``, re-expressing resource paths relative to it.

        Resource paths are authored relative to ``source_dir`` (default cwd, i.e.
        where the wizard ran). They are rewritten relative to ``root`` so they stay
        correct when ``root`` differs from that directory (as with
        ``deet project new``). The stored values remain relative, never absolute.
        """
        source = source_dir or Path.cwd()
        self.gold_standard_data_path = (
            source / self.gold_standard_data_path
        ).relative_to(root, walk_up=True)
        if self.pdf_dir is not None:
            self.pdf_dir = (source / self.pdf_dir).relative_to(root, walk_up=True)
        self._root = root

    def validate_resources(self) -> None:
        """Check that the project's resource paths exist on disk."""
        if not self.gold_standard_data_abspath.exists():
            missing = (
                f"Gold standard data not found at {self.gold_standard_data_abspath} "
                f"(stored as '{self.gold_standard_data_path}', relative to {self.root})"
            )
            raise FileNotFoundError(missing)
        if self.pdf_dir_abspath is not None and not self.pdf_dir_abspath.is_dir():
            missing_pdfs = (
                f"PDF directory not found at {self.pdf_dir_abspath} "
                f"(stored as '{self.pdf_dir}', relative to {self.root})"
            )
            raise FileNotFoundError(missing_pdfs)

    def setup(self) -> None:
        """
        Set a project up.

        Create directory structure, process gold-standard data, and create
            prompt csv and link map
        """
        self.root.mkdir(parents=True, exist_ok=True)
        self.validate_resources()

        processed_data = self.process_data()
        notify("Successfully parsed processed data.", level=LogLevel.SUCCESS)

        processed_data.export_attributes_csv_file(filepath=self.prompt_csv_path)
        notify("Initialised prompt definition file.", level=LogLevel.SUCCESS)

        processed_data.export_linkage_mapper_csv(
            file_path=self.link_map_path, document_base_dir=self.pdf_dir_abspath
        )
        notify("Initialised reference-pdf link mapping file.", level=LogLevel.SUCCESS)

        self.export_config_template()
        notify("Exported default config template", level=LogLevel.SUCCESS)

        self.experiments_dir.mkdir(parents=True, exist_ok=True)
        self.linked_documents_path.mkdir(parents=True, exist_ok=True)

        self.dump_to_yaml()

    def dump_to_yaml(self) -> None:
        """
        Write a minimal ``project.yaml`` file to save project options.

        Written to the project root by default. Resource paths are stored as their
        relative values, never resolved to absolute, so the file stays portable.
        """
        target = self.root / PROJECT_FILE
        target.parent.mkdir(parents=True, exist_ok=True)
        data = {"project": self.model_dump(mode="json")}
        with target.open("w", encoding="utf-8") as f:
            yaml.safe_dump(data, f)

    def export_config_template(self) -> None:
        """Export a default config template."""
        from deet.extractors.llm_data_extractor import DataExtractionConfig

        config = DataExtractionConfig()
        self.config_path.write_text(
            yaml.safe_dump(config.model_dump(mode="json"), sort_keys=False),
            encoding="utf-8",
        )

    @classmethod
    def load(cls, project_dir: Path | None = None) -> DeetProject:
        """
        Load the project from ``project_dir`` (default: the current directory).

        The project root is anchored to that directory; stored resource paths stay
        relative and are resolved against it. deet commands are run from the project
        directory.
        """
        root = (project_dir or Path.cwd()).resolve()
        project_file = root / PROJECT_FILE
        if not project_file.is_file():
            not_found = f"No {PROJECT_FILE} found in {root}"
            raise FileNotFoundError(not_found)
        data = yaml.safe_load(project_file.read_text())
        project = cls.model_validate(data["project"])
        project._root = root  # noqa: SLF001
        return project

    def process_data(self) -> ProcessedAnnotationData:
        """Process the project's gold standard data."""
        converter = self.gold_standard_data_format.get_annotation_converter()
        return converter.process_annotation_file(self.gold_standard_data_abspath)

    def get_all_doc_ids(self) -> list[int]:
        """Process the full dataset and return all document IDs."""
        processed_data = self.process_data()
        return processed_data.all_doc_ids

    def load_evaluation_strategy(
        self,
    ) -> BaseEvaluationStrategy[BaseSplits]:
        """Load split state."""
        from deet.data_models.evaluation_strategies import STRATEGY_REGISTRY

        return STRATEGY_REGISTRY[self.evaluation_strategy](self)

completed_experiments property

Return all completed experiments.

config_path property

Return path to config file.

evaluation_splits_path property

Return path to file recording evaluation split state.

experiments_dir property

Return path to experiments directory.

Each time we run data extraction in this project, the results of the experiment will be stored here.

gold_standard_data_abspath property

Return a usable path to the gold-standard data.

gold_standard_data_path is stored relative to the project root (and never persisted as an absolute path, so project.yaml stays portable). This joins it with the root only for I/O.

gold_standard_data_format pydantic-field

Format of gold standard annotations

gold_standard_data_path pydantic-field

Path to gold standard annotated data

link_map_path property

Return path to link map.

linked_documents_path property

Return path to linked documents folder.

name pydantic-field

The name of a deet project

pdf_dir = None pydantic-field

Path to folder containing PDFs

pdf_dir_abspath property

Return a usable path to the pdf directory, or None if unset.

pdf_dir is stored relative to the project root and joined with it here only for I/O; it is never persisted as an absolute path.

prompt_csv_path property

Return path to prompt definition file.

root property

Return project root.

anchor_to(root, source_dir=None)

Anchor the project to root, re-expressing resource paths relative to it.

Resource paths are authored relative to source_dir (default cwd, i.e. where the wizard ran). They are rewritten relative to root so they stay correct when root differs from that directory (as with deet project new). The stored values remain relative, never absolute.

Source code in deet/data_models/project.py

def anchor_to(self, root: Path, source_dir: Path | None = None) -> None:
    """
    Anchor the project to ``root``, re-expressing resource paths relative to it.

    Resource paths are authored relative to ``source_dir`` (default cwd, i.e.
    where the wizard ran). They are rewritten relative to ``root`` so they stay
    correct when ``root`` differs from that directory (as with
    ``deet project new``). The stored values remain relative, never absolute.
    """
    source = source_dir or Path.cwd()
    self.gold_standard_data_path = (
        source / self.gold_standard_data_path
    ).relative_to(root, walk_up=True)
    if self.pdf_dir is not None:
        self.pdf_dir = (source / self.pdf_dir).relative_to(root, walk_up=True)
    self._root = root

check_suffix(value) pydantic-validator

Check if extension is supported.

Source code in deet/data_models/project.py

@field_validator("gold_standard_data_path", mode="after")
@classmethod
def check_suffix(cls, value: Path) -> Path:
    """Check if extension is supported."""
    if value.suffix not in SUPPORTED_EXTENSIONS:
        unsupported_ext = f"Unsupported extension, allowed: {SUPPORTED_EXTENSIONS}"
        raise ValueError(unsupported_ext)
    return value

dump_to_yaml()

Write a minimal project.yaml file to save project options.

Written to the project root by default. Resource paths are stored as their relative values, never resolved to absolute, so the file stays portable.

Source code in deet/data_models/project.py

def dump_to_yaml(self) -> None:
    """
    Write a minimal ``project.yaml`` file to save project options.

    Written to the project root by default. Resource paths are stored as their
    relative values, never resolved to absolute, so the file stays portable.
    """
    target = self.root / PROJECT_FILE
    target.parent.mkdir(parents=True, exist_ok=True)
    data = {"project": self.model_dump(mode="json")}
    with target.open("w", encoding="utf-8") as f:
        yaml.safe_dump(data, f)

export_config_template()

Export a default config template.

Source code in deet/data_models/project.py

def export_config_template(self) -> None:
    """Export a default config template."""
    from deet.extractors.llm_data_extractor import DataExtractionConfig

    config = DataExtractionConfig()
    self.config_path.write_text(
        yaml.safe_dump(config.model_dump(mode="json"), sort_keys=False),
        encoding="utf-8",
    )

get_all_doc_ids()

Process the full dataset and return all document IDs.

Source code in deet/data_models/project.py

def get_all_doc_ids(self) -> list[int]:
    """Process the full dataset and return all document IDs."""
    processed_data = self.process_data()
    return processed_data.all_doc_ids

load(project_dir=None) classmethod

Load the project from project_dir (default: the current directory).

The project root is anchored to that directory; stored resource paths stay relative and are resolved against it. deet commands are run from the project directory.

Source code in deet/data_models/project.py

@classmethod
def load(cls, project_dir: Path | None = None) -> DeetProject:
    """
    Load the project from ``project_dir`` (default: the current directory).

    The project root is anchored to that directory; stored resource paths stay
    relative and are resolved against it. deet commands are run from the project
    directory.
    """
    root = (project_dir or Path.cwd()).resolve()
    project_file = root / PROJECT_FILE
    if not project_file.is_file():
        not_found = f"No {PROJECT_FILE} found in {root}"
        raise FileNotFoundError(not_found)
    data = yaml.safe_load(project_file.read_text())
    project = cls.model_validate(data["project"])
    project._root = root  # noqa: SLF001
    return project

load_evaluation_strategy()

Load split state.

Source code in deet/data_models/project.py

def load_evaluation_strategy(
    self,
) -> BaseEvaluationStrategy[BaseSplits]:
    """Load split state."""
    from deet.data_models.evaluation_strategies import STRATEGY_REGISTRY

    return STRATEGY_REGISTRY[self.evaluation_strategy](self)

process_data()

Process the project's gold standard data.

Source code in deet/data_models/project.py

def process_data(self) -> ProcessedAnnotationData:
    """Process the project's gold standard data."""
    converter = self.gold_standard_data_format.get_annotation_converter()
    return converter.process_annotation_file(self.gold_standard_data_abspath)

setup()

Set a project up.

Create directory structure, process gold-standard data, and create prompt csv and link map

Source code in deet/data_models/project.py

def setup(self) -> None:
    """
    Set a project up.

    Create directory structure, process gold-standard data, and create
        prompt csv and link map
    """
    self.root.mkdir(parents=True, exist_ok=True)
    self.validate_resources()

    processed_data = self.process_data()
    notify("Successfully parsed processed data.", level=LogLevel.SUCCESS)

    processed_data.export_attributes_csv_file(filepath=self.prompt_csv_path)
    notify("Initialised prompt definition file.", level=LogLevel.SUCCESS)

    processed_data.export_linkage_mapper_csv(
        file_path=self.link_map_path, document_base_dir=self.pdf_dir_abspath
    )
    notify("Initialised reference-pdf link mapping file.", level=LogLevel.SUCCESS)

    self.export_config_template()
    notify("Exported default config template", level=LogLevel.SUCCESS)

    self.experiments_dir.mkdir(parents=True, exist_ok=True)
    self.linked_documents_path.mkdir(parents=True, exist_ok=True)

    self.dump_to_yaml()

validate_resources()

Check that the project's resource paths exist on disk.

Source code in deet/data_models/project.py

def validate_resources(self) -> None:
    """Check that the project's resource paths exist on disk."""
    if not self.gold_standard_data_abspath.exists():
        missing = (
            f"Gold standard data not found at {self.gold_standard_data_abspath} "
            f"(stored as '{self.gold_standard_data_path}', relative to {self.root})"
        )
        raise FileNotFoundError(missing)
    if self.pdf_dir_abspath is not None and not self.pdf_dir_abspath.is_dir():
        missing_pdfs = (
            f"PDF directory not found at {self.pdf_dir_abspath} "
            f"(stored as '{self.pdf_dir}', relative to {self.root})"
        )
        raise FileNotFoundError(missing_pdfs)

ExperimentArtefacts dataclass

Defines the structure of a data extraction experiment directory.

Source code in deet/data_models/project.py

@dataclass(frozen=True)
class ExperimentArtefacts:
    """Defines the structure of a data extraction experiment directory."""

    base_dir: Path

    @classmethod
    def create(cls, experiments_dir: Path, run_name: str) -> ExperimentArtefacts:
        """Initialise and create a new experiments directory."""
        extraction_run_id = (
            datetime.now(tz=UTC).strftime("%Y-%m-%d_%H-%M-%S") + f"_{run_name}"
        )

        experiment_out_dir = experiments_dir / extraction_run_id
        experiment_out_dir.mkdir(parents=True)

        logger.add(experiment_out_dir / "deet.log", level="DEBUG")

        return cls(base_dir=experiment_out_dir)

    @property
    def is_complete(self) -> bool:
        """Verify the experiment directory contains a completed and evaluated run."""
        return self.config_snapshot.exists() and self.metrics.exists()

    @property
    def run_id(self) -> str:
        """Return identifier (based on the directory where the experiment lives)."""
        return self.base_dir.name

    @property
    def metrics(self) -> Path:
        """Return location of experiment metrics."""
        return self.base_dir / "metrics.csv"

    @property
    def metrics_json(self) -> Path:
        """Return location of experiment metrics JSON."""
        return self.base_dir / "metrics.json"

    @property
    def comparison(self) -> Path:
        """Return location of csv comparing goldstandard to llm extractions."""
        return self.base_dir / "goldstandard_llm_comparison.csv"

    @property
    def prompts_snapshot(self) -> Path:
        """Return location of csv capturing prompts used."""
        return self.base_dir / "prompts_used.csv"

    @property
    def config_snapshot(self) -> Path:
        """Return location of csv capturing config used."""
        return self.base_dir / "config.yaml"

    @property
    def evaluation_splits_snapshot(self) -> Path:
        """Return location of json capturing how docs were split for evaluation."""
        return self.base_dir / "evaluation_splits.json"

    @property
    def llm_annotations(self) -> Path:
        """Return location of json containing llm extractions."""
        return self.base_dir / "llm_annotations.json"

    @property
    def llm_annotation_csv(self) -> Path:
        """Return location of csv containing llm extractions."""
        return self.base_dir / "llm_annotations.csv"

    @property
    def extraction_metadata(self) -> Path:
        """Return path to extraction metadata JSON (cost, tokens, timing)."""
        return self.base_dir / "extraction_metadata.json"

comparison property

Return location of csv comparing goldstandard to llm extractions.

config_snapshot property

Return location of csv capturing config used.

evaluation_splits_snapshot property

Return location of json capturing how docs were split for evaluation.

extraction_metadata property

Return path to extraction metadata JSON (cost, tokens, timing).

is_complete property

Verify the experiment directory contains a completed and evaluated run.

llm_annotation_csv property

Return location of csv containing llm extractions.

llm_annotations property

Return location of json containing llm extractions.

metrics property

Return location of experiment metrics.

metrics_json property

Return location of experiment metrics JSON.

prompts_snapshot property

Return location of csv capturing prompts used.

run_id property

Return identifier (based on the directory where the experiment lives).

create(experiments_dir, run_name) classmethod

Initialise and create a new experiments directory.

Source code in deet/data_models/project.py

@classmethod
def create(cls, experiments_dir: Path, run_name: str) -> ExperimentArtefacts:
    """Initialise and create a new experiments directory."""
    extraction_run_id = (
        datetime.now(tz=UTC).strftime("%Y-%m-%d_%H-%M-%S") + f"_{run_name}"
    )

    experiment_out_dir = experiments_dir / extraction_run_id
    experiment_out_dir.mkdir(parents=True)

    logger.add(experiment_out_dir / "deet.log", level="DEBUG")

    return cls(base_dir=experiment_out_dir)

ui_schema

Definition of data model to add annotations to pydantic model that can be used in automatic wizard generation.

UI dataclass

Metadata for automatic UI generation. Used with Annotated[Type, UI(help="...")].

Source code in deet/data_models/ui_schema.py

@dataclass(frozen=True)
class UI:
    """
    Metadata for automatic UI generation.
    Used with Annotated[Type, UI(help="...")].
    """

    help: str = ""
    label: str | None = None
    placeholder: str | None = None
    hidden: bool = False
    valid: str = "Invalid input"
    instructions: str = ""

evaluators

gold_standard_llm_evaluator

Generalisable evaluation module for comparing data extracted by LLMs with data extracted by hand.

EvaluationRow dataclass

Per-document values for extraction scoring and comparison export.

Source code in deet/evaluators/gold_standard_llm_evaluator.py

@dataclass(slots=True)
class EvaluationRow:
    """Per-document values for extraction scoring and comparison export."""

    document_id: int | str | None
    gold_value: Any
    predicted_value: Any | None
    citation_haystack: str
    context: str | None
    gold_in_citation: bool | None
    gold_in_context: bool | None
    match_status: str | None

GoldStandardLLMEvaluator

A class to manage the evaluation of LLM-extracted data against "gold-standard" ground truth data.

Source code in deet/evaluators/gold_standard_llm_evaluator.py

class GoldStandardLLMEvaluator:
    """
    A class to manage the evaluation of LLM-extracted data against
    "gold-standard" ground truth data.
    """

    def __init__(  # noqa: PLR0913
        self,
        gold_standard_annotated_documents: Sequence[GoldStandardAnnotatedDocument],
        llm_annotated_documents: Sequence[GoldStandardAnnotatedDocument],
        attributes: Sequence[Attribute],
        extraction_run_id: str,
        custom_metrics: list[str] | None = None,
        metric_settings: EvaluationMetricSettings | None = None,
    ) -> None:
        """
        Initialise GoldStandardLLMEvaluator with a list of ground truth and
        LLM-generated data to compare, along with the attributes you want to
        compare.

        Args:
            gold_standard_annotated_documents: Human / gold annotations.
            llm_annotated_documents: LLM annotations to score.
            attributes: Attributes to evaluate.
            extraction_run_id: Run identifier written into metric rows.
            custom_metrics: Optional sklearn metric names to merge in.
            metric_settings: Thresholds for extraction metrics (e.g. edit
                distance). Defaults to :class:`EvaluationMetricSettings`.

        """
        self.gold_standard_annotated_documents = gold_standard_annotated_documents
        self.llm_annotated_documents = GoldStandardAnnotatedDocumentList(
            gold_standard_annotations=llm_annotated_documents
        )
        self.attributes = attributes
        self.extraction_run_id = extraction_run_id
        self.metric_settings = metric_settings or EvaluationMetricSettings()
        self.metrics_config: dict[str, MetricFunction] = METRICS
        self.custom_metrics: dict[str, MetricFunction] = {}
        self.calculated_metrics: list[AttributeMetric] = []
        self._evaluation_rows_by_attribute: dict[int, list[EvaluationRow]] = {}
        if custom_metrics is not None:
            self.add_custom_metrics(custom_metrics)

    def add_custom_metrics(self, custom_metrics: list[str]) -> None:
        """Add custom metrics. These must be valid metrics from sklearn.metrics."""
        for custom_metric_name in custom_metrics:
            custom_metric = getattr(sklearn.metrics, custom_metric_name, None)
            if custom_metric is not None:
                if check_metric_returns_float(custom_metric):
                    self.metrics_config[custom_metric_name] = custom_metric
                    self.custom_metrics[custom_metric_name] = custom_metric
                else:
                    logger.warning(
                        f"Tried to add {custom_metric_name} to"
                        " evaluation metrics, but it does not return a float."
                    )
            else:
                logger.warning(
                    f"Tried to add {custom_metric_name} to"
                    " evaluation metrics, but it does not exist"
                )

    def evaluate_llm_annotations(
        self,
    ) -> None:
        """
        Compare a list of human annotations to those generated by llms.
        Return a list of AttributeMetric objects.
        """
        if self.calculated_metrics:
            logger.warning("Already calculated metrics, deleting and overwriting.")
            self.calculated_metrics = []
            self._evaluation_rows_by_attribute = {}
        for attribute in self.attributes:
            logger.debug(
                f"Calculating metric for attribute: {attribute.attribute_label}"
            )
            rows = self._collect_attribute_rows(attribute)
            self._evaluation_rows_by_attribute[attribute.attribute_id] = rows
            y_true = [row.gold_value for row in rows]
            y_pred = [row.predicted_value for row in rows]

            applicable_metrics = get_metrics_for_attribute_type(
                attribute.output_data_type,
                settings=self.metric_settings,
            )
            combined_metrics = {**applicable_metrics, **self.custom_metrics}

            self._append_count_metric(
                attribute=attribute,
                metric_name="n_gold_instances",
                metric_value=len(rows),
            )
            self._append_score_metrics(
                attribute=attribute,
                y_true=y_true,
                y_pred=y_pred,
                metrics=combined_metrics,
                suffix="",
            )

            if attribute.output_data_type not in SOURCE_FIDELITY_ATTRIBUTE_TYPES:
                continue

            good_indices = [
                i for i, row in enumerate(rows) if row.gold_in_context is True
            ]
            bad_indices = [
                i for i, row in enumerate(rows) if row.gold_in_context is False
            ]

            self._append_count_metric(
                attribute=attribute,
                metric_name="n_good_source_instances",
                metric_value=len(good_indices),
            )
            self._append_count_metric(
                attribute=attribute,
                metric_name="n_good_citation_instances",
                metric_value=sum(1 for row in rows if row.gold_in_citation is True),
            )

            self._append_score_metrics(
                attribute=attribute,
                y_true=[y_true[i] for i in good_indices],
                y_pred=[y_pred[i] for i in good_indices],
                metrics=combined_metrics,
                suffix="_given_good_source",
            )
            self._append_score_metrics(
                attribute=attribute,
                y_true=[y_true[i] for i in bad_indices],
                y_pred=[y_pred[i] for i in bad_indices],
                metrics=combined_metrics,
                suffix="_given_bad_source",
            )

    def _append_count_metric(
        self,
        *,
        attribute: Attribute,
        metric_name: str,
        metric_value: int,
    ) -> None:
        """Append an :class:`AttributeCountMetric` onto ``calculated_metrics``."""
        self.calculated_metrics.append(
            AttributeCountMetric(
                attribute=attribute,
                metric_name=metric_name,
                value=metric_value,
                extraction_run_id=self.extraction_run_id,
            )
        )

    def _append_score_metrics(
        self,
        *,
        attribute: Attribute,
        y_true: list[Any],
        y_pred: list[Any | None],
        metrics: dict[str, MetricFunction],
        suffix: str,
    ) -> None:
        """
        Apply score metrics to ``y_true`` / ``y_pred`` and append result rows.

        Appends one :class:`AttributeScoreMetric` per metric onto
        ``self.calculated_metrics``. The lists may be the full attribute set or
        any filtered subset (e.g. good/bad source). When a metric raises on
        empty or invalid inputs, ``value`` is recorded as ``None`` (blank CSV
        cell).

        Args:
            attribute: Attribute being scored.
            y_true: Gold values for this (sub)set.
            y_pred: Predictions aligned with ``y_true``.
            metrics: Metric name → callable map to apply.
            suffix: Appended to each metric name (e.g. ``_given_good_source``).

        """
        for metric_name, metric_fn in metrics.items():
            metric_key = f"{metric_name}{suffix}"
            try:
                value = float(metric_fn(y_true, y_pred))
            except (ValueError, TypeError) as error:
                logger.warning(
                    f"Metric '{metric_key}' not applicable for "
                    f"attribute '{attribute.attribute_label}' "
                    f"(type={attribute.output_data_type}): {error}"
                )
                value = None
            self.calculated_metrics.append(
                AttributeScoreMetric(
                    attribute=attribute,
                    metric_name=metric_key,
                    value=value,
                    extraction_run_id=self.extraction_run_id,
                )
            )

    def _source_fidelity_fields(
        self,
        *,
        attribute: Attribute,
        gold_value: object,
        predicted_value: object | None,
        citation_haystack: str,
        context: str | None,
    ) -> tuple[bool | None, bool | None, str | None]:
        """
        Compute citation/context presence and match status when in scope.

        Returns:
            ``(gold_in_citation, gold_in_context, match_status)``, or three
            ``None`` values for attribute types outside source fidelity.

        """
        if attribute.output_data_type not in SOURCE_FIDELITY_ATTRIBUTE_TYPES:
            return None, None, None
        threshold = self.metric_settings.edit_distance_match_threshold
        gold_in_citation = is_gold_value_in_text(
            gold_value=gold_value,
            haystack_text=citation_haystack,
            attribute_type=attribute.output_data_type,
            edit_distance_threshold=threshold,
            allow_string_near_match=True,
        )
        gold_in_context = is_gold_value_in_text(
            gold_value=gold_value,
            haystack_text=context,
            attribute_type=attribute.output_data_type,
            edit_distance_threshold=threshold,
            allow_string_near_match=False,
        )
        match_status = classify_match_status(
            gold_value=gold_value,
            predicted_value=predicted_value,
            gold_in_context=gold_in_context,
            attribute_type=attribute.output_data_type,
            edit_distance_threshold=threshold,
        )
        return gold_in_citation, gold_in_context, match_status

    def _collect_attribute_rows(self, attribute: Attribute) -> list[EvaluationRow]:
        """Collect gold/prediction/source fidelity rows for one attribute."""
        rows: list[EvaluationRow] = []
        for document in self.gold_standard_annotated_documents:
            doc_id = document.document.safe_identity.document_id
            logger.debug(f"Extracting gold/LLM values for doc {doc_id}")
            gs_val = document.get_attribute_annotation(attribute).output_data
            gold_real = next(
                (
                    ann
                    for ann in document.annotations
                    if ann.attribute.attribute_id == attribute.attribute_id
                ),
                None,
            )
            citation_haystack = (
                _citation_haystack_from_annotation(gold_real)
                if gold_real is not None
                else ""
            )

            context: str | None = None
            llm_val: Any | None = None
            try:
                llm_doc = self.llm_annotated_documents.get_by_id(doc_id)
                context = (
                    str(text)
                    if (text := llm_doc.document.context) not in (None, "")
                    else None
                )
            except MissingDocumentError:
                logger.warning(f"LLM annotated doc not found - ID: {doc_id}")
                gold_in_citation, gold_in_context, match_status = (
                    self._source_fidelity_fields(
                        attribute=attribute,
                        gold_value=gs_val,
                        predicted_value=None,
                        citation_haystack=citation_haystack,
                        context=context,
                    )
                )
                rows.append(
                    EvaluationRow(
                        document_id=doc_id,
                        gold_value=gs_val,
                        predicted_value=None,
                        citation_haystack=citation_haystack,
                        context=context,
                        gold_in_citation=gold_in_citation,
                        gold_in_context=gold_in_context,
                        match_status=match_status,
                    )
                )
                continue

            try:
                llm_val = llm_doc.get_attribute_annotation(attribute).output_data
            except DuplicateAnnotationError:
                logger.warning(
                    f"LLM produced multiple annotations for a single"
                    f" attribute with doc: {doc_id}"
                )
                llm_val = None

            gold_in_citation, gold_in_context, match_status = (
                self._source_fidelity_fields(
                    attribute=attribute,
                    gold_value=gs_val,
                    predicted_value=llm_val,
                    citation_haystack=citation_haystack,
                    context=context,
                )
            )
            rows.append(
                EvaluationRow(
                    document_id=doc_id,
                    gold_value=gs_val,
                    predicted_value=llm_val,
                    citation_haystack=citation_haystack,
                    context=context,
                    gold_in_citation=gold_in_citation,
                    gold_in_context=gold_in_context,
                    match_status=match_status,
                )
            )
        return rows

    def _rows_by_document_id(
        self, attribute: Attribute
    ) -> dict[int | str | None, EvaluationRow]:
        """
        Return evaluation rows for ``attribute``, collecting if needed.

        Used by comparison export so source-fidelity fields are not recomputed.
        """
        rows = self._evaluation_rows_by_attribute.get(attribute.attribute_id)
        if rows is None:
            rows = self._collect_attribute_rows(attribute)
            self._evaluation_rows_by_attribute[attribute.attribute_id] = rows
        return {row.document_id: row for row in rows}

    def display_metrics(self) -> None:
        """Print metrics in a nice table to the command line."""
        console = Console()

        table = Table(title="Evaluation Results")

        metric_names = sorted({str(m.metric_name) for m in self.calculated_metrics})

        # Create table with metrics as columns
        table = Table(title="Evaluation Metrics")
        table.add_column("Attribute")
        for name in metric_names:
            table.add_column(name, justify="right")

        # Group metrics by attribute
        metrics_sorted = sorted(
            self.calculated_metrics, key=lambda m: m.attribute.attribute_label
        )
        for attribute, group in groupby(
            metrics_sorted, key=lambda m: m.attribute.attribute_label
        ):
            row = [attribute]
            group_metrics: dict[str, float | int | None] = {}
            for metric in group:
                if isinstance(metric, AttributeCountMetric | AttributeScoreMetric):
                    group_metrics[str(metric.metric_name)] = metric.value
            # fill cells in order of metric_names
            formatted: list[str] = []
            for name in metric_names:
                value = group_metrics.get(name)
                if value is None:
                    formatted.append("N/A")
                elif isinstance(value, int) and not isinstance(value, bool):
                    formatted.append(str(value))
                else:
                    formatted.append(f"{float(value):.4f}")
            row += formatted
            table.add_row(*row)

        console.print(table)

    def write_metrics_to_csv(self, filepath: Path) -> None:
        """Save metrics to csv in wide format via :class:`RunMetricsReport`."""
        if filepath.suffix != ".csv":
            bad_filetype = "file ending must be .csv"
            raise ValueError(bad_filetype)
        report = RunMetricsReport.from_attribute_metrics(
            extraction_run_id=self.extraction_run_id,
            calculated_metrics=self.calculated_metrics,
        )
        report.to_csv(filepath)

    def write_metrics_to_json(self, filepath: Path) -> None:
        """Save metrics to JSON via :class:`RunMetricsReport`."""
        if filepath.suffix != ".json":
            bad_filetype = "file ending must be .json"
            raise ValueError(bad_filetype)
        report = RunMetricsReport.from_attribute_metrics(
            extraction_run_id=self.extraction_run_id,
            calculated_metrics=self.calculated_metrics,
        )
        report.to_json(filepath)

    def export_llm_comparison(
        self,
        filepath: Path,
    ) -> None:
        """
        Export a comparison CSV for gold vs LLM per document and attribute.

        Columns include identifiers, EPPI-oriented fields, extractions, LLM verbatim,
        fuzzy grounding scores (against the LLM annotated document's ``context``),
        and run id.

        Column semantics:

        - ``attribute_presence``: Whether the gold annotation is present.
        - ``human_additional_text`` / ``item_attribute_full_text_details``: Taken from
          the eppi json file when present; empty when absent.
        - ``citation_page`` / ``citation_highlight_text``: Parsed from raw EPPI
          citation markup (``Page N:`` / ``[¬s]...[¬e]``); multiple fragments joined
          with ``": "``. Empty when markup is absent.
        - ``human_extraction``: Actual ground truth to be extracted.
        - ``human_verbatim_fuzzy_match_pct``: Grounding of ``human_additional_text``
          against the LLM annotated document's ``context``.
        - ``llm_verbatim_text`` / ``llm_verbatim_fuzzy_match_pct``: LLM
          ``additional_text`` and its grounding against the same ``context``.

        Example row (illustrative types): ``attribute_presence`` is the string
        ``"True"`` or ``"False"``; ``human_verbatim_fuzzy_match_pct`` and
        ``llm_verbatim_fuzzy_match_pct`` are decimal strings (e.g. ``"100.00"``,
        ``"87.50"``); ``human_extraction`` / ``llm_extraction`` serialize according to
        the attribute's coerced value (e.g. bool, int, or str) as written by
        :class:`csv.DictWriter`.
        """
        with filepath.open("w", newline="", encoding="utf-8") as f:
            writer = csv.DictWriter(
                f,
                fieldnames=[
                    "document_id",
                    "external_id",
                    "document_name",
                    "attribute_id",
                    "attribute_label",
                    "attribute_presence",
                    "human_additional_text",
                    "item_attribute_full_text_details",
                    "citation_page",
                    "citation_highlight_text",
                    "human_extraction",
                    "llm_extraction",
                    "llm_reasoning",
                    "llm_verbatim_text",
                    "human_verbatim_fuzzy_match_pct",
                    "llm_verbatim_fuzzy_match_pct",
                    "gold_value_in_citation",
                    "gold_value_in_context",
                    "match_status",
                    "extraction_run_id",
                ],
            )
            writer.writeheader()
            rows_by_attr: dict[int, dict[int | str | None, EvaluationRow]] = {
                attribute.attribute_id: self._rows_by_document_id(attribute)
                for attribute in self.attributes
            }
            for doc in self.gold_standard_annotated_documents:
                try:
                    llm_annotated_doc = self.llm_annotated_documents.get_by_id(
                        doc.document.safe_identity.document_id
                    )
                except MissingDocumentError:
                    llm_annotated_doc = None

                context: str | None = (
                    None
                    if llm_annotated_doc is None
                    else (str(t) if (t := llm_annotated_doc.document.context) else None)
                )

                for attribute in self.attributes:
                    eval_row = rows_by_attr[attribute.attribute_id].get(
                        doc.document.safe_identity.document_id
                    )

                    human_ann = doc.get_attribute_annotation(attribute)
                    gold_real = next(
                        (
                            ann
                            for ann in doc.annotations
                            if ann.attribute.attribute_id == attribute.attribute_id
                        ),
                        None,
                    )
                    if gold_real is not None:
                        human_additional_text: str = gold_real.additional_text or ""
                        item_attr_full: str = _eppi_full_text_details_colon_separated(
                            gold_real
                        )
                        citation_page, citation_highlight = (
                            _citation_fields_from_annotation(gold_real)
                        )
                    else:
                        human_additional_text = ""
                        item_attr_full = ""
                        citation_page = ""
                        citation_highlight = ""
                    present = gold_real is not None
                    human_fuzzy = _verbatim_fuzzy_match_pct(
                        human_additional_text, context
                    )

                    llm_extraction: Any = None
                    llm_reasoning: str | None = None
                    llm_verbatim: str | None = None
                    llm_fuzzy = 0.0

                    if llm_annotated_doc is None:
                        llm_reasoning = (
                            "LLM did not produce an output for this document."
                            " Check the logs carefully to find out why"
                        )
                    else:
                        try:
                            llm_annotation = llm_annotated_doc.get_attribute_annotation(
                                attribute
                            )
                            llm_extraction = llm_annotation.output_data
                            llm_reasoning = llm_annotation.reasoning
                            llm_verbatim = llm_annotation.additional_text
                            llm_fuzzy = _verbatim_fuzzy_match_pct(llm_verbatim, context)
                        except DuplicateAnnotationError:
                            llm_reasoning = (
                                "The LLM produced multiple annotations"
                                "for this single attribute"
                            )

                    if eval_row is not None:
                        gold_in_citation = eval_row.gold_in_citation
                        gold_in_context = eval_row.gold_in_context
                        match_status = eval_row.match_status
                    else:
                        gold_in_citation = None
                        gold_in_context = None
                        match_status = None

                    writer.writerow(
                        {
                            "document_id": doc.document.safe_identity.document_id,
                            "external_id": doc.document.safe_identity.external_id,
                            "document_name": doc.document.name,
                            "attribute_id": attribute.attribute_id,
                            "attribute_label": attribute.attribute_label,
                            "attribute_presence": str(present),
                            "human_additional_text": human_additional_text,
                            "item_attribute_full_text_details": item_attr_full,
                            "citation_page": citation_page,
                            "citation_highlight_text": citation_highlight,
                            "human_extraction": human_ann.output_data,
                            "llm_extraction": llm_extraction,
                            "llm_reasoning": llm_reasoning,
                            "llm_verbatim_text": llm_verbatim,
                            "human_verbatim_fuzzy_match_pct": f"{human_fuzzy:.2f}",
                            "llm_verbatim_fuzzy_match_pct": f"{llm_fuzzy:.2f}",
                            "gold_value_in_citation": (
                                ""
                                if gold_in_citation is None
                                else str(gold_in_citation)
                            ),
                            "gold_value_in_context": (
                                "" if gold_in_context is None else str(gold_in_context)
                            ),
                            "match_status": match_status or "",
                            "extraction_run_id": self.extraction_run_id,
                        }
                    )

    def export_llm_csv(self, filepath: Path) -> None:
        """Write the LLM output to csv."""
        with filepath.open("w", newline="", encoding="utf-8") as f:
            writer = csv.DictWriter(
                f,
                fieldnames=[
                    "document_id",
                    "external_id",
                    "document_name",
                    "attribute_id",
                    "attribute_label",
                    "llm_extraction",
                    "llm_reasoning",
                    "llm_verbatim_text",
                    "extraction_run_id",
                ],
            )
            writer.writeheader()
            for (
                llm_annotated_doc
            ) in self.llm_annotated_documents.gold_standard_annotations:
                for attribute in self.attributes:
                    try:
                        llm_annotation = llm_annotated_doc.get_attribute_annotation(
                            attribute
                        )
                        llm_extraction = llm_annotation.output_data
                        llm_reasoning = llm_annotation.reasoning
                        llm_verbatim = llm_annotation.additional_text
                    except DuplicateAnnotationError:
                        llm_extraction = None
                        llm_reasoning = (
                            "The LLM produced multiple annotations"
                            "for this single attribute"
                        )
                        llm_verbatim = None

                    document = llm_annotated_doc.document

                    writer.writerow(
                        {
                            "document_id": document.safe_identity.document_id,
                            "external_id": document.safe_identity.external_id,
                            "document_name": document.name,
                            "attribute_id": attribute.attribute_id,
                            "attribute_label": attribute.attribute_label,
                            "llm_extraction": llm_extraction,
                            "llm_reasoning": llm_reasoning,
                            "llm_verbatim_text": llm_verbatim,
                            "extraction_run_id": self.extraction_run_id,
                        }
                    )

__init__(gold_standard_annotated_documents, llm_annotated_documents, attributes, extraction_run_id, custom_metrics=None, metric_settings=None)

Initialise GoldStandardLLMEvaluator with a list of ground truth and LLM-generated data to compare, along with the attributes you want to compare.

Parameters:

Name	Type	Description	Default
gold_standard_annotated_documents	Sequence[GoldStandardAnnotatedDocument]	Human / gold annotations.	required
llm_annotated_documents	Sequence[GoldStandardAnnotatedDocument]	LLM annotations to score.	required
attributes	Sequence[Attribute]	Attributes to evaluate.	required
extraction_run_id	str	Run identifier written into metric rows.	required
custom_metrics	list[str] | None	Optional sklearn metric names to merge in.	None
metric_settings	EvaluationMetricSettings | None	Thresholds for extraction metrics (e.g. edit distance). Defaults to :class:EvaluationMetricSettings.	None

Source code in deet/evaluators/gold_standard_llm_evaluator.py

def __init__(  # noqa: PLR0913
    self,
    gold_standard_annotated_documents: Sequence[GoldStandardAnnotatedDocument],
    llm_annotated_documents: Sequence[GoldStandardAnnotatedDocument],
    attributes: Sequence[Attribute],
    extraction_run_id: str,
    custom_metrics: list[str] | None = None,
    metric_settings: EvaluationMetricSettings | None = None,
) -> None:
    """
    Initialise GoldStandardLLMEvaluator with a list of ground truth and
    LLM-generated data to compare, along with the attributes you want to
    compare.

    Args:
        gold_standard_annotated_documents: Human / gold annotations.
        llm_annotated_documents: LLM annotations to score.
        attributes: Attributes to evaluate.
        extraction_run_id: Run identifier written into metric rows.
        custom_metrics: Optional sklearn metric names to merge in.
        metric_settings: Thresholds for extraction metrics (e.g. edit
            distance). Defaults to :class:`EvaluationMetricSettings`.

    """
    self.gold_standard_annotated_documents = gold_standard_annotated_documents
    self.llm_annotated_documents = GoldStandardAnnotatedDocumentList(
        gold_standard_annotations=llm_annotated_documents
    )
    self.attributes = attributes
    self.extraction_run_id = extraction_run_id
    self.metric_settings = metric_settings or EvaluationMetricSettings()
    self.metrics_config: dict[str, MetricFunction] = METRICS
    self.custom_metrics: dict[str, MetricFunction] = {}
    self.calculated_metrics: list[AttributeMetric] = []
    self._evaluation_rows_by_attribute: dict[int, list[EvaluationRow]] = {}
    if custom_metrics is not None:
        self.add_custom_metrics(custom_metrics)

add_custom_metrics(custom_metrics)

Add custom metrics. These must be valid metrics from sklearn.metrics.

Source code in deet/evaluators/gold_standard_llm_evaluator.py

def add_custom_metrics(self, custom_metrics: list[str]) -> None:
    """Add custom metrics. These must be valid metrics from sklearn.metrics."""
    for custom_metric_name in custom_metrics:
        custom_metric = getattr(sklearn.metrics, custom_metric_name, None)
        if custom_metric is not None:
            if check_metric_returns_float(custom_metric):
                self.metrics_config[custom_metric_name] = custom_metric
                self.custom_metrics[custom_metric_name] = custom_metric
            else:
                logger.warning(
                    f"Tried to add {custom_metric_name} to"
                    " evaluation metrics, but it does not return a float."
                )
        else:
            logger.warning(
                f"Tried to add {custom_metric_name} to"
                " evaluation metrics, but it does not exist"
            )

display_metrics()

Print metrics in a nice table to the command line.

Source code in deet/evaluators/gold_standard_llm_evaluator.py

def display_metrics(self) -> None:
    """Print metrics in a nice table to the command line."""
    console = Console()

    table = Table(title="Evaluation Results")

    metric_names = sorted({str(m.metric_name) for m in self.calculated_metrics})

    # Create table with metrics as columns
    table = Table(title="Evaluation Metrics")
    table.add_column("Attribute")
    for name in metric_names:
        table.add_column(name, justify="right")

    # Group metrics by attribute
    metrics_sorted = sorted(
        self.calculated_metrics, key=lambda m: m.attribute.attribute_label
    )
    for attribute, group in groupby(
        metrics_sorted, key=lambda m: m.attribute.attribute_label
    ):
        row = [attribute]
        group_metrics: dict[str, float | int | None] = {}
        for metric in group:
            if isinstance(metric, AttributeCountMetric | AttributeScoreMetric):
                group_metrics[str(metric.metric_name)] = metric.value
        # fill cells in order of metric_names
        formatted: list[str] = []
        for name in metric_names:
            value = group_metrics.get(name)
            if value is None:
                formatted.append("N/A")
            elif isinstance(value, int) and not isinstance(value, bool):
                formatted.append(str(value))
            else:
                formatted.append(f"{float(value):.4f}")
        row += formatted
        table.add_row(*row)

    console.print(table)

evaluate_llm_annotations()

Compare a list of human annotations to those generated by llms. Return a list of AttributeMetric objects.

Source code in deet/evaluators/gold_standard_llm_evaluator.py

def evaluate_llm_annotations(
    self,
) -> None:
    """
    Compare a list of human annotations to those generated by llms.
    Return a list of AttributeMetric objects.
    """
    if self.calculated_metrics:
        logger.warning("Already calculated metrics, deleting and overwriting.")
        self.calculated_metrics = []
        self._evaluation_rows_by_attribute = {}
    for attribute in self.attributes:
        logger.debug(
            f"Calculating metric for attribute: {attribute.attribute_label}"
        )
        rows = self._collect_attribute_rows(attribute)
        self._evaluation_rows_by_attribute[attribute.attribute_id] = rows
        y_true = [row.gold_value for row in rows]
        y_pred = [row.predicted_value for row in rows]

        applicable_metrics = get_metrics_for_attribute_type(
            attribute.output_data_type,
            settings=self.metric_settings,
        )
        combined_metrics = {**applicable_metrics, **self.custom_metrics}

        self._append_count_metric(
            attribute=attribute,
            metric_name="n_gold_instances",
            metric_value=len(rows),
        )
        self._append_score_metrics(
            attribute=attribute,
            y_true=y_true,
            y_pred=y_pred,
            metrics=combined_metrics,
            suffix="",
        )

        if attribute.output_data_type not in SOURCE_FIDELITY_ATTRIBUTE_TYPES:
            continue

        good_indices = [
            i for i, row in enumerate(rows) if row.gold_in_context is True
        ]
        bad_indices = [
            i for i, row in enumerate(rows) if row.gold_in_context is False
        ]

        self._append_count_metric(
            attribute=attribute,
            metric_name="n_good_source_instances",
            metric_value=len(good_indices),
        )
        self._append_count_metric(
            attribute=attribute,
            metric_name="n_good_citation_instances",
            metric_value=sum(1 for row in rows if row.gold_in_citation is True),
        )

        self._append_score_metrics(
            attribute=attribute,
            y_true=[y_true[i] for i in good_indices],
            y_pred=[y_pred[i] for i in good_indices],
            metrics=combined_metrics,
            suffix="_given_good_source",
        )
        self._append_score_metrics(
            attribute=attribute,
            y_true=[y_true[i] for i in bad_indices],
            y_pred=[y_pred[i] for i in bad_indices],
            metrics=combined_metrics,
            suffix="_given_bad_source",
        )

export_llm_comparison(filepath)

Export a comparison CSV for gold vs LLM per document and attribute.

Columns include identifiers, EPPI-oriented fields, extractions, LLM verbatim, fuzzy grounding scores (against the LLM annotated document's context), and run id.

Column semantics:

• attribute_presence: Whether the gold annotation is present.
• human_additional_text / item_attribute_full_text_details: Taken from the eppi json file when present; empty when absent.
• citation_page / citation_highlight_text: Parsed from raw EPPI citation markup (Page N: / [¬s]...[¬e]); multiple fragments joined with ": ". Empty when markup is absent.
• human_extraction: Actual ground truth to be extracted.
• human_verbatim_fuzzy_match_pct: Grounding of human_additional_text against the LLM annotated document's context.
• llm_verbatim_text / llm_verbatim_fuzzy_match_pct: LLM additional_text and its grounding against the same context.

Example row (illustrative types): attribute_presence is the string "True" or "False"; human_verbatim_fuzzy_match_pct and llm_verbatim_fuzzy_match_pct are decimal strings (e.g. "100.00", "87.50"); human_extraction / llm_extraction serialize according to the attribute's coerced value (e.g. bool, int, or str) as written by :class:csv.DictWriter.

Source code in deet/evaluators/gold_standard_llm_evaluator.py

def export_llm_comparison(
    self,
    filepath: Path,
) -> None:
    """
    Export a comparison CSV for gold vs LLM per document and attribute.

    Columns include identifiers, EPPI-oriented fields, extractions, LLM verbatim,
    fuzzy grounding scores (against the LLM annotated document's ``context``),
    and run id.

    Column semantics:

    - ``attribute_presence``: Whether the gold annotation is present.
    - ``human_additional_text`` / ``item_attribute_full_text_details``: Taken from
      the eppi json file when present; empty when absent.
    - ``citation_page`` / ``citation_highlight_text``: Parsed from raw EPPI
      citation markup (``Page N:`` / ``[¬s]...[¬e]``); multiple fragments joined
      with ``": "``. Empty when markup is absent.
    - ``human_extraction``: Actual ground truth to be extracted.
    - ``human_verbatim_fuzzy_match_pct``: Grounding of ``human_additional_text``
      against the LLM annotated document's ``context``.
    - ``llm_verbatim_text`` / ``llm_verbatim_fuzzy_match_pct``: LLM
      ``additional_text`` and its grounding against the same ``context``.

    Example row (illustrative types): ``attribute_presence`` is the string
    ``"True"`` or ``"False"``; ``human_verbatim_fuzzy_match_pct`` and
    ``llm_verbatim_fuzzy_match_pct`` are decimal strings (e.g. ``"100.00"``,
    ``"87.50"``); ``human_extraction`` / ``llm_extraction`` serialize according to
    the attribute's coerced value (e.g. bool, int, or str) as written by
    :class:`csv.DictWriter`.
    """
    with filepath.open("w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(
            f,
            fieldnames=[
                "document_id",
                "external_id",
                "document_name",
                "attribute_id",
                "attribute_label",
                "attribute_presence",
                "human_additional_text",
                "item_attribute_full_text_details",
                "citation_page",
                "citation_highlight_text",
                "human_extraction",
                "llm_extraction",
                "llm_reasoning",
                "llm_verbatim_text",
                "human_verbatim_fuzzy_match_pct",
                "llm_verbatim_fuzzy_match_pct",
                "gold_value_in_citation",
                "gold_value_in_context",
                "match_status",
                "extraction_run_id",
            ],
        )
        writer.writeheader()
        rows_by_attr: dict[int, dict[int | str | None, EvaluationRow]] = {
            attribute.attribute_id: self._rows_by_document_id(attribute)
            for attribute in self.attributes
        }
        for doc in self.gold_standard_annotated_documents:
            try:
                llm_annotated_doc = self.llm_annotated_documents.get_by_id(
                    doc.document.safe_identity.document_id
                )
            except MissingDocumentError:
                llm_annotated_doc = None

            context: str | None = (
                None
                if llm_annotated_doc is None
                else (str(t) if (t := llm_annotated_doc.document.context) else None)
            )

            for attribute in self.attributes:
                eval_row = rows_by_attr[attribute.attribute_id].get(
                    doc.document.safe_identity.document_id
                )

                human_ann = doc.get_attribute_annotation(attribute)
                gold_real = next(
                    (
                        ann
                        for ann in doc.annotations
                        if ann.attribute.attribute_id == attribute.attribute_id
                    ),
                    None,
                )
                if gold_real is not None:
                    human_additional_text: str = gold_real.additional_text or ""
                    item_attr_full: str = _eppi_full_text_details_colon_separated(
                        gold_real
                    )
                    citation_page, citation_highlight = (
                        _citation_fields_from_annotation(gold_real)
                    )
                else:
                    human_additional_text = ""
                    item_attr_full = ""
                    citation_page = ""
                    citation_highlight = ""
                present = gold_real is not None
                human_fuzzy = _verbatim_fuzzy_match_pct(
                    human_additional_text, context
                )

                llm_extraction: Any = None
                llm_reasoning: str | None = None
                llm_verbatim: str | None = None
                llm_fuzzy = 0.0

                if llm_annotated_doc is None:
                    llm_reasoning = (
                        "LLM did not produce an output for this document."
                        " Check the logs carefully to find out why"
                    )
                else:
                    try:
                        llm_annotation = llm_annotated_doc.get_attribute_annotation(
                            attribute
                        )
                        llm_extraction = llm_annotation.output_data
                        llm_reasoning = llm_annotation.reasoning
                        llm_verbatim = llm_annotation.additional_text
                        llm_fuzzy = _verbatim_fuzzy_match_pct(llm_verbatim, context)
                    except DuplicateAnnotationError:
                        llm_reasoning = (
                            "The LLM produced multiple annotations"
                            "for this single attribute"
                        )

                if eval_row is not None:
                    gold_in_citation = eval_row.gold_in_citation
                    gold_in_context = eval_row.gold_in_context
                    match_status = eval_row.match_status
                else:
                    gold_in_citation = None
                    gold_in_context = None
                    match_status = None

                writer.writerow(
                    {
                        "document_id": doc.document.safe_identity.document_id,
                        "external_id": doc.document.safe_identity.external_id,
                        "document_name": doc.document.name,
                        "attribute_id": attribute.attribute_id,
                        "attribute_label": attribute.attribute_label,
                        "attribute_presence": str(present),
                        "human_additional_text": human_additional_text,
                        "item_attribute_full_text_details": item_attr_full,
                        "citation_page": citation_page,
                        "citation_highlight_text": citation_highlight,
                        "human_extraction": human_ann.output_data,
                        "llm_extraction": llm_extraction,
                        "llm_reasoning": llm_reasoning,
                        "llm_verbatim_text": llm_verbatim,
                        "human_verbatim_fuzzy_match_pct": f"{human_fuzzy:.2f}",
                        "llm_verbatim_fuzzy_match_pct": f"{llm_fuzzy:.2f}",
                        "gold_value_in_citation": (
                            ""
                            if gold_in_citation is None
                            else str(gold_in_citation)
                        ),
                        "gold_value_in_context": (
                            "" if gold_in_context is None else str(gold_in_context)
                        ),
                        "match_status": match_status or "",
                        "extraction_run_id": self.extraction_run_id,
                    }
                )

export_llm_csv(filepath)

Write the LLM output to csv.

Source code in deet/evaluators/gold_standard_llm_evaluator.py

def export_llm_csv(self, filepath: Path) -> None:
    """Write the LLM output to csv."""
    with filepath.open("w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(
            f,
            fieldnames=[
                "document_id",
                "external_id",
                "document_name",
                "attribute_id",
                "attribute_label",
                "llm_extraction",
                "llm_reasoning",
                "llm_verbatim_text",
                "extraction_run_id",
            ],
        )
        writer.writeheader()
        for (
            llm_annotated_doc
        ) in self.llm_annotated_documents.gold_standard_annotations:
            for attribute in self.attributes:
                try:
                    llm_annotation = llm_annotated_doc.get_attribute_annotation(
                        attribute
                    )
                    llm_extraction = llm_annotation.output_data
                    llm_reasoning = llm_annotation.reasoning
                    llm_verbatim = llm_annotation.additional_text
                except DuplicateAnnotationError:
                    llm_extraction = None
                    llm_reasoning = (
                        "The LLM produced multiple annotations"
                        "for this single attribute"
                    )
                    llm_verbatim = None

                document = llm_annotated_doc.document

                writer.writerow(
                    {
                        "document_id": document.safe_identity.document_id,
                        "external_id": document.safe_identity.external_id,
                        "document_name": document.name,
                        "attribute_id": attribute.attribute_id,
                        "attribute_label": attribute.attribute_label,
                        "llm_extraction": llm_extraction,
                        "llm_reasoning": llm_reasoning,
                        "llm_verbatim_text": llm_verbatim,
                        "extraction_run_id": self.extraction_run_id,
                    }
                )

write_metrics_to_csv(filepath)

Save metrics to csv in wide format via :class:RunMetricsReport.

Source code in deet/evaluators/gold_standard_llm_evaluator.py

def write_metrics_to_csv(self, filepath: Path) -> None:
    """Save metrics to csv in wide format via :class:`RunMetricsReport`."""
    if filepath.suffix != ".csv":
        bad_filetype = "file ending must be .csv"
        raise ValueError(bad_filetype)
    report = RunMetricsReport.from_attribute_metrics(
        extraction_run_id=self.extraction_run_id,
        calculated_metrics=self.calculated_metrics,
    )
    report.to_csv(filepath)

write_metrics_to_json(filepath)

Save metrics to JSON via :class:RunMetricsReport.

Source code in deet/evaluators/gold_standard_llm_evaluator.py

def write_metrics_to_json(self, filepath: Path) -> None:
    """Save metrics to JSON via :class:`RunMetricsReport`."""
    if filepath.suffix != ".json":
        bad_filetype = "file ending must be .json"
        raise ValueError(bad_filetype)
    report = RunMetricsReport.from_attribute_metrics(
        extraction_run_id=self.extraction_run_id,
        calculated_metrics=self.calculated_metrics,
    )
    report.to_json(filepath)

metrics

Metric functions and registries for gold-vs-LLM evaluation.

EvaluationMetricSettings pydantic-model

Bases: BaseModel

Configurable thresholds for extraction evaluation metrics.

Fields:

• edit_distance_match_threshold (float)

Source code in deet/evaluators/metrics.py

class EvaluationMetricSettings(BaseModel):
    """Configurable thresholds for extraction evaluation metrics."""

    edit_distance_match_threshold: float = Field(
        default=DEFAULT_EDIT_DISTANCE_MATCH_THRESHOLD,
        ge=0.0,
        le=1.0,
        description=(
            "Minimum normalised Levenshtein similarity (0-1) for a string pair "
            "to count as a match in edit_distance_match_rate."
        ),
    )

edit_distance_match_threshold = DEFAULT_EDIT_DISTANCE_MATCH_THRESHOLD pydantic-field

Minimum normalised Levenshtein similarity (0-1) for a string pair to count as a match in edit_distance_match_rate.

check_metric_returns_float(metric)

Check whether a metric returns a scalar.

Source code in deet/evaluators/metrics.py

def check_metric_returns_float(metric: MetricFunction) -> bool:
    """Check whether a metric returns a scalar."""
    y_true = [1, 0, 0, 1]
    y_pred = [1, 0, 0, 0]
    result = metric(y_true, y_pred)
    return isinstance(result, float)

edit_distance_match_rate(y_true, y_pred, *, threshold=DEFAULT_EDIT_DISTANCE_MATCH_THRESHOLD)

Fraction of pairs whose normalised Levenshtein similarity meets a threshold.

Values are normalised with :func:normalize_string_for_match before comparison. Missing predictions are not dropped: any None in y_pred raises, matching binary-metric behaviour (the evaluator then records value=None for the metric).

Parameters:

Name	Type	Description	Default
y_true	Sequence[Any]	Gold-standard values.	required
y_pred	Sequence[Any]	Predicted values.	required
threshold	float	Minimum normalised similarity in [0, 1] to count as a match. Defaults to 0.90.	DEFAULT_EDIT_DISTANCE_MATCH_THRESHOLD

Returns:

Type	Description
float	Match rate in [0.0, 1.0].

Raises:

Type	Description
ValueError	If y_true and y_pred have different lengths, or either list is empty (mirroring sklearn empty-input behaviour).
TypeError	If any prediction is None.

Source code in deet/evaluators/metrics.py

def edit_distance_match_rate(
    y_true: Sequence[Any],
    y_pred: Sequence[Any],
    *,
    threshold: float = DEFAULT_EDIT_DISTANCE_MATCH_THRESHOLD,
) -> float:
    """
    Fraction of pairs whose normalised Levenshtein similarity meets a threshold.

    Values are normalised with :func:`normalize_string_for_match` before
    comparison. Missing predictions are not dropped: any ``None`` in
    ``y_pred`` raises, matching binary-metric behaviour (the evaluator then
    records ``value=None`` for the metric).

    Args:
        y_true: Gold-standard values.
        y_pred: Predicted values.
        threshold: Minimum normalised similarity in ``[0, 1]`` to count as a
            match. Defaults to ``0.90``.

    Returns:
        Match rate in ``[0.0, 1.0]``.

    Raises:
        ValueError: If ``y_true`` and ``y_pred`` have different lengths, or
            either list is empty (mirroring sklearn empty-input behaviour).
        TypeError: If any prediction is ``None``.

    """
    if len(y_true) != len(y_pred):
        msg = (
            f"y_true and y_pred must have the same length "
            f"(got {len(y_true)} and {len(y_pred)})"
        )
        raise ValueError(msg)
    if any(pred_val is None for pred_val in y_pred):
        msg = "edit_distance_match_rate does not accept None predictions"
        raise TypeError(msg)
    if not y_true:
        msg = "edit_distance_match_rate requires non-empty y_true and y_pred"
        raise ValueError(msg)

    matches = 0
    for true_val, pred_val in zip(y_true, y_pred, strict=True):
        true_norm = normalize_string_for_match(str(true_val))
        pred_norm = normalize_string_for_match(str(pred_val))
        similarity = Levenshtein.normalized_similarity(true_norm, pred_norm)
        if similarity >= threshold:
            matches += 1
    return matches / len(y_true)

get_metrics_for_attribute_type(attribute_type, settings=None)

Return the metric set registered for the given attribute data type.

For STRING attributes, edit_distance_match_rate is rebuilt from settings.edit_distance_match_threshold (defaults to 0.90).

Some types map to an empty dict when no suitable default metrics are implemented yet (list, dict); callers may still merge in custom metrics.

Parameters:

Name	Type	Description	Default
attribute_type	AttributeType	Attribute output data type.	required
settings	EvaluationMetricSettings | None	Optional metric settings; defaults used when None.	None

Returns:

Type	Description
dict[str, MetricFunction]	Mapping of metric name to callable.

Source code in deet/evaluators/metrics.py

def get_metrics_for_attribute_type(
    attribute_type: AttributeType,
    settings: EvaluationMetricSettings | None = None,
) -> dict[str, MetricFunction]:
    """
    Return the metric set registered for the given attribute data type.

    For STRING attributes, ``edit_distance_match_rate`` is rebuilt from
    ``settings.edit_distance_match_threshold`` (defaults to 0.90).

    Some types map to an empty dict when no suitable default metrics are
    implemented yet (list, dict); callers may still merge in custom metrics.

    Args:
        attribute_type: Attribute output data type.
        settings: Optional metric settings; defaults used when ``None``.

    Returns:
        Mapping of metric name to callable.

    """
    resolved_settings = settings or EvaluationMetricSettings()
    metrics = dict(METRICS_BY_ATTRIBUTE_TYPE[attribute_type])
    if attribute_type == AttributeType.STRING:
        metrics["edit_distance_match_rate"] = partial(
            edit_distance_match_rate,
            threshold=resolved_settings.edit_distance_match_threshold,
        )
    return metrics

n_labels(y_true, y_pred)

Count the number of positive instances of the class in gold data.

Source code in deet/evaluators/metrics.py

def n_labels(y_true: list[int], y_pred: list[int]) -> float:  # noqa: ARG001
    """Count the number of positive instances of the class in gold data."""
    return sum(y_true)

source_fidelity

Helpers for source-fidelity checks and row-level match status.

classify_match_status(*, gold_value, predicted_value, gold_in_context, attribute_type, edit_distance_threshold)

Classify row-level match status for STRING / INTEGER / FLOAT attributes.

Parameters:

Name	Type	Description	Default
gold_value	object	Gold-standard value.	required
predicted_value	object | None	Model predicted value.	required
gold_in_context	bool	Whether gold is found in parsed context.	required
attribute_type	AttributeType	Attribute output type.	required
edit_distance_threshold	float	Similarity threshold for STRING near-match.	required

Returns:

Type	Description
str | None	One of exact_match, near_match, missing_prediction,
str | None	extraction_error_bad_source, or extraction_error_good_source
str | None	for STRING/INTEGER/FLOAT, else None.

Source code in deet/evaluators/source_fidelity.py

def classify_match_status(
    *,
    gold_value: object,
    predicted_value: object | None,
    gold_in_context: bool,
    attribute_type: AttributeType,
    edit_distance_threshold: float,
) -> str | None:
    """
    Classify row-level match status for STRING / INTEGER / FLOAT attributes.

    Args:
        gold_value: Gold-standard value.
        predicted_value: Model predicted value.
        gold_in_context: Whether gold is found in parsed context.
        attribute_type: Attribute output type.
        edit_distance_threshold: Similarity threshold for STRING near-match.

    Returns:
        One of ``exact_match``, ``near_match``, ``missing_prediction``,
        ``extraction_error_bad_source``, or ``extraction_error_good_source``
        for STRING/INTEGER/FLOAT, else ``None``.

    """
    if attribute_type not in SOURCE_FIDELITY_ATTRIBUTE_TYPES:
        return None
    if predicted_value is None:
        return "missing_prediction"
    if predicted_value == gold_value:
        return "exact_match"
    if attribute_type == AttributeType.STRING:
        gold_text = normalize_string_for_match(str(gold_value))
        pred_text = normalize_string_for_match(str(predicted_value))
        similarity = Levenshtein.normalized_similarity(gold_text, pred_text)
        if similarity >= edit_distance_threshold:
            return "near_match"
    if not gold_in_context:
        return "extraction_error_bad_source"
    return "extraction_error_good_source"

is_gold_value_in_text(*, gold_value, haystack_text, attribute_type, edit_distance_threshold, allow_string_near_match)

Check whether a gold value can be found in source text.

Parameters:

Name	Type	Description	Default
gold_value	object	Gold-standard output_data.	required
haystack_text	str | None	Citation/context text to search.	required
attribute_type	AttributeType	Attribute output type.	required
edit_distance_threshold	float	Similarity threshold used for optional string near-match.	required
allow_string_near_match	bool	Whether approximate match fallback is enabled.	required

Returns:

Type	Description
bool	True when the gold value is considered present in the haystack.

Source code in deet/evaluators/source_fidelity.py

def is_gold_value_in_text(
    *,
    gold_value: object,
    haystack_text: str | None,
    attribute_type: AttributeType,
    edit_distance_threshold: float,
    allow_string_near_match: bool,
) -> bool:
    """
    Check whether a gold value can be found in source text.

    Args:
        gold_value: Gold-standard output_data.
        haystack_text: Citation/context text to search.
        attribute_type: Attribute output type.
        edit_distance_threshold: Similarity threshold used for optional
            string near-match.
        allow_string_near_match: Whether approximate match fallback is enabled.

    Returns:
        True when the gold value is considered present in the haystack.

    """
    if haystack_text is None or haystack_text.strip() == "":
        return False

    if attribute_type == AttributeType.STRING:
        gold_text = normalize_string_for_match(str(gold_value))
        haystack_normalised = normalize_string_for_match(haystack_text)
        if not gold_text:
            present = False
        elif gold_text in haystack_normalised:
            present = True
        elif allow_string_near_match:
            present = _string_near_match_in_haystack(
                gold_text, haystack_normalised, edit_distance_threshold
            )
        else:
            present = False
        return present

    if attribute_type in {AttributeType.INTEGER, AttributeType.FLOAT}:
        try:
            gold_float = float(str(gold_value))
        except (TypeError, ValueError):
            return False
        return any(value == gold_float for value in parse_numeric_tokens(haystack_text))

    return False

parse_numeric_tokens(text)

Parse standalone numeric tokens from free text.

Tokens must be digit sequences optionally with a decimal point (e.g. 1000, 0.11). Thousand separators such as 1,000 are not recognised: comma-separated forms are treated as separate tokens, which is locale-ambiguous (1,015 may mean one thousand fifteen or one point zero one five).

Parameters:

Name	Type	Description	Default
text	str | None	Source text to scan.	required

Returns:

Type	Description
list[float]	List of parsed float values. Invalid tokens are ignored.

Source code in deet/evaluators/source_fidelity.py

def parse_numeric_tokens(text: str | None) -> list[float]:
    """
    Parse standalone numeric tokens from free text.

    Tokens must be digit sequences optionally with a decimal point (e.g.
    ``1000``, ``0.11``). Thousand separators such as ``1,000`` are **not**
    recognised: comma-separated forms are treated as separate tokens, which
    is locale-ambiguous (``1,015`` may mean one thousand fifteen or one
    point zero one five).

    Args:
        text: Source text to scan.

    Returns:
        List of parsed float values. Invalid tokens are ignored.

    """
    values: list[float] = []
    if not text:
        return values
    for token in _NUMERIC_TOKEN_PATTERN.findall(text):
        try:
            values.append(float(token))
        except ValueError:
            continue
    return values

exceptions

Custom exceptions.

BadDocumentIdError

Bases: Exception

Raise when our Document.document_id field doesn't satisfy our criteria.

Parameters:

Name	Type	Description	Default
Exception	_type_	description	required

Source code in deet/exceptions.py

class BadDocumentIdError(Exception):
    """
    Raise when our `Document.document_id` field
    doesn't satisfy our criteria.

    Args:
        Exception (_type_): _description_

    """

DuplicateAnnotationError

Bases: Exception

Raise when multiple annotations for a single attribute are present.

Parameters:

Name	Type	Description	Default
Exception	_type_		required

Source code in deet/exceptions.py

class DuplicateAnnotationError(Exception):
    """
    Raise when multiple annotations for a single attribute are present.

    Args:
        Exception (_type_):

    """

EmptyPdfExtractionError

Bases: Exception

Raise when PDF parsing yields no extractable text.

Occurs when the PDF has no mappable text (e.g. image-only) or when text is represented in a way pdfminer cannot decode.

Source code in deet/exceptions.py

class EmptyPdfExtractionError(Exception):
    """
    Raise when PDF parsing yields no extractable text.

    Occurs when the PDF has no mappable text (e.g. image-only) or when text
    is represented in a way pdfminer cannot decode.

    """

    DEFAULT_MESSAGE = (
        "PDF contained no extractable text (e.g. image-only or text in "
        "unsupported encoding)."
    )

FileParserMismatchError

Bases: Exception

Raise when we have an input-file <> parser mismatch.

Parameters:

Name	Type	Description	Default
Exception	_type_		required

Source code in deet/exceptions.py

class FileParserMismatchError(Exception):
    """
    Raise when we have an input-file <> parser mismatch.

    Args:
        Exception (_type_):

    """

InvalidFileTypeError

Bases: Exception

Raise when user supplies a not permitted file.

Parameters:

Name	Type	Description	Default
Exception	_type_		required

Source code in deet/exceptions.py

class InvalidFileTypeError(Exception):
    """
    Raise when user supplies a not permitted file.

    Args:
        Exception (_type_):

    """

InvalidInputFileTypeError

Bases: Exception

Raise when user supplies a not permitted input file type.

Parameters:

Name	Type	Description	Default
Exception	_type_		required

Source code in deet/exceptions.py

class InvalidInputFileTypeError(Exception):
    """
    Raise when user supplies a not permitted input file type.

    Args:
        Exception (_type_):

    """

InvalidOutputFileTypeError

Bases: Exception

Raise when user supplies a not permitted output file.

Parameters:

Name	Type	Description	Default
Exception	_type_		required

Source code in deet/exceptions.py

class InvalidOutputFileTypeError(Exception):
    """
    Raise when user supplies a not permitted output file.

    Args:
        Exception (_type_):

    """

JsonStyleError

Bases: Exception

Raise when a json containing document-reference-linkages is incorrectly formatted.

Parameters:

Name	Type	Description	Default
Exception	_type_	description	required

Source code in deet/exceptions.py

class JsonStyleError(Exception):
    """
    Raise when a json containing document-reference-linkages
    is incorrectly formatted.

    Args:
        Exception (_type_): _description_

    """

LitellmModelNotMappedError

Bases: Exception

Raised when litellm reports the model is missing from its registry.

litellm.get_max_tokens can raise a bare Exception with a characteristic message; we translate that to this type so callers can handle it without a broad except Exception.

Source code in deet/exceptions.py

class LitellmModelNotMappedError(Exception):
    """
    Raised when litellm reports the model is missing from its registry.

    ``litellm.get_max_tokens`` can raise a bare ``Exception`` with a
    characteristic message; we translate that to this type so callers can
    handle it without a broad ``except Exception``.
    """

MalformedLanguageError

Bases: Exception

Raise when language checker fails.

Parameters:

Name	Type	Description	Default
Exception	_type_	description	required

Source code in deet/exceptions.py

class MalformedLanguageError(Exception):
    """
    Raise when language checker fails.

    Args:
        Exception (_type_): _description_

    """

MissingCitationElementError

Bases: Exception

Raise when required element of citation is missing.

Parameters:

Name	Type	Description	Default
Exception	_type_	description	required

Source code in deet/exceptions.py

class MissingCitationElementError(Exception):
    """
    Raise when required element of citation is missing.

    Args:
        Exception (_type_): _description_

    """

MissingDocumentError

Bases: Exception

Raise when looking up a document by id fails.

Parameters:

Name	Type	Description	Default
Exception	_type_		required

Source code in deet/exceptions.py

class MissingDocumentError(Exception):
    """
    Raise when looking up a document by id fails.

    Args:
        Exception (_type_):

    """

NoAbstractError

Bases: Exception

Raise when we can't find an abstract in our citation info.

Parameters:

Name	Type	Description	Default
Exception	_type_	description	required

Source code in deet/exceptions.py

class NoAbstractError(Exception):
    """
    Raise when we can't find an abstract in our citation info.

    Args:
        Exception (_type_): _description_

    """

SplitsValidationError

Bases: Exception

Raised when invalid allocation of documents to splits is attemped.

Source code in deet/exceptions.py

class SplitsValidationError(Exception):
    """Raised when invalid allocation of documents to splits is attemped."""

UnsupportedEppiAttributeTypeError

Bases: ValueError

Raised when EPPI AdditionalText / Codes mapping cannot handle an attribute type.

Subclasses :class:ValueError so existing except ValueError call sites remain valid while allowing targeted handling via UnsupportedEppiAttributeTypeError.

Parameters:

Name	Type	Description	Default
output_data_type	AttributeType	The :class:~deet.data_models.base.AttributeType that has no mapping in the EPPI raw_data pipeline (see deet.processors.eppi_annotation_converter.eppi_output_data_from_eppi_fields).	required

Source code in deet/exceptions.py

class UnsupportedEppiAttributeTypeError(ValueError):
    """
    Raised when EPPI ``AdditionalText`` / Codes mapping cannot handle an attribute type.

    Subclasses :class:`ValueError` so existing ``except ValueError`` call sites remain
    valid while allowing targeted handling via ``UnsupportedEppiAttributeTypeError``.

    Args:
        output_data_type: The :class:`~deet.data_models.base.AttributeType` that has
            no mapping in the EPPI ``raw_data`` pipeline (see
            ``deet.processors.eppi_annotation_converter.eppi_output_data_from_eppi_fields``).

    """

    def __init__(
        self,
        output_data_type: AttributeType,
        *,
        message: str | None = None,
    ) -> None:
        """Store ``output_data_type`` and build a default message when omitted."""
        self.output_data_type = output_data_type
        text = message or (
            f"Unsupported AttributeType for EPPI mapping: {output_data_type!s}"
        )
        super().__init__(text)

__init__(output_data_type, *, message=None)

Store output_data_type and build a default message when omitted.

Source code in deet/exceptions.py

def __init__(
    self,
    output_data_type: AttributeType,
    *,
    message: str | None = None,
) -> None:
    """Store ``output_data_type`` and build a default message when omitted."""
    self.output_data_type = output_data_type
    text = message or (
        f"Unsupported AttributeType for EPPI mapping: {output_data_type!s}"
    )
    super().__init__(text)

extractors

cli_helpers

Helper functions to run extraction via the CLI.

evaluate_extraction_pipeline(processed_annotation_data, run_output, experiment_artefacts)

Evaluate results of an extraction pipeline.

Source code in deet/extractors/cli_helpers.py

def evaluate_extraction_pipeline(
    processed_annotation_data: ProcessedAnnotationData,
    run_output: ExtractionRunOutput,
    experiment_artefacts: ExperimentArtefacts,
) -> None:
    """Evaluate results of an extraction pipeline."""
    evaluator = GoldStandardLLMEvaluator(
        gold_standard_annotated_documents=processed_annotation_data.annotated_documents,
        llm_annotated_documents=run_output.annotated_documents,
        attributes=processed_annotation_data.attributes,
        extraction_run_id=experiment_artefacts.run_id,
    )
    evaluator.evaluate_llm_annotations()
    evaluator.write_metrics_to_csv(experiment_artefacts.metrics)
    evaluator.export_llm_comparison(experiment_artefacts.comparison)
    evaluator.display_metrics()

load_or_init_config(config_path)

Load config from project context or path, or fail informatively.

Source code in deet/extractors/cli_helpers.py

def load_or_init_config(config_path: Path | None) -> DataExtractionConfig:
    """Load config from project context or path, or fail informatively."""
    if config_path is None:
        console.clear()
        console.print(
            info_panel(
                render_template("extraction/config_init"),
                "Data extraction config wizard",
            )
        )
        continue_after_key()
        return run_model_wizard(DataExtractionConfig)
    try:
        return DataExtractionConfig.from_yaml(config_path)
    except FileNotFoundError:
        fail_with_message(f"Config file not found: {config_path}")
    except yaml.YAMLError as e:
        fail_with_message(f"YAML Syntax Error in {config_path}:\n{e}")
    except ValidationError as e:
        fail_with_message(f"Config validation error in {config_path}:\n{e}")

prepare_documents(documents, config, linked_document_path, pdf_dir, link_map_path)

Load documents depending on the context type we want.

NOTE: while there are no arg-defaults defined here, when used in cli.py, we populate defaults via typer arg defaults.

If fulltext, try to load linked documents, or create them if not.

Source code in deet/extractors/cli_helpers.py

def prepare_documents(
    documents: Sequence[Document],
    config: DataExtractionConfig,
    linked_document_path: Path,
    pdf_dir: Path | None,
    link_map_path: Path | None,
) -> tuple[Sequence[Document], dict[str, DocumentParsingStats]]:
    """
    Load documents depending on the context type we want.

    NOTE: while there are no arg-defaults defined here,
    when used in cli.py, we populate defaults via
    typer arg defaults.

    If fulltext, try to load linked documents, or create them if not.
    """
    if config.default_context_type == ContextType.ABSTRACT_ONLY:
        return documents, {}
    if config.default_context_type == ContextType.FULL_DOCUMENT:
        if linked_document_path.exists():
            notify(f"Loading linked documents from {linked_document_path}")
            linked_documents = []
            for document in documents:
                document_id = document.safe_identity.document_id
                document_path = linked_document_path / f"{document_id}.json"
                linked_documents.append(Document.load(document_path))
            if linked_documents:
                return linked_documents, _cached_parsing_stats(documents)

            notify(f"Couldn't find linked documents in {linked_document_path}")
        if pdf_dir is None:
            no_linked_docs_no_pdf = (
                "Full text extraction specified but"
                " linked document path does not contain documents,"
                " and no pdf dir supplied"
            )
            fail_with_message(no_linked_docs_no_pdf)

        if link_map_path is None:
            fail_with_message(
                "No link map supplied"
                f" and no linked documents in {linked_document_path}"
            )
        else:
            notify(f"Linking documents using link map: {link_map_path}")
            linker = DocumentReferenceLinker(
                references=documents,
                document_base_dir=pdf_dir,
                document_reference_mapping=link_map_path,
                linking_strategies=[LinkingStrategy.MAPPING_FILE],
            )
            documents = linker.link_many_references_parsed_documents()
            for linked_document in documents:
                file_path = (
                    linked_document_path
                    / f"{linked_document.safe_identity.document_id}.json"
                )
                linked_document.save(file_path)

            if not documents:
                no_links = (
                    f"context type {config.default_context_type} selected"
                    " but no linked documents could be found or created"
                )
                fail_with_message(no_links)

            return documents, linker.document_parsing_stats

    else:
        message = f"context type {config.default_context_type} not supported"
        fail_with_message(message)

    return None, {}

run_extraction_pipeline(deet_project, prompt_csv_path, config_path=None, prompt_population=CustomPromptPopulationMethod.FILE, run_name='', *, ignore_references=False)

Run the standard data extraction pipeline from the CLI.

Source code in deet/extractors/cli_helpers.py

def run_extraction_pipeline(  # noqa: PLR0913
    deet_project: DeetProject,
    prompt_csv_path: Path | None,
    config_path: Path | None = None,
    prompt_population: (
        CustomPromptPopulationMethod | None
    ) = CustomPromptPopulationMethod.FILE,
    run_name: str = "",
    *,
    ignore_references: bool = False,
) -> tuple[
    ExtractionRunOutput,
    ProcessedAnnotationData,
    ExperimentArtefacts,
    DataExtractionConfig,
]:
    """Run the standard data extraction pipeline from the CLI."""
    pipeline_start = time.perf_counter()
    stage_durations: dict[str, float] = {}

    with measure_elapsed() as stage_timer:
        processed_annotation_data = deet_project.process_data()
    stage_durations[ExtractionPipelineStage.annotation_conversion] = stage_timer.seconds

    config = load_or_init_config(config_path)

    experiment_artefacts = ExperimentArtefacts.create(
        deet_project.experiments_dir, run_name=run_name
    )

    with measure_elapsed() as stage_timer:
        if prompt_population is not None:
            if (
                prompt_population == CustomPromptPopulationMethod.FILE
                and prompt_csv_path is not None
                and not prompt_csv_path.exists()
            ):
                fail_with_message(f"Prompt csv {prompt_csv_path} cannot be found")
            processed_annotation_data.populate_custom_prompts(
                method=prompt_population,
                filepath=prompt_csv_path or deet_project.prompt_csv_path,
            )
            if not processed_annotation_data.attributes:
                fail_with_message(
                    "No attributes selected. "
                    "Perhaps you forgot to edit your prompt file"
                )
    stage_durations[ExtractionPipelineStage.prompt_population] = stage_timer.seconds

    if not processed_annotation_data.documents:
        no_documents = "No documents found in project"
        fail_with_message(no_documents)

    evaluation_strategy = deet_project.load_evaluation_strategy()
    evaluation_strategy.snapshot(experiment_artefacts)
    active_ids = evaluation_strategy.get_active_ids(deet_project)
    processed_annotation_data.filter_documents_by_ids(active_ids)
    if not processed_annotation_data.documents:
        no_documents_in_stage = (
            "No documents in evaluation stage"
            f" '{evaluation_strategy.splits.current_stage}'."
            "\n\nManage evaluation splits with `deet experiments splits`"
        )
        fail_with_message(no_documents_in_stage)

    data_extractor = LLMDataExtractor(config=config)

    document_parsing: dict[str, DocumentParsingStats] = {}
    with measure_elapsed() as stage_timer:
        if ignore_references:
            if deet_project.pdf_dir_abspath is None:
                fail_with_message(
                    "This project doesn't specify a pdf directory. "
                    "Either edit the yaml file to create one or "
                    "re-initialise the project."
                )
            documents, document_parsing = create_documents_from_directory(
                deet_project.pdf_dir_abspath
            )
        else:
            documents, document_parsing = prepare_documents(
                processed_annotation_data.documents,
                config,
                linked_document_path=deet_project.linked_documents_path,
                pdf_dir=deet_project.pdf_dir_abspath,
                link_map_path=deet_project.link_map_path,
            )
    stage_durations[ExtractionPipelineStage.document_preparation] = stage_timer.seconds

    with measure_elapsed() as stage_timer:
        run_output = data_extractor.extract_from_documents(
            attributes=processed_annotation_data.attributes,
            documents=documents,
            context_type=data_extractor.config.default_context_type,
            output_file=experiment_artefacts.llm_annotations,
            document_parsing=document_parsing,
            show_progress=True,
        )
    stage_durations[ExtractionPipelineStage.llm_extraction] = stage_timer.seconds

    with measure_elapsed() as stage_timer:
        processed_annotation_data.export_attributes_csv_file(
            experiment_artefacts.prompts_snapshot
        )

        experiment_artefacts.config_snapshot.write_text(
            yaml.safe_dump(
                data_extractor.config.model_dump(mode="json"), sort_keys=False
            ),
            encoding="utf-8",
        )
    stage_durations[ExtractionPipelineStage.artifact_export] = stage_timer.seconds

    run_output.metadata.total_pipeline_duration_seconds = round(
        time.perf_counter() - pipeline_start, 3
    )
    run_output.metadata.stage_durations_seconds = stage_durations

    experiment_artefacts.extraction_metadata.write_text(
        run_output.metadata.model_dump_json(indent=2),
        encoding="utf-8",
    )

    logger.info(f"Run metadata saved to: {experiment_artefacts.extraction_metadata}")

    return run_output, processed_annotation_data, experiment_artefacts, config

llm_data_extractor

Generalisable data extraction module for LLM-based document analysis.

DataExtractionConfig pydantic-model

Bases: BaseModel

Configuration for data extraction tasks.

Fields:

• provider (LLMProvider)
• model (str)
• temperature (float)
• max_tokens (int | None)
• max_context_tokens (int | None)
• default_context_type (ContextType)
• truncate_on_overflow (bool)
• prompt_config (PromptConfig)
• dynamic_json_schema (bool)
• include_reasoning (bool)
• include_additional_text (bool)
• edit_distance_match_threshold (float)

Validators:

• populate_max_context_tokens_from_model

Source code in deet/extractors/llm_data_extractor.py

class DataExtractionConfig(BaseModel):
    """Configuration for data extraction tasks."""

    model_config = ConfigDict()

    # LLM
    provider: Annotated[
        LLMProvider, UI(help="Choose from a list of supported LLM providers.")
    ] = Field(default=LLMProvider.AZURE, description="LLM Provider")
    model: Annotated[str, UI(help="The name of the LLM model you want to use.")] = (
        Field(
            default="gpt-4o-mini",
            description="LLM model identifier used for completions.",
        )
    )
    temperature: float = Field(
        default=0.1,
        description="Sampling temperature for the LLM.",
        ge=0.0,
    )
    max_tokens: Annotated[
        int | None,
        UI(
            help=(
                "The maximum number of tokens in the LLM response. "
                "Leave blank for the provider default."
            )
        ),
    ] = Field(
        default=None,
        description=(
            "Maximum number of tokens to generate (Leave blank for provider default)."
        ),
    )

    max_context_tokens: Annotated[
        int | None,
        UI(help=("Maximum input context length (Leave blank for provider default).")),
    ] = Field(
        default=None,
        description=(
            "Maximum input context length in tokens (system + attributes + "
            "document). None = infer from model (litellm registry), else "
            f"{DEFAULT_LLM_MAX_CONTEXT_TOKENS_FALLBACK} via "
            "DEFAULT_LLM_MAX_CONTEXT_TOKENS_FALLBACK. Override to manage costs."
        ),
    )

    # Context
    default_context_type: Annotated[
        ContextType, UI(help="Where to extract data from.")
    ] = Field(
        default=ContextType.FULL_DOCUMENT, description="Type of context to provide"
    )

    truncate_on_overflow: Annotated[
        bool,
        UI(
            help=(
                "Select true to truncate documents longer than max_context_tokens. "
                "This will ensure extraction runs without crashing, but may mean"
                " some parts of the document are not seen by the LLM."
            )
        ),
    ] = Field(
        default=False,
        description=(
            "When True, automatically truncate context that exceeds "
            "max_context_tokens. When False (default), raise ValueError."
        ),
    )

    # Prompt
    prompt_config: PromptConfig = Field(
        default_factory=PromptConfig, description="Prompt configuration"
    )

    dynamic_json_schema: bool = Field(
        default=True,
        description=(
            "If True, produce dynamic json schema with a key for each attribute"
            " and where each attribute response is typed (marginally more expensive but"
            " may be more likely to produce valid output)."
        ),
    )

    # Output
    include_reasoning: bool = Field(
        default=True, description="Include reasoning in output"
    )
    include_additional_text: bool = Field(
        default=True, description="Include additional text/citations in output"
    )

    # Evaluation
    edit_distance_match_threshold: float = Field(
        default=DEFAULT_EDIT_DISTANCE_MATCH_THRESHOLD,
        ge=0.0,
        le=1.0,
        description=(
            "Minimum normalised Levenshtein similarity (0-1) for a string pair "
            "to count as a match in edit_distance_match_rate. Omit from the "
            "config YAML to use the default."
        ),
        json_schema_extra={"skip_prompt": True},
    )

    @model_validator(mode="after")
    def populate_max_context_tokens_from_model(self) -> "DataExtractionConfig":
        """Populate max_context_tokens from model when not set."""
        if self.max_context_tokens is not None:
            return self
        model_str = _model_string_for_tokenisation(self.provider, self.model)
        try:
            inferred = get_model_max_tokens(model_str)
        except LitellmModelNotMappedError:
            inferred = None
        if inferred is not None:
            self.max_context_tokens = inferred
        else:
            # Use shared fallback when model max tokens cannot be inferred.
            self.max_context_tokens = DEFAULT_LLM_MAX_CONTEXT_TOKENS_FALLBACK
        return self

    @classmethod
    def from_yaml(cls, path: Path) -> "DataExtractionConfig":
        """Load config object from a yaml file."""
        if not path.exists():
            not_found = f"Config file not found at: {path}"
            raise FileNotFoundError(not_found)

        return cls.model_validate(yaml.safe_load(path.read_text()))

default_context_type = ContextType.FULL_DOCUMENT pydantic-field

Type of context to provide

dynamic_json_schema = True pydantic-field

If True, produce dynamic json schema with a key for each attribute and where each attribute response is typed (marginally more expensive but may be more likely to produce valid output).

edit_distance_match_threshold = DEFAULT_EDIT_DISTANCE_MATCH_THRESHOLD pydantic-field

Minimum normalised Levenshtein similarity (0-1) for a string pair to count as a match in edit_distance_match_rate. Omit from the config YAML to use the default.

include_additional_text = True pydantic-field

Include additional text/citations in output

include_reasoning = True pydantic-field

Include reasoning in output

max_tokens = None pydantic-field

Maximum number of tokens to generate (Leave blank for provider default).

model = 'gpt-4o-mini' pydantic-field

LLM model identifier used for completions.

prompt_config pydantic-field

Prompt configuration

provider = LLMProvider.AZURE pydantic-field

LLM Provider

temperature = 0.1 pydantic-field

Sampling temperature for the LLM.

truncate_on_overflow = False pydantic-field

When True, automatically truncate context that exceeds max_context_tokens. When False (default), raise ValueError.

from_yaml(path) classmethod

Load config object from a yaml file.

Source code in deet/extractors/llm_data_extractor.py

@classmethod
def from_yaml(cls, path: Path) -> "DataExtractionConfig":
    """Load config object from a yaml file."""
    if not path.exists():
        not_found = f"Config file not found at: {path}"
        raise FileNotFoundError(not_found)

    return cls.model_validate(yaml.safe_load(path.read_text()))

populate_max_context_tokens_from_model() pydantic-validator

Populate max_context_tokens from model when not set.

Source code in deet/extractors/llm_data_extractor.py

@model_validator(mode="after")
def populate_max_context_tokens_from_model(self) -> "DataExtractionConfig":
    """Populate max_context_tokens from model when not set."""
    if self.max_context_tokens is not None:
        return self
    model_str = _model_string_for_tokenisation(self.provider, self.model)
    try:
        inferred = get_model_max_tokens(model_str)
    except LitellmModelNotMappedError:
        inferred = None
    if inferred is not None:
        self.max_context_tokens = inferred
    else:
        # Use shared fallback when model max tokens cannot be inferred.
        self.max_context_tokens = DEFAULT_LLM_MAX_CONTEXT_TOKENS_FALLBACK
    return self

LLMDataExtractor

Generalisable module for LLM-based data extraction from documents.

This module provides a flexible interface for extracting structured data from documents using LLMs, with support for different context types and customizable prompts.

Source code in deet/extractors/llm_data_extractor.py

class LLMDataExtractor:
    """
    Generalisable module for LLM-based data extraction from documents.

    This module provides a flexible interface for extracting structured data
    from documents using LLMs, with support for different context types and
    customizable prompts.
    """

    def __init__(
        self,
        config: DataExtractionConfig,
        custom_system_prompt_file: Path | None = None,
        *,
        show_litellm_debug_messages: bool = False,
    ) -> None:
        """
        Initialise the data extraction module.

        Args:
            config (DataExtractionConfig): config obj for data extraction run
            custom_system_prompt_file (Path | None, optional): path to non-defualt
            sys prompt file. Defaults to None.
            show_litellm_debug_messages (bool, optional): show verbose litellm logs.
            Defaults to False.

        """
        self.config = config
        self.custom_system_prompt_file = custom_system_prompt_file
        if self.config.provider == LLMProvider.AZURE:
            self.model = f"azure/{self.config.model}"
            self.llm_api_key = settings.azure_api_key.get_secret_value()  # type: ignore[union-attr]
            self.api_base = settings.azure_api_base.get_secret_value()  # type: ignore[union-attr]
        elif self.config.provider == LLMProvider.OLLAMA:
            self.model = f"ollama/{self.config.model}"
            self.llm_api_key = None
            self.api_base = None
        else:
            error_message = f"Unsupported LLM provider: {self.config.provider}"
            raise ValueError(error_message)

        logger.info(f"Using {self.config.provider} with model: {self.model}")
        if self.config.max_tokens is not None:
            logger.info(f"max_tokens={self.config.max_tokens}")

        if show_litellm_debug_messages:
            litellm._turn_on_debug()  # noqa: SLF001

        if (
            self.custom_system_prompt_file
            and self.custom_system_prompt_file
            != self.config.prompt_config.system_prompt
        ):
            logger.debug("found custom sys prompt. loading...")
            self.config.prompt_config.system_prompt = (
                self.custom_system_prompt_file.read_text()
            )

    def extract_from_document(
        self,
        attributes: list[Attribute],
        filter_attribute_ids: list[int] | None = None,
        *,
        payload: str | None = None,
        md_path: Path | None = None,
        context_type: ContextType | None = None,
    ) -> DocumentExtractionResult:
        """
        Extract data from a single document.

        Call with either payload (document text) or md_path (path to markdown file).
        If md_path is provided, the file is read and used as the payload.
        Prompt payloads are not written here; the batch entry point
        extract_from_documents writes them to prompt_outfile when provided.

        Args:
            attributes: List of attributes to extract.
            payload: Document text to extract from. Required if md_path not set.
            md_path: Path to a markdown file to read as payload.
                Required if payload not set.
            context_type: Override config context type; if None, use config default.

        Returns:
            DocumentExtractionResult with annotations, messages, token counts,
            cost, model name, LLM call duration, and timestamp.

        Raises:
            ValueError: If no attributes are selected for extraction after filtering.
            ValueError: If neither payload nor md_path provided, or both provided.

        """
        if (payload is None and md_path is None) or (
            payload is not None and md_path is not None
        ):
            msg = "Exactly one of payload or md_path must be provided"
            raise ValueError(msg)
        if md_path is not None:
            if not md_path.exists():
                msg = f"Markdown file not found: {md_path}"
                raise FileNotFoundError(msg)
            payload = md_path.read_text(encoding="utf-8")
        payload = cast("str", payload)

        selected_attributes = attributes
        if filter_attribute_ids and len(filter_attribute_ids) > 0:
            try:
                selected_attributes = self._filter_attributes(
                    selected_attributes, filter_ids=filter_attribute_ids
                )
            except (ValueError, TypeError):
                logger.warning(
                    f"Invalid attribute IDs in config: "
                    f"{filter_attribute_ids}. "
                    "No attributes will be selected."
                )

        if not selected_attributes:
            msg = "No attributes selected for extraction"
            logger.warning(msg)
            raise ValueError(msg)

        context = self._prepare_context(payload=payload, context_type=context_type)
        prompt = self._generate_user_message_json(
            payload=context, attributes=selected_attributes
        )

        response_model: type[BaseLLMResponse]

        if self.config.dynamic_json_schema:
            response_model = build_llm_response_model(selected_attributes)
        else:
            response_model = LLMResponseSchema

        with measure_elapsed() as llm_timer:
            llm_response, messages, output_tokens, input_tokens = self._call_llm(
                prompt=prompt, response_model=response_model
            )

        annotations = self._parse_llm_response(
            response_content=llm_response,
            response_model=response_model,
            attributes=selected_attributes,
        )

        return DocumentExtractionResult(
            annotations=annotations,
            messages=messages,
            input_tokens=input_tokens,
            output_tokens=output_tokens,
            model=self.model,
            llm_call_seconds=llm_timer.seconds,
        )

    def extract_from_documents(  # noqa: PLR0913
        self,
        attributes: list[Attribute],
        documents: Sequence[Document],
        filter_attribute_ids: list[int] | None = None,
        output_file: Path | None = None,
        context_type: ContextType | None = None,
        prompt_outfile: Path | None = None,
        document_parsing: dict[str, DocumentParsingStats] | None = None,
        *,
        show_progress: bool = False,
    ) -> ExtractionRunOutput:
        """
        Extract data from all documents.

        Loops over documents and extracts data using list of attributes.
        A document that's missing what it needs for the chosen context_type
        (e.g. no abstract when using ABSTRACT_ONLY) is skipped with a warning,
        not raised.

        Args:
            attributes: List of attributes to extract.
            documents: Sequence of Document instances (required).
            filter_attribute_ids: Optional list of attribute IDs to filter by.
            output_file: Optional path to save combined results JSON.
            context_type: Override config context type; if None, use config default.
            prompt_outfile: Optional path to write a single JSON object:
                keys are document IDs, values are prompt payload (messages).
            document_parsing: Optional per-document parsing stats from preparation.
            show_progress: Whether to show a progress bar.

        Returns:
            ExtractionRunOutput containing annotated documents and run metadata.

        """
        if context_type is None:
            context_type = self.config.default_context_type

        parsing_by_doc = document_parsing or {}
        prompt_payloads: dict[str, Any] = {}
        per_document: dict[str, PerDocumentExtractionStats] = {}

        llm_annotated_docs: list[GoldStandardAnnotatedDocument] = []
        total_input_tokens = 0
        total_output_tokens = 0
        total_cost: float | None = None

        with optional_progress(
            documents, show_progress=show_progress
        ) as iterable_documents:
            for document in iterable_documents:
                logger.info(f"Processing document: {document.name}")

                try:
                    if context_type == ContextType.ABSTRACT_ONLY:
                        document.set_abstract_context()
                    elif context_type == ContextType.FULL_DOCUMENT:
                        document.context = document.safe_parsed_document.text

                    result = self.extract_from_document(
                        attributes=attributes,
                        filter_attribute_ids=filter_attribute_ids,
                        payload=document.context,
                        context_type=context_type,
                    )

                    llm_annotated_docs.append(
                        GoldStandardAnnotatedDocument(
                            document=document, annotations=result.annotations
                        )
                    )
                    doc_id_str = str(document.safe_identity.document_id)
                    prompt_payloads[doc_id_str] = result.messages
                    parsing = parsing_by_doc.get(doc_id_str, DocumentParsingStats())
                    per_document[doc_id_str] = PerDocumentExtractionStats(
                        input_tokens=result.input_tokens,
                        output_tokens=result.output_tokens,
                        parsing_seconds=parsing.parsing_seconds,
                        parsing_skipped=parsing.parsing_skipped,
                        llm_call_seconds=result.llm_call_seconds,
                    )
                    total_input_tokens += result.input_tokens
                    total_output_tokens += result.output_tokens
                    if result.total_cost_usd is not None:
                        total_cost = (total_cost or 0.0) + result.total_cost_usd

                except NoAbstractError as e:
                    logger.warning(f"Skipping {document.name}: {e}")
                except Exception as e:  # noqa: BLE001
                    logger.error(f"Failed to process {document.name}: {e}")
                    logger.debug("Error details", exc_info=True)

        run_metadata = ExtractionRunMetadata(
            model=self.model,
            total_input_tokens=total_input_tokens,
            total_output_tokens=total_output_tokens,
            total_cost_usd=round(total_cost, 6) if total_cost is not None else None,
            per_document=per_document,
        )
        run_output = ExtractionRunOutput(
            annotated_documents=llm_annotated_docs,
            metadata=run_metadata,
        )

        if output_file is not None:
            self._save_results(run_output, output_file)
            logger.info(f"Combined LLM classifications written to: {output_file}")

        if prompt_outfile is not None:
            prompt_outfile.write_text(
                json.dumps(prompt_payloads, indent=2), encoding="utf-8"
            )
            logger.info(f"Prompt payloads saved to: {prompt_outfile}")

        return run_output

    def _write_json_if_path(
        self, data: dict[str, Any] | list[Any], path: Path | None
    ) -> None:
        """
        Write data as JSON to path if path is not None; otherwise no-op.

        Args:
            data: Dict or list to serialize as JSON.
            path: Optional file path; when None, nothing is written.

        """
        if path is not None:
            path.write_text(json.dumps(data, indent=2), encoding="utf-8")

    def _filter_attributes(
        self, attributes: list[Attribute], filter_ids: list[int] | None = None
    ) -> list[Attribute]:
        """
        Filter attributes using provided attribute IDs.

        Args:
            attributes: List of attributes to filter
            filter_ids: Optional list of attribute IDs (ints) to filter by.
                        If None, returns all attributes.
                        If empty list, returns empty list.

        Returns:
            Filtered list of attributes matching the provided IDs, or all attributes
            if filter_ids is None, or empty list if filter_ids is empty.

        """
        if filter_ids is None:
            logger.debug(
                f"No attribute filtering applied, "
                f"using all {len(attributes)} attributes"
            )
            return attributes

        filtered = [attr for attr in attributes if attr.attribute_id in filter_ids]
        logger.debug(
            f"Filtered {len(attributes)} attributes to {len(filtered)} "
            f"using filter_ids: {filter_ids}"
        )
        return filtered

    def _prepare_context(
        self,
        payload: str,
        context_type: ContextType | None = None,
    ) -> str:
        """Prepare context based on context type."""
        ctx = (
            context_type
            if context_type is not None
            else self.config.default_context_type
        )
        logger.debug(f"Using context type: {ctx}")
        if ctx == ContextType.FULL_DOCUMENT:
            context = payload
            logger.debug(f"Using full document context (length: {len(str(context))})")
        elif ctx == ContextType.ABSTRACT_ONLY:
            context = payload
            logger.debug(f"Using abstract context (length: {len(str(context))})")
        else:
            other_not_allowed = f"{ctx} context type is not allowed."
            raise ValueError(other_not_allowed)

        if isinstance(context, list):
            logger.debug(f"Converting list context to string (items: {len(context)})")
            context = " ".join(context)

        return context

    def _generate_user_message_json(
        self,
        payload: str,
        attributes: list[Attribute],
    ) -> str:
        """
        Generate structured JSON input for the LLM user message.

        The payload contains the prepared document context and an array
        `LLMInputSchema` objects, containing the attribute id, prompt and
        target output data type.

        NOTE: If `prompt` field is not populated in incoming data,
        LLMInputSchema will populate from `attribute_label`
        field, or fail.

        Args:
            payload: Prepared document context string.
            attributes: List of Attribute objects to extract.

        Returns:
            JSON string containing `context` and `attributes`.

        """
        logger.debug(f"Generating prompt for {len(attributes)} attributes")
        attributes_payload = []
        for attr in attributes:
            # validate schema & fill prompt if not yet filled
            # Use exclude_none=False to ensure prompt field is included even if None
            attr_dict = attr.model_dump(exclude_none=False)
            llm_input_attr = LLMInputSchema(**attr_dict)
            attributes_payload.append(llm_input_attr.model_dump())

        unserialised_prompt = {
            "context": payload,
            "attributes": attributes_payload,
        }

        logger.debug(f"attributes payload: {attributes_payload}")
        prompt_json = json.dumps(unserialised_prompt, ensure_ascii=False)
        logger.debug(f"Generated prompt JSON ({len(prompt_json)} characters)")

        return prompt_json

    def _enforce_context_limit(
        self,
        messages: list[dict[str, Any]],
        prompt: str,
        system_prompt: str,
    ) -> None:
        """
        Enforce max_context_tokens on the messages payload.

        When the total token count exceeds max_context_tokens:
        - If ``truncate_on_overflow`` is False (default), raises ValueError.
        - If ``truncate_on_overflow`` is True, truncates the "context" field
          inside the user prompt JSON to fit. Mutates messages in place.

        Args:
            messages: List of message dicts with "role" and "content";
                messages[1] is the user message (prompt JSON).
            prompt: Current user prompt JSON string.
            system_prompt: System prompt text (for token counting).

        Raises:
            ValueError: When payload exceeds max_context_tokens and
                truncate_on_overflow is False, or when truncate_on_overflow is
                True but the prompt JSON cannot be parsed or messages are not
                in the expected shape so truncation cannot be applied.

        """
        max_ctx = self.config.max_context_tokens
        if max_ctx is None:
            return
        messages_text = " ".join(str(m.get("content", "")) for m in messages)
        total_tokens = count_tokens(self.model, messages_text)
        if total_tokens <= max_ctx:
            return

        if not self.config.truncate_on_overflow:
            msg = (
                f"Payload ({total_tokens} tokens) exceeds "
                f"max_context_tokens ({max_ctx} tokens). "
                "Set truncate_on_overflow=True in your config to "
                "automatically truncate, or increase max_context_tokens."
            )
            raise ValueError(msg)

        try:
            prompt_data = json.loads(prompt)
            context = prompt_data.get("context", "")
            attributes_payload = prompt_data.get("attributes", [])
            attributes_part = json.dumps(
                {"context": "", "attributes": attributes_payload},
                ensure_ascii=False,
            )
            system_tokens = count_tokens(self.model, str(system_prompt))
            attributes_tokens = count_tokens(self.model, attributes_part)
            # Buffer for token-count discrepancies or extra tokens from
            # serialization/whitespace that LLM APIs may add.
            buffer = 50
            context_limit = max_ctx - system_tokens - attributes_tokens - buffer
            if context_limit > 0:
                context = truncate_to_token_limit(context, self.model, context_limit)
                prompt_data["context"] = context
                truncated_prompt = json.dumps(prompt_data, ensure_ascii=False)
                messages[1]["content"] = truncated_prompt
                logger.warning(
                    f"Truncated context to fit {max_ctx} "
                    "tokens. Edit `max_context_tokens` in your config."
                )
            else:
                logger.warning(
                    "System prompt and attributes exceed "
                    "max_context_tokens; context will be empty."
                )
                prompt_data["context"] = ""
                messages[1]["content"] = json.dumps(prompt_data, ensure_ascii=False)
        except (json.JSONDecodeError, KeyError, IndexError) as e:
            logger.debug(f"Could not truncate by tokens: {e}")
            logger.warning(
                "Could not enforce max_context_tokens by truncating the prompt JSON; "
                "prompt appears to be invalid or not in the expected shape."
            )
            msg = (
                "Failed to enforce max_context_tokens: invalid or unexpected prompt "
                "JSON structure while truncate_on_overflow=True."
            )
            raise ValueError(msg) from e

    def _call_llm(
        self,
        prompt: str,
        response_model: type[BaseLLMResponse],
    ) -> tuple[str, list[dict[str, Any]], int, int]:
        """
        Call the LLM with the given prompt.

        Args:
            prompt: The user prompt (with context and attributes).
            response_model: Dynamic Pydantic model describing the expected
                response; its JSON schema is passed as the structured-output
                schema so the LLM must return one typed entry per attribute.

        Returns:
            Tuple of (LLM response text, messages list, output token count,
            input/prompt token count from the response usage).

        """
        system_prompt = self.config.prompt_config.system_prompt
        messages: list[dict[str, Any]] = [
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": prompt},
        ]

        self._enforce_context_limit(messages, prompt, str(system_prompt))

        logger.debug(f"Model: {self.model}")
        logger.debug(f"Temperature: {self.config.temperature}")
        logger.debug(f" sys message: {messages[0]['content'][:1000]}")
        logger.debug(f" user message: {messages[1]['content'][:1000]}")

        messages_text = " ".join(str(m.get("content", "")) for m in messages)
        input_tokens = count_tokens(self.model, messages_text)
        prompt_cost, _ = estimate_cost_usd(
            self.model,
            prompt_tokens=input_tokens,
            completion_tokens=0,
        )
        if prompt_cost is not None:
            logger.info(
                f"Estimated input cost: ${prompt_cost:.6f} USD ({input_tokens} tokens)"
            )

        response = litellm.completion(
            model=self.model,
            api_key=self.llm_api_key,
            api_base=self.api_base,
            messages=messages,
            temperature=self.config.temperature,
            response_format={
                "type": "json_schema",
                "json_schema": {
                    "name": "llm_annotation_response",
                    "schema": response_model.model_json_schema(),
                    "strict": True,
                },
            },
            max_tokens=self.config.max_tokens,
        )

        msg = response.choices[0].message

        if getattr(msg, "content", None) is not None:
            response_content = msg.content
        elif getattr(msg, "tool_calls", None):
            response_content = msg.tool_calls[0].function.arguments
        logger.debug(f"raw response: {response_content}")

        input_tokens = 0
        output_tokens = 0
        if response.usage is not None:
            if hasattr(response.usage, "prompt_tokens"):
                input_tokens = response.usage.prompt_tokens or 0
            if hasattr(response.usage, "completion_tokens"):
                output_tokens = response.usage.completion_tokens or 0

        return response_content, messages, output_tokens, input_tokens

    def _create_gold_standard_annotation(
        self, llm_annotation_response: LLMAnnotationResponse, attribute: Attribute
    ) -> GoldStandardAnnotation:
        """Create GoldStandardAnnotation from LLMAnnotationResponse."""
        additional_text = (
            llm_annotation_response.additional_text
            if self.config.include_additional_text
            else None
        )
        reasoning = (
            llm_annotation_response.reasoning if self.config.include_reasoning else None
        )
        return GoldStandardAnnotation(
            attribute=attribute,
            raw_data=llm_annotation_response.output_data,
            annotation_type=AnnotationType.LLM,
            additional_text=additional_text,
            reasoning=reasoning,
        )

    def _parse_static_llm_response_annotations(
        self,
        validated_response: LLMResponseSchema,
        attributes_by_id: dict[int, Attribute],
    ) -> list[GoldStandardAnnotation]:
        """Convert each response from a list of responses into an annotation."""
        logger.debug(
            f"Parsing LLM response with {len(validated_response.annotations)} "
            f"attribute responses"
        )
        annotations: list[GoldStandardAnnotation] = []
        for llm_annotation in validated_response.annotations:
            attribute = attributes_by_id.get(llm_annotation.attribute_id)
            if attribute is None:
                logger.warning(
                    f"No attribute found for ID: {llm_annotation.attribute_id}"
                )
                continue

            annotations.append(
                self._create_gold_standard_annotation(llm_annotation, attribute)
            )

        return annotations

    def _parse_dynamic_llm_response_annotations(
        self,
        response_model: type[DynamicLLMResponseBase],
        validated_response: DynamicLLMResponseBase,
        attributes_by_id: dict[int, Attribute],
    ) -> list[GoldStandardAnnotation]:
        """Convert each field of a Dynamic response model into an annotation."""
        logger.debug(
            f"Parsing LLM response with {len(response_model.model_fields)} "
            f"attribute responses"
        )
        annotations = []
        for (
            field_name,
            attribute_response,
        ) in validated_response.iter_attribute_responses():
            attribute_id = attribute_id_from_response_key(field_name)
            attribute = attributes_by_id.get(attribute_id)
            if attribute is None:
                msg = (
                    f"No attribute found for ID: {attribute_id}. "
                    "The dynamic response model and attribute list are out of sync."
                )
                raise ValueError(msg)

            annotations.append(
                self._create_gold_standard_annotation(attribute_response, attribute)
            )
            logger.debug(
                f"Created annotation for attribute {attribute.attribute_id}: "
                f"output_data={attribute_response.output_data}"
            )

        logger.debug(f"Successfully parsed {len(annotations)} annotations")
        return annotations

    def _parse_llm_response(
        self,
        response_content: str,
        response_model: type[BaseLLMResponse],
        attributes: list[Attribute],
    ) -> list[GoldStandardAnnotation]:
        """
        Parse and validate LLM response against the dynamic response schema.

        The response is validated against ``response_model`` (built from the
        selected attributes), which guarantees one correctly typed entry per
        attribute. Each ``attribute_<id>`` field is then mapped back to its
        full Attribute and converted to a GoldStandardAnnotation.

        Args:
            response_content: Raw JSON string response from LLM.
            response_model: Dynamic model the response was requested against.
            attributes: List of attributes to resolve ids against.

        Returns:
            List of GoldStandardAnnotation objects.

        Raises:
            ValidationError: If response fails schema validation (e.g. a missing
                attribute, an extra attribute, or a mistyped ``output_data``).
            ValueError: If JSON parsing fails.

        """
        try:
            validated_response = response_model.model_validate_json(response_content)
        except ValidationError as ve:
            logger.error(f"LLM response failed schema validation: {ve}")
            logger.debug(f"Response content: {response_content}")
            raise
        except json.JSONDecodeError as je:
            error_msg = f"Invalid JSON in LLM response: {je}"
            logger.error(f"Failed to parse LLM response as JSON: {je}")
            raise ValueError(error_msg) from je

        attributes_by_id = {attr.attribute_id: attr for attr in attributes}

        if self.config.dynamic_json_schema:
            dynamic_model = cast("type[DynamicLLMResponseBase]", response_model)
            dynamic_response = cast("DynamicLLMResponseBase", validated_response)
            return self._parse_dynamic_llm_response_annotations(
                response_model=dynamic_model,
                validated_response=dynamic_response,
                attributes_by_id=attributes_by_id,
            )

        static_response = cast("LLMResponseSchema", validated_response)
        return self._parse_static_llm_response_annotations(
            validated_response=static_response, attributes_by_id=attributes_by_id
        )

    def _save_results(
        self,
        run_output: ExtractionRunOutput,
        output_file: Path,
    ) -> None:
        """
        Serialize an ExtractionRunOutput to JSON and write it to disk.

        Args:
            run_output: The complete extraction run output to persist.
            output_file: Destination file path.

        """
        output_file.write_text(
            run_output.model_dump_json(indent=2),
            encoding="utf-8",
        )
        logger.info(f"Results saved to: {output_file}")

__init__(config, custom_system_prompt_file=None, *, show_litellm_debug_messages=False)

Initialise the data extraction module.

Parameters:

Name	Type	Description	Default
config	DataExtractionConfig	config obj for data extraction run	required
custom_system_prompt_file	Path | None	path to non-defualt	None
show_litellm_debug_messages	bool	show verbose litellm logs.	False

Source code in deet/extractors/llm_data_extractor.py

def __init__(
    self,
    config: DataExtractionConfig,
    custom_system_prompt_file: Path | None = None,
    *,
    show_litellm_debug_messages: bool = False,
) -> None:
    """
    Initialise the data extraction module.

    Args:
        config (DataExtractionConfig): config obj for data extraction run
        custom_system_prompt_file (Path | None, optional): path to non-defualt
        sys prompt file. Defaults to None.
        show_litellm_debug_messages (bool, optional): show verbose litellm logs.
        Defaults to False.

    """
    self.config = config
    self.custom_system_prompt_file = custom_system_prompt_file
    if self.config.provider == LLMProvider.AZURE:
        self.model = f"azure/{self.config.model}"
        self.llm_api_key = settings.azure_api_key.get_secret_value()  # type: ignore[union-attr]
        self.api_base = settings.azure_api_base.get_secret_value()  # type: ignore[union-attr]
    elif self.config.provider == LLMProvider.OLLAMA:
        self.model = f"ollama/{self.config.model}"
        self.llm_api_key = None
        self.api_base = None
    else:
        error_message = f"Unsupported LLM provider: {self.config.provider}"
        raise ValueError(error_message)

    logger.info(f"Using {self.config.provider} with model: {self.model}")
    if self.config.max_tokens is not None:
        logger.info(f"max_tokens={self.config.max_tokens}")

    if show_litellm_debug_messages:
        litellm._turn_on_debug()  # noqa: SLF001

    if (
        self.custom_system_prompt_file
        and self.custom_system_prompt_file
        != self.config.prompt_config.system_prompt
    ):
        logger.debug("found custom sys prompt. loading...")
        self.config.prompt_config.system_prompt = (
            self.custom_system_prompt_file.read_text()
        )

extract_from_document(attributes, filter_attribute_ids=None, *, payload=None, md_path=None, context_type=None)

Extract data from a single document.

Call with either payload (document text) or md_path (path to markdown file). If md_path is provided, the file is read and used as the payload. Prompt payloads are not written here; the batch entry point extract_from_documents writes them to prompt_outfile when provided.

Parameters:

Name	Type	Description	Default
attributes	list[Attribute]	List of attributes to extract.	required
payload	str | None	Document text to extract from. Required if md_path not set.	None
md_path	Path | None	Path to a markdown file to read as payload. Required if payload not set.	None
context_type	ContextType | None	Override config context type; if None, use config default.	None

Returns:

Type	Description
DocumentExtractionResult	DocumentExtractionResult with annotations, messages, token counts,
DocumentExtractionResult	cost, model name, LLM call duration, and timestamp.

Raises:

Type	Description
ValueError	If no attributes are selected for extraction after filtering.
ValueError	If neither payload nor md_path provided, or both provided.

Source code in deet/extractors/llm_data_extractor.py

def extract_from_document(
    self,
    attributes: list[Attribute],
    filter_attribute_ids: list[int] | None = None,
    *,
    payload: str | None = None,
    md_path: Path | None = None,
    context_type: ContextType | None = None,
) -> DocumentExtractionResult:
    """
    Extract data from a single document.

    Call with either payload (document text) or md_path (path to markdown file).
    If md_path is provided, the file is read and used as the payload.
    Prompt payloads are not written here; the batch entry point
    extract_from_documents writes them to prompt_outfile when provided.

    Args:
        attributes: List of attributes to extract.
        payload: Document text to extract from. Required if md_path not set.
        md_path: Path to a markdown file to read as payload.
            Required if payload not set.
        context_type: Override config context type; if None, use config default.

    Returns:
        DocumentExtractionResult with annotations, messages, token counts,
        cost, model name, LLM call duration, and timestamp.

    Raises:
        ValueError: If no attributes are selected for extraction after filtering.
        ValueError: If neither payload nor md_path provided, or both provided.

    """
    if (payload is None and md_path is None) or (
        payload is not None and md_path is not None
    ):
        msg = "Exactly one of payload or md_path must be provided"
        raise ValueError(msg)
    if md_path is not None:
        if not md_path.exists():
            msg = f"Markdown file not found: {md_path}"
            raise FileNotFoundError(msg)
        payload = md_path.read_text(encoding="utf-8")
    payload = cast("str", payload)

    selected_attributes = attributes
    if filter_attribute_ids and len(filter_attribute_ids) > 0:
        try:
            selected_attributes = self._filter_attributes(
                selected_attributes, filter_ids=filter_attribute_ids
            )
        except (ValueError, TypeError):
            logger.warning(
                f"Invalid attribute IDs in config: "
                f"{filter_attribute_ids}. "
                "No attributes will be selected."
            )

    if not selected_attributes:
        msg = "No attributes selected for extraction"
        logger.warning(msg)
        raise ValueError(msg)

    context = self._prepare_context(payload=payload, context_type=context_type)
    prompt = self._generate_user_message_json(
        payload=context, attributes=selected_attributes
    )

    response_model: type[BaseLLMResponse]

    if self.config.dynamic_json_schema:
        response_model = build_llm_response_model(selected_attributes)
    else:
        response_model = LLMResponseSchema

    with measure_elapsed() as llm_timer:
        llm_response, messages, output_tokens, input_tokens = self._call_llm(
            prompt=prompt, response_model=response_model
        )

    annotations = self._parse_llm_response(
        response_content=llm_response,
        response_model=response_model,
        attributes=selected_attributes,
    )

    return DocumentExtractionResult(
        annotations=annotations,
        messages=messages,
        input_tokens=input_tokens,
        output_tokens=output_tokens,
        model=self.model,
        llm_call_seconds=llm_timer.seconds,
    )

extract_from_documents(attributes, documents, filter_attribute_ids=None, output_file=None, context_type=None, prompt_outfile=None, document_parsing=None, *, show_progress=False)

Extract data from all documents.

Loops over documents and extracts data using list of attributes. A document that's missing what it needs for the chosen context_type (e.g. no abstract when using ABSTRACT_ONLY) is skipped with a warning, not raised.

Parameters:

Name	Type	Description	Default
attributes	list[Attribute]	List of attributes to extract.	required
documents	Sequence[Document]	Sequence of Document instances (required).	required
filter_attribute_ids	list[int] | None	Optional list of attribute IDs to filter by.	None
output_file	Path | None	Optional path to save combined results JSON.	None
context_type	ContextType | None	Override config context type; if None, use config default.	None
prompt_outfile	Path | None	Optional path to write a single JSON object: keys are document IDs, values are prompt payload (messages).	None
document_parsing	dict[str, DocumentParsingStats] | None	Optional per-document parsing stats from preparation.	None
show_progress	bool	Whether to show a progress bar.	False

Returns:

Type	Description
ExtractionRunOutput	ExtractionRunOutput containing annotated documents and run metadata.

Source code in deet/extractors/llm_data_extractor.py

def extract_from_documents(  # noqa: PLR0913
    self,
    attributes: list[Attribute],
    documents: Sequence[Document],
    filter_attribute_ids: list[int] | None = None,
    output_file: Path | None = None,
    context_type: ContextType | None = None,
    prompt_outfile: Path | None = None,
    document_parsing: dict[str, DocumentParsingStats] | None = None,
    *,
    show_progress: bool = False,
) -> ExtractionRunOutput:
    """
    Extract data from all documents.

    Loops over documents and extracts data using list of attributes.
    A document that's missing what it needs for the chosen context_type
    (e.g. no abstract when using ABSTRACT_ONLY) is skipped with a warning,
    not raised.

    Args:
        attributes: List of attributes to extract.
        documents: Sequence of Document instances (required).
        filter_attribute_ids: Optional list of attribute IDs to filter by.
        output_file: Optional path to save combined results JSON.
        context_type: Override config context type; if None, use config default.
        prompt_outfile: Optional path to write a single JSON object:
            keys are document IDs, values are prompt payload (messages).
        document_parsing: Optional per-document parsing stats from preparation.
        show_progress: Whether to show a progress bar.

    Returns:
        ExtractionRunOutput containing annotated documents and run metadata.

    """
    if context_type is None:
        context_type = self.config.default_context_type

    parsing_by_doc = document_parsing or {}
    prompt_payloads: dict[str, Any] = {}
    per_document: dict[str, PerDocumentExtractionStats] = {}

    llm_annotated_docs: list[GoldStandardAnnotatedDocument] = []
    total_input_tokens = 0
    total_output_tokens = 0
    total_cost: float | None = None

    with optional_progress(
        documents, show_progress=show_progress
    ) as iterable_documents:
        for document in iterable_documents:
            logger.info(f"Processing document: {document.name}")

            try:
                if context_type == ContextType.ABSTRACT_ONLY:
                    document.set_abstract_context()
                elif context_type == ContextType.FULL_DOCUMENT:
                    document.context = document.safe_parsed_document.text

                result = self.extract_from_document(
                    attributes=attributes,
                    filter_attribute_ids=filter_attribute_ids,
                    payload=document.context,
                    context_type=context_type,
                )

                llm_annotated_docs.append(
                    GoldStandardAnnotatedDocument(
                        document=document, annotations=result.annotations
                    )
                )
                doc_id_str = str(document.safe_identity.document_id)
                prompt_payloads[doc_id_str] = result.messages
                parsing = parsing_by_doc.get(doc_id_str, DocumentParsingStats())
                per_document[doc_id_str] = PerDocumentExtractionStats(
                    input_tokens=result.input_tokens,
                    output_tokens=result.output_tokens,
                    parsing_seconds=parsing.parsing_seconds,
                    parsing_skipped=parsing.parsing_skipped,
                    llm_call_seconds=result.llm_call_seconds,
                )
                total_input_tokens += result.input_tokens
                total_output_tokens += result.output_tokens
                if result.total_cost_usd is not None:
                    total_cost = (total_cost or 0.0) + result.total_cost_usd

            except NoAbstractError as e:
                logger.warning(f"Skipping {document.name}: {e}")
            except Exception as e:  # noqa: BLE001
                logger.error(f"Failed to process {document.name}: {e}")
                logger.debug("Error details", exc_info=True)

    run_metadata = ExtractionRunMetadata(
        model=self.model,
        total_input_tokens=total_input_tokens,
        total_output_tokens=total_output_tokens,
        total_cost_usd=round(total_cost, 6) if total_cost is not None else None,
        per_document=per_document,
    )
    run_output = ExtractionRunOutput(
        annotated_documents=llm_annotated_docs,
        metadata=run_metadata,
    )

    if output_file is not None:
        self._save_results(run_output, output_file)
        logger.info(f"Combined LLM classifications written to: {output_file}")

    if prompt_outfile is not None:
        prompt_outfile.write_text(
            json.dumps(prompt_payloads, indent=2), encoding="utf-8"
        )
        logger.info(f"Prompt payloads saved to: {prompt_outfile}")

    return run_output

PromptConfig pydantic-model

Bases: BaseModel

Configuration for prompts used in data extraction.

Fields:

• system_prompt (str | Path)

Validators:

• load_system_prompt_file

Source code in deet/extractors/llm_data_extractor.py

class PromptConfig(BaseModel):
    """Configuration for prompts used in data extraction."""

    model_config = ConfigDict()

    system_prompt: str | Path = Field(
        description="System prompt that defines the task and role",
        default_factory=default_system_prompt,
    )

    @model_validator(mode="after")
    def load_system_prompt_file(self) -> "PromptConfig":
        """Load system prompt from file if Path provided."""
        if isinstance(self.system_prompt, Path):
            if not self.system_prompt.exists():
                sys_prompt_missing = f"sys prompt {self.system_prompt} not found."
                raise ValueError(sys_prompt_missing)
            logger.debug(f"Reading system prompt from {self.system_prompt}")
            self.system_prompt = self.system_prompt.read_text()

        return self

system_prompt pydantic-field

System prompt that defines the task and role

load_system_prompt_file() pydantic-validator

Load system prompt from file if Path provided.

Source code in deet/extractors/llm_data_extractor.py

@model_validator(mode="after")
def load_system_prompt_file(self) -> "PromptConfig":
    """Load system prompt from file if Path provided."""
    if isinstance(self.system_prompt, Path):
        if not self.system_prompt.exists():
            sys_prompt_missing = f"sys prompt {self.system_prompt} not found."
            raise ValueError(sys_prompt_missing)
        logger.debug(f"Reading system prompt from {self.system_prompt}")
        self.system_prompt = self.system_prompt.read_text()

    return self

default_system_prompt()

Get default system prompt included in the package.

Source code in deet/extractors/llm_data_extractor.py

def default_system_prompt() -> str:
    """Get default system prompt included in the package."""
    return (files("deet.prompts") / "system_prompt.txt").read_text()

logger

Customisations on the loguru logger.

processors

base_converter

Generic classes and functions for converters.

AnnotationConverter

Bases: ABC

Abstract base class to define expected behaviour of an annotationconverter.

OUTFILE_LOADERS maps the outfiles to be read/written to a filename, and a TypeAdapter defining the type of Pydantic Model to read back in.

Source code in deet/processors/base_converter.py

class AnnotationConverter[
    DocumentType: Document,
    AttributeT: Attribute,
    GoldStandardAnnotationType: GoldStandardAnnotation,
    GoldStandardAnnotatedDocumentType: GoldStandardAnnotatedDocument,
](ABC):
    """
    Abstract base class to define expected behaviour of an annotationconverter.

    OUTFILE_LOADERS maps the outfiles to be read/written to a filename, and a
    TypeAdapter defining the type of Pydantic Model to read back in.
    """

    OUTFILE_LOADERS: dict[Outfiles, tuple[str, TypeAdapter]]

    @property
    def outfile_names(self) -> dict[Outfiles, str]:
        """Return a dictionary of outfiles to names, using outfile_loaders."""
        return {k: v[0] for k, v in self.OUTFILE_LOADERS.items()}

    def __init__(
        self,
        base_output_dir: str | Path | None = DEFAULT_BASE_OUTPUT_DIR,
    ) -> None:
        """
        Initialise the converter with configurable output paths.

        Args:
            output_dir: Base directory for saving processed files
            attributes_filename: Filename for attributes output
            documents_filename: Filename for documents output
            annotated_documents_filename: Filename for annotated documents output
            attribute_mapping_filename: Filename for attribute ID to label mapping

        """
        if base_output_dir is None:
            logger.debug(
                "`base_output_dir` set to None; "
                "converting to empty string for compatibility."
            )
            base_output_dir = ""
        self.base_output_dir = Path(base_output_dir)

    @property
    @abstractmethod
    def processed_data_type(
        self,
    ) -> type[
        ProcessedAnnotationData[
            AttributeT,
            DocumentType,
            GoldStandardAnnotationType,
            GoldStandardAnnotatedDocumentType,
        ]
    ]:
        """Return the class to use when instantiating processed data."""
        ...

    @abstractmethod
    def process_annotation_file(
        self,
        file_path: str | Path,
        set_attribute_type: str | AttributeType | None = None,
    ) -> ProcessedAnnotationData:
        """Parse a raw input format into processed data."""
        ...

    def write_processed_data_to_file(
        self,
        processed_data: ProcessedAnnotationData,
        output_dir: str | Path,
        outfiles_to_write: list[Outfiles] | None = None,
    ) -> dict[str, str]:
        """
        Save processed data to structured files using Pydantic model serialisation.

        Args:
            processed_data: The processed data from process_annotation_file
            output_dir: Write all output (json) files from conversion to this
            directory. NOTE: we output files will live in a sub-directory
            `self.base_output_dir`, which by default is `DEFAULT_BASE_OUTPUT_DIR`.
            so, if you want output files to go straight to `output_dir`, set
            `self.base_output_dir` to '' or None.

        Returns:
            Dictionary mapping data types to saved file paths

        """
        file_mappings = {
            Outfiles.ATTRIBUTES: processed_data.attributes,
            Outfiles.DOCUMENTS: processed_data.documents,
            Outfiles.ANNOTATED_DOCUMENTS: processed_data.annotated_documents,
            Outfiles.ATTRIBUTE_LABEL_MAPPING: processed_data.attribute_id_to_label,
        }
        # setting here to avoid mutable default.
        if outfiles_to_write is None:
            outfiles_to_write = [Outfiles.ATTRIBUTES, Outfiles.DOCUMENTS]

        file_mappings = {
            k: v for k, v in file_mappings.items() if k in outfiles_to_write
        }
        files_to_write_message = {",".join(k.value for k in file_mappings)}
        logger.info(f"writing {files_to_write_message} out...")

        user_dir = Path(output_dir)
        target_dir = user_dir / self.base_output_dir
        target_dir.mkdir(parents=True, exist_ok=True)
        logger.info(f"writing files to dir: {target_dir}")

        saved_files = {}

        for file_type, data_list in file_mappings.items():
            file_path = target_dir / self.outfile_names[file_type]
            logger.debug(f"writing file {file_type} to {file_path}")
            if file_type == Outfiles.ATTRIBUTE_LABEL_MAPPING:
                file_path.write_text(json.dumps(data_list), encoding="utf-8")
            else:
                file_path.write_text(
                    json.dumps(
                        [item.model_dump(mode="json") for item in data_list],  # type: ignore[attr-defined]
                        indent=2,
                    ),
                    encoding="utf-8",
                )
            saved_files[file_type.value] = str(file_path)

        return saved_files

    def reload_output(self, file_path: Path) -> ProcessedAnnotationData:
        """
        Read data back in, using OUTFILE_LOADERS and the
        subclass's processed_data_type.
        """
        loaded_data = {}
        for key, (filename, adapter) in self.OUTFILE_LOADERS.items():
            path = file_path / self.base_output_dir / filename
            if not path.exists():
                message = f"Expected {key.value} at {path}"
                raise FileNotFoundError(message)
            loaded_data[key] = adapter.validate_json(path.read_text())

        return self.processed_data_type(
            attributes=loaded_data[Outfiles.ATTRIBUTES],
            documents=loaded_data[Outfiles.DOCUMENTS],
            annotations=[],  # or derive from loaded_data if applicable
            annotated_documents=loaded_data[Outfiles.ANNOTATED_DOCUMENTS],
            attribute_id_to_label=loaded_data[Outfiles.ATTRIBUTE_LABEL_MAPPING],
        )

outfile_names property

Return a dictionary of outfiles to names, using outfile_loaders.

processed_data_type abstractmethod property

Return the class to use when instantiating processed data.

__init__(base_output_dir=DEFAULT_BASE_OUTPUT_DIR)

Initialise the converter with configurable output paths.

Parameters:

Name	Type	Description	Default
output_dir		Base directory for saving processed files	required
attributes_filename		Filename for attributes output	required
documents_filename		Filename for documents output	required
annotated_documents_filename		Filename for annotated documents output	required
attribute_mapping_filename		Filename for attribute ID to label mapping	required

Source code in deet/processors/base_converter.py

def __init__(
    self,
    base_output_dir: str | Path | None = DEFAULT_BASE_OUTPUT_DIR,
) -> None:
    """
    Initialise the converter with configurable output paths.

    Args:
        output_dir: Base directory for saving processed files
        attributes_filename: Filename for attributes output
        documents_filename: Filename for documents output
        annotated_documents_filename: Filename for annotated documents output
        attribute_mapping_filename: Filename for attribute ID to label mapping

    """
    if base_output_dir is None:
        logger.debug(
            "`base_output_dir` set to None; "
            "converting to empty string for compatibility."
        )
        base_output_dir = ""
    self.base_output_dir = Path(base_output_dir)

process_annotation_file(file_path, set_attribute_type=None) abstractmethod

Parse a raw input format into processed data.

Source code in deet/processors/base_converter.py

@abstractmethod
def process_annotation_file(
    self,
    file_path: str | Path,
    set_attribute_type: str | AttributeType | None = None,
) -> ProcessedAnnotationData:
    """Parse a raw input format into processed data."""
    ...

reload_output(file_path)

Read data back in, using OUTFILE_LOADERS and the subclass's processed_data_type.

Source code in deet/processors/base_converter.py

def reload_output(self, file_path: Path) -> ProcessedAnnotationData:
    """
    Read data back in, using OUTFILE_LOADERS and the
    subclass's processed_data_type.
    """
    loaded_data = {}
    for key, (filename, adapter) in self.OUTFILE_LOADERS.items():
        path = file_path / self.base_output_dir / filename
        if not path.exists():
            message = f"Expected {key.value} at {path}"
            raise FileNotFoundError(message)
        loaded_data[key] = adapter.validate_json(path.read_text())

    return self.processed_data_type(
        attributes=loaded_data[Outfiles.ATTRIBUTES],
        documents=loaded_data[Outfiles.DOCUMENTS],
        annotations=[],  # or derive from loaded_data if applicable
        annotated_documents=loaded_data[Outfiles.ANNOTATED_DOCUMENTS],
        attribute_id_to_label=loaded_data[Outfiles.ATTRIBUTE_LABEL_MAPPING],
    )

write_processed_data_to_file(processed_data, output_dir, outfiles_to_write=None)

Save processed data to structured files using Pydantic model serialisation.

Parameters:

Name	Type	Description	Default
processed_data	ProcessedAnnotationData	The processed data from process_annotation_file	required
output_dir	str | Path	Write all output (json) files from conversion to this	required
directory. NOTE		we output files will live in a sub-directory	required

Returns:

Type	Description
dict[str, str]	Dictionary mapping data types to saved file paths

Source code in deet/processors/base_converter.py

def write_processed_data_to_file(
    self,
    processed_data: ProcessedAnnotationData,
    output_dir: str | Path,
    outfiles_to_write: list[Outfiles] | None = None,
) -> dict[str, str]:
    """
    Save processed data to structured files using Pydantic model serialisation.

    Args:
        processed_data: The processed data from process_annotation_file
        output_dir: Write all output (json) files from conversion to this
        directory. NOTE: we output files will live in a sub-directory
        `self.base_output_dir`, which by default is `DEFAULT_BASE_OUTPUT_DIR`.
        so, if you want output files to go straight to `output_dir`, set
        `self.base_output_dir` to '' or None.

    Returns:
        Dictionary mapping data types to saved file paths

    """
    file_mappings = {
        Outfiles.ATTRIBUTES: processed_data.attributes,
        Outfiles.DOCUMENTS: processed_data.documents,
        Outfiles.ANNOTATED_DOCUMENTS: processed_data.annotated_documents,
        Outfiles.ATTRIBUTE_LABEL_MAPPING: processed_data.attribute_id_to_label,
    }
    # setting here to avoid mutable default.
    if outfiles_to_write is None:
        outfiles_to_write = [Outfiles.ATTRIBUTES, Outfiles.DOCUMENTS]

    file_mappings = {
        k: v for k, v in file_mappings.items() if k in outfiles_to_write
    }
    files_to_write_message = {",".join(k.value for k in file_mappings)}
    logger.info(f"writing {files_to_write_message} out...")

    user_dir = Path(output_dir)
    target_dir = user_dir / self.base_output_dir
    target_dir.mkdir(parents=True, exist_ok=True)
    logger.info(f"writing files to dir: {target_dir}")

    saved_files = {}

    for file_type, data_list in file_mappings.items():
        file_path = target_dir / self.outfile_names[file_type]
        logger.debug(f"writing file {file_type} to {file_path}")
        if file_type == Outfiles.ATTRIBUTE_LABEL_MAPPING:
            file_path.write_text(json.dumps(data_list), encoding="utf-8")
        else:
            file_path.write_text(
                json.dumps(
                    [item.model_dump(mode="json") for item in data_list],  # type: ignore[attr-defined]
                    indent=2,
                ),
                encoding="utf-8",
            )
        saved_files[file_type.value] = str(file_path)

    return saved_files

Outfiles

Bases: StrEnum

Enum of all outfiles producable by this module. Extend as required.

Source code in deet/processors/base_converter.py

class Outfiles(StrEnum):
    """Enum of all outfiles producable by this module. Extend as required."""

    ATTRIBUTES = auto()
    DOCUMENTS = auto()
    ANNOTATED_DOCUMENTS = auto()
    ATTRIBUTE_LABEL_MAPPING = auto()

converter_register

A register of supported supported annotation formats and a map to their converters.

SupportedImportFormat

Bases: StrEnum

Supported formats to import gold standard annotation data from.

Source code in deet/processors/converter_register.py

class SupportedImportFormat(StrEnum):
    """Supported formats to import gold standard annotation data from."""

    EPPI_JSON = auto()
    GENERIC_CSV = auto()

    def get_annotation_converter(self) -> AnnotationConverter:
        """
        Return an instance of the converter for the given data type.

        Converters are imported lazily here, rather than at module load, because
        they pull in the heavy document/SDK stack. Keeping this module import-light
        keeps CLI startup fast for commands that only need the enum.
        """
        from deet.processors.csv_annotation_converter import (
            CSVAnnotationConverter,
        )
        from deet.processors.eppi_annotation_converter import (
            EppiAnnotationConverter,
        )

        registry: dict[SupportedImportFormat, type[AnnotationConverter]] = {
            SupportedImportFormat.EPPI_JSON: EppiAnnotationConverter,
            SupportedImportFormat.GENERIC_CSV: CSVAnnotationConverter,
        }
        return registry[self]()

get_annotation_converter()

Return an instance of the converter for the given data type.

Converters are imported lazily here, rather than at module load, because they pull in the heavy document/SDK stack. Keeping this module import-light keeps CLI startup fast for commands that only need the enum.

Source code in deet/processors/converter_register.py

def get_annotation_converter(self) -> AnnotationConverter:
    """
    Return an instance of the converter for the given data type.

    Converters are imported lazily here, rather than at module load, because
    they pull in the heavy document/SDK stack. Keeping this module import-light
    keeps CLI startup fast for commands that only need the enum.
    """
    from deet.processors.csv_annotation_converter import (
        CSVAnnotationConverter,
    )
    from deet.processors.eppi_annotation_converter import (
        EppiAnnotationConverter,
    )

    registry: dict[SupportedImportFormat, type[AnnotationConverter]] = {
        SupportedImportFormat.EPPI_JSON: EppiAnnotationConverter,
        SupportedImportFormat.GENERIC_CSV: CSVAnnotationConverter,
    }
    return registry[self]()

csv_annotation_converter

Convert annotation CSV files to Pydantic models.

CSVAnnotationConverter

Bases: AnnotationConverter

A class to convert raw CSV (e.g. Covidence) annotations into structured Pydantic models.

This converter operates on flat CSV columns, infers field/column types, and produces attributes, documents, and annotated document records.

Source code in deet/processors/csv_annotation_converter.py

class CSVAnnotationConverter(AnnotationConverter):
    """
    A class to convert raw CSV (e.g. Covidence) annotations
    into structured Pydantic models.

     This converter operates on flat CSV columns, infers field/column types,
    and produces attributes, documents, and annotated document records.
    """

    def __init__(  # noqa: PLR0913
        self,
        base_output_dir: str | Path | None = DEFAULT_BASE_OUTPUT_DIR,
        attributes_filename: str = DEFAULT_ATTRIBUTES_FILENAME,
        documents_filename: str = DEFAULT_DOCUMENTS_FILENAME,
        annotated_documents_filename: str = DEFAULT_ANNOTATED_DOCUMENTS_FILENAME,
        attribute_mapping_filename: str = DEFAULT_ATTRIBUTE_MAPPING_FILENAME,
        config: CSVParserConfig | None = None,
    ) -> None:
        """
        Initialize the converter output configurations (base directory + filenames) to
        save the multiple files created during csv processing.

        Args:
            base_output_dir: Base directory for saving processed files
            attributes_filename: Filename for attributes output
            documents_filename: Filename for documents output
            annotated_documents_filename: Filename for annotated documents output
            attribute_mapping_filename: Filename for attribute ID to label mapping

        """
        self.config = config or CSVParserConfig()
        # If no directory given, write everything relative to current working directory
        if base_output_dir is None:
            logger.debug(
                "`base_output_dir` set to None; "
                "converting to empty string for compatibility."
            )
            base_output_dir = ""
        self.base_output_dir = Path(base_output_dir)

        # Output registry:
        # For each output type, provides the filename and how to serialize/validate it.
        # Used when writing and reloading the results

        self.OUTFILE_LOADERS: dict[Outfiles, tuple[str, TypeAdapter]] = {
            Outfiles.ATTRIBUTES: (
                attributes_filename,
                TypeAdapter(list[Attribute]),
            ),
            Outfiles.DOCUMENTS: (
                documents_filename,
                TypeAdapter(list[Document]),
            ),
            Outfiles.ANNOTATED_DOCUMENTS: (
                annotated_documents_filename,
                TypeAdapter(list[GoldStandardAnnotatedDocument]),
            ),
            Outfiles.ATTRIBUTE_LABEL_MAPPING: (
                attribute_mapping_filename,
                TypeAdapter(dict[int, str]),
            ),
        }

    @property
    def processed_data_type(self) -> type[ProcessedAnnotationData]:
        """Return ProcessedAnnotationData."""
        return ProcessedAnnotationData

    @staticmethod
    def _find_duplicate_column_names(column_names: list[str]) -> list[dict]:
        """
        Find duplicate column names and return their positions.

        Given a list of field/column names, returns a list of dictionaries
        containing the duplicated column name and the indices where it appears.

        Example:
            column_names = ["id", "name", "title", "name", "title"] ->
            {"name": [1, 3], "title": [2, 4]}

        """
        positions = defaultdict(list)
        for idx, name in enumerate(column_names):
            positions[name].append(idx)

        return [{name: ids} for name, ids in positions.items() if len(ids) > 1]

    @staticmethod
    def _infer_type(value: str) -> type | None:
        """
        Given a string value infer its type.
        Returns bool, int, float, or str depending on the value.
        Empty or null-like values return None.
        """
        if value is None or str(value).strip() == "":
            return None

        v = str(value).strip().lower()

        if v in {"true", "false", "t", "f"}:
            return bool
        try:
            float(v)
            if v.lstrip("-+").isdigit():
                return int
            return float  # noqa: TRY300
        except ValueError:
            return str

    # TODO: Extend support for list and dict AttributeType values.
    # Discussed use case: 'tags' associated with a document ([USA, Germnay, India])

    @staticmethod
    def _resolve_types(types: list) -> AttributeType:
        """
        Given a list of types, resolve to a single type and return an
        AttributeType object.

        Examples:
        [int, int, None, int, None, int] -> int -> AttributeType.INTEGER
        [int, int, None, float, None, int] -> float -> AttributeType.FLOAT

        """
        unique = set(types) - {None}
        if len(unique) == 0:
            msg = "Type not inferred. Sample is null."
            raise ValueError(msg)
        if unique == {str}:
            return AttributeType.STRING
        if unique == {bool}:
            return AttributeType.BOOL
        if unique == {int}:
            return AttributeType.INTEGER
        if unique.issubset({float, int}) and bool not in unique:
            return AttributeType.FLOAT

        msg = f"Multiple incompatible types found: {unique}"
        raise ColumnTypeInferenceError(msg)

    def _infer_column_types(
        self,
        rows: list[dict],
        column_names: list[str],
        sample_size: int | None = None,
    ) -> dict[str, AttributeType]:
        """
        Infer the data type of each column using up to `sample_size` non-null samples.

        By default, `sample_size` is the total number of rows. Specifying a smaller
        sample can speed up inference but may bias results if columns are sparse.

        Args:
            rows: List of dictionaries representing CSV rows.
            column_names: List of column names for which to infer data types.
            sample_size: Maximum number of non-null samples to use per column.

        Returns:
        - Dictionary mapping each column name to its inferred `AttributeType`.

        Example:
            rows = [
                {"id": "1", "age": "42", "gender": "F"},
                {"id": "2", "age": "", "gender": "M"},
                {"id": "3", "age": "36", "gender": ""},
            ]
            fieldnames = ["id", "age", "gender"]

            converter = CovidenceAnnotationConverter()
            converter._infer_column_types(rows, fieldnames, sample_size=2)
            # Output: {"id": int, "age": int, "gender": str}

        """
        n_rows = len(rows)

        # Collect observed value types per column (skipping missing values).
        # Used as evidence for downstream column type inference.
        # Example: column_type_observations = {age: [int, int], gender: [str, str]}
        column_type_evidence: defaultdict[str, list] = defaultdict(list)

        if sample_size is None:
            sample_size = n_rows

        for col_name in column_names:
            row_num = 0

            while (
                len(column_type_evidence[col_name]) < sample_size and row_num < n_rows
            ):
                att_type = self._infer_type(rows[row_num][col_name])

                if att_type is not None:
                    column_type_evidence[col_name].append(att_type)

                row_num += 1

        resolved_column_types: dict[str, AttributeType] = {}
        for col_name, col_types in column_type_evidence.items():
            resolved_column_types[col_name] = self._resolve_types(col_types)

        return resolved_column_types

    @staticmethod
    def _build_destiny_reference_dict_from_row(
        mapping: dict[str, str],
        row: dict[str, Any],
    ) -> dict[str, Any]:
        """
        Build a nested dictionary of reference data from a CSV row.

        Maps flat CSV columns to the nested structure expected by
        destiny_sdk.enhancements models, The resulting dictionary is used to create
        `EnhancementFileInput` and `ReferenceFileInput` objects.

        Args:
            mapping: Dictionary mapping nested keys(dot-separated strings) to CSV
            column names.
            row: Dictionary representing a CSV row.

        Returns:
            Nested dictionary representing the reference data. Example:

        -----------------------------------------------
        USER GUIDE
        -----------
        The `mapping` argument defines how CSV columns map to Destiny SDK fields.

        Format:
            mapping = {
                "<destiny_field>": "<csv_column_name>",
            }
        Rules:
            - Keys are dot-separated Destiny SDK field paths.
            - Values are CSV column names.

        SUPPORTED FIELDS
        ----------------
        - abstract
        - authorship
        - cited_by_count
        - created_date
        - updated_date
        - publication_date
        - publication_year
        - publisher
        - title
        - pagination.volume
        - pagination.issue
        - pagination.first_page
        - pagination.last_page
        - publication_venue.display_name
        - publication_venue.issn
        - publication_venue.venue_type
        - publication_venue.issn_l
        - publication_venue.host_organization_name

        -----------------------------------------------

        Example:
        mapping = {"publication_venue.display_name": "journal", "abstract": "abstract"}
        row = {"journal": "Nature Climate Change", "abstract": "ABCDEFXXXXXX"}
        # Output
            {
            "publication_venue": {"display_name": "Nature Climate Change"},
            "abstract": "ABCDEFXXXXXX"
            }


        """
        result: dict[str, Any] = {}

        for key, column_name in mapping.items():
            value = row[column_name].strip()
            if value == "":
                value = None

            parts = key.split(".")
            d = result

            for part in parts[:-1]:
                d = d.setdefault(part, {})

            d[parts[-1]] = value

        return result

    # TODO: Allow users to provide seperator
    def _build_destiny_authorship_list(self, authors: str) -> list[Authorship]:
        """
        Build a list of a `Authorship` objects  (as defined in destiny_sdk.enhancements)
        from a string. The returned list is in the format expected by
        `BibliographicMetadataEnhancement` for authorship.

        Args:
            authors: A semicolon-separated string of author names. eg:("Alice; Bob; Mo")

        Returns:
            List of `Authorship` objects representing each author with their position.

        """
        # Split on semicolons, strip whitespace, and remove any empty entries
        sep: str = self.config.author_separator
        clean_authors = [
            author.strip() for author in authors.split(sep) if author.strip()
        ]

        authorship: list[Authorship] = []
        if not clean_authors:
            return authorship

        for i, author in enumerate(clean_authors):
            position = AuthorPosition.MIDDLE
            if i == 0:
                position = AuthorPosition.FIRST
            elif i == len(clean_authors) - 1:
                position = AuthorPosition.LAST
            authorship.append(Authorship(display_name=author, position=position))
        return authorship

    def build_destiny_reference(
        self,
        row: dict,
        mapping: dict[str, Any],
        source: str = "deet CSV converter",
    ) -> ReferenceFileInput:
        """

        Convert a CSV row into a `ReferenceFileInput` for destiny_sdk.reference.

        This method extracts bibliographic metadata and abstract content from the
        given row using the provided reference field mapping. It constructs
        `BibliographicMetadataEnhancement` and `AbstractContentEnhancement`
        objects, then wraps them in `EnhancementFileInput` objects, and finally
        returns a `ReferenceFileInput`.

        Args:
            row: Dictionary representing a single CSV row.
            mapping: Dictionary mapping nested keys (dot-separated strings) to CSV
                 column names.
            source: Optional string indicating the source of the data; defaults to
                "deet CSV converter"
        Returns:
            ReferenceFileInput object containing all extracted enhancements

        """
        reference_dict = self._build_destiny_reference_dict_from_row(mapping, row)

        abstract = None
        if "abstract" in reference_dict:
            if reference_dict["abstract"]:
                abstract = AbstractContentEnhancement(
                    abstract=reference_dict["abstract"],
                    process=AbstractProcessType.UNINVERTED,
                )
            else:
                logger.warning(
                    f"document_id={row.get('document_id')!r} "
                    f"name={row.get('name')!r} has a blank abstract; "
                    "skipping AbstractContentEnhancement for this row."
                )

        reference_dict["authorship"] = (
            self._build_destiny_authorship_list(reference_dict["authorship"])
            if "authorship" in reference_dict
            and isinstance(reference_dict["authorship"], str)
            else None
        )

        bibliographic_data = BibliographicMetadataEnhancement(**reference_dict)

        enhancements = [
            EnhancementFileInput(
                source=source,
                visibility=Visibility.PUBLIC,
                content=content,
            )
            for content in [abstract, bibliographic_data]
            if content is not None
        ]

        return ReferenceFileInput(
            visibility=Visibility.PUBLIC,
            enhancements=enhancements,
        )

    def load_csv(
        self,
        file_path: Path,
        attribute_fields: list[str] | None = None,
        reference_fields: dict | None = None,
    ) -> tuple[list[str], dict[str, str], list[str], list[dict[str, Any]]]:
        """
        Load a CSV, normalize headers, and return all column names, attribute names,
        reference names, and rows.
        """
        path = Path(file_path)
        with path.open(newline="", encoding="utf-8-sig") as f:
            csv_reader = csv.DictReader(f)

            # normalize headers BEFORE reading rows
            raw_headers = csv_reader.fieldnames or []
            colnames: list[str] = [h.strip().lower() for h in raw_headers]
            csv_reader.fieldnames = colnames

            # --- validate duplicates ---
            dup_fields = self._find_duplicate_column_names(colnames)
            if dup_fields:
                msg = f"{len(dup_fields)} Duplicate fieldnames found: {dup_fields}"
                raise ValueError(msg)

            # --- validate required fields ---
            meta_fields = {"name", "document_id"}
            missing = meta_fields - set(colnames)
            if missing:
                msg = f"Required columns missing: {missing}"
                raise ValueError(msg)

            # --- validate reference fields ---
            # If reference_fields is not povided....
            if reference_fields is None:
                if self.config.auto_assign_reference_fields:
                    logger.info("Auto assigning reference fields")
                    reference_fields = {
                        ref_field: ref_field
                        for ref_field in ALLOWED_REFERENCE_MAPPING_KEYS
                        if ref_field in colnames
                    }
                else:
                    logger.info("No reference fields provided and auto assign is False")
                    reference_fields = {}

            # If reference_fields is povided....
            if reference_fields:
                invalid_keys = set(reference_fields) - ALLOWED_REFERENCE_MAPPING_KEYS
                if invalid_keys:
                    msg = f"Invalid mapping keys: {invalid_keys}"
                    raise ValueError(msg)

                # normalize reference fields
                reference_fields = {
                    k: v.strip().lower() for k, v in reference_fields.items()
                }

                unknown = set(reference_fields.values()) - set(colnames)
                if unknown:
                    msg = f"Reference fields not found in CSV: {unknown}"
                    raise ValueError(msg)

            # --- validate attribute fields ---
            # normalize and validate provided attribute fields
            if attribute_fields is None:
                logger.info("No attribute fields provided")
                excluded_fields = meta_fields | set(reference_fields.values())
                resolved_attribute_fields = [
                    h for h in colnames if h not in excluded_fields
                ]
            else:
                resolved_attribute_fields = [
                    field.strip().lower() for field in attribute_fields
                ]
                unknown_attributes = set(resolved_attribute_fields) - set(colnames)
                if unknown_attributes:
                    msg = f"Attribute fields not found in CSV: {unknown_attributes}"
                    raise ValueError(msg)

            # Read rows into mem once
            rows = list(csv_reader)

        return colnames, reference_fields, resolved_attribute_fields, rows

    def build_attributes(
        self,
        attribute_fields: list[str],
        rows: list[dict],
    ) -> list[Attribute]:
        """
        Build a list of `Attribute` objects from CSV rows and specified attribute
        columns.

        Infers the `AttributeType` for each attribute field and assigns a unique ID
        to each `Attribute`.
        """
        attribute_types = self._infer_column_types(rows, attribute_fields)
        attributes = []
        for idx, (k, v) in enumerate(attribute_types.items()):
            attribute = Attribute(
                output_data_type=v,
                attribute_id=idx,
                attribute_label=k,
            )
            attributes.append(attribute)

        return attributes

    def build_documents_and_annotations(
        self,
        attributes: list[Attribute],
        reference_fields: dict,
        rows: list[dict],
    ) -> tuple[list[Document], list[GoldStandardAnnotatedDocument]]:
        """
        Build `Document` objects and their corresponding GoldStandardAnnotatedDocument`s
        from CSV rows.

        Args:
            attributes:
            reference_fields: Dictionary mapping reference field labels (as defined in
                destiny_sdk.enhancements) to CSV column names.

            rows: List of dictionaries representing all CSV rows.

        Returns:
            A tuple containing: Document and List[GoldStandardAnnotatedDocument]

        """
        annotated_documents: list[GoldStandardAnnotatedDocument] = []
        documents: list[Document] = []

        # --- To acces attributes by their names ---
        attr_by_label = {a.attribute_label: a for a in attributes}

        for row in rows:
            # --- Build destiny Reference given row ---
            row_reference = self.build_destiny_reference(row, reference_fields)
            # --- Build Document ---
            document = Document(
                name=row["name"],
                citation=row_reference,
                document_id=row["document_id"],
            )
            document.init_document_identity()
            documents.append(document)

            # --- Build Document Annotations ---
            annotations = []
            for label, attr in attr_by_label.items():
                raw_value = row[label].strip()

                annotation = GoldStandardAnnotation(
                    attribute=attr,
                    raw_data=raw_value,
                    annotation_type=AnnotationType.HUMAN,
                )

                annotations.append(annotation)

            # --- Build Annotated Documents = Attach annotations to document ---
            annotated_doc = GoldStandardAnnotatedDocument(
                document=document,
                annotations=annotations,
            )

            annotated_documents.append(annotated_doc)

        return documents, annotated_documents

    def process_annotation_file(
        self,
        file_path: str | Path,
        set_attribute_type: str | AttributeType | None = None,
        attribute_fields: list | None = None,
        reference_fields: dict | None = None,
    ) -> ProcessedAnnotationData:
        """
        Process a complete CSV annotation file and return structured data.

        Each row is assumed to represent a document, and columns correspond to
        different types of fields (metadata, reference information, and attributes).

        Args:
            file_path: Path to the CSV annotation file.
            attribute_fields: List of column names to be treated as document attributes.
            reference_fields: Dictionary mapping reference field labels(as defined in
            destiny_sdk.enhancements) to corresponding CSV column names.


        Returns:
            ProcessedAnnotationData containing all processed data.

        """
        file_path = Path(file_path)

        if set_attribute_type is not None:
            msg = (
                "CsvAnnotationConverter does not support set_attribute_type; "
                "use attribute_fields instead."
            )
            raise NotImplementedError(msg)
        logger.info(f"Processing annotation file: {file_path}")

        colnames, reference_fields, attribute_fields, rows = self.load_csv(
            file_path, attribute_fields, reference_fields
        )

        attributes = self.build_attributes(attribute_fields, rows)

        documents, annotated_documents = self.build_documents_and_annotations(
            attributes, reference_fields, rows
        )
        attribute_id_to_label = {
            attr.attribute_id: attr.attribute_label for attr in attributes
        }

        logger.info(
            f"Processed {len(attributes)} attributes, {len(documents)} documents, "
            f" {len(annotated_documents)} annotated documents"
        )

        return ProcessedAnnotationData(
            attributes=attributes,
            documents=documents,
            annotated_documents=annotated_documents,
            attribute_id_to_label=attribute_id_to_label,
            annotations=[],  # keep type compatible
        )

processed_data_type property

Return ProcessedAnnotationData.

__init__(base_output_dir=DEFAULT_BASE_OUTPUT_DIR, attributes_filename=DEFAULT_ATTRIBUTES_FILENAME, documents_filename=DEFAULT_DOCUMENTS_FILENAME, annotated_documents_filename=DEFAULT_ANNOTATED_DOCUMENTS_FILENAME, attribute_mapping_filename=DEFAULT_ATTRIBUTE_MAPPING_FILENAME, config=None)

Initialize the converter output configurations (base directory + filenames) to save the multiple files created during csv processing.

Parameters:

Name	Type	Description	Default
base_output_dir	str | Path | None	Base directory for saving processed files	DEFAULT_BASE_OUTPUT_DIR
attributes_filename	str	Filename for attributes output	DEFAULT_ATTRIBUTES_FILENAME
documents_filename	str	Filename for documents output	DEFAULT_DOCUMENTS_FILENAME
annotated_documents_filename	str	Filename for annotated documents output	DEFAULT_ANNOTATED_DOCUMENTS_FILENAME
attribute_mapping_filename	str	Filename for attribute ID to label mapping	DEFAULT_ATTRIBUTE_MAPPING_FILENAME

Source code in deet/processors/csv_annotation_converter.py

def __init__(  # noqa: PLR0913
    self,
    base_output_dir: str | Path | None = DEFAULT_BASE_OUTPUT_DIR,
    attributes_filename: str = DEFAULT_ATTRIBUTES_FILENAME,
    documents_filename: str = DEFAULT_DOCUMENTS_FILENAME,
    annotated_documents_filename: str = DEFAULT_ANNOTATED_DOCUMENTS_FILENAME,
    attribute_mapping_filename: str = DEFAULT_ATTRIBUTE_MAPPING_FILENAME,
    config: CSVParserConfig | None = None,
) -> None:
    """
    Initialize the converter output configurations (base directory + filenames) to
    save the multiple files created during csv processing.

    Args:
        base_output_dir: Base directory for saving processed files
        attributes_filename: Filename for attributes output
        documents_filename: Filename for documents output
        annotated_documents_filename: Filename for annotated documents output
        attribute_mapping_filename: Filename for attribute ID to label mapping

    """
    self.config = config or CSVParserConfig()
    # If no directory given, write everything relative to current working directory
    if base_output_dir is None:
        logger.debug(
            "`base_output_dir` set to None; "
            "converting to empty string for compatibility."
        )
        base_output_dir = ""
    self.base_output_dir = Path(base_output_dir)

    # Output registry:
    # For each output type, provides the filename and how to serialize/validate it.
    # Used when writing and reloading the results

    self.OUTFILE_LOADERS: dict[Outfiles, tuple[str, TypeAdapter]] = {
        Outfiles.ATTRIBUTES: (
            attributes_filename,
            TypeAdapter(list[Attribute]),
        ),
        Outfiles.DOCUMENTS: (
            documents_filename,
            TypeAdapter(list[Document]),
        ),
        Outfiles.ANNOTATED_DOCUMENTS: (
            annotated_documents_filename,
            TypeAdapter(list[GoldStandardAnnotatedDocument]),
        ),
        Outfiles.ATTRIBUTE_LABEL_MAPPING: (
            attribute_mapping_filename,
            TypeAdapter(dict[int, str]),
        ),
    }

build_attributes(attribute_fields, rows)

Build a list of Attribute objects from CSV rows and specified attribute columns.

Infers the AttributeType for each attribute field and assigns a unique ID to each Attribute.

Source code in deet/processors/csv_annotation_converter.py

def build_attributes(
    self,
    attribute_fields: list[str],
    rows: list[dict],
) -> list[Attribute]:
    """
    Build a list of `Attribute` objects from CSV rows and specified attribute
    columns.

    Infers the `AttributeType` for each attribute field and assigns a unique ID
    to each `Attribute`.
    """
    attribute_types = self._infer_column_types(rows, attribute_fields)
    attributes = []
    for idx, (k, v) in enumerate(attribute_types.items()):
        attribute = Attribute(
            output_data_type=v,
            attribute_id=idx,
            attribute_label=k,
        )
        attributes.append(attribute)

    return attributes

build_destiny_reference(row, mapping, source='deet CSV converter')

Convert a CSV row into a ReferenceFileInput for destiny_sdk.reference.

This method extracts bibliographic metadata and abstract content from the given row using the provided reference field mapping. It constructs BibliographicMetadataEnhancement and AbstractContentEnhancement objects, then wraps them in EnhancementFileInput objects, and finally returns a ReferenceFileInput.

Parameters:

Name	Type	Description	Default
row	dict	Dictionary representing a single CSV row.	required
mapping	dict[str, Any]	Dictionary mapping nested keys (dot-separated strings) to CSV column names.	required
source	str	Optional string indicating the source of the data; defaults to "deet CSV converter"	'deet CSV converter'

Returns: ReferenceFileInput object containing all extracted enhancements

Source code in deet/processors/csv_annotation_converter.py

def build_destiny_reference(
    self,
    row: dict,
    mapping: dict[str, Any],
    source: str = "deet CSV converter",
) -> ReferenceFileInput:
    """

    Convert a CSV row into a `ReferenceFileInput` for destiny_sdk.reference.

    This method extracts bibliographic metadata and abstract content from the
    given row using the provided reference field mapping. It constructs
    `BibliographicMetadataEnhancement` and `AbstractContentEnhancement`
    objects, then wraps them in `EnhancementFileInput` objects, and finally
    returns a `ReferenceFileInput`.

    Args:
        row: Dictionary representing a single CSV row.
        mapping: Dictionary mapping nested keys (dot-separated strings) to CSV
             column names.
        source: Optional string indicating the source of the data; defaults to
            "deet CSV converter"
    Returns:
        ReferenceFileInput object containing all extracted enhancements

    """
    reference_dict = self._build_destiny_reference_dict_from_row(mapping, row)

    abstract = None
    if "abstract" in reference_dict:
        if reference_dict["abstract"]:
            abstract = AbstractContentEnhancement(
                abstract=reference_dict["abstract"],
                process=AbstractProcessType.UNINVERTED,
            )
        else:
            logger.warning(
                f"document_id={row.get('document_id')!r} "
                f"name={row.get('name')!r} has a blank abstract; "
                "skipping AbstractContentEnhancement for this row."
            )

    reference_dict["authorship"] = (
        self._build_destiny_authorship_list(reference_dict["authorship"])
        if "authorship" in reference_dict
        and isinstance(reference_dict["authorship"], str)
        else None
    )

    bibliographic_data = BibliographicMetadataEnhancement(**reference_dict)

    enhancements = [
        EnhancementFileInput(
            source=source,
            visibility=Visibility.PUBLIC,
            content=content,
        )
        for content in [abstract, bibliographic_data]
        if content is not None
    ]

    return ReferenceFileInput(
        visibility=Visibility.PUBLIC,
        enhancements=enhancements,
    )

build_documents_and_annotations(attributes, reference_fields, rows)

Build Document objects and their corresponding GoldStandardAnnotatedDocument`s from CSV rows.

Parameters:

Name	Type	Description	Default
attributes	list[Attribute]		required
reference_fields	dict	Dictionary mapping reference field labels (as defined in destiny_sdk.enhancements) to CSV column names.	required
rows	list[dict]	List of dictionaries representing all CSV rows.	required

Returns:

Type	Description
tuple[list[Document], list[GoldStandardAnnotatedDocument]]	A tuple containing: Document and List[GoldStandardAnnotatedDocument]

Source code in deet/processors/csv_annotation_converter.py

def build_documents_and_annotations(
    self,
    attributes: list[Attribute],
    reference_fields: dict,
    rows: list[dict],
) -> tuple[list[Document], list[GoldStandardAnnotatedDocument]]:
    """
    Build `Document` objects and their corresponding GoldStandardAnnotatedDocument`s
    from CSV rows.

    Args:
        attributes:
        reference_fields: Dictionary mapping reference field labels (as defined in
            destiny_sdk.enhancements) to CSV column names.

        rows: List of dictionaries representing all CSV rows.

    Returns:
        A tuple containing: Document and List[GoldStandardAnnotatedDocument]

    """
    annotated_documents: list[GoldStandardAnnotatedDocument] = []
    documents: list[Document] = []

    # --- To acces attributes by their names ---
    attr_by_label = {a.attribute_label: a for a in attributes}

    for row in rows:
        # --- Build destiny Reference given row ---
        row_reference = self.build_destiny_reference(row, reference_fields)
        # --- Build Document ---
        document = Document(
            name=row["name"],
            citation=row_reference,
            document_id=row["document_id"],
        )
        document.init_document_identity()
        documents.append(document)

        # --- Build Document Annotations ---
        annotations = []
        for label, attr in attr_by_label.items():
            raw_value = row[label].strip()

            annotation = GoldStandardAnnotation(
                attribute=attr,
                raw_data=raw_value,
                annotation_type=AnnotationType.HUMAN,
            )

            annotations.append(annotation)

        # --- Build Annotated Documents = Attach annotations to document ---
        annotated_doc = GoldStandardAnnotatedDocument(
            document=document,
            annotations=annotations,
        )

        annotated_documents.append(annotated_doc)

    return documents, annotated_documents

load_csv(file_path, attribute_fields=None, reference_fields=None)

Load a CSV, normalize headers, and return all column names, attribute names, reference names, and rows.

Source code in deet/processors/csv_annotation_converter.py

def load_csv(
    self,
    file_path: Path,
    attribute_fields: list[str] | None = None,
    reference_fields: dict | None = None,
) -> tuple[list[str], dict[str, str], list[str], list[dict[str, Any]]]:
    """
    Load a CSV, normalize headers, and return all column names, attribute names,
    reference names, and rows.
    """
    path = Path(file_path)
    with path.open(newline="", encoding="utf-8-sig") as f:
        csv_reader = csv.DictReader(f)

        # normalize headers BEFORE reading rows
        raw_headers = csv_reader.fieldnames or []
        colnames: list[str] = [h.strip().lower() for h in raw_headers]
        csv_reader.fieldnames = colnames

        # --- validate duplicates ---
        dup_fields = self._find_duplicate_column_names(colnames)
        if dup_fields:
            msg = f"{len(dup_fields)} Duplicate fieldnames found: {dup_fields}"
            raise ValueError(msg)

        # --- validate required fields ---
        meta_fields = {"name", "document_id"}
        missing = meta_fields - set(colnames)
        if missing:
            msg = f"Required columns missing: {missing}"
            raise ValueError(msg)

        # --- validate reference fields ---
        # If reference_fields is not povided....
        if reference_fields is None:
            if self.config.auto_assign_reference_fields:
                logger.info("Auto assigning reference fields")
                reference_fields = {
                    ref_field: ref_field
                    for ref_field in ALLOWED_REFERENCE_MAPPING_KEYS
                    if ref_field in colnames
                }
            else:
                logger.info("No reference fields provided and auto assign is False")
                reference_fields = {}

        # If reference_fields is povided....
        if reference_fields:
            invalid_keys = set(reference_fields) - ALLOWED_REFERENCE_MAPPING_KEYS
            if invalid_keys:
                msg = f"Invalid mapping keys: {invalid_keys}"
                raise ValueError(msg)

            # normalize reference fields
            reference_fields = {
                k: v.strip().lower() for k, v in reference_fields.items()
            }

            unknown = set(reference_fields.values()) - set(colnames)
            if unknown:
                msg = f"Reference fields not found in CSV: {unknown}"
                raise ValueError(msg)

        # --- validate attribute fields ---
        # normalize and validate provided attribute fields
        if attribute_fields is None:
            logger.info("No attribute fields provided")
            excluded_fields = meta_fields | set(reference_fields.values())
            resolved_attribute_fields = [
                h for h in colnames if h not in excluded_fields
            ]
        else:
            resolved_attribute_fields = [
                field.strip().lower() for field in attribute_fields
            ]
            unknown_attributes = set(resolved_attribute_fields) - set(colnames)
            if unknown_attributes:
                msg = f"Attribute fields not found in CSV: {unknown_attributes}"
                raise ValueError(msg)

        # Read rows into mem once
        rows = list(csv_reader)

    return colnames, reference_fields, resolved_attribute_fields, rows

process_annotation_file(file_path, set_attribute_type=None, attribute_fields=None, reference_fields=None)

Process a complete CSV annotation file and return structured data.

Each row is assumed to represent a document, and columns correspond to different types of fields (metadata, reference information, and attributes).

Parameters:

Name	Type	Description	Default
file_path	str | Path	Path to the CSV annotation file.	required
attribute_fields	list | None	List of column names to be treated as document attributes.	None
reference_fields	dict | None	Dictionary mapping reference field labels(as defined in	None

Returns:

Type	Description
ProcessedAnnotationData	ProcessedAnnotationData containing all processed data.

Source code in deet/processors/csv_annotation_converter.py

def process_annotation_file(
    self,
    file_path: str | Path,
    set_attribute_type: str | AttributeType | None = None,
    attribute_fields: list | None = None,
    reference_fields: dict | None = None,
) -> ProcessedAnnotationData:
    """
    Process a complete CSV annotation file and return structured data.

    Each row is assumed to represent a document, and columns correspond to
    different types of fields (metadata, reference information, and attributes).

    Args:
        file_path: Path to the CSV annotation file.
        attribute_fields: List of column names to be treated as document attributes.
        reference_fields: Dictionary mapping reference field labels(as defined in
        destiny_sdk.enhancements) to corresponding CSV column names.


    Returns:
        ProcessedAnnotationData containing all processed data.

    """
    file_path = Path(file_path)

    if set_attribute_type is not None:
        msg = (
            "CsvAnnotationConverter does not support set_attribute_type; "
            "use attribute_fields instead."
        )
        raise NotImplementedError(msg)
    logger.info(f"Processing annotation file: {file_path}")

    colnames, reference_fields, attribute_fields, rows = self.load_csv(
        file_path, attribute_fields, reference_fields
    )

    attributes = self.build_attributes(attribute_fields, rows)

    documents, annotated_documents = self.build_documents_and_annotations(
        attributes, reference_fields, rows
    )
    attribute_id_to_label = {
        attr.attribute_id: attr.attribute_label for attr in attributes
    }

    logger.info(
        f"Processed {len(attributes)} attributes, {len(documents)} documents, "
        f" {len(annotated_documents)} annotated documents"
    )

    return ProcessedAnnotationData(
        attributes=attributes,
        documents=documents,
        annotated_documents=annotated_documents,
        attribute_id_to_label=attribute_id_to_label,
        annotations=[],  # keep type compatible
    )

CSVParserConfig pydantic-model

Bases: BaseModel

Configuration Seetings for parsing CSV.

Fields:

• author_separator (str)
• auto_assign_reference_fields (Boolean)

Source code in deet/processors/csv_annotation_converter.py

class CSVParserConfig(BaseModel):
    """Configuration Seetings for parsing CSV."""

    author_separator: str = ";"
    auto_assign_reference_fields: Boolean = True

ColumnTypeInferenceError

Bases: Exception

Raised when column type inference fails due to incompatible types.

Source code in deet/processors/csv_annotation_converter.py

class ColumnTypeInferenceError(Exception):
    """Raised when column type inference fails due to incompatible types."""

directory_processor

Tools to create Documents directly from pdf or md files.

create_documents_from_directory(directory_path)

Collect markdown and PDF files and return linked documents with parsing stats.

Collects .md files and .pdf files. PDFs whose stem already has a matching .md are skipped.

Source code in deet/processors/directory_processor.py

def create_documents_from_directory(
    directory_path: Path,
) -> tuple[Sequence[Document], dict[str, DocumentParsingStats]]:
    """
    Collect markdown and PDF files and return linked documents with parsing stats.

    Collects ``.md`` files and ``.pdf`` files. PDFs whose stem already has a
    matching ``.md`` are skipped.
    """
    target_files = list(directory_path.glob("*.md"))
    md_stems = {p.stem for p in target_files}

    target_files.extend(
        p for p in directory_path.glob("*.pdf") if p.stem not in md_stems
    )

    mock_references = []
    mock_mappings = []

    for file_path in target_files:
        document_id = hash_n_strings_to_document_id([file_path.stem])

        mock_ref = Document(
            name=file_path.name,
            document_id=document_id,
            is_linked=False,
            is_final=False,
            citation=ReferenceFileInput(),
        )
        mock_ref.init_document_identity()

        mock_references.append(mock_ref)
        mock_mappings.append(
            DocumentReferenceMapping(document_id=document_id, file_path=file_path)
        )

    if not mock_references:
        return [], {}

    linker = DocumentReferenceLinker(
        references=mock_references,
        document_reference_mapping=mock_mappings,
        document_base_dir=directory_path,
    )
    documents = linker.link_many_references_parsed_documents()
    return documents, linker.document_parsing_stats

eppi_annotation_converter

Convert annotation JSON files to Pydantic models.

EppiAnnotationConverter

Bases: AnnotationConverter

A class to convert raw EPPI-Reviewer JSON annotations into structured Pydantic models.

This converter handles the complex hierarchical structure of EPPI attributes by flattening them while preserving parent-child relationships through path information.

Source code in deet/processors/eppi_annotation_converter.py

class EppiAnnotationConverter(AnnotationConverter):
    """
    A class to convert raw EPPI-Reviewer JSON annotations
    into structured Pydantic models.

    This converter handles the complex hierarchical
    structure of EPPI attributes by flattening
    them while preserving parent-child relationships
    through path information.
    """

    def __init__(
        self,
        base_output_dir: str | Path | None = DEFAULT_BASE_OUTPUT_DIR,
        attributes_filename: str = DEFAULT_ATTRIBUTES_FILENAME,
        documents_filename: str = DEFAULT_DOCUMENTS_FILENAME,
        annotated_documents_filename: str = DEFAULT_ANNOTATED_DOCUMENTS_FILENAME,
        attribute_mapping_filename: str = DEFAULT_ATTRIBUTE_MAPPING_FILENAME,
    ) -> None:
        """
        Initialise the converter with configurable output paths.
        Set self.OUTFILE_LOADERS mapping the outfiles to be read/written to a
        filename, and a TypeAdapter defining the type of Pydantic Model to
        read back in when deserialising.

        Args:
            output_dir: Base directory for saving processed files
            attributes_filename: Filename for attributes output
            documents_filename: Filename for documents output
            annotated_documents_filename: Filename for annotated documents output
            attribute_mapping_filename: Filename for attribute ID to label mapping

        """
        if base_output_dir is None:
            logger.debug(
                "`base_output_dir` set to None; "
                "converting to empty string for compatibility."
            )
            base_output_dir = ""
        self.base_output_dir = Path(base_output_dir)

        # extend below if adding more output files in `Outfiles`.
        self.OUTFILE_LOADERS: dict[Outfiles, tuple[str, TypeAdapter]] = {
            Outfiles.ATTRIBUTES: (
                attributes_filename,
                TypeAdapter(list[EppiAttribute]),
            ),
            Outfiles.DOCUMENTS: (
                documents_filename,
                TypeAdapter(list[EppiDocument]),
            ),
            Outfiles.ANNOTATED_DOCUMENTS: (
                annotated_documents_filename,
                TypeAdapter(list[EppiGoldStandardAnnotatedDocument]),
            ),
            Outfiles.ATTRIBUTE_LABEL_MAPPING: (
                attribute_mapping_filename,
                TypeAdapter(dict[int, str]),
            ),
        }

    @property
    def processed_data_type(self) -> type[ProcessedEppiAnnotationData]:
        """Return ProcessedEppiAnnotationData."""
        return ProcessedEppiAnnotationData

    def flatten_attributes_hierarchy(
        self, attributes_list: list[dict[str, Any]], parent_path: str = ""
    ) -> list[dict[str, Any]]:
        """
        Recursively flatten the hierarchical attributes structure.

        Args:
            attributes_list: List of attribute dictionaries from the JSON
            parent_path: Path to the parent attribute (for hierarchy tracking)

        Returns:
            List of flattened attribute dictionaries with hierarchy information

        """
        flattened = []

        for attr in attributes_list:
            # extract children before modifying  dict
            child_attributes = attr.get("Attributes", {}).get("AttributesList", [])

            attr["hierarchy_path"] = parent_path
            attr["hierarchy_level"] = (
                len(parent_path.split(" > ")) if parent_path else 0
            )
            attr["is_leaf"] = not bool(child_attributes)

            flattened.append(attr)

            # recursive extension
            if child_attributes:
                current_path = (
                    f"{parent_path} > {attr.get('AttributeName', '')}"
                    if parent_path
                    else attr.get("AttributeName", "")
                )
                flattened.extend(
                    self.flatten_attributes_hierarchy(child_attributes, current_path)
                )

        return flattened

    def _extract_attributes_from_codesets(
        self, raw_data: EppiRawData
    ) -> list[dict[str, Any]]:
        """Extract and flatten attributes from CodeSets using structured models."""
        return raw_data.extract_all_attributes(self.flatten_attributes_hierarchy)

    def convert_to_eppi_attributes(
        self,
        flattened_attributes: list[dict[str, Any]],
        set_attribute_type: AttributeType | None = None,
    ) -> list[EppiAttribute]:
        """
        Convert flattened attribute data to EppiAttribute models.

        Args:
            flattened_attributes: List of flattened attribute dictionaries

        Returns:
            List of EppiAttribute models

        """
        out = []
        for att_dict in flattened_attributes:
            if "AttributeId" not in att_dict:
                att_dict["AttributeId"] = 0
            new_attribute = EppiAttribute(**att_dict)
            if set_attribute_type:
                logger.debug(
                    f"setting custom attribute type {set_attribute_type.value} "
                    f"for attribute {new_attribute.attribute_id}"
                )
                new_attribute.output_data_type = set_attribute_type
            out.append(new_attribute)
        return out

    def _process_text_details(
        self, text_details: list[dict[str, Any]]
    ) -> tuple[list[str], list[EppiItemAttributeFullTextDetails]]:
        """
        Process ItemAttributeFullTextDetails to extract texts and create detail objects.

        Args:
            text_details: List of text detail dictionaries from EPPI JSON

        Returns:
            Tuple of (extracted_texts, item_attribute_details)

        """
        extracted_texts = []
        item_attribute_details = []

        for text_detail in text_details:
            text = text_detail.get("Text", "")
            if text:
                extracted_texts.append(text)

            detail = EppiItemAttributeFullTextDetails(
                item_document_id=text_detail.get("ItemDocumentId"),
                text=text,
                item_arm=text_detail.get("ItemArm", ""),
            )
            item_attribute_details.append(detail)

        return extracted_texts, item_attribute_details

    def _convert_single_annotation(
        self,
        annotation: dict[str, Any],
        attributes_lookup: dict[int, EppiAttribute],
        attribute_id_to_label: dict[int, str] | None = None,
    ) -> EppiGoldStandardAnnotation:
        """
        Convert a single annotation dictionary to EppiGoldStandardAnnotation.

        Args:
            annotation: Single annotation dictionary from EPPI JSON
            attributes_lookup: Lookup dictionary for attributes
            attribute_id_to_label: Mapping from attribute ID to label

        Returns:
            EppiGoldStandardAnnotation model

        Note:
            If attribute is not found in lookup, creates a basic attribute using
            the attribute_id_to_label mapping. All annotations from JSON are
            marked as HUMAN type. ``raw_data`` follows
            :func:`eppi_output_data_from_eppi_fields` (booleans from code presence;
            other types from ``AdditionalText`` only).

        """
        text_details = annotation.get("ItemAttributeFullTextDetails", [])
        _extracted_texts, item_attribute_details = self._process_text_details(
            text_details
        )

        # Look up the attribute from the attributes list
        if (attribute_id := annotation.get("AttributeId")) is None:
            missing_attr_id_msg = (
                "Annotation is missing required field 'AttributeId'. "
                "All annotations must have an AttributeId."
            )
            raise ValueError(missing_attr_id_msg)

        # find attribute in attributes_lookup
        if (attribute := attributes_lookup.get(attribute_id)) is None:
            attr_not_found_msg = (
                f"Attribute with ID {attribute_id} not found in attributes list. "
                "All annotations must reference a valid attribute from the CodeSets."
            )
            raise ValueError(attr_not_found_msg)

        # ensure the attribute has the correct label from the mapping if available
        if attribute_id_to_label is not None and attribute_id in attribute_id_to_label:
            attribute.attribute_label = attribute_id_to_label[attribute_id]

        additional_text = str(annotation.get("AdditionalText", "") or "")
        typed_raw_data: bool | str | int | float | list[Any] | dict[str, Any] = (
            eppi_output_data_from_eppi_fields(
                attribute.output_data_type, additional_text=additional_text
            )
        )

        return EppiGoldStandardAnnotation(
            attribute=attribute,
            additional_text=annotation.get("AdditionalText", ""),
            arm_id=annotation.get("ArmId"),
            arm_title=annotation.get("ArmTitle", ""),
            arm_description=annotation.get("ArmDescription", ""),
            raw_data=typed_raw_data,
            annotation_type=AnnotationType.HUMAN,
            item_attribute_full_text_details=item_attribute_details,
        )

    def _merge_raw_values(
        self, existing: SUPPORTED_TYPES, new: SUPPORTED_TYPES
    ) -> SUPPORTED_TYPES:
        """Merge the raw_data of duplicated annotations."""
        if existing is None:
            return new
        if new is None:
            return existing
        if isinstance(existing, list) and isinstance(new, list):
            return existing + new
        if isinstance(existing, dict) and isinstance(new, dict):
            return {**existing, **new}

        return f"{existing};;; {new}"

    def convert_to_eppi_annotations(
        self,
        annotations_data: list[dict[str, Any]],
        attributes_lookup: dict[int, EppiAttribute],
        attribute_id_to_label: dict[int, str] | None = None,
    ) -> list[EppiGoldStandardAnnotation]:
        """
        Convert several dicts to a list of EppiGoldStandardAnnotations.

        Args:
            annotations_data: List of human, gold standard
                annotation dicts from EPPI JSON
            document: The document these annotations belong to
            attributes_lookup: Lookup dictionary for attributes
            attribute_id_to_label: Mapping from attribute ID to label

        Returns:
            List of EppiGoldStandardAnnotation models

        """
        results = []
        for annotation in annotations_data:
            try:
                converted = self._convert_single_annotation(
                    annotation, attributes_lookup, attribute_id_to_label
                )
                results.append(converted)
            except ValueError as e:
                logger.warning(f"Skipping annotation due to error: {e}")
                continue
        return results

    def dedup_annotations(
        self, annotations: list[EppiGoldStandardAnnotation]
    ) -> list[EppiGoldStandardAnnotation]:
        """Merge annotations with the same attribute id."""
        merged: dict[int, EppiGoldStandardAnnotation] = {}

        for ann in annotations:
            attr_id = ann.attribute.attribute_id

            if attr_id not in merged:
                merged[attr_id] = ann
                continue

            target = merged[attr_id]

            target.raw_data = self._merge_raw_values(target.raw_data, ann.raw_data)

            if ann.additional_text:
                existing_text = target.additional_text or ""
                target.additional_text = f"{existing_text};;; {ann.additional_text}"

        return list(merged.values())

    def process_annotation_file(
        self,
        file_path: str | Path,
        set_attribute_type: str | AttributeType | None = None,
    ) -> ProcessedEppiAnnotationData:
        """
        Process a complete annotation file and return structured data.

        Args:
            file_path: Path to the JSON annotation file
            set_attribute_type: custom AttributeType to set for incoming annotations.

        Returns:
            ProcessedAnnotationData containing all processed data

        """
        logger.info(f"Processing annotation file: {file_path}")

        with Path(file_path).open("r", encoding="utf-8") as f:
            data: dict = json.load(f)

        raw_data = EppiRawData.model_validate(data)

        all_attributes_raw = self._extract_attributes_from_codesets(raw_data)

        if isinstance(set_attribute_type, str):
            set_attribute_type = AttributeType(set_attribute_type)
        attributes = self.convert_to_eppi_attributes(
            flattened_attributes=all_attributes_raw,
            set_attribute_type=set_attribute_type,
        )

        attributes_lookup: dict[int, EppiAttribute] = {
            attr.attribute_id: attr for attr in attributes
        }

        attribute_id_to_label: dict[int, str] = {
            attr.attribute_id: attr.attribute_label for attr in attributes
        }

        annotated_documents = []
        all_annotations = []
        documents_by_item_id: dict[int, EppiDocument] = {}

        # Process each reference with its annotations directly
        # Annotations are already nested within their parent reference
        for reference in data.get("References", []):
            item_id = reference.get("ItemId")
            if item_id is None:
                continue

            # Create or retrieve document using ItemId as unique identifier
            if item_id not in documents_by_item_id:
                document = EppiDocument.model_validate(reference)
                document.init_document_identity()
                documents_by_item_id[item_id] = document
            else:
                document = documents_by_item_id[item_id]

            # Get annotations directly from this reference's Codes array
            reference_codes = reference.get("Codes", [])
            if reference_codes:
                annotations = self.convert_to_eppi_annotations(
                    reference_codes,
                    attributes_lookup,
                    attribute_id_to_label,
                )

                annotations = self.dedup_annotations(annotations)

                annotated_doc = EppiGoldStandardAnnotatedDocument(
                    document=document, annotations=annotations
                )

                annotated_documents.append(annotated_doc)
                all_annotations.extend(annotations)

        logger.info(
            f"Processed {len(attributes)} attributes,"
            f" {len(documents_by_item_id)} documents, "
            f"{len(all_annotations)} annotations,"
            f" {len(annotated_documents)} annotated documents"
        )

        return ProcessedEppiAnnotationData(
            attributes=attributes,
            documents=list(documents_by_item_id.values()),
            annotations=all_annotations,
            annotated_documents=annotated_documents,
            attribute_id_to_label=attribute_id_to_label,
            raw_data=raw_data,
        )

processed_data_type property

Return ProcessedEppiAnnotationData.

__init__(base_output_dir=DEFAULT_BASE_OUTPUT_DIR, attributes_filename=DEFAULT_ATTRIBUTES_FILENAME, documents_filename=DEFAULT_DOCUMENTS_FILENAME, annotated_documents_filename=DEFAULT_ANNOTATED_DOCUMENTS_FILENAME, attribute_mapping_filename=DEFAULT_ATTRIBUTE_MAPPING_FILENAME)

Initialise the converter with configurable output paths. Set self.OUTFILE_LOADERS mapping the outfiles to be read/written to a filename, and a TypeAdapter defining the type of Pydantic Model to read back in when deserialising.

Parameters:

Name	Type	Description	Default
output_dir		Base directory for saving processed files	required
attributes_filename	str	Filename for attributes output	DEFAULT_ATTRIBUTES_FILENAME
documents_filename	str	Filename for documents output	DEFAULT_DOCUMENTS_FILENAME
annotated_documents_filename	str	Filename for annotated documents output	DEFAULT_ANNOTATED_DOCUMENTS_FILENAME
attribute_mapping_filename	str	Filename for attribute ID to label mapping	DEFAULT_ATTRIBUTE_MAPPING_FILENAME

Source code in deet/processors/eppi_annotation_converter.py

def __init__(
    self,
    base_output_dir: str | Path | None = DEFAULT_BASE_OUTPUT_DIR,
    attributes_filename: str = DEFAULT_ATTRIBUTES_FILENAME,
    documents_filename: str = DEFAULT_DOCUMENTS_FILENAME,
    annotated_documents_filename: str = DEFAULT_ANNOTATED_DOCUMENTS_FILENAME,
    attribute_mapping_filename: str = DEFAULT_ATTRIBUTE_MAPPING_FILENAME,
) -> None:
    """
    Initialise the converter with configurable output paths.
    Set self.OUTFILE_LOADERS mapping the outfiles to be read/written to a
    filename, and a TypeAdapter defining the type of Pydantic Model to
    read back in when deserialising.

    Args:
        output_dir: Base directory for saving processed files
        attributes_filename: Filename for attributes output
        documents_filename: Filename for documents output
        annotated_documents_filename: Filename for annotated documents output
        attribute_mapping_filename: Filename for attribute ID to label mapping

    """
    if base_output_dir is None:
        logger.debug(
            "`base_output_dir` set to None; "
            "converting to empty string for compatibility."
        )
        base_output_dir = ""
    self.base_output_dir = Path(base_output_dir)

    # extend below if adding more output files in `Outfiles`.
    self.OUTFILE_LOADERS: dict[Outfiles, tuple[str, TypeAdapter]] = {
        Outfiles.ATTRIBUTES: (
            attributes_filename,
            TypeAdapter(list[EppiAttribute]),
        ),
        Outfiles.DOCUMENTS: (
            documents_filename,
            TypeAdapter(list[EppiDocument]),
        ),
        Outfiles.ANNOTATED_DOCUMENTS: (
            annotated_documents_filename,
            TypeAdapter(list[EppiGoldStandardAnnotatedDocument]),
        ),
        Outfiles.ATTRIBUTE_LABEL_MAPPING: (
            attribute_mapping_filename,
            TypeAdapter(dict[int, str]),
        ),
    }

convert_to_eppi_annotations(annotations_data, attributes_lookup, attribute_id_to_label=None)

Convert several dicts to a list of EppiGoldStandardAnnotations.

Parameters:

Name	Type	Description	Default
annotations_data	list[dict[str, Any]]	List of human, gold standard annotation dicts from EPPI JSON	required
document		The document these annotations belong to	required
attributes_lookup	dict[int, EppiAttribute]	Lookup dictionary for attributes	required
attribute_id_to_label	dict[int, str] | None	Mapping from attribute ID to label	None

Returns:

Type	Description
list[EppiGoldStandardAnnotation]	List of EppiGoldStandardAnnotation models

Source code in deet/processors/eppi_annotation_converter.py

def convert_to_eppi_annotations(
    self,
    annotations_data: list[dict[str, Any]],
    attributes_lookup: dict[int, EppiAttribute],
    attribute_id_to_label: dict[int, str] | None = None,
) -> list[EppiGoldStandardAnnotation]:
    """
    Convert several dicts to a list of EppiGoldStandardAnnotations.

    Args:
        annotations_data: List of human, gold standard
            annotation dicts from EPPI JSON
        document: The document these annotations belong to
        attributes_lookup: Lookup dictionary for attributes
        attribute_id_to_label: Mapping from attribute ID to label

    Returns:
        List of EppiGoldStandardAnnotation models

    """
    results = []
    for annotation in annotations_data:
        try:
            converted = self._convert_single_annotation(
                annotation, attributes_lookup, attribute_id_to_label
            )
            results.append(converted)
        except ValueError as e:
            logger.warning(f"Skipping annotation due to error: {e}")
            continue
    return results

convert_to_eppi_attributes(flattened_attributes, set_attribute_type=None)

Convert flattened attribute data to EppiAttribute models.

Parameters:

Name	Type	Description	Default
flattened_attributes	list[dict[str, Any]]	List of flattened attribute dictionaries	required

Returns:

Type	Description
list[EppiAttribute]	List of EppiAttribute models

Source code in deet/processors/eppi_annotation_converter.py

def convert_to_eppi_attributes(
    self,
    flattened_attributes: list[dict[str, Any]],
    set_attribute_type: AttributeType | None = None,
) -> list[EppiAttribute]:
    """
    Convert flattened attribute data to EppiAttribute models.

    Args:
        flattened_attributes: List of flattened attribute dictionaries

    Returns:
        List of EppiAttribute models

    """
    out = []
    for att_dict in flattened_attributes:
        if "AttributeId" not in att_dict:
            att_dict["AttributeId"] = 0
        new_attribute = EppiAttribute(**att_dict)
        if set_attribute_type:
            logger.debug(
                f"setting custom attribute type {set_attribute_type.value} "
                f"for attribute {new_attribute.attribute_id}"
            )
            new_attribute.output_data_type = set_attribute_type
        out.append(new_attribute)
    return out

dedup_annotations(annotations)

Merge annotations with the same attribute id.

Source code in deet/processors/eppi_annotation_converter.py

def dedup_annotations(
    self, annotations: list[EppiGoldStandardAnnotation]
) -> list[EppiGoldStandardAnnotation]:
    """Merge annotations with the same attribute id."""
    merged: dict[int, EppiGoldStandardAnnotation] = {}

    for ann in annotations:
        attr_id = ann.attribute.attribute_id

        if attr_id not in merged:
            merged[attr_id] = ann
            continue

        target = merged[attr_id]

        target.raw_data = self._merge_raw_values(target.raw_data, ann.raw_data)

        if ann.additional_text:
            existing_text = target.additional_text or ""
            target.additional_text = f"{existing_text};;; {ann.additional_text}"

    return list(merged.values())

flatten_attributes_hierarchy(attributes_list, parent_path='')

Recursively flatten the hierarchical attributes structure.

Parameters:

Name	Type	Description	Default
attributes_list	list[dict[str, Any]]	List of attribute dictionaries from the JSON	required
parent_path	str	Path to the parent attribute (for hierarchy tracking)	''

Returns:

Type	Description
list[dict[str, Any]]	List of flattened attribute dictionaries with hierarchy information

Source code in deet/processors/eppi_annotation_converter.py

def flatten_attributes_hierarchy(
    self, attributes_list: list[dict[str, Any]], parent_path: str = ""
) -> list[dict[str, Any]]:
    """
    Recursively flatten the hierarchical attributes structure.

    Args:
        attributes_list: List of attribute dictionaries from the JSON
        parent_path: Path to the parent attribute (for hierarchy tracking)

    Returns:
        List of flattened attribute dictionaries with hierarchy information

    """
    flattened = []

    for attr in attributes_list:
        # extract children before modifying  dict
        child_attributes = attr.get("Attributes", {}).get("AttributesList", [])

        attr["hierarchy_path"] = parent_path
        attr["hierarchy_level"] = (
            len(parent_path.split(" > ")) if parent_path else 0
        )
        attr["is_leaf"] = not bool(child_attributes)

        flattened.append(attr)

        # recursive extension
        if child_attributes:
            current_path = (
                f"{parent_path} > {attr.get('AttributeName', '')}"
                if parent_path
                else attr.get("AttributeName", "")
            )
            flattened.extend(
                self.flatten_attributes_hierarchy(child_attributes, current_path)
            )

    return flattened

process_annotation_file(file_path, set_attribute_type=None)

Process a complete annotation file and return structured data.

Parameters:

Name	Type	Description	Default
file_path	str | Path	Path to the JSON annotation file	required
set_attribute_type	str | AttributeType | None	custom AttributeType to set for incoming annotations.	None

Returns:

Type	Description
ProcessedEppiAnnotationData	ProcessedAnnotationData containing all processed data

Source code in deet/processors/eppi_annotation_converter.py

def process_annotation_file(
    self,
    file_path: str | Path,
    set_attribute_type: str | AttributeType | None = None,
) -> ProcessedEppiAnnotationData:
    """
    Process a complete annotation file and return structured data.

    Args:
        file_path: Path to the JSON annotation file
        set_attribute_type: custom AttributeType to set for incoming annotations.

    Returns:
        ProcessedAnnotationData containing all processed data

    """
    logger.info(f"Processing annotation file: {file_path}")

    with Path(file_path).open("r", encoding="utf-8") as f:
        data: dict = json.load(f)

    raw_data = EppiRawData.model_validate(data)

    all_attributes_raw = self._extract_attributes_from_codesets(raw_data)

    if isinstance(set_attribute_type, str):
        set_attribute_type = AttributeType(set_attribute_type)
    attributes = self.convert_to_eppi_attributes(
        flattened_attributes=all_attributes_raw,
        set_attribute_type=set_attribute_type,
    )

    attributes_lookup: dict[int, EppiAttribute] = {
        attr.attribute_id: attr for attr in attributes
    }

    attribute_id_to_label: dict[int, str] = {
        attr.attribute_id: attr.attribute_label for attr in attributes
    }

    annotated_documents = []
    all_annotations = []
    documents_by_item_id: dict[int, EppiDocument] = {}

    # Process each reference with its annotations directly
    # Annotations are already nested within their parent reference
    for reference in data.get("References", []):
        item_id = reference.get("ItemId")
        if item_id is None:
            continue

        # Create or retrieve document using ItemId as unique identifier
        if item_id not in documents_by_item_id:
            document = EppiDocument.model_validate(reference)
            document.init_document_identity()
            documents_by_item_id[item_id] = document
        else:
            document = documents_by_item_id[item_id]

        # Get annotations directly from this reference's Codes array
        reference_codes = reference.get("Codes", [])
        if reference_codes:
            annotations = self.convert_to_eppi_annotations(
                reference_codes,
                attributes_lookup,
                attribute_id_to_label,
            )

            annotations = self.dedup_annotations(annotations)

            annotated_doc = EppiGoldStandardAnnotatedDocument(
                document=document, annotations=annotations
            )

            annotated_documents.append(annotated_doc)
            all_annotations.extend(annotations)

    logger.info(
        f"Processed {len(attributes)} attributes,"
        f" {len(documents_by_item_id)} documents, "
        f"{len(all_annotations)} annotations,"
        f" {len(annotated_documents)} annotated documents"
    )

    return ProcessedEppiAnnotationData(
        attributes=attributes,
        documents=list(documents_by_item_id.values()),
        annotations=all_annotations,
        annotated_documents=annotated_documents,
        attribute_id_to_label=attribute_id_to_label,
        raw_data=raw_data,
    )

eppi_output_data_from_eppi_fields(output_data_type, *, additional_text)

Map EPPI evidence onto typed raw_data for coerced output_data.

Glossary

• Codes: Rows under References[].Codes in EPPI export JSON. Each row means the reviewer applied that code for the reference (e.g. ticked a box).
• raw_data: The value stored on GoldStandardAnnotation before / during coercion to the Python type implied by the attribute.
• output_data: The coerced, typed value used in evaluation (derived from raw_data). For EPPI ingest, booleans reflect code presence; other types come from the AdditionalText field.

A Code row exists means the attribute was applied. For boolean attributes that is True even when AdditionalText is empty (the checkbox alone carries the positive annotation).

For every non-boolean type, only the info-box AdditionalText is used. ItemAttributeFullTextDetails is not used for the stored value (it may still be attached to the model for other uses).

Parameters:

Name	Type	Description	Default
output_data_type	AttributeType	Target attribute type (from codeset or prompt CSV).	required
additional_text	str	EPPI AdditionalText / info-box value.	required

Returns:

Type	Description
EppiRawDataValue	Value to store in GoldStandardAnnotation.raw_data (then coerced via
EppiRawDataValue	output_data). Never None; see module-level note on
EppiRawDataValue	EppiRawDataValue.

Source code in deet/processors/eppi_annotation_converter.py

def eppi_output_data_from_eppi_fields(
    output_data_type: AttributeType,
    *,
    additional_text: str,
) -> EppiRawDataValue:
    """
    Map EPPI evidence onto typed ``raw_data`` for coerced ``output_data``.

    **Glossary**

    - **Codes:** Rows under ``References[].Codes`` in EPPI export JSON. Each row
      means the reviewer applied that code for the reference (e.g. ticked a box).
    - **raw_data:** The value stored on ``GoldStandardAnnotation`` before / during
      coercion to the Python type implied by the attribute.
    - **output_data:** The coerced, typed value used in evaluation (derived from
      ``raw_data``). For EPPI ingest, booleans reflect **code presence**; other types
      come from the ``AdditionalText`` field.

    A Code row exists means the attribute was applied. For boolean attributes that is
    ``True`` even when ``AdditionalText`` is empty (the checkbox alone carries the
    positive annotation).

    For every non-boolean type, only the info-box ``AdditionalText`` is used.
    ``ItemAttributeFullTextDetails`` is not used for the stored value (it may still be
    attached to the model for other uses).

    Args:
        output_data_type: Target attribute type (from codeset or prompt CSV).
        additional_text: EPPI ``AdditionalText`` / info-box value.

    Returns:
        Value to store in ``GoldStandardAnnotation.raw_data`` (then coerced via
        ``output_data``). Never ``None``; see module-level note on
        ``EppiRawDataValue``.

    """
    additional = (additional_text or "").strip()

    if output_data_type == AttributeType.BOOL:
        return True
    if output_data_type == AttributeType.STRING:
        return additional
    if output_data_type == AttributeType.INTEGER:
        return _parse_eppi_integer(additional, output_data_type)
    if output_data_type == AttributeType.FLOAT:
        return _parse_eppi_float(additional, output_data_type)
    if output_data_type in (AttributeType.LIST, AttributeType.DICT):
        return _parse_eppi_list_or_dict(additional, output_data_type)

    raise UnsupportedEppiAttributeTypeError(output_data_type)

eppi_citation_parser

Parse raw EPPI citation markup into page and highlight fields.

ParsedEppiCitation dataclass

Structured fields extracted from a raw EPPI Text citation block.

Attributes:

Name	Type	Description
page	int | None	Page number from a Page N: prefix, or None if absent/invalid.
highlight_text	str	Cleaned highlight content (quotes/residual markup removed).
raw	str	Original input string retained for audit.

Source code in deet/processors/eppi_citation_parser.py

@dataclass(frozen=True, slots=True)
class ParsedEppiCitation:
    """
    Structured fields extracted from a raw EPPI ``Text`` citation block.

    Attributes:
        page: Page number from a ``Page N:`` prefix, or ``None`` if absent/invalid.
        highlight_text: Cleaned highlight content (quotes/residual markup removed).
        raw: Original input string retained for audit.

    """

    page: int | None
    highlight_text: str
    raw: str

format_parsed_citations(citations)

Join multiple parsed citations for CSV export.

Uses ": " as the separator (same convention as the raw item_attribute_full_text_details column).

Parameters:

Name	Type	Description	Default
citations	list[ParsedEppiCitation]	Parsed citation list (may be empty).	required

Returns:

Type	Description
str	(citation_page, citation_highlight_text) strings. Missing pages are
str	omitted from the page string; empty highlights are omitted from the
tuple[str, str]	highlight string.

Source code in deet/processors/eppi_citation_parser.py

def format_parsed_citations(
    citations: list[ParsedEppiCitation],
) -> tuple[str, str]:
    """
    Join multiple parsed citations for CSV export.

    Uses ``": "`` as the separator (same convention as the raw
    ``item_attribute_full_text_details`` column).

    Args:
        citations: Parsed citation list (may be empty).

    Returns:
        ``(citation_page, citation_highlight_text)`` strings. Missing pages are
        omitted from the page string; empty highlights are omitted from the
        highlight string.

    """
    pages = [str(c.page) for c in citations if c.page is not None]
    highlights = [c.highlight_text for c in citations if c.highlight_text]
    return ": ".join(pages), ": ".join(highlights)

parse_eppi_citation_text(raw_text)

Parse raw EPPI item_attribute_full_text_details markup.

Operates on raw markup so [¬s] / [¬e] can be used as structure. Extracted highlight fragments are cleaned via :func:clean_extracted_text.

Example::

Page 7:
[¬s]"Odds ratio[¬e]"

yields page=7, highlight_text="Odds ratio".

Parameters:

Name	Type	Description	Default
raw_text	str	Raw EPPI Text field value.	required

Returns:

Type	Description
ParsedEppiCitation	class:ParsedEppiCitation with best-effort page and highlight fields.

Source code in deet/processors/eppi_citation_parser.py

def parse_eppi_citation_text(raw_text: str) -> ParsedEppiCitation:
    """
    Parse raw EPPI ``item_attribute_full_text_details`` markup.

    Operates on **raw** markup so ``[¬s]`` / ``[¬e]`` can be used as structure.
    Extracted highlight fragments are cleaned via :func:`clean_extracted_text`.

    Example::

        Page 7:
        [¬s]"Odds ratio[¬e]"

    yields ``page=7``, ``highlight_text="Odds ratio"``.

    Args:
        raw_text: Raw EPPI ``Text`` field value.

    Returns:
        :class:`ParsedEppiCitation` with best-effort page and highlight fields.

    """
    raw = raw_text or ""
    page = _extract_page(raw)
    highlight_text = _extract_highlight(raw)
    return ParsedEppiCitation(page=page, highlight_text=highlight_text, raw=raw)

parse_eppi_citations_from_details(details)

Parse each EPPI full-text detail entry independently.

Parameters:

Name	Type	Description	Default
details	list[EppiItemAttributeFullTextDetails]	List of :class:EppiItemAttributeFullTextDetails (use [] when absent).	required

Returns:

Name	Type	Description
One	list[ParsedEppiCitation]	class:ParsedEppiCitation per detail with a non-empty text.
	list[ParsedEppiCitation]	Empty input yields [].

Source code in deet/processors/eppi_citation_parser.py

def parse_eppi_citations_from_details(
    details: list[EppiItemAttributeFullTextDetails],
) -> list[ParsedEppiCitation]:
    """
    Parse each EPPI full-text detail entry independently.

    Args:
        details: List of :class:`EppiItemAttributeFullTextDetails` (use ``[]`` when
            absent).

    Returns:
        One :class:`ParsedEppiCitation` per detail with a non-empty ``text``.
        Empty input yields ``[]``.

    """
    parsed: list[ParsedEppiCitation] = []
    for detail in details:
        text = detail.text
        if text is not None and text.strip():
            parsed.append(parse_eppi_citation_text(text))
    return parsed

linker

Tools for linking references/citations with parsed documents.

DocumentReferenceLinker

Core class for linking references/citations with parsed document text.

Source code in deet/processors/linker.py

class DocumentReferenceLinker:
    """Core class for linking references/citations with parsed document text."""

    LINKING_STRATEGY_HIERARCHY = [
        LinkingStrategy.MAPPING_FILE,
        LinkingStrategy.FILENAME_ID,
        LinkingStrategy.FILENAME_EXTERNAL_ID,
    ]

    def __init__(
        self,
        references: Sequence[Document],
        document_reference_mapping: list[DocumentReferenceMapping] | Path | None = None,
        document_base_dir: Path | None = None,
        parser: DocumentParser = parser,
        linking_strategies: list[LinkingStrategy] | None = None,
    ) -> None:
        """Initialise DocumentReferenceLinker class."""
        # deep copies to ensure lookup tables don't modify validators
        # on our existing reference docuemnts.
        tmp_refs = [doc.model_copy(deep=True) for doc in references]
        # initialise identity only where missing, to ensure heuristic strategies work.
        for doc in tmp_refs:
            if (
                doc.document_identity is None
                or doc.document_identity.document_id is None
            ):
                doc.init_document_identity(return_id=False)
        self.documents_references = references

        # lookup dics for O(1) id & author_year based lookup
        self._references_by_id: dict[int, Document] = {
            doc.document_identity.document_id: doc  # type:ignore[union-attr]
            for doc in tmp_refs
            if doc.document_identity.document_id is not None  # type:ignore[union-attr]
        }
        logger.debug(self._references_by_id.keys())

        logger.debug("populating _references_by_external_id lookup...")
        self._references_by_external_id: dict[str, Document] = {
            str(doc.document_identity.external_id): doc  # type:ignore[union-attr]
            for doc in tmp_refs
            if doc.document_identity.external_id is not None  # type:ignore[union-attr]
        }
        logger.debug(self._references_by_external_id.keys())

        try:
            logger.debug("populating _references_by_author_year lookup (longest)...")
            self._references_by_author_year_longest: dict[str, Document] = {
                doc.author_year_from_document_identity(
                    substring_strategy="longest"
                ): doc
                for doc in tmp_refs
            }
        except ValueError:
            logger.warning("author or year missing, returning empty dict.")
            self._references_by_author_year_longest = {}

        try:
            logger.debug("populating _references_by_author_year lookup (last)...")
            self._references_by_author_year_last: dict[str, Document] = {
                doc.author_year_from_document_identity(substring_strategy="last"): doc
                for doc in tmp_refs
            }
        except ValueError:
            logger.warning("author or year missing, returning empty dict.")
            self._references_by_author_year_last = {}

        self.document_base_dir = document_base_dir
        if isinstance(document_reference_mapping, Path):
            logger.debug(
                f"document_reference_mapping type: {type(document_reference_mapping)}"
            )
            document_reference_mapping = MappingImporter(
                mapping_file_path=document_reference_mapping,
                document_base_dir=document_base_dir,
            ).import_mapping()
        self.document_reference_mapping = document_reference_mapping
        self.parser = parser

        self.linking_strategies = linking_strategies or self.LINKING_STRATEGY_HIERARCHY
        self.document_parsing_stats: dict[str, DocumentParsingStats] = {}

    def _create_linking_factory(
        self, linking_strategy: LinkingStrategy
    ) -> Callable[[], Generator[LinkedInterimPayload, None, None]]:
        """
        Create a match function contingent on linking strategy.


        Args:
            linking_strategy (LinkingStrategy): see the enum for available ones.


        Returns:
            function: the function to run to retrieve objects to be linked,
            which will always yield a LinkedInterimPayload.


        """
        linking_strategy_map: dict[LinkingStrategy, Callable] = {  # extend as needed.
            LinkingStrategy.MAPPING_FILE: self._get_linkages_mapping_file,
            LinkingStrategy.FILENAME_AUTHOR_YEAR_LONGEST: partial(
                self._get_linkages_filename_author_year, "longest"
            ),
            LinkingStrategy.FILENAME_AUTHOR_YEAR_LAST: partial(
                self._get_linkages_filename_author_year, "last"
            ),
            LinkingStrategy.FILENAME_ID: self._get_linkages_filename_id,
            LinkingStrategy.FILENAME_EXTERNAL_ID: self._get_linkages_filename_external_id,  # noqa:E501
        }

        return linking_strategy_map[linking_strategy]

    def _get_linkages_mapping_file(self) -> Generator[LinkedInterimPayload]:
        """
        Yield linkages between files (pdf/md) and document_ids.


        This is contingent on
        - successful reading of a mapping file (csv/json) and
          storing in self.document_reference_mapping
        - there being at least 1 document that matches the ids
          therein.


        Yields
             Generator[LinkedInterimPayload].

        """
        if (
            self.document_reference_mapping is not None
            and False
            in [
                isinstance(x, DocumentReferenceMapping)
                for x in self.document_reference_mapping
            ]
        ) or self.document_reference_mapping is None:
            bad_mapping = (
                "self.document_reference_mapping needs to be a list of "
                " DocumentReferenceMapping objects."
                f"actual: {type(self.document_reference_mapping)}"
            )
            raise TypeError(bad_mapping)

        for mapping in self.document_reference_mapping:
            # get the right unlinked doc (reference)
            unlinked_doc = self._references_by_id.get(mapping.document_id)
            if unlinked_doc is None:
                logger.debug(
                    f"no reference document found for id {mapping.document_id}. next!"
                )
                continue
            interim_payload_dict = {
                **mapping.model_dump(),
                "unlinked_document": unlinked_doc,
            }
            yield LinkedInterimPayload(**interim_payload_dict)

    def _get_linkages_filename_author_year(
        self, substring_strategy: Literal["longest", "last"]
    ) -> Generator[LinkedInterimPayload]:
        """
        Yield linkages between files and reference-documents based on
        best guess at `author_year.pdf` filename structure.


        Yields
            Generator[LinkedInterimPayload]:


        """
        # we could type-check or existence-check self.document_reference_mapping
        # here; but there might be an instance in which the user chooses this
        # regardless of other situation, so really this should be handled elsewhere.
        # necessary condition for this method to atempt is only availability of
        # files.
        if not self.document_base_dir or not self.document_base_dir.is_dir():
            bad_base_dir = "self.document_base_dir needs to be a valid directory."
            raise ValueError(bad_base_dir)

        lookup_dict: dict[str, Document]
        if substring_strategy == "longest":
            lookup_dict = self._references_by_author_year_longest
        elif substring_strategy == "last":
            lookup_dict = self._references_by_author_year_last
        else:
            bad_strat_err = "unimnplemented substring strategy for finding author"
            raise NotImplementedError(bad_strat_err)
        logger.debug(f"last name strategy is {substring_strategy}")

        for file in self.document_base_dir.iterdir():
            if file.suffix not in [".md", ".pdf"]:
                logger.warning(f"file {file} is not pdf/md. next!")
                continue
            author_year_guess = file.name.split(".")[0].lower()
            unlinked_doc = lookup_dict.get(author_year_guess)
            if unlinked_doc is None:
                logger.debug(
                    f"no reference document found for id {author_year_guess}. next!"
                )
                continue
            if unlinked_doc.document_identity is None:
                unlinked_doc.init_document_identity()

            yield LinkedInterimPayload(
                document_id=unlinked_doc.document_identity.document_id,  # type:ignore[union-attr, arg-type]
                file_path=file,
                format=cast("Literal['md', 'pdf']", file.suffix[1:]),
                unlinked_document=unlinked_doc,
            )

    def _get_linkages_filename_id(self) -> Generator[LinkedInterimPayload]:
        """
        Yield linkages between files and reference-documents based on
        assumption that files are named `id.pdf`, e.g `12345678.pdf`.


        Yields
            Generator[LinkedInterimPayload]:


        """
        if not self.document_base_dir or not self.document_base_dir.is_dir():
            bad_base_dir = "self.document_base_dir needs to be a valid directory."
            raise ValueError(bad_base_dir)

        for file in self.document_base_dir.iterdir():
            if file.suffix not in [".md", ".pdf"]:
                logger.warning(f"file {file} is not pdf/md. next!")
                continue
            try:
                id_guess = int(file.name.split(".")[0])
            except ValueError:
                logger.debug(f"{file.name} isn't convertible to int. next!")
                continue
            unlinked_doc = self._references_by_id.get(id_guess)
            if unlinked_doc is None:
                logger.debug(f"no reference document found for id {id_guess}. next!")
                continue
            if unlinked_doc.document_identity is None:
                unlinked_doc.init_document_identity()
            yield LinkedInterimPayload(
                document_id=unlinked_doc.document_identity.document_id,  # type:ignore[union-attr, arg-type]
                file_path=file,
                format=cast("Literal['md', 'pdf']", file.suffix[1:]),
                unlinked_document=unlinked_doc,
            )

    def _get_linkages_filename_external_id(self) -> Generator[LinkedInterimPayload]:
        """
        Yield linkages between files and reference-documents based on
        assumption that files are named `external_id.pdf`, e.g `ABC123.pdf`.

        This strategy looks up documents using their external_id field instead of
        the internal document_id field.

        Yields
            Generator[LinkedInterimPayload]:

        """
        if not self.document_base_dir or not self.document_base_dir.is_dir():
            bad_base_dir = "self.document_base_dir needs to be a valid directory."
            raise ValueError(bad_base_dir)

        for file in self.document_base_dir.iterdir():
            if file.suffix not in [".md", ".pdf"]:
                logger.warning(f"file {file} is not pdf/md. next!")
                continue
            # ID from filename
            id_guess = str(file.stem)

            # Try to find document by external ID
            unlinked_doc = self._references_by_external_id.get(id_guess)
            if unlinked_doc is None:
                logger.debug(
                    f"no reference document found for external id {id_guess}. next!"
                )
                continue

            if unlinked_doc.document_identity is None:
                unlinked_doc.init_document_identity()

            yield LinkedInterimPayload(
                document_id=unlinked_doc.document_identity.document_id,  # type:ignore[union-attr, arg-type]
                file_path=file,
                format=cast("Literal['md', 'pdf']", file.suffix[1:]),
                unlinked_document=unlinked_doc,
            )

    def guess_file_paths(
        self,
        strategies: list[LinkingStrategy] | None = None,
    ) -> dict[int, Path]:
        """
        Attempt to pre-fill doc-file mappings
        using LinkingStrategies.

        Returns
            a dict of {document_id: matched_file_path}
            for documents where a match was found.
            Unmatched documents are absent from the result.

            NOTE: MAPPING_FILE strategy is excluded, obviously.

        """
        default_strategies = [
            LinkingStrategy.FILENAME_ID,
            LinkingStrategy.FILENAME_EXTERNAL_ID,
            LinkingStrategy.FILENAME_AUTHOR_YEAR_LONGEST,
            LinkingStrategy.FILENAME_AUTHOR_YEAR_LAST,
        ]
        strategies = [
            s
            for s in (strategies or default_strategies)
            if s != LinkingStrategy.MAPPING_FILE
        ]
        logger.debug(f"strategies for guessing file paths: {', '.join(strategies)}")

        result: dict[int, Path] = {}

        for strategy in strategies:
            try:
                factory = self._create_linking_factory(strategy)
                for payload in factory():
                    if payload.document_id not in result:  # first-match wins
                        result[payload.document_id] = payload.file_path
            except (TypeError, ValueError) as e:
                logger.warning(f"pre-fill strategy {strategy} failed: {e}")
                continue

        return result

    @staticmethod
    def _parse_pdf(
        path_to_pdf: Path, *, return_images: bool = False, return_metadata: bool = False
    ) -> ParsedOutput:
        """
        Parse a pdf to ParsedOutput.


        NOTE: wrapper around DocumentParser.parse().


        Args:
            path_to_pdf (Path): where to find the file
            return_images (bool, optional): store images or not. Defaults to False.
            return_metadata (bool, optional): store metadata or not. Defaults to False.


        Raises:
            TypeError: when it's not a pdf file.


        Returns:
            ParsedOutput: a container for markdown, images and metadata.


        """
        if path_to_pdf.suffix != ".pdf":
            not_pdf = "need a valid pdf file."
            raise TypeError(not_pdf)
        return parser(
            input_=path_to_pdf,
            return_images=return_images,
            return_metadata=return_metadata,
        )

    @staticmethod
    def link_reference_parsed_document(
        reference: Document,
        parsed_output: ParsedOutput,
        original_filepath: Path | None = None,
    ) -> Document:
        """
        Link a reference, in `Document` format with a parsed document,
        in ParsedOutput format.


        Args:
            reference (Document): the reference, e.g. 'document' from eppi json.
            parsed_output (ParsedOutput): parser output.
            original_filepath (Path | None, optional): Defaults to None.


        Returns:
            Document: a linked Document with
            all required fields populated, and is_linked==True,
            and required fields' presence validated.


        """
        if not reference.document_identity:
            logger.debug(
                "initialising document identity for "
                f"reference with id {reference.document_id}."
            )

            reference.init_document_identity()

        logger.debug(
            f"adding parsed_output to reference with id {reference.document_id}."
        )
        reference.link_parsed_document(
            parsed_document=parsed_output,
            original_doc_filepath=original_filepath,
        )

        # set context & context-type
        reference.set_context_from_parsed()
        # reference.context_type = ContextType.FULL_DOCUMENT

        # set is_linked -- this will raise a ValidationError
        # if min requirements are not met
        reference.is_final = True
        reference.is_linked = True

        return reference

    def link_many_references_parsed_documents(
        self,
        *,
        return_images: bool = False,
        return_metadata: bool = False,
    ) -> list[Document]:
        """
        Link multiple references to parsed
        documents using available LinkingStrategy(s).


        Iterates over linking strategies in
        hierarchical order, attempting to link
        each reference-doc to its corresponding file.


        If required, parses files.
        Creates Document objects where is_linked=True.


        Args:
            return_images: Whether to include images in parsed output
            return_metadata: Whether to include metadata in parsed output


        Returns:
            List of Document objects successfully linked and parsed


        """
        linked_documents: list[Document] = []
        processed_doc_ids = set()
        n_docs_to_link = len(self.documents_references)
        self.document_parsing_stats = {}

        for strategy in self.linking_strategies:
            if len(linked_documents) == n_docs_to_link:
                logger.info("all linking jobs completed.")
                break
            logger.info(f"Attempting linking strategy: {strategy}")

            try:
                linking_function = self._create_linking_factory(strategy)  # type:ignore[arg-type] #mypy is wrong here...

                for interim_payload in linking_function():
                    if interim_payload.document_id in processed_doc_ids:
                        logger.debug(
                            f"document {interim_payload.document_id} "
                            " already linked, next"
                        )
                        continue

                    doc_id_str = str(interim_payload.document_id)
                    if interim_payload.format == "pdf":
                        with measure_elapsed() as parse_timer:
                            parsed_output = self._parse_pdf(
                                interim_payload.file_path,
                                return_images=return_images,
                                return_metadata=return_metadata,
                            )
                        self.document_parsing_stats[doc_id_str] = DocumentParsingStats(
                            parsing_seconds=parse_timer.seconds,
                            parsing_skipped=False,
                        )
                    elif interim_payload.format == "md":
                        with measure_elapsed() as parse_timer:
                            md_text = interim_payload.file_path.read_text(
                                encoding="utf-8"
                            )
                        parsed_output = ParsedOutput(
                            text=md_text,
                            parser_library="unknown",
                        )
                        self.document_parsing_stats[doc_id_str] = DocumentParsingStats(
                            parsing_seconds=parse_timer.seconds,
                            parsing_skipped=False,
                        )
                    else:
                        logger.warning(
                            f"unsupported format {interim_payload.format} for "
                            f"document {interim_payload.document_id}, skipping"
                        )
                        continue

                    # link!
                    linked_doc = self.link_reference_parsed_document(
                        reference=interim_payload.unlinked_document,
                        parsed_output=parsed_output,
                        original_filepath=interim_payload.file_path,
                    )

                    linked_documents.append(linked_doc)
                    processed_doc_ids.add(interim_payload.document_id)

                    logger.info(
                        f"successfully linked document {interim_payload.document_id} "
                        f"with file {interim_payload.file_path} "
                        f"using {strategy}"
                    )

            except (TypeError, ValueError) as e:
                # if i + 1 < len(self.linking_strategies):
                logger.error(f"Error with linking strategy {strategy}: {e}")
                continue
                # raise

        total_refs = len(self.documents_references)
        linked_count = len(linked_documents)
        logger.info(
            f"linking complete: {linked_count}/{total_refs} "
            "references successfully linked"
        )

        if linked_count < total_refs:
            unlinked_ids = [
                doc.document_identity.document_id  # type:ignore[union-attr]
                for doc in self.documents_references
                if doc.document_identity.document_id not in processed_doc_ids  # type:ignore[operator, union-attr]
            ]
            logger.warning(f"Unlinked document IDs: {unlinked_ids}")

        return linked_documents

__init__(references, document_reference_mapping=None, document_base_dir=None, parser=parser, linking_strategies=None)

Initialise DocumentReferenceLinker class.

Source code in deet/processors/linker.py

def __init__(
    self,
    references: Sequence[Document],
    document_reference_mapping: list[DocumentReferenceMapping] | Path | None = None,
    document_base_dir: Path | None = None,
    parser: DocumentParser = parser,
    linking_strategies: list[LinkingStrategy] | None = None,
) -> None:
    """Initialise DocumentReferenceLinker class."""
    # deep copies to ensure lookup tables don't modify validators
    # on our existing reference docuemnts.
    tmp_refs = [doc.model_copy(deep=True) for doc in references]
    # initialise identity only where missing, to ensure heuristic strategies work.
    for doc in tmp_refs:
        if (
            doc.document_identity is None
            or doc.document_identity.document_id is None
        ):
            doc.init_document_identity(return_id=False)
    self.documents_references = references

    # lookup dics for O(1) id & author_year based lookup
    self._references_by_id: dict[int, Document] = {
        doc.document_identity.document_id: doc  # type:ignore[union-attr]
        for doc in tmp_refs
        if doc.document_identity.document_id is not None  # type:ignore[union-attr]
    }
    logger.debug(self._references_by_id.keys())

    logger.debug("populating _references_by_external_id lookup...")
    self._references_by_external_id: dict[str, Document] = {
        str(doc.document_identity.external_id): doc  # type:ignore[union-attr]
        for doc in tmp_refs
        if doc.document_identity.external_id is not None  # type:ignore[union-attr]
    }
    logger.debug(self._references_by_external_id.keys())

    try:
        logger.debug("populating _references_by_author_year lookup (longest)...")
        self._references_by_author_year_longest: dict[str, Document] = {
            doc.author_year_from_document_identity(
                substring_strategy="longest"
            ): doc
            for doc in tmp_refs
        }
    except ValueError:
        logger.warning("author or year missing, returning empty dict.")
        self._references_by_author_year_longest = {}

    try:
        logger.debug("populating _references_by_author_year lookup (last)...")
        self._references_by_author_year_last: dict[str, Document] = {
            doc.author_year_from_document_identity(substring_strategy="last"): doc
            for doc in tmp_refs
        }
    except ValueError:
        logger.warning("author or year missing, returning empty dict.")
        self._references_by_author_year_last = {}

    self.document_base_dir = document_base_dir
    if isinstance(document_reference_mapping, Path):
        logger.debug(
            f"document_reference_mapping type: {type(document_reference_mapping)}"
        )
        document_reference_mapping = MappingImporter(
            mapping_file_path=document_reference_mapping,
            document_base_dir=document_base_dir,
        ).import_mapping()
    self.document_reference_mapping = document_reference_mapping
    self.parser = parser

    self.linking_strategies = linking_strategies or self.LINKING_STRATEGY_HIERARCHY
    self.document_parsing_stats: dict[str, DocumentParsingStats] = {}

guess_file_paths(strategies=None)

Attempt to pre-fill doc-file mappings using LinkingStrategies.

Returns a dict of {document_id: matched_file_path} for documents where a match was found. Unmatched documents are absent from the result.

NOTE: MAPPING_FILE strategy is excluded, obviously.

Source code in deet/processors/linker.py

def guess_file_paths(
    self,
    strategies: list[LinkingStrategy] | None = None,
) -> dict[int, Path]:
    """
    Attempt to pre-fill doc-file mappings
    using LinkingStrategies.

    Returns
        a dict of {document_id: matched_file_path}
        for documents where a match was found.
        Unmatched documents are absent from the result.

        NOTE: MAPPING_FILE strategy is excluded, obviously.

    """
    default_strategies = [
        LinkingStrategy.FILENAME_ID,
        LinkingStrategy.FILENAME_EXTERNAL_ID,
        LinkingStrategy.FILENAME_AUTHOR_YEAR_LONGEST,
        LinkingStrategy.FILENAME_AUTHOR_YEAR_LAST,
    ]
    strategies = [
        s
        for s in (strategies or default_strategies)
        if s != LinkingStrategy.MAPPING_FILE
    ]
    logger.debug(f"strategies for guessing file paths: {', '.join(strategies)}")

    result: dict[int, Path] = {}

    for strategy in strategies:
        try:
            factory = self._create_linking_factory(strategy)
            for payload in factory():
                if payload.document_id not in result:  # first-match wins
                    result[payload.document_id] = payload.file_path
        except (TypeError, ValueError) as e:
            logger.warning(f"pre-fill strategy {strategy} failed: {e}")
            continue

    return result

link_many_references_parsed_documents(*, return_images=False, return_metadata=False)

Link multiple references to parsed documents using available LinkingStrategy(s).

Iterates over linking strategies in hierarchical order, attempting to link each reference-doc to its corresponding file.

If required, parses files. Creates Document objects where is_linked=True.

Parameters:

Name	Type	Description	Default
return_images	bool	Whether to include images in parsed output	False
return_metadata	bool	Whether to include metadata in parsed output	False

Returns:

Type	Description
list[Document]	List of Document objects successfully linked and parsed

Source code in deet/processors/linker.py

def link_many_references_parsed_documents(
    self,
    *,
    return_images: bool = False,
    return_metadata: bool = False,
) -> list[Document]:
    """
    Link multiple references to parsed
    documents using available LinkingStrategy(s).


    Iterates over linking strategies in
    hierarchical order, attempting to link
    each reference-doc to its corresponding file.


    If required, parses files.
    Creates Document objects where is_linked=True.


    Args:
        return_images: Whether to include images in parsed output
        return_metadata: Whether to include metadata in parsed output


    Returns:
        List of Document objects successfully linked and parsed


    """
    linked_documents: list[Document] = []
    processed_doc_ids = set()
    n_docs_to_link = len(self.documents_references)
    self.document_parsing_stats = {}

    for strategy in self.linking_strategies:
        if len(linked_documents) == n_docs_to_link:
            logger.info("all linking jobs completed.")
            break
        logger.info(f"Attempting linking strategy: {strategy}")

        try:
            linking_function = self._create_linking_factory(strategy)  # type:ignore[arg-type] #mypy is wrong here...

            for interim_payload in linking_function():
                if interim_payload.document_id in processed_doc_ids:
                    logger.debug(
                        f"document {interim_payload.document_id} "
                        " already linked, next"
                    )
                    continue

                doc_id_str = str(interim_payload.document_id)
                if interim_payload.format == "pdf":
                    with measure_elapsed() as parse_timer:
                        parsed_output = self._parse_pdf(
                            interim_payload.file_path,
                            return_images=return_images,
                            return_metadata=return_metadata,
                        )
                    self.document_parsing_stats[doc_id_str] = DocumentParsingStats(
                        parsing_seconds=parse_timer.seconds,
                        parsing_skipped=False,
                    )
                elif interim_payload.format == "md":
                    with measure_elapsed() as parse_timer:
                        md_text = interim_payload.file_path.read_text(
                            encoding="utf-8"
                        )
                    parsed_output = ParsedOutput(
                        text=md_text,
                        parser_library="unknown",
                    )
                    self.document_parsing_stats[doc_id_str] = DocumentParsingStats(
                        parsing_seconds=parse_timer.seconds,
                        parsing_skipped=False,
                    )
                else:
                    logger.warning(
                        f"unsupported format {interim_payload.format} for "
                        f"document {interim_payload.document_id}, skipping"
                    )
                    continue

                # link!
                linked_doc = self.link_reference_parsed_document(
                    reference=interim_payload.unlinked_document,
                    parsed_output=parsed_output,
                    original_filepath=interim_payload.file_path,
                )

                linked_documents.append(linked_doc)
                processed_doc_ids.add(interim_payload.document_id)

                logger.info(
                    f"successfully linked document {interim_payload.document_id} "
                    f"with file {interim_payload.file_path} "
                    f"using {strategy}"
                )

        except (TypeError, ValueError) as e:
            # if i + 1 < len(self.linking_strategies):
            logger.error(f"Error with linking strategy {strategy}: {e}")
            continue
            # raise

    total_refs = len(self.documents_references)
    linked_count = len(linked_documents)
    logger.info(
        f"linking complete: {linked_count}/{total_refs} "
        "references successfully linked"
    )

    if linked_count < total_refs:
        unlinked_ids = [
            doc.document_identity.document_id  # type:ignore[union-attr]
            for doc in self.documents_references
            if doc.document_identity.document_id not in processed_doc_ids  # type:ignore[operator, union-attr]
        ]
        logger.warning(f"Unlinked document IDs: {unlinked_ids}")

    return linked_documents

link_reference_parsed_document(reference, parsed_output, original_filepath=None) staticmethod

Link a reference, in Document format with a parsed document, in ParsedOutput format.

Parameters:

Name	Type	Description	Default
reference	Document	the reference, e.g. 'document' from eppi json.	required
parsed_output	ParsedOutput	parser output.	required
original_filepath	Path | None	Defaults to None.	None

Returns:

Name	Type	Description
Document	Document	a linked Document with
	Document	all required fields populated, and is_linked==True,
	Document	and required fields' presence validated.

Source code in deet/processors/linker.py

@staticmethod
def link_reference_parsed_document(
    reference: Document,
    parsed_output: ParsedOutput,
    original_filepath: Path | None = None,
) -> Document:
    """
    Link a reference, in `Document` format with a parsed document,
    in ParsedOutput format.


    Args:
        reference (Document): the reference, e.g. 'document' from eppi json.
        parsed_output (ParsedOutput): parser output.
        original_filepath (Path | None, optional): Defaults to None.


    Returns:
        Document: a linked Document with
        all required fields populated, and is_linked==True,
        and required fields' presence validated.


    """
    if not reference.document_identity:
        logger.debug(
            "initialising document identity for "
            f"reference with id {reference.document_id}."
        )

        reference.init_document_identity()

    logger.debug(
        f"adding parsed_output to reference with id {reference.document_id}."
    )
    reference.link_parsed_document(
        parsed_document=parsed_output,
        original_doc_filepath=original_filepath,
    )

    # set context & context-type
    reference.set_context_from_parsed()
    # reference.context_type = ContextType.FULL_DOCUMENT

    # set is_linked -- this will raise a ValidationError
    # if min requirements are not met
    reference.is_final = True
    reference.is_linked = True

    return reference

DocumentReferenceMapping pydantic-model

Bases: BaseModel

Data model for incoming, manual mappings of references (via integer ids) to documents, via filename.

Fields:

• document_id (int)
• file_path (Path)
• format (Literal['md', 'pdf'] | None)

Validators:

• ensure_valid_doc_id → document_id
• ensure_file_exists

Source code in deet/processors/linker.py

class DocumentReferenceMapping(BaseModel):
    """
    Data model for incoming, manual mappings
    of references (via integer ids) to
    documents, via filename.
    """

    document_id: int
    file_path: Path
    # NOTE: may want to inherit this from supported
    # formats in parser.py
    format: Literal["md", "pdf"] | None = None

    @field_validator("document_id", mode="before")
    @classmethod
    def ensure_valid_doc_id(cls, value: int) -> int:
        """Ensure supplied document_id has a valid number of digits."""
        int_value = int(value)
        if int_value <= 0:
            invalid_int_id_error = (
                f"`document_id` must be a positive integer. Supplied: {int_value}"
            )
            raise ValueError(invalid_int_id_error)
        num_digits = len(str(abs(int_value)))
        if not (MIN_DOCUMENT_ID_DIGITS <= num_digits <= MAX_DOCUMENT_ID_DIGITS):
            val_err = (
                f"`document_id` must be between {MIN_DOCUMENT_ID_DIGITS} "
                f"and {MAX_DOCUMENT_ID_DIGITS} digits. Supplied: {int_value}"
            )
            raise ValueError(val_err)
        return int_value

    @model_validator(mode="after")
    def ensure_file_exists(self) -> Self:
        """
        Ensure either md_path or pdf_path are populated,
        and the associated file exists.
        """
        # check path resolves
        if not self.file_path.is_file():
            not_a_file = f"{self.file_path} is not a file."
            raise ValueError(not_a_file)
        # check suffix
        if self.file_path.suffix not in [".md", ".pdf"]:
            unsupported = (
                f"{self.file_path.suffix} is not supported. either .md or .pdf"
            )
            raise ValueError(unsupported)
        self.format = self.file_path.suffix[1:]  # type:ignore[assignment] # omit .

        return self

ensure_file_exists() pydantic-validator

Ensure either md_path or pdf_path are populated, and the associated file exists.

Source code in deet/processors/linker.py

@model_validator(mode="after")
def ensure_file_exists(self) -> Self:
    """
    Ensure either md_path or pdf_path are populated,
    and the associated file exists.
    """
    # check path resolves
    if not self.file_path.is_file():
        not_a_file = f"{self.file_path} is not a file."
        raise ValueError(not_a_file)
    # check suffix
    if self.file_path.suffix not in [".md", ".pdf"]:
        unsupported = (
            f"{self.file_path.suffix} is not supported. either .md or .pdf"
        )
        raise ValueError(unsupported)
    self.format = self.file_path.suffix[1:]  # type:ignore[assignment] # omit .

    return self

ensure_valid_doc_id(value) pydantic-validator

Ensure supplied document_id has a valid number of digits.

Source code in deet/processors/linker.py

@field_validator("document_id", mode="before")
@classmethod
def ensure_valid_doc_id(cls, value: int) -> int:
    """Ensure supplied document_id has a valid number of digits."""
    int_value = int(value)
    if int_value <= 0:
        invalid_int_id_error = (
            f"`document_id` must be a positive integer. Supplied: {int_value}"
        )
        raise ValueError(invalid_int_id_error)
    num_digits = len(str(abs(int_value)))
    if not (MIN_DOCUMENT_ID_DIGITS <= num_digits <= MAX_DOCUMENT_ID_DIGITS):
        val_err = (
            f"`document_id` must be between {MIN_DOCUMENT_ID_DIGITS} "
            f"and {MAX_DOCUMENT_ID_DIGITS} digits. Supplied: {int_value}"
        )
        raise ValueError(val_err)
    return int_value

LinkedInterimPayload pydantic-model

Bases: DocumentReferenceMapping

Interim output from a linking factory method; extending DocumentReferenceMapping.

Interim as the document may a) still need to be parsed, and b) still needs to be coerced into ParsedOutput.

Fields:

• document_id (int)
• file_path (Path)
• format (Literal['md', 'pdf'] | None)
• unlinked_document (Document)

Validators:

• ensure_valid_doc_id → document_id
• ensure_file_exists

Source code in deet/processors/linker.py

class LinkedInterimPayload(DocumentReferenceMapping):
    """
    Interim output from a linking factory method; extending
    DocumentReferenceMapping.


    Interim as the document may
    a) still need to be parsed, and
    b) still needs to be coerced into ParsedOutput.
    """

    unlinked_document: Document

LinkingStrategy

Bases: StrEnum

Enum of permitted/implemented ref<>parsed_doc linking strategies.

Source code in deet/processors/linker.py

class LinkingStrategy(StrEnum):
    """Enum of permitted/implemented ref<>parsed_doc linking strategies."""

    MAPPING_FILE = auto()
    FILENAME_AUTHOR_YEAR_LONGEST = auto()
    FILENAME_AUTHOR_YEAR_LAST = auto()
    FILENAME_ID = auto()
    FILENAME_EXTERNAL_ID = auto()

MappingImporter

Tool for importing manual mappings from csv/json to list[DocumentReferenceMapping].

Source code in deet/processors/linker.py

class MappingImporter:
    """
    Tool for importing manual mappings from csv/json
    to list[DocumentReferenceMapping].
    """

    def __init__(
        self, mapping_file_path: Path, document_base_dir: Path | None = None
    ) -> None:
        """
        Initialise MappingImporter instance.


        Args:
            mapping_file_path (Path): Path to csv/json file containing mappings.
            document_base_dir (Path | None, optional): Optional directory path
            where documents (pdf/md) live. Defaults to None.


        """
        if mapping_file_path.suffix not in [".csv", ".json"]:
            bad_file = "mapping file needs to be supplied in csv or json format."
            raise ValueError(bad_file)

        if document_base_dir and not document_base_dir.is_dir():
            bad_dir = "document_base_dir is optional, but must be a dir if supplied."
            raise ValueError(bad_dir)

        self.mapping_file_path = mapping_file_path
        self.mapping_file_type = mapping_file_path.suffix[1:]
        self.document_base_dir = document_base_dir

    def import_mapping(self) -> list[DocumentReferenceMapping]:
        """Parse a csv/json file to a list od DocumentReferenceMapping objects."""
        if self.mapping_file_type == "json":
            logger.debug("importing mapping from json")
            payload = self._load_json()
        elif self.mapping_file_type == "csv":
            logger.debug("importing mapping from csv")
            payload = self._load_csv()
        else:
            bad_file = "only json or csv files are supported."
            raise ValueError(bad_file)

        return [
            DocumentReferenceMapping(document_id=doc_id, file_path=file_path)
            for doc_id, file_path in payload.items()
        ]

    def _load_json(self) -> dict[int, Path]:
        """
        Load json file with mappings to dict.


        NOTE: json can be in...
        - array style:
        [
        {"document_id": 12345678, "file_path": "path/to/file.pdf"},
        {"document_id": 87654321, "file_path": "path/to/other.md"}
        ]


        - dict style:
        {
        "12345678": "path/to/file.pdf",
        "87654321": "path/to/other.md"
        }


        Raises
            JsonStyleError: if bad json style.


        Returns
            dict[int, Path]: the pre-validation object.


        """
        with self.mapping_file_path.open() as f:
            data = json.load(f)

        result = {}

        # list[dict] (array) style
        if isinstance(data, list):
            logger.debug("json format is array.")
            for item in data:
                doc_id = int(item["document_id"])
                file_path = self._resolve_file_path(item["file_path"])
                if file_path:
                    logger.debug(f"file path {file_path} resolved, adding dict entry.")
                    result[doc_id] = Path(file_path)
                logger.debug(
                    f"file path {file_path} not resoved resolved, adding dict entry."
                )

        # dict style
        elif isinstance(data, dict):
            logger.debug("json format is dict.")
            for doc_id_str, file_path in data.items():
                doc_id = int(doc_id_str)
                file_path_out = self._resolve_file_path(file_path)
                if file_path_out:
                    logger.debug(
                        f"file path {file_path_out} resolved, adding dict entry."
                    )
                    result[doc_id] = Path(file_path_out)
                logger.debug(
                    f"file path {file_path_out} not resolved, not adding dict entry."
                )

        else:
            bad_json = "json must be either a list(array) or dict format."
            raise JsonStyleError(bad_json)

        return result

    def _load_csv(self) -> dict[int, Path]:
        result = {}

        with self.mapping_file_path.open(newline="", encoding="utf-8") as f:
            reader = csv.DictReader(f)

            # ensure required columns exist
            if (
                reader.fieldnames is None
                or "document_id" not in reader.fieldnames
                or "file_path" not in reader.fieldnames
            ):
                required_cols = "csv must contain 'document_id' and 'file_path' columns"
                raise ValueError(required_cols)

            for row in reader:
                doc_id = int(row["document_id"])
                file_path = self._resolve_file_path(row["file_path"])
                if file_path:
                    logger.debug(f"file path {file_path} resolved, adding dict entry.")
                    result[doc_id] = Path(file_path)
                logger.debug(
                    f"file path not {file_path} resolved, not adding dict entry."
                )

        return result

    @staticmethod
    def merge_partial_paths(parts_a: list[str], parts_b: list[str]) -> list[str]:
        """
        Merge partial file path components.

        Implements the Knuth-Morris-Pratt (KMP) algorithm. Nice!
        https://en.wikipedia.org/wiki/Knuth%E2%80%93Morris%E2%80%93Pratt_algorithm

        Args:
            parts_a (list[str]): the longer of the path parts
            parts_b (list[str]): the shorter of the path parts.

        Returns:
            list[str]: the merged combined path

        """
        # build combined list with sentinel
        sentinel_combined = [*parts_b, None, *parts_a]

        # prefix function (KMP)
        pi = [0] * len(sentinel_combined)  # this is the nomenclature KMP uses
        for i in range(1, len(sentinel_combined)):
            j = pi[i - 1]
            while j > 0 and sentinel_combined[i] != sentinel_combined[j]:
                j = pi[j - 1]
            if sentinel_combined[i] == sentinel_combined[j]:
                j += 1
            pi[i] = j

        overlap = pi[-1]  # length of prefix of b matching suffix of a

        return parts_a[: len(parts_a) - overlap] + parts_b

    def _resolve_file_path(self, file_path: str | None | Path) -> Path | None:
        """
        Resolve file path, handling absolute
        paths and document_base_dir.


        Args:
            file_path: File path from mapping file
            (can be relative or absolute)


        Returns:
            Resolved Path object


        Raises:
            FileNotFoundError: If resolved path doesn't exist


        """
        if file_path == "" or file_path is None:
            return None
        file_path = Path(file_path)

        # if already findable, use as-is
        if file_path.exists():
            logger.debug(f"Using absolute path: {file_path.absolute()!s}")
            return file_path.absolute()

        # if document_base_dir provided, prepend it -- if appropriate
        if not self.document_base_dir or not self.document_base_dir.is_dir():
            dir_missing = (
                f"self.document_base_dir {self.document_base_dir!s}"
                " does not exist or is not defined."
            )
            raise FileNotFoundError(dir_missing)

        # try finding a file that simply appends to base path and exists
        if (self.document_base_dir / file_path).exists():
            return (self.document_base_dir / file_path).absolute()

        # now we're facing a situation where we try to resolve it all...
        base_dir_parts = list(self.document_base_dir.absolute().parts)
        file_path_parts = list(file_path.parts)
        merged_partials = self.merge_partial_paths(base_dir_parts, file_path_parts)
        merged_partial_path = Path("/".join(merged_partials))
        if merged_partial_path.exists():
            return merged_partial_path
        cant_resolve = f"merged file path {merged_partial_path!s} cant be resolved."
        raise FileNotFoundError(cant_resolve)

__init__(mapping_file_path, document_base_dir=None)

Initialise MappingImporter instance.

Parameters:

Name	Type	Description	Default
mapping_file_path	Path	Path to csv/json file containing mappings.	required
document_base_dir	Path | None	Optional directory path	None

Source code in deet/processors/linker.py

def __init__(
    self, mapping_file_path: Path, document_base_dir: Path | None = None
) -> None:
    """
    Initialise MappingImporter instance.


    Args:
        mapping_file_path (Path): Path to csv/json file containing mappings.
        document_base_dir (Path | None, optional): Optional directory path
        where documents (pdf/md) live. Defaults to None.


    """
    if mapping_file_path.suffix not in [".csv", ".json"]:
        bad_file = "mapping file needs to be supplied in csv or json format."
        raise ValueError(bad_file)

    if document_base_dir and not document_base_dir.is_dir():
        bad_dir = "document_base_dir is optional, but must be a dir if supplied."
        raise ValueError(bad_dir)

    self.mapping_file_path = mapping_file_path
    self.mapping_file_type = mapping_file_path.suffix[1:]
    self.document_base_dir = document_base_dir

import_mapping()

Parse a csv/json file to a list od DocumentReferenceMapping objects.

Source code in deet/processors/linker.py

def import_mapping(self) -> list[DocumentReferenceMapping]:
    """Parse a csv/json file to a list od DocumentReferenceMapping objects."""
    if self.mapping_file_type == "json":
        logger.debug("importing mapping from json")
        payload = self._load_json()
    elif self.mapping_file_type == "csv":
        logger.debug("importing mapping from csv")
        payload = self._load_csv()
    else:
        bad_file = "only json or csv files are supported."
        raise ValueError(bad_file)

    return [
        DocumentReferenceMapping(document_id=doc_id, file_path=file_path)
        for doc_id, file_path in payload.items()
    ]

merge_partial_paths(parts_a, parts_b) staticmethod

Merge partial file path components.

Implements the Knuth-Morris-Pratt (KMP) algorithm. Nice! https://en.wikipedia.org/wiki/Knuth%E2%80%93Morris%E2%80%93Pratt_algorithm

Parameters:

Name	Type	Description	Default
parts_a	list[str]	the longer of the path parts	required
parts_b	list[str]	the shorter of the path parts.	required

Returns:

Type	Description
list[str]	list[str]: the merged combined path

Source code in deet/processors/linker.py

@staticmethod
def merge_partial_paths(parts_a: list[str], parts_b: list[str]) -> list[str]:
    """
    Merge partial file path components.

    Implements the Knuth-Morris-Pratt (KMP) algorithm. Nice!
    https://en.wikipedia.org/wiki/Knuth%E2%80%93Morris%E2%80%93Pratt_algorithm

    Args:
        parts_a (list[str]): the longer of the path parts
        parts_b (list[str]): the shorter of the path parts.

    Returns:
        list[str]: the merged combined path

    """
    # build combined list with sentinel
    sentinel_combined = [*parts_b, None, *parts_a]

    # prefix function (KMP)
    pi = [0] * len(sentinel_combined)  # this is the nomenclature KMP uses
    for i in range(1, len(sentinel_combined)):
        j = pi[i - 1]
        while j > 0 and sentinel_combined[i] != sentinel_combined[j]:
            j = pi[j - 1]
        if sentinel_combined[i] == sentinel_combined[j]:
            j += 1
        pi[i] = j

    overlap = pi[-1]  # length of prefix of b matching suffix of a

    return parts_a[: len(parts_a) - overlap] + parts_b

parser

Utilities for parsing input files (e.g. pdf) into output files (e.g. md).

DocumentParser

Parse documents from target format to other target format.

Source code in deet/processors/parser.py

class DocumentParser:
    """Parse documents from target format to other target format."""

    DEFAULT_PARSERS: dict[str, type[ParserLibrary]] = {
        "pdf": PdfminerParser,
        "epub": PandocParser,
        "html": PandocParser,
        "xml": PandocParser,
    }

    def __init__(
        self, parsers: dict[str, type[ParserLibrary]] = DEFAULT_PARSERS
    ) -> None:
        """
        Initialise instance of DocumentParser with default parsers.
        Default parsers are in dict.

        """
        self.parsers = parsers

        if self.parsers is not None and isinstance(self.parsers, dict):
            for parser_name, parser in self.parsers.items():
                logger.debug(f"default {parser_name} parser: {parser.name}")

    def __call__(  # noqa: PLR0913 - img/meta needs to be explicit (non-kwargs) here.
        self,
        input_: str | PathLike,
        out_path: str | PathLike | None = None,
        parser: type[ParserLibrary] | None = None,
        input_type: InputFileType | str | None = None,
        *,
        return_images: bool = False,
        return_metadata: bool = False,
        **kwargs,
    ) -> ParsedOutput:
        """
        Run the parser on one input_.

        Args:
            input_ (str | PathLike): File(path) or str of input_.
            out_path (str | PathLike | None): If None, return parsed content as str.
            parser (ParserLibrary | None, optional): Defaults to None.
                If None, uses the default parser.
            input_type (InputFileType | None, optional): Defaults to None.
                If None, infers file type using `detect_filetype`.
            return_images (bool): Defaults to False. Whether to write
                parsed images (JPEG) to file, or not. `out_path`. can't be None.
            return_metadata (bool): Defaults to None. Whether to write
                parsed metadata (json).

        Returns:
            str: ParsedOutput object.

        """
        logger.debug(f"kwargs: {kwargs}")
        if input_type is None:
            logger.debug(
                "no input file type provided. using `detect_filetype` to infer."
            )
            try:
                input_type = InputFileType(
                    self.detect_filetype(
                        file=input_,
                        permitted_file_enum_list=list(InputFileType),
                    )
                )
            except ValueError as ve:
                raise InvalidInputFileTypeError(ve) from ve
        logger.debug(f"input file type: {input_type}.")

        if parser is not None and (
            (not isinstance(parser, type)) or (not issubclass(parser, ParserLibrary))
        ):
            bad_parser_err = f"parser {parser} is not a valid ParserLibrary."
            raise FileParserMismatchError(bad_parser_err)
        if parser is None and input_type is not None:
            logger.debug("parser not supplied. selecting default parser for file_type.")
            if isinstance(input_type, str):
                try:
                    input_type = InputFileType(input_type)
                except ValueError as ve:
                    if "is not a valid InputFileType" in str(ve):
                        invalid_input_ft = f"{input_type} is not a valid InputFileType"
                        raise InvalidInputFileTypeError(invalid_input_ft) from ve
            if (
                self.parsers is None
                or (isinstance(input_type, str) and input_type not in self.parsers)
                or (
                    isinstance(input_type, InputFileType)
                    and input_type.value not in self.parsers
                )
            ):
                missing_parser = "no parser supplied."
                raise ValueError(missing_parser)
            if isinstance(input_type, InputFileType):
                parser = self.parsers[input_type.value]
            elif isinstance(input_type, str):
                parser = self.parsers[input_type]
            else:
                missing_parser = "no parser supplied."
                raise ValueError(missing_parser)
        logger.debug(f"parser: {parser}.")
        kwargs["input_type"] = input_type

        parsed = self.parse(
            input_=input_,
            parser=parser,
            return_images=return_images,
            return_metadata=return_metadata,
            **kwargs,
        )

        if out_path:
            self.write_files(
                out_path=out_path,
                parser=parser,
                write_metadata=return_metadata,
                write_images=return_images,
                text=parsed.text,
                metadata=parsed.metadata,
                images=parsed.images,
            )

        return parsed

    def parse(
        self,
        input_: str | PathLike,
        parser: type[ParserLibrary],
        *,
        return_metadata: bool = False,
        return_images: bool = False,
        **kwargs,
    ) -> ParsedOutput:
        """
        Parse target file.
        Wraps around specific parser methods.

        Args:
            input_ (str | PathLike):
            input_type (InputFileType):
            parser (ParserLibrary):
            parse_method (Callable[[str  |  PathLike, ParserLibrary], str]):

        Returns:
            str: _description_

        """
        logger.debug(f"kwargs: {kwargs}")
        if return_metadata and OutputFileType.JSON not in parser.output_file_types:
            metadata_not_allowed = (
                f"metadata out not permitted for parser {parser.name}."
            )
            raise InvalidOutputFileTypeError(metadata_not_allowed)
        if return_images and OutputFileType.JPEG not in parser.output_file_types:
            images_not_allowed = f"images out not permitted for parser {parser.name}."
            raise InvalidOutputFileTypeError(images_not_allowed)

        return parser.parse(
            input_=input_,
            return_metadata=return_metadata,
            return_images=return_images,
            **kwargs,
        )

    @staticmethod
    def detect_filetype(
        file: str | PathLike,
        permitted_file_enum_list: list[InputFileType]
        | list[OutputFileType]
        | list[str]
        | None = None,
    ) -> str:
        """
        Detect file type from a file_path.

        Args:
            file (str | PathLike): _description_

        Raises:
            InvalidInputFileTypeError: If file extension isn't permitted.

        Returns:
            InputFileType: _description_

        """
        if permitted_file_enum_list is None:
            permitted_file_enum_list = list(InputFileType) + list(OutputFileType)
        permitted_extensions_str = {
            x.value
            for x in permitted_file_enum_list
            if isinstance(x, (InputFileType | OutputFileType))
        }

        extension = str(file).split(".")[-1]
        if extension not in permitted_extensions_str:
            has_input = any(
                isinstance(ft, InputFileType) for ft in permitted_file_enum_list
            )
            has_output = any(
                isinstance(ft, OutputFileType) for ft in permitted_file_enum_list
            )
            target_error: type[Exception]
            if has_input and not has_output:
                target_error = InvalidInputFileTypeError
            elif not has_input and has_output:
                target_error = InvalidOutputFileTypeError
            else:
                target_error = InvalidFileTypeError

            forbidden_file_type = (
                f"file type {extension} is not permitted. "
                f" Use one of {permitted_extensions_str}."
            )
            raise target_error(forbidden_file_type)

        logger.debug(f"filetype is: {extension}.")
        return extension

    @staticmethod
    def write_files(  # noqa: PLR0913
        out_path: str | PathLike,
        parser: type[ParserLibrary],
        *,
        write_metadata: bool,
        write_images: bool,
        text: str,
        metadata: dict | None = None,
        images: dict[str, Image] | None = None,
    ) -> None:
        """
        Write parsed content to file(s).

        NOTE: we are taking existence of `out_path` as an intention to
        write all requested objects to file. out_path can be a file or a dir.
        if out_path is a file, we write remaining files to parent dir.

        Args:
            out_path (str | PathLike): _description_
            write_metadata (bool): _description_
            write_images (bool): _description_
            text (str): _description_
            metadata (dict | None, optional): _description_. Defaults to None.
            images (dict[str, Image] | None, optional): _description_. Defaults to None.

        """
        extension = (
            DocumentParser.detect_filetype(  # should raise error if not permitted
                out_path, permitted_file_enum_list=parser.output_file_types
            )
        )

        required_outfiles = ["md"]
        if write_images:
            required_outfiles.append("jpeg")
            if images is None:  # or raise something?
                logger.warning(
                    "`write_images` set to True, but no images obj supplied."
                )
        if write_metadata:
            required_outfiles.append("json")
            if metadata is None:  # or raise something?
                logger.warning(
                    "`write_metadata` set to True, but no metadata obj supplied."
                )
        logger.debug(f"required outfiles: {required_outfiles}")
        if False in [ft in parser.output_file_types for ft in required_outfiles]:
            raise InvalidOutputFileTypeError

        Path(out_path).parent.mkdir(parents=True, exist_ok=True)

        if Path(out_path).is_file() or (extension in required_outfiles):
            logger.debug(f"`out_path` {out_path} points to a file.")

            dir_base = Path(out_path).parent
            filename_base = "".join(Path(out_path).name.split(".")[0:-1])

        if Path(out_path).is_dir():
            logger.debug(f"`out_path` {out_path} points to a dir.")
            # we now have to get our filename base from somewhere...
            dir_base = Path(out_path)
            filename_base = text.split("\n")[0][:15].replace(" ", "_").lower()

        for ext in required_outfiles:
            out = dir_base / (filename_base + "." + ext)
            logger.debug(f"writing out {ext} to {out}.")
            if ext == "md":
                out.write_text(text, encoding="utf-8")
            if ext == "json" and metadata is not None:
                out.write_text(json.dumps(metadata), encoding="utf-8")
            if ext == "jpeg" and images is not None:
                for img_name, img in images.items():
                    img_out = dir_base / (filename_base + "_" + img_name)
                    img.save(img_out, ext)

__call__(input_, out_path=None, parser=None, input_type=None, *, return_images=False, return_metadata=False, **kwargs)

Run the parser on one input_.

Parameters:

Name	Type	Description	Default
input_	str | PathLike	File(path) or str of input_.	required
out_path	str | PathLike | None	If None, return parsed content as str.	None
parser	ParserLibrary | None	Defaults to None. If None, uses the default parser.	None
input_type	InputFileType | None	Defaults to None. If None, infers file type using detect_filetype.	None
return_images	bool	Defaults to False. Whether to write parsed images (JPEG) to file, or not. out_path. can't be None.	False
return_metadata	bool	Defaults to None. Whether to write parsed metadata (json).	False

Returns:

Name	Type	Description
str	ParsedOutput	ParsedOutput object.

Source code in deet/processors/parser.py

def __call__(  # noqa: PLR0913 - img/meta needs to be explicit (non-kwargs) here.
    self,
    input_: str | PathLike,
    out_path: str | PathLike | None = None,
    parser: type[ParserLibrary] | None = None,
    input_type: InputFileType | str | None = None,
    *,
    return_images: bool = False,
    return_metadata: bool = False,
    **kwargs,
) -> ParsedOutput:
    """
    Run the parser on one input_.

    Args:
        input_ (str | PathLike): File(path) or str of input_.
        out_path (str | PathLike | None): If None, return parsed content as str.
        parser (ParserLibrary | None, optional): Defaults to None.
            If None, uses the default parser.
        input_type (InputFileType | None, optional): Defaults to None.
            If None, infers file type using `detect_filetype`.
        return_images (bool): Defaults to False. Whether to write
            parsed images (JPEG) to file, or not. `out_path`. can't be None.
        return_metadata (bool): Defaults to None. Whether to write
            parsed metadata (json).

    Returns:
        str: ParsedOutput object.

    """
    logger.debug(f"kwargs: {kwargs}")
    if input_type is None:
        logger.debug(
            "no input file type provided. using `detect_filetype` to infer."
        )
        try:
            input_type = InputFileType(
                self.detect_filetype(
                    file=input_,
                    permitted_file_enum_list=list(InputFileType),
                )
            )
        except ValueError as ve:
            raise InvalidInputFileTypeError(ve) from ve
    logger.debug(f"input file type: {input_type}.")

    if parser is not None and (
        (not isinstance(parser, type)) or (not issubclass(parser, ParserLibrary))
    ):
        bad_parser_err = f"parser {parser} is not a valid ParserLibrary."
        raise FileParserMismatchError(bad_parser_err)
    if parser is None and input_type is not None:
        logger.debug("parser not supplied. selecting default parser for file_type.")
        if isinstance(input_type, str):
            try:
                input_type = InputFileType(input_type)
            except ValueError as ve:
                if "is not a valid InputFileType" in str(ve):
                    invalid_input_ft = f"{input_type} is not a valid InputFileType"
                    raise InvalidInputFileTypeError(invalid_input_ft) from ve
        if (
            self.parsers is None
            or (isinstance(input_type, str) and input_type not in self.parsers)
            or (
                isinstance(input_type, InputFileType)
                and input_type.value not in self.parsers
            )
        ):
            missing_parser = "no parser supplied."
            raise ValueError(missing_parser)
        if isinstance(input_type, InputFileType):
            parser = self.parsers[input_type.value]
        elif isinstance(input_type, str):
            parser = self.parsers[input_type]
        else:
            missing_parser = "no parser supplied."
            raise ValueError(missing_parser)
    logger.debug(f"parser: {parser}.")
    kwargs["input_type"] = input_type

    parsed = self.parse(
        input_=input_,
        parser=parser,
        return_images=return_images,
        return_metadata=return_metadata,
        **kwargs,
    )

    if out_path:
        self.write_files(
            out_path=out_path,
            parser=parser,
            write_metadata=return_metadata,
            write_images=return_images,
            text=parsed.text,
            metadata=parsed.metadata,
            images=parsed.images,
        )

    return parsed

__init__(parsers=DEFAULT_PARSERS)

Initialise instance of DocumentParser with default parsers. Default parsers are in dict.

Source code in deet/processors/parser.py

def __init__(
    self, parsers: dict[str, type[ParserLibrary]] = DEFAULT_PARSERS
) -> None:
    """
    Initialise instance of DocumentParser with default parsers.
    Default parsers are in dict.

    """
    self.parsers = parsers

    if self.parsers is not None and isinstance(self.parsers, dict):
        for parser_name, parser in self.parsers.items():
            logger.debug(f"default {parser_name} parser: {parser.name}")

detect_filetype(file, permitted_file_enum_list=None) staticmethod

Detect file type from a file_path.

Parameters:

Name	Type	Description	Default
file	str | PathLike	description	required

Raises:

Type	Description
InvalidInputFileTypeError	If file extension isn't permitted.

Returns:

Name	Type	Description
InputFileType	str	description

Source code in deet/processors/parser.py

@staticmethod
def detect_filetype(
    file: str | PathLike,
    permitted_file_enum_list: list[InputFileType]
    | list[OutputFileType]
    | list[str]
    | None = None,
) -> str:
    """
    Detect file type from a file_path.

    Args:
        file (str | PathLike): _description_

    Raises:
        InvalidInputFileTypeError: If file extension isn't permitted.

    Returns:
        InputFileType: _description_

    """
    if permitted_file_enum_list is None:
        permitted_file_enum_list = list(InputFileType) + list(OutputFileType)
    permitted_extensions_str = {
        x.value
        for x in permitted_file_enum_list
        if isinstance(x, (InputFileType | OutputFileType))
    }

    extension = str(file).split(".")[-1]
    if extension not in permitted_extensions_str:
        has_input = any(
            isinstance(ft, InputFileType) for ft in permitted_file_enum_list
        )
        has_output = any(
            isinstance(ft, OutputFileType) for ft in permitted_file_enum_list
        )
        target_error: type[Exception]
        if has_input and not has_output:
            target_error = InvalidInputFileTypeError
        elif not has_input and has_output:
            target_error = InvalidOutputFileTypeError
        else:
            target_error = InvalidFileTypeError

        forbidden_file_type = (
            f"file type {extension} is not permitted. "
            f" Use one of {permitted_extensions_str}."
        )
        raise target_error(forbidden_file_type)

    logger.debug(f"filetype is: {extension}.")
    return extension

parse(input_, parser, *, return_metadata=False, return_images=False, **kwargs)

Parse target file. Wraps around specific parser methods.

Parameters:

Name	Type	Description	Default
input_	str | PathLike		required
input_type	InputFileType		required
parser	ParserLibrary		required
parse_method	Callable[[str | PathLike, ParserLibrary], str]		required

Returns:

Name	Type	Description
str	ParsedOutput	description

Source code in deet/processors/parser.py

def parse(
    self,
    input_: str | PathLike,
    parser: type[ParserLibrary],
    *,
    return_metadata: bool = False,
    return_images: bool = False,
    **kwargs,
) -> ParsedOutput:
    """
    Parse target file.
    Wraps around specific parser methods.

    Args:
        input_ (str | PathLike):
        input_type (InputFileType):
        parser (ParserLibrary):
        parse_method (Callable[[str  |  PathLike, ParserLibrary], str]):

    Returns:
        str: _description_

    """
    logger.debug(f"kwargs: {kwargs}")
    if return_metadata and OutputFileType.JSON not in parser.output_file_types:
        metadata_not_allowed = (
            f"metadata out not permitted for parser {parser.name}."
        )
        raise InvalidOutputFileTypeError(metadata_not_allowed)
    if return_images and OutputFileType.JPEG not in parser.output_file_types:
        images_not_allowed = f"images out not permitted for parser {parser.name}."
        raise InvalidOutputFileTypeError(images_not_allowed)

    return parser.parse(
        input_=input_,
        return_metadata=return_metadata,
        return_images=return_images,
        **kwargs,
    )

write_files(out_path, parser, *, write_metadata, write_images, text, metadata=None, images=None) staticmethod

Write parsed content to file(s).

NOTE: we are taking existence of out_path as an intention to write all requested objects to file. out_path can be a file or a dir. if out_path is a file, we write remaining files to parent dir.

Parameters:

Name	Type	Description	Default
out_path	str | PathLike	description	required
write_metadata	bool	description	required
write_images	bool	description	required
text	str	description	required
metadata	dict | None	description. Defaults to None.	None
images	dict[str, Image] | None	description. Defaults to None.	None

Source code in deet/processors/parser.py

@staticmethod
def write_files(  # noqa: PLR0913
    out_path: str | PathLike,
    parser: type[ParserLibrary],
    *,
    write_metadata: bool,
    write_images: bool,
    text: str,
    metadata: dict | None = None,
    images: dict[str, Image] | None = None,
) -> None:
    """
    Write parsed content to file(s).

    NOTE: we are taking existence of `out_path` as an intention to
    write all requested objects to file. out_path can be a file or a dir.
    if out_path is a file, we write remaining files to parent dir.

    Args:
        out_path (str | PathLike): _description_
        write_metadata (bool): _description_
        write_images (bool): _description_
        text (str): _description_
        metadata (dict | None, optional): _description_. Defaults to None.
        images (dict[str, Image] | None, optional): _description_. Defaults to None.

    """
    extension = (
        DocumentParser.detect_filetype(  # should raise error if not permitted
            out_path, permitted_file_enum_list=parser.output_file_types
        )
    )

    required_outfiles = ["md"]
    if write_images:
        required_outfiles.append("jpeg")
        if images is None:  # or raise something?
            logger.warning(
                "`write_images` set to True, but no images obj supplied."
            )
    if write_metadata:
        required_outfiles.append("json")
        if metadata is None:  # or raise something?
            logger.warning(
                "`write_metadata` set to True, but no metadata obj supplied."
            )
    logger.debug(f"required outfiles: {required_outfiles}")
    if False in [ft in parser.output_file_types for ft in required_outfiles]:
        raise InvalidOutputFileTypeError

    Path(out_path).parent.mkdir(parents=True, exist_ok=True)

    if Path(out_path).is_file() or (extension in required_outfiles):
        logger.debug(f"`out_path` {out_path} points to a file.")

        dir_base = Path(out_path).parent
        filename_base = "".join(Path(out_path).name.split(".")[0:-1])

    if Path(out_path).is_dir():
        logger.debug(f"`out_path` {out_path} points to a dir.")
        # we now have to get our filename base from somewhere...
        dir_base = Path(out_path)
        filename_base = text.split("\n")[0][:15].replace(" ", "_").lower()

    for ext in required_outfiles:
        out = dir_base / (filename_base + "." + ext)
        logger.debug(f"writing out {ext} to {out}.")
        if ext == "md":
            out.write_text(text, encoding="utf-8")
        if ext == "json" and metadata is not None:
            out.write_text(json.dumps(metadata), encoding="utf-8")
        if ext == "jpeg" and images is not None:
            for img_name, img in images.items():
                img_out = dir_base / (filename_base + "_" + img_name)
                img.save(img_out, ext)

InputFileType

Bases: StrEnum

Enumeration of permitted input file types.

Parameters:

Name	Type	Description	Default
StrEnum	_type_		required

Source code in deet/processors/parser.py

class InputFileType(StrEnum):
    """
    Enumeration of permitted input file types.

    Args:
        StrEnum (_type_):

    """

    PDF = auto()
    EPUB = auto()
    HTML = auto()
    XML = auto()  # NOTE - this only covers JATS xml.

MarkerParser

Bases: ParserLibrary

Parser with marker backend.

Source code in deet/processors/parser.py

class MarkerParser(ParserLibrary):
    """Parser with `marker` backend."""

    name: Literal["marker"] = "marker"
    input_types = [InputFileType.PDF]
    output_file_types = [OutputFileType.MD, OutputFileType.JPEG, OutputFileType.JSON]

    @classmethod
    @parser_cache.memoize(typed=True, expire=None, tag="marker-converter")
    def _get_converter(cls):  # noqa: ANN206 no return type hint as we don't know PdfConverter yet
        """Lazy initialization of marker converter with disk caching."""
        logger.debug("Initializing marker converter...")
        from marker.converters.pdf import PdfConverter
        from marker.models import create_model_dict

        artifact_dict = create_model_dict()
        return PdfConverter(artifact_dict=artifact_dict)

    @classmethod
    def parse(
        cls,
        input_: str | PathLike,
        *,
        return_metadata: bool = False,
        return_images: bool = False,
        **kwargs,  # noqa: ARG003
    ) -> ParsedOutput:
        """Parse file using marker."""
        from marker.output import text_from_rendered

        converter = cls._get_converter()
        rendered = converter(str(input_))
        text, extension, images = text_from_rendered(rendered)
        out = {"text": text}
        if return_metadata:
            out["metadata"] = rendered.metadata
        if return_images:
            out["images"] = images
        return ParsedOutput(**out, parser_library=cls.name)

    @classmethod
    def clear_cache(cls) -> None:
        """Clear the cached converter."""
        parser_cache.clear()
        logger.info("Marker converter cache cleared")

clear_cache() classmethod

Clear the cached converter.

Source code in deet/processors/parser.py

@classmethod
def clear_cache(cls) -> None:
    """Clear the cached converter."""
    parser_cache.clear()
    logger.info("Marker converter cache cleared")

parse(input_, *, return_metadata=False, return_images=False, **kwargs) classmethod

Parse file using marker.

Source code in deet/processors/parser.py

@classmethod
def parse(
    cls,
    input_: str | PathLike,
    *,
    return_metadata: bool = False,
    return_images: bool = False,
    **kwargs,  # noqa: ARG003
) -> ParsedOutput:
    """Parse file using marker."""
    from marker.output import text_from_rendered

    converter = cls._get_converter()
    rendered = converter(str(input_))
    text, extension, images = text_from_rendered(rendered)
    out = {"text": text}
    if return_metadata:
        out["metadata"] = rendered.metadata
    if return_images:
        out["images"] = images
    return ParsedOutput(**out, parser_library=cls.name)

OutputFileType

Bases: StrEnum

Enumeration of permitted output file types.

Parameters:

Name	Type	Description	Default
StrEnum	_type_		required

Source code in deet/processors/parser.py

class OutputFileType(StrEnum):
    """
    Enumeration of permitted output file types.

    Args:
        StrEnum (_type_):

    """

    MD = auto()
    JPEG = auto()
    JSON = auto()

PandocParser

Bases: ParserLibrary

Parser with pandoc backend.

Source code in deet/processors/parser.py

class PandocParser(ParserLibrary):
    """Parser with `pandoc` backend."""

    name: Literal["pandoc"] = "pandoc"
    input_types = [InputFileType.EPUB, InputFileType.HTML, InputFileType.XML]
    output_file_types = [OutputFileType.MD]

    @classmethod
    def parse(
        cls,
        input_: str | PathLike,
        input_type: InputFileType | str | None = None,
        *,
        input_is_string: bool = False,
        return_metadata: bool = False,
        return_images: bool = False,
        **kwargs,  # noqa: ARG003
    ) -> ParsedOutput:
        """Parse file using pandoc."""
        if True in [return_images, return_metadata]:
            image_meta_erro = "PandocParser can't produce images or metadata."
            raise InvalidOutputFileTypeError(image_meta_erro)
        if input_is_string and not input_type:
            missing_filetype = (
                "if input is str in memory, provide format as `input_type`."
            )
            raise InvalidInputFileTypeError(missing_filetype)
        if not input_type:
            input_type = DocumentParser.detect_filetype(input_, cls.input_types)
        if isinstance(input_type, InputFileType):
            input_type = input_type.value
        if input_type == "xml":
            input_type = "jats"

        if input_is_string:
            parse_method = pypandoc.convert_text
        else:
            parse_method = pypandoc.convert_file

        out = {
            "text": parse_method(
                input_,
                to="md",
                format=input_type,
            )
        }
        return ParsedOutput(**out, parser_library=cls.name)

parse(input_, input_type=None, *, input_is_string=False, return_metadata=False, return_images=False, **kwargs) classmethod

Parse file using pandoc.

Source code in deet/processors/parser.py

@classmethod
def parse(
    cls,
    input_: str | PathLike,
    input_type: InputFileType | str | None = None,
    *,
    input_is_string: bool = False,
    return_metadata: bool = False,
    return_images: bool = False,
    **kwargs,  # noqa: ARG003
) -> ParsedOutput:
    """Parse file using pandoc."""
    if True in [return_images, return_metadata]:
        image_meta_erro = "PandocParser can't produce images or metadata."
        raise InvalidOutputFileTypeError(image_meta_erro)
    if input_is_string and not input_type:
        missing_filetype = (
            "if input is str in memory, provide format as `input_type`."
        )
        raise InvalidInputFileTypeError(missing_filetype)
    if not input_type:
        input_type = DocumentParser.detect_filetype(input_, cls.input_types)
    if isinstance(input_type, InputFileType):
        input_type = input_type.value
    if input_type == "xml":
        input_type = "jats"

    if input_is_string:
        parse_method = pypandoc.convert_text
    else:
        parse_method = pypandoc.convert_file

    out = {
        "text": parse_method(
            input_,
            to="md",
            format=input_type,
        )
    }
    return ParsedOutput(**out, parser_library=cls.name)

ParsedOutput pydantic-model

Bases: BaseModel

Output returned from the parser() method of subclasses of ParserLibrary.

Contains

text, str: md-formatted parsed text (required) images, pillow.img: pillow-formatted image(s) (optional) metadata, dict: metadata json (optional) timestamp: datetime: auto-populates with now parser_library: str: name of the ParserLibrary implementation used

Config:

• arbitrary_types_allowed: True

Fields:

• text (str)
• images (dict[str, Image] | None)
• metadata (dict | None)
• timestamp (datetime)
• parser_library (Literal['pandoc', 'marker', 'pdfminer', 'unknown'])

Validators:

• assess_language_quality → text

Source code in deet/processors/parser.py

class ParsedOutput(BaseModel):
    """
    Output returned from the `parser()` method of subclasses of ParserLibrary.

    Contains:
        text, str: md-formatted parsed text (required)
        images, pillow.img: pillow-formatted image(s) (optional)
        metadata, dict: metadata json (optional)
        timestamp: datetime: auto-populates with _now_
        parser_library: str: name of the ParserLibrary implementation used
    """

    text: str
    images: dict[str, Image] | None = None
    metadata: dict | None = None
    timestamp: datetime = Field(default_factory=lambda: datetime.now(tz=UTC))
    parser_library: Literal[
        "pandoc", "marker", "pdfminer", "unknown"
    ]  # extend when adding new parsers

    model_config = ConfigDict(
        arbitrary_types_allowed=True
    )  # this is to allow our Executor class as a type.

    @field_validator("text", mode="after")
    @classmethod
    def assess_language_quality(cls, value: str) -> str:
        """
        Assess language quality.

        Args:
            text (str): Parsed text.

        Raises:
            MalformedLanguageError: If threshold not met.

        Returns:
            str: parsed text.

        """
        if not check_language(value):
            logger.debug("check lang failed")
            bad_language = "Supplied text didn't pass quality check."
            raise ValueError(bad_language)
        return value

assess_language_quality(value) pydantic-validator

Assess language quality.

Parameters:

Name	Type	Description	Default
text	str	Parsed text.	required

Raises:

Type	Description
MalformedLanguageError	If threshold not met.

Returns:

Name	Type	Description
str	str	parsed text.

Source code in deet/processors/parser.py

@field_validator("text", mode="after")
@classmethod
def assess_language_quality(cls, value: str) -> str:
    """
    Assess language quality.

    Args:
        text (str): Parsed text.

    Raises:
        MalformedLanguageError: If threshold not met.

    Returns:
        str: parsed text.

    """
    if not check_language(value):
        logger.debug("check lang failed")
        bad_language = "Supplied text didn't pass quality check."
        raise ValueError(bad_language)
    return value

ParserLibrary

Bases: ABC

Base parser class.

Source code in deet/processors/parser.py

class ParserLibrary(ABC):
    """Base parser class."""

    name: str
    input_types: list[InputFileType]
    output_file_types: list[OutputFileType]

    @classmethod
    @abstractmethod
    def parse(
        cls,
        input_: str | PathLike,
        *,
        return_metadata: bool = False,
        return_images: bool = False,
        **kwargs,
    ) -> ParsedOutput:
        """
        Parse a document.
        Intentionelly left blank as this should be populated in sub-classes.

        Args:
            input_ (str | PathLike): Path to input file or string of input string.
            return_metadata (bool, optional): Return json metadata. Defaults to False.
            return_images (bool, optional): Return images in doc. Defaults to False.

        Raises:
            NotImplementedError: The default, should never actually come.

        Returns:
            str | tuple[str, Any, Any]: There will always be str, but sometimes more.

        """
        raise NotImplementedError

parse(input_, *, return_metadata=False, return_images=False, **kwargs) abstractmethod classmethod

Parse a document. Intentionelly left blank as this should be populated in sub-classes.

Parameters:

Name	Type	Description	Default
input_	str | PathLike	Path to input file or string of input string.	required
return_metadata	bool	Return json metadata. Defaults to False.	False
return_images	bool	Return images in doc. Defaults to False.	False

Raises:

Type	Description
NotImplementedError	The default, should never actually come.

Returns:

Type	Description
ParsedOutput	str | tuple[str, Any, Any]: There will always be str, but sometimes more.

Source code in deet/processors/parser.py

@classmethod
@abstractmethod
def parse(
    cls,
    input_: str | PathLike,
    *,
    return_metadata: bool = False,
    return_images: bool = False,
    **kwargs,
) -> ParsedOutput:
    """
    Parse a document.
    Intentionelly left blank as this should be populated in sub-classes.

    Args:
        input_ (str | PathLike): Path to input file or string of input string.
        return_metadata (bool, optional): Return json metadata. Defaults to False.
        return_images (bool, optional): Return images in doc. Defaults to False.

    Raises:
        NotImplementedError: The default, should never actually come.

    Returns:
        str | tuple[str, Any, Any]: There will always be str, but sometimes more.

    """
    raise NotImplementedError

PdfminerParser

Bases: ParserLibrary

Parser with pdfminer.six backend. Fast text extraction, no images or metadata.

Source code in deet/processors/parser.py

class PdfminerParser(ParserLibrary):
    """Parser with pdfminer.six backend. Fast text extraction, no images or metadata."""

    name: Literal["pdfminer"] = "pdfminer"
    input_types = [InputFileType.PDF]
    output_file_types = [OutputFileType.MD]
    _LAPARAMS = LAParams()

    @classmethod
    def parse(
        cls,
        input_: str | PathLike,
        *,
        return_metadata: bool = False,
        return_images: bool = False,
        **kwargs,  # noqa: ARG003
    ) -> ParsedOutput:
        """Parse file using pdfminer.six (no OCR)."""
        if return_metadata or return_images:
            msg = "PdfminerParser can't produce images or metadata."
            raise InvalidOutputFileTypeError(msg)
        rsrcmgr = PDFResourceManager()
        out = StringIO()
        device = TextConverter(rsrcmgr, out, laparams=cls._LAPARAMS)
        with Path(input_).open("rb") as f:
            interpreter = PDFPageInterpreter(rsrcmgr, device)
            for page in PDFPage.get_pages(f):
                interpreter.process_page(page)
        device.close()
        text = out.getvalue() or ""
        if not text.strip():
            raise EmptyPdfExtractionError(EmptyPdfExtractionError.DEFAULT_MESSAGE)
        return ParsedOutput(text=text, parser_library=cls.name)

parse(input_, *, return_metadata=False, return_images=False, **kwargs) classmethod

Parse file using pdfminer.six (no OCR).

Source code in deet/processors/parser.py

@classmethod
def parse(
    cls,
    input_: str | PathLike,
    *,
    return_metadata: bool = False,
    return_images: bool = False,
    **kwargs,  # noqa: ARG003
) -> ParsedOutput:
    """Parse file using pdfminer.six (no OCR)."""
    if return_metadata or return_images:
        msg = "PdfminerParser can't produce images or metadata."
        raise InvalidOutputFileTypeError(msg)
    rsrcmgr = PDFResourceManager()
    out = StringIO()
    device = TextConverter(rsrcmgr, out, laparams=cls._LAPARAMS)
    with Path(input_).open("rb") as f:
        interpreter = PDFPageInterpreter(rsrcmgr, device)
        for page in PDFPage.get_pages(f):
            interpreter.process_page(page)
    device.close()
    text = out.getvalue() or ""
    if not text.strip():
        raise EmptyPdfExtractionError(EmptyPdfExtractionError.DEFAULT_MESSAGE)
    return ParsedOutput(text=text, parser_library=cls.name)

scripts

Scripts for data extraction evaluation toolkit.

cli

A CLI app to run deet pipelines.

global_options(typer_context, *, verbose=typer.Option(default=False, help='Display verbose logs.'))

Set global options for all deet commands.

Source code in deet/scripts/cli.py

@app.callback(invoke_without_command=True)
def global_options(
    typer_context: typer.Context,
    *,
    verbose: bool = typer.Option(default=False, help="Display verbose logs."),
) -> None:
    """Set global options for all deet commands."""
    from deet.data_models.project import DeetProject

    log_level = "DEBUG" if verbose else "INFO"
    logger.add(
        typer.echo,
        colorize=True,
        level=log_level,
        filter=lambda record: "is_echo" not in record["extra"],
    )
    if not verbose:
        warnings.filterwarnings("ignore", message=".*is ill-defined.*")

    cli_state = CLIState()

    with contextlib.suppress(FileNotFoundError):
        cli_state.project = DeetProject.load()

    typer_context.obj = cli_state

    if typer_context.invoked_subcommand is None:
        md_text = render_template("welcome", project=cli_state.project)
        console.clear()
        console.print(info_panel(md_text))

main()

Run CLI app.

Source code in deet/scripts/cli.py

def main() -> None:
    """Run CLI app."""
    app()

commands

deprecated

Registry of deprecated commands with instructions on how to use successors.

export_config_template_legacy()

Return deprecation warning for old export config command.

Source code in deet/scripts/commands/deprecated.py

def export_config_template_legacy() -> None:
    """Return deprecation warning for old export config command."""
    body = Group(
        "[bold red]deprecated:[/bold red] the command `export-config-template`"
        " has been deprecated.",
        " \nThe config file is automatically created on setting up a project \n",
        Syntax("deet project init", "bash"),
        " \nTo re-create this file from an existing project, run\n",
        Syntax("deet project regenerate-config-template", "bash"),
    )

    console.print(deprecation_panel(body))

extract_data_legacy()

Return deprecation warning for old extract data command.

Source code in deet/scripts/commands/deprecated.py

def extract_data_legacy() -> None:
    """Return deprecation warning for old extract data command."""
    body = Group(
        "[bold red]deprecated:[/bold red] the command `extract-data`"
        " has been deprecated.",
        " \nTo extract data, first define a project with \n",
        Syntax("deet project init", "bash"),
        " \nand follow the instructions to set up your project.",
        "\nOnce your project has been set up, you can extract data with\n",
        Syntax("deet experiments evaluate", "bash"),
    )

    console.print(deprecation_panel(body))

init_linkage_mapping_file_legacy()

Return deprecation warning for old link map command.

Source code in deet/scripts/commands/deprecated.py

def init_linkage_mapping_file_legacy() -> None:
    """Return deprecation warning for old link map command."""
    body = Group(
        "[bold red]deprecated:[/bold red] the command `init-linkage-mapping-file`"
        " has been deprecated.",
        " \nThe link map file is automatically created on setting up a project \n",
        Syntax("deet project init", "bash"),
        " \nTo re-create this file from an existing project, run\n",
        Syntax("deet project regenerate-link-map", "bash"),
    )

    console.print(deprecation_panel(body))

init_prompt_csv_legacy()

Return deprecation warning for old init prompt command.

Source code in deet/scripts/commands/deprecated.py

def init_prompt_csv_legacy() -> None:
    """Return deprecation warning for old init prompt command."""
    body = Group(
        "[bold red]deprecated:[/bold red] the command `init-prompt-csv`"
        " has been deprecated.",
        " \nThe prompt csv file is automatically created on setting up a project \n",
        Syntax("deet project init", "bash"),
        " \nTo re-create this file from an existing project, run\n",
        Syntax("deet project regenerate-prompt-csv", "bash"),
    )

    console.print(deprecation_panel(body))

link_documents_fulltexts_legacy()

Return deprecation warning for old export link documents command.

Source code in deet/scripts/commands/deprecated.py

def link_documents_fulltexts_legacy() -> None:
    """Return deprecation warning for old export link documents command."""
    body = Group(
        "[bold red]deprecated:[/bold red] the command `link-documents-fulltexts`"
        " has been deprecated.",
        " \nTo link documents to fulltexts, first create a project \n",
        Syntax("deet project init", "bash"),
        " \nAnd then run, run\n",
        Syntax("deet project link", "bash"),
    )

    console.print(deprecation_panel(body))

test_llm_config_legacy()

Return deprecation warning for old test llm command.

Source code in deet/scripts/commands/deprecated.py

def test_llm_config_legacy() -> None:
    """Return deprecation warning for old test llm command."""
    body = Group(
        "[bold red]deprecated:[/bold red] the command `test-llm-config`"
        " has been deprecated.",
        " \nTo test your llm config, first create a project: \n",
        Syntax("deet project init", "bash"),
        " \nAnd then run, run\n",
        Syntax("deet project test-llm-config", "bash"),
    )

    console.print(deprecation_panel(body))

experiments

CLI sub-commands for running data extraction experiments (and evaluating them).

evaluate(typer_context, config_path=None, prompt_population=CustomPromptPopulationMethod.FILE, prompt_csv_path=None, run_name='', custom_evaluation_metrics=None)

Extract data from documents and evaluate.

Load gold standard annotation data, and use an LLM to extract data from the documents in your dataset. Evaluate by comparing the results to the gold standard data.

Source code in deet/scripts/commands/experiments.py

@app.command()
@project_required
def evaluate(  # noqa: PLR0913
    typer_context: typer.Context,
    config_path: ConfigPathOption = None,
    prompt_population: PromptPopulationOption = CustomPromptPopulationMethod.FILE,
    prompt_csv_path: PromptPathOption = None,
    run_name: RunNameOption = "",
    custom_evaluation_metrics: Annotated[
        list[str] | None,
        typer.Option(
            help="A list of additional sklearn metrics that you wish to "
            " calculate. Use this option for each additional metric you "
            " would like to add, e.g. `deet extract-data "
            "--custom-evaluation-metrics brier_score_loss "
            "--custom-evaluation-metrics cohen_kappa_score`"
        ),
    ] = None,
) -> None:
    """
    Extract data from documents and evaluate.

    Load gold standard annotation data, and use an LLM to extract data from the
    documents in your dataset. Evaluate by comparing the results to the gold
    standard data.
    """
    from deet.evaluators.gold_standard_llm_evaluator import GoldStandardLLMEvaluator
    from deet.evaluators.metrics import EvaluationMetricSettings
    from deet.extractors.cli_helpers import run_extraction_pipeline

    deet_project: DeetProject = typer_context.obj.project

    run_output, processed_annotation_data, experiment_artefacts, config = (
        run_extraction_pipeline(
            deet_project=deet_project,
            prompt_csv_path=prompt_csv_path,
            config_path=config_path,
            prompt_population=prompt_population,
            run_name=run_name,
        )
    )

    evaluator = GoldStandardLLMEvaluator(
        gold_standard_annotated_documents=processed_annotation_data.annotated_documents,
        llm_annotated_documents=run_output.annotated_documents,
        attributes=processed_annotation_data.attributes,
        custom_metrics=custom_evaluation_metrics,
        extraction_run_id=experiment_artefacts.run_id,
        metric_settings=EvaluationMetricSettings(
            edit_distance_match_threshold=config.edit_distance_match_threshold,
        ),
    )
    evaluator.evaluate_llm_annotations()
    evaluator.write_metrics_to_csv(experiment_artefacts.metrics)
    evaluator.write_metrics_to_json(experiment_artefacts.metrics_json)
    evaluator.export_llm_comparison(experiment_artefacts.comparison)
    evaluator.display_metrics()

predict(typer_context, config_path=None, prompt_population=CustomPromptPopulationMethod.FILE, prompt_csv_path=None, run_name='', ignore_references=typer.Option(default=False, help='Ignore references in gold standard data and justextract from whatever is in your pdf_dir'))

Extract data from documents without evaluating.

Load gold standard annotation data, and use an LLM to extract data from the documents in your dataset. When used with ignore_references = True, documents are created directly from the files contained in pdf_dir.

Source code in deet/scripts/commands/experiments.py

@app.command()
@project_required
def predict(  # noqa: PLR0913
    typer_context: typer.Context,
    config_path: ConfigPathOption = None,
    prompt_population: PromptPopulationOption = CustomPromptPopulationMethod.FILE,
    prompt_csv_path: PromptPathOption = None,
    run_name: RunNameOption = "",
    ignore_references: bool = typer.Option(  # noqa: FBT001
        default=False,
        help=(
            "Ignore references in gold standard data and just"
            "extract from whatever is in your pdf_dir"
        ),
    ),
) -> None:
    """
    Extract data from documents without evaluating.

    Load gold standard annotation data, and use an LLM to extract data from the
    documents in your dataset. When used with ignore_references = True,
    documents are created directly from the files contained in pdf_dir.
    """
    from deet.evaluators.gold_standard_llm_evaluator import GoldStandardLLMEvaluator
    from deet.extractors.cli_helpers import run_extraction_pipeline

    deet_project: DeetProject = typer_context.obj.project

    (run_output, processed_annotation_data, experiment_artefacts, _config) = (
        run_extraction_pipeline(
            deet_project=deet_project,
            prompt_csv_path=prompt_csv_path,
            config_path=config_path,
            prompt_population=prompt_population,
            run_name=run_name,
            ignore_references=ignore_references,
        )
    )

    evaluator = GoldStandardLLMEvaluator(
        gold_standard_annotated_documents=[],
        llm_annotated_documents=run_output.annotated_documents,
        attributes=processed_annotation_data.attributes,
        extraction_run_id=experiment_artefacts.run_id,
    )
    evaluator.export_llm_csv(experiment_artefacts.llm_annotation_csv)

splits(typer_context, action=None, size=None)

Manage evaluation splits for this project.

Source code in deet/scripts/commands/experiments.py

@app.command()
@project_required
def splits(
    typer_context: typer.Context,
    action: Annotated[
        str | None,
        typer.Option("--action", "-a", help="add-dev | validate"),
    ] = None,
    size: Annotated[
        int | None,
        typer.Option("--size", "-s", help="Documents to sample (bypasses prompt)."),
    ] = None,
) -> None:
    """Manage evaluation splits for this project."""
    deet_project: DeetProject = typer_context.obj.project
    deet_project.load_evaluation_strategy().run_splits_wizard(
        project=deet_project, action=action, size=size
    )

project

Sub-commands for project initialisation and configuration.

edit(typer_context, field=None)

Edit an existing project's configuration.

Re-collects the project fields (pre-filled with the current values) and rewrites project.yaml WITHOUT regenerating artefacts (prompt CSV, link map, experiment dirs). Pass a field name to edit just that field; the full edit also lets you update credentials.

Source code in deet/scripts/commands/project.py

@app.command()
@project_required
def edit(
    typer_context: typer.Context,
    field: Annotated[
        str | None,
        typer.Argument(help="Optionally edit a single field instead of all of them."),
    ] = None,
) -> None:
    """
    Edit an existing project's configuration.

    Re-collects the project fields (pre-filled with the current values) and rewrites
    ``project.yaml`` WITHOUT regenerating artefacts (prompt CSV, link map, experiment
    dirs). Pass a field name to edit just that field; the full edit also lets you
    update credentials.
    """
    run_edit(typer_context.obj.project, field)

init(*, data_path=None, data_type=SupportedImportFormat.EPPI_JSON, pdf_dir=None, force_overwrite=False)

Initialise a new project in the current directory.

The project name is taken from the current directory. Leave the data and pdf options empty to enter the interactive wizard. Use deet project new to create a project in a new directory.

Source code in deet/scripts/commands/project.py

@app.command()
def init(
    *,
    data_path: DataPathOption = None,
    data_type: DataFormatOption = SupportedImportFormat.EPPI_JSON,
    pdf_dir: PdfDirOption = None,
    force_overwrite: ForceOption = False,
) -> None:
    """
    Initialise a new project in the current directory.

    The project name is taken from the current directory. Leave the data and pdf
    options empty to enter the interactive wizard.
    Use deet project new to create a project in a new directory.
    """
    root = Path.cwd()
    guard_overwrite(
        root, force=force_overwrite, interactive=not any([data_path, pdf_dir])
    )
    create_project(
        root, root.name, data_path=data_path, data_type=data_type, pdf_dir=pdf_dir
    )

link(typer_context)

Link documents to their fulltexts.

This creates a document with the parsed output of the corresponding fulltext for each of the documents in your project.

Linking will be attempted using your project's link_map.csv. See deet.processors.linker for more details.

Source code in deet/scripts/commands/project.py

@app.command()
@project_required
def link(typer_context: typer.Context) -> None:
    """
    Link documents to their fulltexts.

    This creates a document with the parsed output of the corresponding fulltext
    for each of the documents in your project.

    Linking will be attempted using your project's link_map.csv.
    See `deet.processors.linker` for more details.

    """
    from deet.processors.linker import DocumentReferenceLinker, LinkingStrategy

    deet_project: DeetProject = typer_context.obj.project
    processed_annotation_data = deet_project.process_data()

    linker = DocumentReferenceLinker(
        references=processed_annotation_data.documents,
        document_base_dir=deet_project.pdf_dir_abspath,
        document_reference_mapping=deet_project.link_map_path,
        linking_strategies=[LinkingStrategy.MAPPING_FILE],
    )
    linked_documents = linker.link_many_references_parsed_documents()

    if not deet_project.linked_documents_path.exists():
        deet_project.linked_documents_path.mkdir()

    if len(linked_documents) == 0:
        fail_with_message("Error. Could not link any documents!")

    for linked_document in linked_documents:
        file_path = (
            deet_project.linked_documents_path
            / f"{linked_document.safe_identity.document_id}.json"
        )
        linked_document.save(file_path)

    # TODO: Nice message explaining what happened
    console.print(
        info_panel(
            render_template(
                "project/linked",
                linked_documents=linked_documents,
                documents=processed_annotation_data.documents,
            )
        )
    )

new(*, name=None, data_path=None, data_type=SupportedImportFormat.EPPI_JSON, pdf_dir=None, force_overwrite=False)

Create a new project in its own directory, named after name.

Leave the data and pdf options empty to enter the interactive wizard. Use deet project init to turn the current directory into a project.

Source code in deet/scripts/commands/project.py

@app.command()
def new(
    *,
    name: Annotated[
        str | None, typer.Option(help="Project name (prompted if omitted).")
    ] = None,
    data_path: DataPathOption = None,
    data_type: DataFormatOption = SupportedImportFormat.EPPI_JSON,
    pdf_dir: PdfDirOption = None,
    force_overwrite: ForceOption = False,
) -> None:
    """
    Create a new project in its own directory, named after ``name``.

    Leave the data and pdf options
    empty to enter the interactive wizard. Use `deet project init` to turn the
    current directory into a project.
    """
    if name is None:
        name = prompt_name()

    target = Path.cwd() / slugify(name)
    guard_overwrite(
        target, force=force_overwrite, interactive=not any([data_path, pdf_dir])
    )
    target.mkdir(parents=True, exist_ok=True)
    notify(f"Creating project in {target}", level=LogLevel.INFO)
    create_project(
        target, name, data_path=data_path, data_type=data_type, pdf_dir=pdf_dir
    )

regenerate_config_template(typer_context)

Regenerate config template from a project.

A config template with defaults for each option is created on project.setup(); this re-creates it.

Source code in deet/scripts/commands/project.py

@app.command()
@project_required
def regenerate_config_template(typer_context: typer.Context) -> None:
    """
    Regenerate config template from a project.

    A config template with defaults for each option is created on project.setup();
    this re-creates it.
    """
    if not inquirer.confirm(
        "Overwrite existing config template? Make sure you have saved your work."
    ).execute():
        fail_with_message("Exiting..")
    deet_project: DeetProject = typer_context.obj.project
    deet_project.export_config_template()

regenerate_link_map(typer_context)

Regenerate a "link map" from a project.

A link map is created on project.setup(); this re-creates it.

Source code in deet/scripts/commands/project.py

@app.command()
@project_required
def regenerate_link_map(typer_context: typer.Context) -> None:
    """
    Regenerate a "link map" from a project.

    A link map is created on project.setup(); this re-creates it.
    """
    if not inquirer.confirm(
        "Overwrite existing link map? Make sure you have saved your work."
    ).execute():
        fail_with_message("Exiting..")
    deet_project: DeetProject = typer_context.obj.project
    processed_annotation_data = deet_project.process_data()

    processed_annotation_data.export_linkage_mapper_csv(
        file_path=deet_project.link_map_path,
        document_base_dir=deet_project.pdf_dir_abspath,
    )

regenerate_prompt_csv(typer_context)

Regenerate a prompt csv from a project.

A prompt csv is created on project.setup(); this re-creates it.

Source code in deet/scripts/commands/project.py

@app.command()
@project_required
def regenerate_prompt_csv(typer_context: typer.Context) -> None:
    """
    Regenerate a prompt csv from a project.

    A prompt csv is created on project.setup(); this re-creates it.
    """
    if not inquirer.confirm(
        "Overwrite existing prompt csv? Make sure you have saved your work."
    ).execute():
        fail_with_message("Exiting..")
    deet_project: DeetProject = typer_context.obj.project
    processed_annotation_data = deet_project.process_data()

    processed_annotation_data.export_attributes_csv_file(
        filepath=deet_project.prompt_csv_path
    )

test_llm_config(config_path=None)

Test llm config.

Source code in deet/scripts/commands/project.py

@app.command()
def test_llm_config(
    config_path: Annotated[
        Path | None,
        typer.Option(
            help="A path to a config file containing options for data "
            "extraction config. Leave empty to test the project config."
        ),
    ] = None,
) -> None:
    """Test llm config."""
    from deet.data_models.base import Attribute, AttributeType
    from deet.extractors.cli_helpers import (
        load_or_init_config,
    )
    from deet.extractors.llm_data_extractor import LLMDataExtractor

    config = load_or_init_config(config_path)
    data_extractor = LLMDataExtractor(config=config)
    attr = Attribute(
        output_data_type=AttributeType.BOOL,
        attribute_id=1234,
        attribute_label="Test Attribute",
        prompt="Is the document about climate and health? Return a BOOL",
    )
    context = (
        "This is document, extract data from me please. I am about climate and health"
    )
    response = data_extractor.extract_from_document(
        attributes=[attr],
        payload=context,
        context_type=None,
    )
    if response.annotations:
        notify(
            (
                f"Successfully returned {len(response.annotations)} annotation: "
                f"{response.annotations}"
            ),
            level=LogLevel.SUCCESS,
        )

project_utils

CLI helpers for the deet project setup/creation commands.

create_project(root, name, *, data_path, data_type, pdf_dir)

Create and set a project named name up at root.

When resource paths are supplied the project is built headlessly from them; otherwise the interactive wizard collects the remaining fields. The project is anchored to root and its directory structure written there.

Source code in deet/scripts/project_utils.py

def create_project(
    root: Path,
    name: str,
    *,
    data_path: Path | None,
    data_type: SupportedImportFormat,
    pdf_dir: Path | None,
) -> None:
    """
    Create and set a project named ``name`` up at ``root``.

    When resource paths are supplied the project is built headlessly from them;
    otherwise the interactive wizard collects the remaining fields. The
    project is anchored to ``root`` and its directory structure written there.
    """
    from deet.data_models.project import DeetProject

    if any([data_path, pdf_dir]):
        try:
            if data_path is None:
                fail_with_message(
                    "Gold-standard data (--data) is required to create a project"
                    " non-interactively"
                )
            project = DeetProject(
                name=name,
                gold_standard_data_path=data_path,
                gold_standard_data_format=data_type,
                pdf_dir=pdf_dir,
            )
        except ValidationError as e:
            fail_with_message(f"Invalid project configuration:\n{e}")
        project.anchor_to(root)
        project.setup()
    else:
        run_init_wizard(root, name)

guard_overwrite(target_dir, *, force, interactive)

Guard against overwriting an existing project at target_dir.

Does nothing if force is set or no project exists there. Otherwise prompts to overwrite when running interactively, or exits with guidance when headless (where prompting is impossible).

Source code in deet/scripts/project_utils.py

def guard_overwrite(target_dir: Path, *, force: bool, interactive: bool) -> None:
    """
    Guard against overwriting an existing project at ``target_dir``.

    Does nothing if ``force`` is set or no project exists there. Otherwise prompts
    to overwrite when running interactively, or exits with guidance when headless
    (where prompting is impossible).
    """
    from deet.data_models.project import PROJECT_FILE, DeetProject

    if force or not (target_dir / PROJECT_FILE).exists():
        return
    if not interactive:
        fail_with_message(
            f"A project already exists at {target_dir}. Use --force to overwrite."
        )
    existing_project = DeetProject.load(target_dir)
    notify(
        (
            f"Project {existing_project.name} already exists in {target_dir}. "
            "Continuing could overwrite data and settings"
        ),
        level=LogLevel.WARNING,
    )
    if not inquirer.confirm("Overwrite existing project?").execute():
        fail_with_message("Exiting..")

prompt_name()

Prompt for a project name.

Source code in deet/scripts/project_utils.py

def prompt_name() -> str:
    """Prompt for a project name."""
    from deet.data_models.project import DeetProject

    info = DeetProject.model_fields["name"]
    ui = get_ui_metadata(info)
    if ui is None:
        no_ui = "No UI component for name"
        raise ValueError(no_ui)
    ui_help = ui.help + (
        ". The name you enter will be standardised and used to create a directory"
    )
    console.clear()
    console.print(wizard_field_help("name", ui_help))
    return str(inquire_pydantic_field(DeetProject, "name", info, ui))

run_edit(project, field)

Re-collect a project's fields (pre-filled) and rewrite project.yaml.

Does NOT run setup(), so existing artefacts (prompt CSV, link map, experiment dirs) are preserved. field edits a single field; otherwise the full wizard runs and credentials may be updated too. Does not validate that resource paths exist on disk; invalid paths will fail when derived artefacts are regenerated or the extraction pipeline runs.

Source code in deet/scripts/project_utils.py

def run_edit(project: "DeetProject", field: str | None) -> None:
    """
    Re-collect a project's fields (pre-filled) and rewrite ``project.yaml``.

    Does NOT run ``setup()``, so existing artefacts (prompt CSV, link map, experiment
    dirs) are preserved. ``field`` edits a single field; otherwise the full wizard
    runs and credentials may be updated too. Does not validate that resource paths
    exist on disk; invalid paths will fail when derived artefacts are regenerated
    or the extraction pipeline runs.
    """
    from deet.data_models.project import DeetProject

    original_data_path = project.gold_standard_data_path
    original_format = project.gold_standard_data_format
    original_pdf_dir = project.pdf_dir
    display = _project_display_defaults(project)

    if field is None:
        updated = run_model_wizard(DeetProject, defaults=display)
    else:
        info = DeetProject.model_fields.get(field)
        ui = get_ui_metadata(info) if info is not None else None
        if info is None or ui is None:
            editable = ", ".join(_editable_field_names(DeetProject))
            fail_with_message(
                f"'{field}' is not an editable field. Choose one of: {editable}."
            )
        console.clear()
        console.print(wizard_field_help(field, ui.help))
        new_value = inquire_pydantic_field(
            DeetProject, field, info, ui, default_override=display[field]
        )
        updated = DeetProject.model_validate({**display, field: new_value})

    updated.anchor_to(project.root)
    updated.created_at = project.created_at
    updated.dump_to_yaml()

    if field is None:
        _configure_credentials(project.root / ".env")

    if (
        updated.gold_standard_data_path != original_data_path
        or updated.gold_standard_data_format != original_format
        or updated.pdf_dir != original_pdf_dir
    ):
        notify(
            "Project data sources changed. Derived artefacts may be stale -- "
            "regenerate the prompt CSV and link map with "
            "`deet project regenerate-prompt-csv` and "
            "`deet project regenerate-link-map` if needed, and re-run "
            "`deet project link` if the PDF directory changed.",
            level=LogLevel.WARNING,
        )

    notify(f"Project '{updated.name}' updated.", level=LogLevel.SUCCESS)

run_init_wizard(root, name)

Run the interactive project + credentials wizards and set the project up.

Prompts for every project field except name (supplied here), anchors the project to root (re-expressing resource paths relative to it), then writes the project structure and credentials into root.

Source code in deet/scripts/project_utils.py

def run_init_wizard(root: Path, name: str) -> None:
    """
    Run the interactive project + credentials wizards and set the project up.

    Prompts for every project field except ``name`` (supplied here), anchors the
    project to ``root`` (re-expressing resource paths relative to it), then writes
    the project structure and credentials into ``root``.
    """
    from deet.data_models.project import DeetProject

    console.clear()
    init_md = render_template("project/init")
    console.print(info_panel(init_md, title=":speedboat: project set-up"))
    continue_after_key()

    project = run_model_wizard(DeetProject, prefill={"name": name})
    project.anchor_to(root)
    project.setup()

    _configure_credentials(root / ".env")

    new_directory = root.relative_to(Path.cwd())

    console.clear()
    console.print(
        info_panel(
            render_template(
                "project/success.md", project=project, new_directory=str(new_directory)
            )
        )
    )

typer_context

decorators to handle typer context in commands.

CLIState dataclass

Structured data store for typer context.

Source code in deet/scripts/typer_context.py

@dataclass
class CLIState:
    """Structured data store for typer context."""

    project: DeetProject | None = None

project_required(f)

Check typer context for existence of a project, and exit if no project exists.

This is used to decorate cli commands which we want to exit gracefully when they are run outside of a project directory.

Source code in deet/scripts/typer_context.py

def project_required(f: Callable[P, R]) -> Callable[P, R]:
    """
    Check typer context for existence of a project, and exit if no project exists.

    This is used to decorate cli commands which we want to exit gracefully
    when they are run outside of a project directory.
    """

    @wraps(f)
    def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
        raw_typer_context = kwargs.get("typer_context")

        typer_context = cast(typer.Context | None, raw_typer_context)

        if not typer_context or not typer_context.obj or not typer_context.obj.project:
            no_project = (
                "This command must be run from a directory that contains a project."
                "\n\nCreate one in this directory by running `deet project init`,"
                "\n\nor create a project in a"
                " new directory by running"
                " `deet project new`."
                "\n\nIf you have already created a project,"
                " you may need to change into the correct directory using `cd`"
                "\n\ne.g. `cd my-project`"
            )
            fail_with_message(no_project)
            raise typer.Exit(1)
        return f(*args, **kwargs)

    return wrapper

settings

App settings using pydantic-settings.

DataExtractionSettings

Bases: BaseSettings

Settings model for data extraction behavior and provider credentials.

All fields are fully typed and documented. Unknown environment variables are forbidden to help catch configuration drift early.

Source code in deet/settings.py

class DataExtractionSettings(BaseSettings):
    """
    Settings model for data extraction behavior and provider credentials.

    All fields are fully typed and documented. Unknown environment variables
    are forbidden to help catch configuration drift early.
    """

    model_config = SettingsConfigDict(
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    # General
    log_level: LogLevel = Field(
        default=LogLevel.DEBUG,
        description="log level for the app logger.",
        json_schema_extra={"skip_prompt": True},
    )

    runtime: Runtime = Field(
        default=Runtime.LOCAL,
        description="Runtime environment.",
        json_schema_extra={"skip_prompt": True},
    )

    # Provider credentials / settings (secrets redacted)
    azure_api_key: Annotated[
        SecretStr | None, UI(help="Press enter to leave this unchanged")
    ] = Field(
        default=None,
        description="Azure OpenAI API key if using Azure provider.",
    )
    azure_api_base: Annotated[
        SecretStr | None, UI(help="Press enter to leave this unchanged")
    ] = Field(default=None, description="Base URL for azure openAI.")

    # disk cache folder
    base_disk_cache_dir: Path = Field(
        default=(Path.home() / ".deet_cache"),
        description="the base directory for disk-based caches.",
        json_schema_extra={"skip_prompt": True},
    )

    def dump_to_env(self, target_path: Path = Path(".env")) -> None:
        """Serialise settings object to a .env file."""
        target_path.parent.mkdir(parents=True, exist_ok=True)

        if not target_path.exists():
            target_path.touch()

        for field_name in type(self).model_fields:
            value = getattr(self, field_name)
            if value is not None:
                if isinstance(value, SecretStr):
                    str_value = value.get_secret_value()
                else:
                    str_value = str(value)

                set_key(
                    str(target_path), field_name.upper(), str_value, quote_mode="always"
                )

dump_to_env(target_path=Path('.env'))

Serialise settings object to a .env file.

Source code in deet/settings.py

def dump_to_env(self, target_path: Path = Path(".env")) -> None:
    """Serialise settings object to a .env file."""
    target_path.parent.mkdir(parents=True, exist_ok=True)

    if not target_path.exists():
        target_path.touch()

    for field_name in type(self).model_fields:
        value = getattr(self, field_name)
        if value is not None:
            if isinstance(value, SecretStr):
                str_value = value.get_secret_value()
            else:
                str_value = str(value)

            set_key(
                str(target_path), field_name.upper(), str_value, quote_mode="always"
            )

LLMProvider

Bases: StrEnum

Supported LLM Providers.

Source code in deet/settings.py

class LLMProvider(StrEnum):
    """Supported LLM Providers."""

    AZURE = auto()
    OLLAMA = auto()

LogLevel

Bases: StrEnum

Supported log levels for logging.

Source code in deet/settings.py

class LogLevel(StrEnum):
    """Supported log levels for logging."""

    TRACE = "TRACE"
    DEBUG = "DEBUG"
    INFO = "INFO"
    SUCCESS = "SUCCESS"
    WARNING = "WARNING"
    ERROR = "ERROR"
    CRITICAL = "CRITICAL"

Runtime

Bases: StrEnum

Permitted runtime environments for DEET.

Source code in deet/settings.py

class Runtime(StrEnum):
    """Permitted runtime environments for DEET."""

    LOCAL = auto()
    DEVELOPMENT = auto()
    STAGING = auto()
    TEST = auto()
    PRODUCTION = auto()

get_settings() cached

Return a cached settings instance for reuse across the process.

Source code in deet/settings.py

@lru_cache(maxsize=1)
def get_settings() -> DataExtractionSettings:
    """Return a cached settings instance for reuse across the process."""
    return DataExtractionSettings()

ui

fail_with_message(message)

Print message and exit CLI.

Source code in deet/ui/messenger.py

def fail_with_message(message: str) -> NoReturn:
    """Print message and exit CLI."""
    notify(message, level=LogLevel.ERROR)
    raise typer.Exit(code=1)

notify(message, level=LogLevel.INFO)

Send messages to logger and to UIs (currently the console).

Source code in deet/ui/messenger.py

def notify(message: str, level: LogLevel = LogLevel.INFO) -> None:
    """Send messages to logger and to UIs (currently the console)."""
    logger.bind(is_echo=True).log(level.value, message)
    render_to_console(message, level)

messenger

Interface to pass messages to UI(s).

fail_with_message(message)

Print message and exit CLI.

Source code in deet/ui/messenger.py

def fail_with_message(message: str) -> NoReturn:
    """Print message and exit CLI."""
    notify(message, level=LogLevel.ERROR)
    raise typer.Exit(code=1)

notify(message, level=LogLevel.INFO)

Send messages to logger and to UIs (currently the console).

Source code in deet/ui/messenger.py

def notify(message: str, level: LogLevel = LogLevel.INFO) -> None:
    """Send messages to logger and to UIs (currently the console)."""
    logger.bind(is_echo=True).log(level.value, message)
    render_to_console(message, level)

terminal

continue_after_key(message='Press Enter to continue...')

Pause execution until the user acknowledges.

Source code in deet/ui/terminal/wizards.py

def continue_after_key(message: str = "Press Enter to continue...") -> None:
    """Pause execution until the user acknowledges."""
    inquirer.secret(
        message=message,
        qmark="⌨️ ",
        transformer=lambda _: "",
    ).execute()

render_template(name, **context)

Load and render a markdown template.

Source code in deet/ui/terminal/render.py

def render_template(name: str, **context) -> str:
    """Load and render a markdown template."""
    filename = f"{name}.md" if not name.endswith(".md") else name

    tmpl = _env.get_template(filename)
    return textwrap.dedent(tmpl.render(**context)).strip()

render_to_console(message, level)

Render message to terminal using Rich.

Source code in deet/ui/terminal/render.py

def render_to_console(message: str, level: LogLevel) -> None:
    """Render message to terminal using Rich."""
    style = level.value.lower()
    panel = Panel(
        message,
        title=f"[{style}]{style.upper()}[/]",
    )
    if level in (LogLevel.ERROR, LogLevel.CRITICAL):
        error_console.print(panel, style=style)
    else:
        console.print(panel, style=style)

run_model_wizard(model_class, *, prefill=None, defaults=None)

Create a wizard from a pydantic model.

Fields present in prefill are not prompted for; their values are injected into the model directly. This lets a caller supply a field (e.g. the project name derived from a directory) instead of asking the user for it.

defaults maps a field name to a display-ready string shown as that field's editable default, so a caller can pre-fill current values while still prompting (e.g. deet project edit).

Every field after the first is back-navigable: pressing Ctrl+Left or Option+Left returns to the previous field, which is re-prompted with the answer already given.

Source code in deet/ui/terminal/wizards.py

def run_model_wizard[T: BaseModel](
    model_class: type[T],
    *,
    prefill: dict[str, object] | None = None,
    defaults: dict[str, str] | None = None,
) -> T:
    """
    Create a wizard from a pydantic model.

    Fields present in ``prefill`` are not prompted for; their values are injected
    into the model directly. This lets a caller supply a field (e.g. the project
    name derived from a directory) instead of asking the user for it.

    ``defaults`` maps a field name to a display-ready string shown as that field's
    editable default, so a caller can pre-fill current values while still prompting
    (e.g. ``deet project edit``).

    Every field after the first is back-navigable: pressing Ctrl+Left or Option+Left
    returns to the previous field, which is re-prompted with the answer already given.
    """
    prefill = prefill or {}
    defaults = defaults or {}
    answers: dict[str, object] = dict(prefill)
    ui_steps: list[tuple[str, FieldInfo, UI]] = [
        (name, info, ui)
        for name, info in model_class.model_fields.items()
        if (ui := get_ui_metadata(info)) is not None and name not in answers
    ]
    total_steps = len(ui_steps)

    index = 0
    while index < len(ui_steps):
        f_name, f_info, f_ui = ui_steps[index]
        console.clear()
        console.print(wizard_header(model_class.__name__, index + 1, total_steps))
        console.print(wizard_field_help(f_name, f_ui.help))
        console.print("Press Ctrl+C to exit", style="dim")
        if index > 0:
            console.print(
                "Press Ctrl+Left/Option+Left to go back",
                style="dim",
            )

        previous_answer = answers.get(f_name)
        seed = (
            previous_answer
            if isinstance(previous_answer, str)
            else defaults.get(f_name)
        )
        result = inquire_pydantic_field(
            model_class,
            f_name,
            f_info,
            f_ui,
            default_override=seed,
            allow_back=index > 0,
        )
        if result is GO_BACK:
            index -= 1
            continue
        answers[f_name] = result
        index += 1

    return model_class.model_validate(answers)

components

Rich UI components for the terminal.

deprecation_panel(content)

Return a panel for deprecation warnings.

Source code in deet/ui/terminal/components.py

def deprecation_panel(content: RenderableType) -> Panel:
    """Return a panel for deprecation warnings."""
    return Panel(
        content,
        title="[bold red]Deprecation warning[/bold red]",
        border_style="red",
        expand=False,
        padding=(1, 2),
    )

info_panel(content, title='INFO')

Return a styled box for displaying Markdown content.

Source code in deet/ui/terminal/components.py

def info_panel(content: str, title: str = "INFO") -> Panel:
    """Return a styled box for displaying Markdown content."""
    return Panel(
        Markdown(content),
        title=title,
        border_style="bright_blue",
        box=box.DOUBLE,
        padding=(1, 2),
        expand=False,
        width=120,
    )

wizard_field_help(field, help_text)

Print help text to the right.

Source code in deet/ui/terminal/components.py

def wizard_field_help(field: str, help_text: str) -> Align:
    """Print help text to the right."""
    help_card = Panel(
        help_text,
        title=f"About: {field}",
        border_style="cyan",
        padding=(1, 2),
        width=40,
        expand=False,
    )
    return Align.right(help_card)

wizard_header(name, current_step, total_steps)

Create a header at the top of a wizard detailing progress through fields.

Source code in deet/ui/terminal/components.py

def wizard_header(name: str, current_step: int, total_steps: int) -> Align:
    """Create a header at the top of a wizard detailing progress through fields."""
    header = Panel(
        f"[bold]Setup {name} {current_step}/{total_steps}[/]",
        border_style="bright_blue",
        width=30,
    )
    return Align.center(header)

prompts

Re-usable inquirerpy prompts.

select_experiment(project)

Select an experiment from a project's completed experiments.

Source code in deet/ui/terminal/prompts.py

def select_experiment(project: DeetProject) -> ExperimentArtefacts:
    """Select an experiment from a project's completed experiments."""
    choices = [
        {"name": exp.run_id, "value": exp} for exp in project.completed_experiments
    ]
    selected_experiment: ExperimentArtefacts = inquirer.select(
        message="Select the experiment configuration to validate:", choices=choices
    ).execute()
    return selected_experiment

select_from_list(items, item_key, selected_value=None, *, display_key='description', prompt_message='Select an option')

Select an item from a list by key, either by prompt or direct value.

Parameters:

Name	Type	Description	Default
items	Sequence[ItemT]	List of dicts to select from.	required

Source code in deet/ui/terminal/prompts.py

def select_from_list[ItemT: Mapping[str, object]](
    items: Sequence[ItemT],
    item_key: str,
    selected_value: str | None = None,
    *,
    display_key: str = "description",
    prompt_message: str = "Select an option",
) -> ItemT:
    """
    Select an item from a list by key, either by prompt or direct value.

    Args:
        items: List of dicts to select from.

    """
    if selected_value is None:
        choices = [
            {"name": item[display_key], "value": item[item_key]} for item in items
        ]
        selected_value = inquirer.select(
            message=prompt_message, choices=choices
        ).execute()
    try:
        return next(item for item in items if item[item_key] == selected_value)
    except StopIteration as e:
        key_not_found = f"'{selected_value}' not found"
        raise KeyError(key_not_found) from e

render

Single import for the rich console, and methods to write to it.

flow(text)

Clean indentation and collapse single newlines into spaces to allow text wrapping in terminal.

Source code in deet/ui/terminal/render.py

def flow(text: str) -> str:
    """
    Clean indentation and collapse single newlines into spaces
    to allow text wrapping in terminal.
    """
    if not text:
        return ""
    # 1. Remove leading indentation
    text = cleandoc(text)
    # 2. Replace single newlines with a space, keep double newlines
    return re.sub(r"(?<!\n)\n(?!\n)", " ", text).strip()

optional_progress(iterable, *, show_progress=False, label='Processing')

Context manager that yields an iterable. If show_progress is True, uses a Rich progress bar that handles logs gracefully.

Source code in deet/ui/terminal/render.py

@contextmanager
def optional_progress(
    iterable: Iterable, *, show_progress: bool = False, label: str = "Processing"
) -> Generator[Iterable, None, None]:
    """
    Context manager that yields an iterable.
    If show_progress is True, uses a Rich progress bar that handles logs gracefully.
    """
    if not show_progress:
        yield iterable
        return

    # Define the "Columns" of your progress bar for a pro look
    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        BarColumn(bar_width=None),
        TaskProgressColumn(),
        TimeRemainingColumn(),
        console=console,
        transient=False,
    ) as progress:
        task = progress.add_task(
            description=label,
            total=len(list(iterable)) if hasattr(iterable, "__len__") else None,
        )

        # We wrap the iterable to update the progress bar automatically
        def progress_iterable() -> Iterator:
            for item in iterable:
                yield item
                progress.advance(task, 1)

        yield progress_iterable()

render_template(name, **context)

Load and render a markdown template.

Source code in deet/ui/terminal/render.py

def render_template(name: str, **context) -> str:
    """Load and render a markdown template."""
    filename = f"{name}.md" if not name.endswith(".md") else name

    tmpl = _env.get_template(filename)
    return textwrap.dedent(tmpl.render(**context)).strip()

render_to_console(message, level)

Render message to terminal using Rich.

Source code in deet/ui/terminal/render.py

def render_to_console(message: str, level: LogLevel) -> None:
    """Render message to terminal using Rich."""
    style = level.value.lower()
    panel = Panel(
        message,
        title=f"[{style}]{style.upper()}[/]",
    )
    if level in (LogLevel.ERROR, LogLevel.CRITICAL):
        error_console.print(panel, style=style)
    else:
        console.print(panel, style=style)

templates

help_text

Help text for CLI commands.

wizards

Module containing interactive wizards for collecting information for the deet cli.

BoolHandler

Bases: WidgetCreator

WidgetCreator to handle boolean fields.

Source code in deet/ui/terminal/wizards.py

class BoolHandler(WidgetCreator):
    """WidgetCreator to handle boolean fields."""

    def can_handle(self, field_info: FieldInfo) -> bool:
        """Check if field is a boolean."""
        return field_info.annotation is bool or bool in get_args(field_info.annotation)

    def execute(
        self, widget_args: dict[str, Any], field_info: FieldInfo, *, allow_back: bool
    ) -> PromptResult:
        """Use a inquirer confirm to return boolean."""
        widget_args.pop("validate", None)
        widget_args.pop("filter", None)
        widget_args.pop("invalid_message", None)

        return _execute(inquirer.confirm(**widget_args), allow_back=allow_back)

can_handle(field_info)

Check if field is a boolean.

Source code in deet/ui/terminal/wizards.py

def can_handle(self, field_info: FieldInfo) -> bool:
    """Check if field is a boolean."""
    return field_info.annotation is bool or bool in get_args(field_info.annotation)

execute(widget_args, field_info, *, allow_back)

Use a inquirer confirm to return boolean.

Source code in deet/ui/terminal/wizards.py

def execute(
    self, widget_args: dict[str, Any], field_info: FieldInfo, *, allow_back: bool
) -> PromptResult:
    """Use a inquirer confirm to return boolean."""
    widget_args.pop("validate", None)
    widget_args.pop("filter", None)
    widget_args.pop("invalid_message", None)

    return _execute(inquirer.confirm(**widget_args), allow_back=allow_back)

DefaultHandler

Bases: WidgetCreator

Fallback handler (simple text prompt).

Should always be last in the strategy list.

Source code in deet/ui/terminal/wizards.py

class DefaultHandler(WidgetCreator):
    """
    Fallback handler (simple text prompt).

    Should always be last in the strategy list.
    """

    def can_handle(self, field_info: FieldInfo) -> bool:
        """Return True, handling whatever is not covered by other strategies."""
        return True

    def execute(
        self, widget_args: dict[str, Any], field_info: FieldInfo, *, allow_back: bool
    ) -> PromptResult:
        """Execute a text prompt."""
        return _execute(inquirer.text(**widget_args), allow_back=allow_back)

can_handle(field_info)

Return True, handling whatever is not covered by other strategies.

Source code in deet/ui/terminal/wizards.py

def can_handle(self, field_info: FieldInfo) -> bool:
    """Return True, handling whatever is not covered by other strategies."""
    return True

execute(widget_args, field_info, *, allow_back)

Execute a text prompt.

Source code in deet/ui/terminal/wizards.py

def execute(
    self, widget_args: dict[str, Any], field_info: FieldInfo, *, allow_back: bool
) -> PromptResult:
    """Execute a text prompt."""
    return _execute(inquirer.text(**widget_args), allow_back=allow_back)

EnumHandler

Bases: WidgetCreator

WidgetCreator to handle enums.

Source code in deet/ui/terminal/wizards.py

class EnumHandler(WidgetCreator):
    """WidgetCreator to handle enums."""

    def can_handle(self, field_info: FieldInfo) -> bool:
        """Check if the field is an enum."""
        return isinstance(field_info.annotation, type) and issubclass(
            field_info.annotation, Enum
        )

    def execute(
        self, widget_args: dict[str, Any], field_info: FieldInfo, *, allow_back: bool
    ) -> PromptResult:
        """Execute an inquirer.select prompt."""
        enum_type = cast("type[Enum]", field_info.annotation)
        widget_args["choices"] = [e.value for e in enum_type]
        return _execute(inquirer.select(**widget_args), allow_back=allow_back)

can_handle(field_info)

Check if the field is an enum.

Source code in deet/ui/terminal/wizards.py

def can_handle(self, field_info: FieldInfo) -> bool:
    """Check if the field is an enum."""
    return isinstance(field_info.annotation, type) and issubclass(
        field_info.annotation, Enum
    )

execute(widget_args, field_info, *, allow_back)

Execute an inquirer.select prompt.

Source code in deet/ui/terminal/wizards.py

def execute(
    self, widget_args: dict[str, Any], field_info: FieldInfo, *, allow_back: bool
) -> PromptResult:
    """Execute an inquirer.select prompt."""
    enum_type = cast("type[Enum]", field_info.annotation)
    widget_args["choices"] = [e.value for e in enum_type]
    return _execute(inquirer.select(**widget_args), allow_back=allow_back)

NumberHandler

Bases: WidgetCreator

WidgetCreator to handle numbers.

Source code in deet/ui/terminal/wizards.py

class NumberHandler(WidgetCreator):
    """WidgetCreator to handle numbers."""

    def can_handle(self, field_info: FieldInfo) -> bool:
        """Check if the field is a number."""
        annotation = field_info.annotation
        return annotation is float or annotation is int or int in get_args(annotation)

    def execute(
        self, widget_args: dict[str, Any], field_info: FieldInfo, *, allow_back: bool
    ) -> PromptResult:
        """
        Execute an inquirer.number prompt, adjusted to whether float or not.

        If it's optional, use a text prompt.
        """
        if type(None) in get_args(field_info.annotation):
            answer = _execute(inquirer.text(**widget_args), allow_back=allow_back)
            if answer is GO_BACK:
                return answer
            return str(answer) if answer else None

        if field_info.annotation is float:
            widget_args["float_allowed"] = True
        return _execute(inquirer.number(**widget_args), allow_back=allow_back)

can_handle(field_info)

Check if the field is a number.

Source code in deet/ui/terminal/wizards.py

def can_handle(self, field_info: FieldInfo) -> bool:
    """Check if the field is a number."""
    annotation = field_info.annotation
    return annotation is float or annotation is int or int in get_args(annotation)

execute(widget_args, field_info, *, allow_back)

Execute an inquirer.number prompt, adjusted to whether float or not.

If it's optional, use a text prompt.

Source code in deet/ui/terminal/wizards.py

def execute(
    self, widget_args: dict[str, Any], field_info: FieldInfo, *, allow_back: bool
) -> PromptResult:
    """
    Execute an inquirer.number prompt, adjusted to whether float or not.

    If it's optional, use a text prompt.
    """
    if type(None) in get_args(field_info.annotation):
        answer = _execute(inquirer.text(**widget_args), allow_back=allow_back)
        if answer is GO_BACK:
            return answer
        return str(answer) if answer else None

    if field_info.annotation is float:
        widget_args["float_allowed"] = True
    return _execute(inquirer.number(**widget_args), allow_back=allow_back)

PathHandler

Bases: WidgetCreator

WidgetCreator to handle paths.

Source code in deet/ui/terminal/wizards.py

class PathHandler(WidgetCreator):
    """WidgetCreator to handle paths."""

    def can_handle(self, field_info: FieldInfo) -> bool:
        """Check if the field is a Path."""
        return field_info.annotation is Path or Path in get_args(field_info.annotation)

    def execute(
        self, widget_args: dict[str, Any], field_info: FieldInfo, *, allow_back: bool
    ) -> PromptResult:
        """Execute an inquirer.filepath prompt."""
        return _execute(inquirer.filepath(**widget_args), allow_back=allow_back)

can_handle(field_info)

Check if the field is a Path.

Source code in deet/ui/terminal/wizards.py

def can_handle(self, field_info: FieldInfo) -> bool:
    """Check if the field is a Path."""
    return field_info.annotation is Path or Path in get_args(field_info.annotation)

execute(widget_args, field_info, *, allow_back)

Execute an inquirer.filepath prompt.

Source code in deet/ui/terminal/wizards.py

def execute(
    self, widget_args: dict[str, Any], field_info: FieldInfo, *, allow_back: bool
) -> PromptResult:
    """Execute an inquirer.filepath prompt."""
    return _execute(inquirer.filepath(**widget_args), allow_back=allow_back)

SecretHandler

Bases: WidgetCreator

Widget creator to handle secrets.

Source code in deet/ui/terminal/wizards.py

class SecretHandler(WidgetCreator):
    """Widget creator to handle secrets."""

    def can_handle(self, field_info: FieldInfo) -> bool:
        """Check if the field is a secretstr."""
        annotation = field_info.annotation
        return annotation is SecretStr or SecretStr in get_args(annotation)

    def execute(
        self, widget_args: dict[str, Any], field_info: FieldInfo, *, allow_back: bool
    ) -> PromptResult:
        """Execute an inquirer.secret prompt. Leave UNCHANGED_SECRET as None."""
        if field_info.get_default() is None:
            widget_args["default"] = UNCHANGED_SECRET
        answer = _execute(inquirer.secret(**widget_args), allow_back=allow_back)
        return None if answer == UNCHANGED_SECRET else answer

can_handle(field_info)

Check if the field is a secretstr.

Source code in deet/ui/terminal/wizards.py

def can_handle(self, field_info: FieldInfo) -> bool:
    """Check if the field is a secretstr."""
    annotation = field_info.annotation
    return annotation is SecretStr or SecretStr in get_args(annotation)

execute(widget_args, field_info, *, allow_back)

Execute an inquirer.secret prompt. Leave UNCHANGED_SECRET as None.

Source code in deet/ui/terminal/wizards.py

def execute(
    self, widget_args: dict[str, Any], field_info: FieldInfo, *, allow_back: bool
) -> PromptResult:
    """Execute an inquirer.secret prompt. Leave UNCHANGED_SECRET as None."""
    if field_info.get_default() is None:
        widget_args["default"] = UNCHANGED_SECRET
    answer = _execute(inquirer.secret(**widget_args), allow_back=allow_back)
    return None if answer == UNCHANGED_SECRET else answer

WidgetCreator

Bases: ABC

Abstract strategy to create Pyinquirer widgets from pydantic fields.

Source code in deet/ui/terminal/wizards.py

class WidgetCreator(ABC):
    """Abstract strategy to create Pyinquirer widgets from pydantic fields."""

    @abstractmethod
    def can_handle(self, field_info: FieldInfo) -> bool:
        """Return True if this handler supports the given field."""
        ...

    @abstractmethod
    def execute(
        self, widget_args: dict[str, Any], field_info: FieldInfo, *, allow_back: bool
    ) -> PromptResult:
        """Execute the InquirerPy widget and return the validated result."""
        ...

can_handle(field_info) abstractmethod

Return True if this handler supports the given field.

Source code in deet/ui/terminal/wizards.py

@abstractmethod
def can_handle(self, field_info: FieldInfo) -> bool:
    """Return True if this handler supports the given field."""
    ...

execute(widget_args, field_info, *, allow_back) abstractmethod

Execute the InquirerPy widget and return the validated result.

Source code in deet/ui/terminal/wizards.py

@abstractmethod
def execute(
    self, widget_args: dict[str, Any], field_info: FieldInfo, *, allow_back: bool
) -> PromptResult:
    """Execute the InquirerPy widget and return the validated result."""
    ...

continue_after_key(message='Press Enter to continue...')

Pause execution until the user acknowledges.

Source code in deet/ui/terminal/wizards.py

def continue_after_key(message: str = "Press Enter to continue...") -> None:
    """Pause execution until the user acknowledges."""
    inquirer.secret(
        message=message,
        qmark="⌨️ ",
        transformer=lambda _: "",
    ).execute()

get_ui_metadata(field_info)

Get UI metadata from pydantic model field.

Source code in deet/ui/terminal/wizards.py

def get_ui_metadata(field_info: FieldInfo) -> UI | None:
    """Get UI metadata from pydantic model field."""
    return next((item for item in field_info.metadata if isinstance(item, UI)), None)

inquire_pydantic_field(model_class, field_name, field_info, ui, default_override=None, *, allow_back=False)

Prompt user to provide data for pydantic field.

default_override (a display-ready string: a path, an enum value, or "" for an unset optional) replaces the field's own default when supplied, so callers can pre-fill the current value while still prompting (e.g. deet project edit).

allow_back makes the prompt skippable with Ctrl+Left / Option+Left, returning GO_BACK so a multi-step wizard can return to the previous field. Standalone prompts leave it False (a single prompt has nowhere to go back to).

Source code in deet/ui/terminal/wizards.py

def inquire_pydantic_field(  # noqa: PLR0913
    model_class: type[BaseModel],
    field_name: str,
    field_info: FieldInfo,
    ui: UI,
    default_override: str | None = None,
    *,
    allow_back: bool = False,
) -> PromptResult:
    """
    Prompt user to provide data for pydantic field.

    ``default_override`` (a display-ready string: a path, an enum value, or "" for
    an unset optional) replaces the field's own default when supplied, so callers
    can pre-fill the current value while still prompting (e.g. ``deet project edit``).

    ``allow_back`` makes the prompt skippable with Ctrl+Left / Option+Left,
    returning GO_BACK so a multi-step wizard can return to the
    previous field. Standalone prompts leave it False (a single prompt has nowhere
    to go back to).
    """

    def pydantic_validate(answer: str) -> bool | str:
        is_optional = type(None) in get_args(field_info.annotation)
        if is_optional and answer.strip() == "":
            return True
        try:
            model_class.__pydantic_validator__.validate_assignment(
                model_class.model_construct(), field_name, answer
            )
        except ValidationError:
            return False
        else:
            return True

    default = (
        default_override if default_override is not None else field_info.get_default()
    )
    default = "" if default in (PydanticUndefined, None) else default

    widget_args: dict[str, Any] = {
        "message": field_info.description,
        "default": default,
        "validate": lambda ans: pydantic_validate(ans),
        "invalid_message": ui.valid,
        "instruction": ui.instructions,
        "filter": lambda ans: ans.strip() if isinstance(ans, str) else ans,
    }

    for strategy in STRATEGIES:
        if strategy.can_handle(field_info):
            return strategy.execute(widget_args, field_info, allow_back=allow_back)

    not_implemented = f"No widget could be created for field: {field_name}"
    raise NotImplementedError(not_implemented)

run_model_wizard(model_class, *, prefill=None, defaults=None)

Create a wizard from a pydantic model.

Fields present in prefill are not prompted for; their values are injected into the model directly. This lets a caller supply a field (e.g. the project name derived from a directory) instead of asking the user for it.

defaults maps a field name to a display-ready string shown as that field's editable default, so a caller can pre-fill current values while still prompting (e.g. deet project edit).

Every field after the first is back-navigable: pressing Ctrl+Left or Option+Left returns to the previous field, which is re-prompted with the answer already given.

Source code in deet/ui/terminal/wizards.py

def run_model_wizard[T: BaseModel](
    model_class: type[T],
    *,
    prefill: dict[str, object] | None = None,
    defaults: dict[str, str] | None = None,
) -> T:
    """
    Create a wizard from a pydantic model.

    Fields present in ``prefill`` are not prompted for; their values are injected
    into the model directly. This lets a caller supply a field (e.g. the project
    name derived from a directory) instead of asking the user for it.

    ``defaults`` maps a field name to a display-ready string shown as that field's
    editable default, so a caller can pre-fill current values while still prompting
    (e.g. ``deet project edit``).

    Every field after the first is back-navigable: pressing Ctrl+Left or Option+Left
    returns to the previous field, which is re-prompted with the answer already given.
    """
    prefill = prefill or {}
    defaults = defaults or {}
    answers: dict[str, object] = dict(prefill)
    ui_steps: list[tuple[str, FieldInfo, UI]] = [
        (name, info, ui)
        for name, info in model_class.model_fields.items()
        if (ui := get_ui_metadata(info)) is not None and name not in answers
    ]
    total_steps = len(ui_steps)

    index = 0
    while index < len(ui_steps):
        f_name, f_info, f_ui = ui_steps[index]
        console.clear()
        console.print(wizard_header(model_class.__name__, index + 1, total_steps))
        console.print(wizard_field_help(f_name, f_ui.help))
        console.print("Press Ctrl+C to exit", style="dim")
        if index > 0:
            console.print(
                "Press Ctrl+Left/Option+Left to go back",
                style="dim",
            )

        previous_answer = answers.get(f_name)
        seed = (
            previous_answer
            if isinstance(previous_answer, str)
            else defaults.get(f_name)
        )
        result = inquire_pydantic_field(
            model_class,
            f_name,
            f_info,
            f_ui,
            default_override=seed,
            allow_back=index > 0,
        )
        if result is GO_BACK:
            index -= 1
            continue
        answers[f_name] = result
        index += 1

    return model_class.model_validate(answers)

utils

assess_text_quality

Tools for text quality assessments.

EmptyTextError

Bases: Exception

Raise when our text passed to is_english is empty.

Parameters:

Name	Type	Description	Default
Exception	_type_		required

Source code in deet/utils/assess_text_quality.py

class EmptyTextError(Exception):
    """
    Raise when our text passed to is_english is empty.

    Args:
        Exception (_type_):

    """

    def __init__(self, msg: str = "Supplied text is empty.", *args, **kwargs) -> None:  # noqa: ANN002
        """Init the exception with default message."""
        super().__init__(msg, *args, **kwargs)

__init__(msg='Supplied text is empty.', *args, **kwargs)

Init the exception with default message.

Source code in deet/utils/assess_text_quality.py

def __init__(self, msg: str = "Supplied text is empty.", *args, **kwargs) -> None:  # noqa: ANN002
    """Init the exception with default message."""
    super().__init__(msg, *args, **kwargs)

Language

Bases: Enum

Enum of languages for quality-checking.

Source code in deet/utils/assess_text_quality.py

class Language(Enum):
    """Enum of languages for quality-checking."""

    ENGLISH = "en"

check_language(text, lang=Language.ENGLISH, threshold=0.2)

Assess if text is in the specified language.

Parameters:

Name	Type	Description	Default
text	str	Text to assess.	required
lang	Language	Language to check against.	ENGLISH
threshold	float	Threshold for word overlap.	0.2

Raises:

Type	Description
EmptyTextError	If text is empty.

Returns:

Name	Type	Description
bool	bool	True if text matches language, False otherwise.

Source code in deet/utils/assess_text_quality.py

def check_language(
    text: str, lang: Language = Language.ENGLISH, threshold: float = 0.2
) -> bool:
    """
    Assess if text is in the specified language.

    Args:
        text (str): Text to assess.
        lang (Language): Language to check against.
        threshold (float): Threshold for word overlap.

    Raises:
        EmptyTextError: If text is empty.

    Returns:
        bool: True if text matches language, False otherwise.

    """
    if text is None or text == "":
        raise EmptyTextError
    tok = set(re.findall(r"\w+", text.lower()))

    if len(tok) <= 0:
        raise EmptyTextError

    if isinstance(lang, str):
        lang = Language(lang)

    if lang == Language.ENGLISH:
        return len(tok & bc_brown) / len(tok) > threshold
    missing_lang = f"Language '{lang.value}' not supported yet."
    raise NotImplementedError(missing_lang)

get_bc_brown_words()

Get bc_brown words, cached.

Source code in deet/utils/assess_text_quality.py

@nltk_cache.memoize(typed=True, expire=None, tag="nltk-brown-words")
def get_bc_brown_words() -> set:
    """Get bc_brown words, cached."""
    from nltk.corpus import brown
    from nltk.data import find

    try:
        find("corpora/brown")
    except LookupError:
        from nltk import download

        download("brown")

    return {x.lower() for x in brown.words()}

is_english(text, threshold=0.2)

Check if text meets minimum English quality.

Parameters:

Name	Type	Description	Default
text	str	the text to check.	required
threshold	float	Defaults to 0.2.	0.2

Returns:

Name	Type	Description
bool	bool	True if 'English', false if not.

Source code in deet/utils/assess_text_quality.py

def is_english(text: str, threshold: float = 0.2) -> bool:
    """
    Check if text meets minimum English quality.

    Args:
        text (str): the text to check.
        threshold (float, optional): Defaults to 0.2.

    Returns:
        bool: True if 'English', false if not.

    """
    return check_language(text, lang=Language.ENGLISH, threshold=threshold)

destiny_utils

Utilities for working with destiny data types.

ReferencePresenter

A class to present references in a Streamlit application.

Source code in deet/utils/destiny_utils.py

class ReferencePresenter:
    """A class to present references in a Streamlit application."""

    def __init__(self, reference: Reference | ReferenceFileInput) -> None:
        """Init the ReferencePresenter class."""
        self.reference = reference

    @property
    def bibliographic_metadata(self) -> BibliographicMetadataEnhancement | None:
        """Returns the bibliographic metadata enhancement of the reference."""
        for e in self.reference.enhancements or []:
            if e.content.enhancement_type == EnhancementType.BIBLIOGRAPHIC:
                return e.content
        return None

    @property
    def location(self) -> LocationEnhancement | None:
        """Returns the location enhancement of the reference."""
        for e in self.reference.enhancements or []:
            if e.content.enhancement_type == EnhancementType.LOCATION:
                return e.content
        return None

    @property
    def title(self) -> str:
        """Returns the title of the reference."""
        if (
            self.bibliographic_metadata is not None
            and self.bibliographic_metadata.title
        ):
            return fix_text(self.bibliographic_metadata.title)
        return ""

    @property
    def authors(self) -> str:
        """Returns the authors of the reference as a semi-colon-separated string."""
        if self.bibliographic_metadata and self.bibliographic_metadata.authorship:
            return "; ".join(
                [a.display_name for a in self.bibliographic_metadata.authorship]
            )
        return ""

    @property
    def orcids(self) -> str:
        """
        Returns all available ORCIDs for each author of
        the reference as a semi-colon-separated string.
        """
        if self.bibliographic_metadata and self.bibliographic_metadata.authorship:
            return "; ".join(
                [str(a.orcid) for a in self.bibliographic_metadata.authorship]
            )
        return ""

    @property
    def first_author(self) -> Authorship | None:
        """Returns the first author of the reference."""
        if self.bibliographic_metadata and self.bibliographic_metadata.authorship:
            for author in self.bibliographic_metadata.authorship:
                if author.position == AuthorPosition.FIRST:
                    return author
        return None

    @property
    def publisher(self) -> str:
        """Returns the publisher of the reference."""
        return (
            self.bibliographic_metadata.publisher or ""
            if self.bibliographic_metadata
            else ""
        )

    @property
    def publication_date(self) -> str:
        """Returns the publication date of the reference in 'dd mmm yyyy' format."""
        if self.bibliographic_metadata:
            pub_date = self.bibliographic_metadata.publication_date
            if pub_date:
                return pub_date.strftime("%d %b %Y") or ""
        return ""

    @property
    def year(self) -> str:
        """Returns the publication year of the reference."""
        return (
            str(
                self.bibliographic_metadata.publication_year
                or (
                    self.bibliographic_metadata.publication_date.year
                    if self.bibliographic_metadata.publication_date
                    else ""
                )
            )
            if self.bibliographic_metadata
            else ""
        )

    @property
    def publication_locations(self) -> str:
        """
        Returns the publication locations of the
        reference as a semi-colon-separated string.
        """
        if self.location:
            return (
                "; ".join(
                    [
                        str(loc.extra["display_name"]) or ""
                        for loc in self.location.locations
                        if loc.extra
                    ]
                )
                or ""
                if self.location.locations
                else ""
            )
        return ""

    @property
    def publication_links(self) -> str:
        """
        Returns the publication location links of the
        reference as a semi-colon-separated string.
        """
        if self.location:
            return (
                "; ".join(
                    [str(loc.landing_page_url) or "" for loc in self.location.locations]
                )
                or ""
                if self.location.locations
                else ""
            )
        return ""

    @property
    def publication_types(self) -> str:
        """
        Returns the publication types of the
        reference as a semi-colon-separated string.
        """
        if self.location:
            return (
                "; ".join(
                    [
                        str(loc.extra["type"]) or ""
                        for loc in self.location.locations
                        if loc.extra
                    ]
                )
                or ""
                if self.location.locations
                else ""
            )
        return ""

    @property
    def doi(self) -> str | None:
        """Returns the DOI of the reference."""
        for i in self.reference.identifiers or []:
            if i.identifier_type == "doi":
                return f"https://doi.org/{i.identifier}"
        return None

    @property
    def abstract(self) -> str:
        """Returns the abstract of the reference."""
        for e in self.reference.enhancements or []:
            if e.content.enhancement_type == EnhancementType.ABSTRACT:
                return fix_text(e.content.abstract)
        return ""

    @property
    def topics(self) -> str:
        """
        Returns the topic labels of the reference and their id links
        as a semi-colon-separated string of tuples.
        """
        a = ""
        for e in self.reference.enhancements or []:
            if e.content.enhancement_type == EnhancementType.ANNOTATION:
                for x in e.content.annotations or []:
                    if x.scheme == "openalex:topic":
                        a += str(((x.label.title()), x.data["id"] if x.data else ""))
                        a += ";"
        return a.strip(";")

    @property
    def topic_domains(self) -> str:
        """
        Returns the topic domain labels of the reference and their id links
        as a semi-colon-separated string of tuples.
        """
        a = ""
        for e in self.reference.enhancements or []:
            if e.content.enhancement_type == EnhancementType.ANNOTATION:
                for x in e.content.annotations or []:
                    if x.scheme == "openalex:topic" and x.data:
                        a += str(
                            (
                                str(x.data["domain"]["display_name"]).title(),
                                x.data["domain"]["id"],
                            )
                        )
                        a += ";"
        return a.strip(";")

    @property
    def topic_fields(self) -> str:
        """
        Returns the topic field labels of the reference and their id links
        as a semi-colon-separated string of tuples.
        """
        a = ""
        for e in self.reference.enhancements or []:
            if e.content.enhancement_type == EnhancementType.ANNOTATION:
                for x in e.content.annotations or []:
                    if x.scheme == "openalex:topic" and x.data:
                        a += str(
                            (
                                str(x.data["field"]["display_name"]).title(),
                                x.data["field"]["id"],
                            )
                        )
                        a += ";"
        return a.strip(";")

    @property
    def topic_sub_fields(self) -> str:
        """
        Returns the topic sub-field labels of the reference and their id links
        as a semi-colon-separated string of tuples.
        """
        a = ""
        for e in self.reference.enhancements or []:
            if e.content.enhancement_type == EnhancementType.ANNOTATION:
                for x in e.content.annotations or []:
                    if x.scheme == "openalex:topic" and x.data:
                        a += str(
                            (
                                str(x.data["subfield"]["display_name"]).title(),
                                x.data["subfield"]["id"],
                            )
                        )
                        a += ";"
        return a.strip(";")

    @property
    def taxonomy(self) -> str:
        """
        Returns the taxonomy labels of the reference as
        a semi-colon-separated string of tuples.
        """
        a = ""
        for e in self.reference.enhancements or []:
            if e.content.enhancement_type == EnhancementType.ANNOTATION:
                for x in e.content.annotations or []:
                    if x.scheme.startswith("classifier:taxonomy") and x.value:  # type:ignore[union-attr]
                        a += str((x.scheme.split(":")[-1], x.label)).title()
                        a += ";"
        return a.strip(";")

    def to_dict(self) -> dict:
        """
        Convert the reference to a dictionary suitable
        for DataFrame representation.
        """
        return {
            "title": self.title,
            "authors": self.authors,
            "orcids": self.orcids,
            "publisher": self.publisher,
            "publication_date": self.publication_date,
            "year": self.year,
            "publication_locations": self.publication_locations,
            "publication_links": self.publication_links,
            "publication_types": self.publication_types,
            "doi": self.doi,
            "abstract": self.abstract,
            "topics": self.topics,
            "topic_domains": self.topic_domains,
            "topic_fields": self.topic_fields,
            "topic_sub_fields": self.topic_sub_fields,
            "taxonomy": self.taxonomy,
        }

abstract property

Returns the abstract of the reference.

authors property

Returns the authors of the reference as a semi-colon-separated string.

bibliographic_metadata property

Returns the bibliographic metadata enhancement of the reference.

doi property

Returns the DOI of the reference.

first_author property

Returns the first author of the reference.

location property

Returns the location enhancement of the reference.

orcids property

Returns all available ORCIDs for each author of the reference as a semi-colon-separated string.

publication_date property

Returns the publication date of the reference in 'dd mmm yyyy' format.

publication_links property

Returns the publication location links of the reference as a semi-colon-separated string.

publication_locations property

Returns the publication locations of the reference as a semi-colon-separated string.

publication_types property

Returns the publication types of the reference as a semi-colon-separated string.

publisher property

Returns the publisher of the reference.

taxonomy property

Returns the taxonomy labels of the reference as a semi-colon-separated string of tuples.

title property

Returns the title of the reference.

topic_domains property

Returns the topic domain labels of the reference and their id links as a semi-colon-separated string of tuples.

topic_fields property

Returns the topic field labels of the reference and their id links as a semi-colon-separated string of tuples.

topic_sub_fields property

Returns the topic sub-field labels of the reference and their id links as a semi-colon-separated string of tuples.

topics property

Returns the topic labels of the reference and their id links as a semi-colon-separated string of tuples.

year property

Returns the publication year of the reference.

__init__(reference)

Init the ReferencePresenter class.

Source code in deet/utils/destiny_utils.py

def __init__(self, reference: Reference | ReferenceFileInput) -> None:
    """Init the ReferencePresenter class."""
    self.reference = reference

to_dict()

Convert the reference to a dictionary suitable for DataFrame representation.

Source code in deet/utils/destiny_utils.py

def to_dict(self) -> dict:
    """
    Convert the reference to a dictionary suitable
    for DataFrame representation.
    """
    return {
        "title": self.title,
        "authors": self.authors,
        "orcids": self.orcids,
        "publisher": self.publisher,
        "publication_date": self.publication_date,
        "year": self.year,
        "publication_locations": self.publication_locations,
        "publication_links": self.publication_links,
        "publication_types": self.publication_types,
        "doi": self.doi,
        "abstract": self.abstract,
        "topics": self.topics,
        "topic_domains": self.topic_domains,
        "topic_fields": self.topic_fields,
        "topic_sub_fields": self.topic_sub_fields,
        "taxonomy": self.taxonomy,
    }

identifier_utils

Utility functions for creating, validating and manipulating identifiers.

check_if_id_exists(new_id, id_list)

Check if target id (int) is in a list of ids (list[int]).

Source code in deet/utils/identifier_utils.py

def check_if_id_exists(new_id: int, id_list: list[int]) -> bool:
    """Check if target id (int) is in a list of ids (list[int])."""
    return new_id in id_list

hash_n_strings_to_document_id(string_list)

Convert n strings into an integer with MIN_DOCUMENT_ID-MAX_DOCUMENT_ID digits (4-10) using hash-based combination.

Parameters:

Name	Type	Description	Default
string_list	list[str]	a list of strings to hash.	required

Returns:

Type	Description
int	An integer with between MIN_DOCUMENT_ID-MAX_DOCUMENT_ID digits
int	(from 1000 to 9999999999).

Source code in deet/utils/identifier_utils.py

def hash_n_strings_to_document_id(string_list: list[str]) -> int:
    """
    Convert n strings into an integer with MIN_DOCUMENT_ID-MAX_DOCUMENT_ID
    digits (4-10) using hash-based combination.

    Args:
        string_list: a list of strings to hash.

    Returns:
        An integer with between MIN_DOCUMENT_ID-MAX_DOCUMENT_ID digits
        (from 1000 to 9999999999).

    """
    combined = "|".join(string_list)
    hash_object = hashlib.sha256(combined.encode())
    hash_hex = hash_object.hexdigest()

    # Use first 8 hex chars to select digit count (4-10)
    hash_int_1 = int(hash_hex[:8], 16)
    n_digits = (hash_int_1 % 7) + 4  # 7 possible values (4-10 inclusive)

    # Use next 8 hex chars to select value within that digit range
    hash_int_2 = int(hash_hex[8:16], 16)

    min_for_digits = 10 ** (n_digits - 1)
    max_for_digits = (10**n_digits) - 1
    range_size = max_for_digits - min_for_digits + 1

    id_ = (hash_int_2 % range_size) + min_for_digits

    # Sanity check
    id_len = len(str(id_))
    if not (MIN_DOCUMENT_ID_DIGITS <= id_len <= MAX_DOCUMENT_ID_DIGITS):
        # NOTE: potentially fix ID at source if this error is ever raised,
        # otherwise remove error.
        bad_id = (
            f"id {id_} is bad, it should have between "
            f"{MIN_DOCUMENT_ID_DIGITS} and {MAX_DOCUMENT_ID_DIGITS} digits!"
        )
        raise ValueError(bad_id)

    return id_

text

Text utilities.

slugify(value)

Turn a string into a filesystem-safe slug (lowercase, hyphen-separated).

Source code in deet/utils/text.py

def slugify(value: str) -> str:
    """Turn a string into a filesystem-safe slug (lowercase, hyphen-separated)."""
    slug = re.sub(r"[^a-z0-9]+", "-", value.strip().lower()).strip("-")
    if not slug:
        invalid = f"Could not derive a slug from {value!r}"
        raise ValueError(invalid)
    return slug

text_normalisation

Shared text normalisation helpers for evaluation and EPPI citation parsing.

clean_extracted_text(text)

Clean an already-parsed highlight or citation fragment.

Removes residual EPPI markup markers ([¬s] / [¬e]), strips surrounding quotes, and normalises whitespace. Intended for fragments extracted after markup-aware parsing — not for raw EPPI Text fields.

Parameters:

Name	Type	Description	Default
text	str	Highlight or citation fragment (already separated from page/markup structure).	required

Returns:

Type	Description
str	Cleaned string ready for display or comparison.

Source code in deet/utils/text_normalisation.py

def clean_extracted_text(text: str) -> str:
    """
    Clean an already-parsed highlight or citation fragment.

    Removes residual EPPI markup markers (``[¬s]`` / ``[¬e]``), strips surrounding
    quotes, and normalises whitespace. Intended for fragments extracted *after*
    markup-aware parsing — not for raw EPPI ``Text`` fields.

    Args:
        text: Highlight or citation fragment (already separated from page/markup
            structure).

    Returns:
        Cleaned string ready for display or comparison.

    """
    cleaned = _EPPI_MARKUP_PATTERN.sub("", text)
    cleaned = normalize_whitespace(cleaned).strip(_SURROUNDING_QUOTES)
    return cleaned.strip()

normalize_list_elements(items)

Apply :func:normalize_string_for_match to each string element.

Non-string elements are passed through unchanged.

Parameters:

Name	Type	Description	Default
items	list[Any]	Mixed list of values (typically strings plus other types).	required

Returns:

Type	Description
list[Any]	New list with string elements normalised (case-insensitive by default).

Source code in deet/utils/text_normalisation.py

def normalize_list_elements(items: list[Any]) -> list[Any]:
    """
    Apply :func:`normalize_string_for_match` to each string element.

    Non-string elements are passed through unchanged.

    Args:
        items: Mixed list of values (typically strings plus other types).

    Returns:
        New list with string elements normalised (case-insensitive by default).

    """
    return [
        normalize_string_for_match(item) if isinstance(item, str) else item
        for item in items
    ]

normalize_string_for_match(text, *, case_insensitive=True)

Normalise a string for comparison or search.

Applies :func:normalize_whitespace and, when case_insensitive is True (the default), lowercases the result.

Parameters:

Name	Type	Description	Default
text	str	Input string to normalise.	required
case_insensitive	bool	If True, lowercase after whitespace normalisation.	True

Returns:

Type	Description
str	Normalised string suitable for matching.

Source code in deet/utils/text_normalisation.py

def normalize_string_for_match(text: str, *, case_insensitive: bool = True) -> str:
    """
    Normalise a string for comparison or search.

    Applies :func:`normalize_whitespace` and, when ``case_insensitive`` is True
    (the default), lowercases the result.

    Args:
        text: Input string to normalise.
        case_insensitive: If True, lowercase after whitespace normalisation.

    Returns:
        Normalised string suitable for matching.

    """
    normalised = normalize_whitespace(text)
    if case_insensitive:
        return normalised.lower()
    return normalised

normalize_whitespace(text)

Collapse runs of whitespace to a single space and strip ends.

Parameters:

Name	Type	Description	Default
text	str	Input string (may contain newlines, tabs, or repeated spaces).	required

Returns:

Type	Description
str	String with internal whitespace collapsed and leading/trailing whitespace
str	removed. Empty input yields "".

Source code in deet/utils/text_normalisation.py

def normalize_whitespace(text: str) -> str:
    """
    Collapse runs of whitespace to a single space and strip ends.

    Args:
        text: Input string (may contain newlines, tabs, or repeated spaces).

    Returns:
        String with internal whitespace collapsed and leading/trailing whitespace
        removed. Empty input yields ``""``.

    """
    return re.sub(r"\s+", " ", text).strip()

timing

Wall-clock timing helpers.

ElapsedSeconds pydantic-model

Bases: BaseModel

Elapsed wall-clock time in seconds, populated when a timing context exits.

Fields:

• seconds (float)

Source code in deet/utils/timing.py

class ElapsedSeconds(BaseModel):
    """Elapsed wall-clock time in seconds, populated when a timing context exits."""

    seconds: float = Field(default=0.0, ge=0.0)

measure_elapsed()

Yield an ElapsedSeconds model with seconds set on context exit.

Source code in deet/utils/timing.py

@contextmanager
def measure_elapsed() -> Generator[ElapsedSeconds, None, None]:
    """Yield an ``ElapsedSeconds`` model with ``seconds`` set on context exit."""
    elapsed = ElapsedSeconds()
    start = time.perf_counter()
    try:
        yield elapsed
    finally:
        elapsed.seconds = round(time.perf_counter() - start, 3)

tokenisation

Token counting and truncation utilities for LLM context management.

count_tokens(model, text)

Count the number of tokens in text for a given model.

Uses litellm.token_counter with messages format. Falls back to a rough character-based estimate (len/4) for unknown models.

Parameters:

Name	Type	Description	Default
model	str	Model identifier for tokenizer selection.	required
text	str	Text to count tokens for.	required

Returns:

Type	Description
int	Number of tokens.

Source code in deet/utils/tokenisation.py

def count_tokens(model: str, text: str) -> int:
    """
    Count the number of tokens in text for a given model.

    Uses litellm.token_counter with messages format. Falls back to
    a rough character-based estimate (len/4) for unknown models.

    Args:
        model: Model identifier for tokenizer selection.
        text: Text to count tokens for.

    Returns:
        Number of tokens.

    """
    try:
        return litellm.token_counter(
            model=model,
            messages=[{"role": "user", "content": text}],
        )
    except _LITELLM_TOKENISER_ERRORS as e:
        logger.warning(
            f"Could not count tokens for model {model}, using char estimate: {e}"
        )
    except Exception as e:  # noqa: BLE001
        # litellm token_counter may raise bare Exception (registry / provider).
        if _LITELLM_UNMAPPED_MODEL_MSG.search(str(e)):
            logger.debug(
                f"Could not count tokens for model {model} (unmapped in registry): {e}"
            )
        else:
            logger.warning(
                f"Could not count tokens for model {model}, using char estimate: {e}"
            )
    return max(1, len(text) // 4)

estimate_cost_usd(model, prompt_tokens=0, completion_tokens=0)

Estimate cost in USD for prompt and completion tokens.

Uses litellm's cost_per_token. Returns (prompt_cost_usd, completion_cost_usd). Either or both may be None if the model is unknown or the call fails. Any other Exception from litellm (e.g. provider HTTP errors) is logged at debug and also yields (None, None) so cost estimation stays best-effort for callers.

Parameters:

Name	Type	Description	Default
model	str	Model identifier (e.g. "gpt-4o-mini", "azure/gpt-4o-mini").	required
prompt_tokens	int	Number of input/prompt tokens.	0
completion_tokens	int	Number of output/completion tokens.	0

Returns:

Type	Description
tuple[float | None, float | None]	Tuple of (prompt_cost_usd, completion_cost_usd). Either can be None.

Source code in deet/utils/tokenisation.py

def estimate_cost_usd(
    model: str,
    prompt_tokens: int = 0,
    completion_tokens: int = 0,
) -> tuple[float | None, float | None]:
    """
    Estimate cost in USD for prompt and completion tokens.

    Uses litellm's cost_per_token. Returns (prompt_cost_usd, completion_cost_usd).
    Either or both may be None if the model is unknown or the call fails.
    Any other ``Exception`` from litellm (e.g. provider HTTP errors) is logged
    at debug and also yields ``(None, None)`` so cost estimation stays
    best-effort for callers.

    Args:
        model: Model identifier (e.g. "gpt-4o-mini", "azure/gpt-4o-mini").
        prompt_tokens: Number of input/prompt tokens.
        completion_tokens: Number of output/completion tokens.

    Returns:
        Tuple of (prompt_cost_usd, completion_cost_usd). Either can be None.

    """
    try:
        prompt_cost, completion_cost = litellm.cost_per_token(
            model=model,
            prompt_tokens=prompt_tokens,
            completion_tokens=completion_tokens,
        )
        return (
            float(prompt_cost) if prompt_cost is not None else None,
            float(completion_cost) if completion_cost is not None else None,
        )
    except _LITELLM_LOOKUP_ERRORS as e:
        logger.debug(f"Could not estimate cost for model {model}: {e}")
        return (None, None)
    except Exception as e:  # noqa: BLE001
        # litellm cost_per_token may raise bare Exception (registry / provider).
        if _LITELLM_UNMAPPED_MODEL_MSG.search(str(e)):
            logger.debug(
                f"Could not estimate cost for model {model} (unmapped in registry): {e}"
            )
        else:
            logger.debug(f"Could not estimate cost for model {model}: {e}")
        return (None, None)

get_model_max_tokens(model)

Get the maximum input tokens allowed for a model.

Uses litellm's model registry. Returns None if the model is unknown.

Parameters:

Name	Type	Description	Default
model	str	Model identifier (e.g. "gpt-4o-mini", "azure/gpt-4o-mini").	required

Returns:

Type	Description
int | None	Maximum input tokens, or None if model not found.

Raises:

Type	Description
LitellmModelNotMappedError	When litellm raises a bare Exception whose message indicates an unmapped model (registry message).
Exception	Any other error from litellm is re-raised.

Source code in deet/utils/tokenisation.py

def get_model_max_tokens(model: str) -> int | None:
    """
    Get the maximum input tokens allowed for a model.

    Uses litellm's model registry. Returns None if the model is unknown.

    Args:
        model: Model identifier (e.g. "gpt-4o-mini", "azure/gpt-4o-mini").

    Returns:
        Maximum input tokens, or None if model not found.

    Raises:
        LitellmModelNotMappedError: When litellm raises a bare ``Exception``
            whose message indicates an unmapped model (registry message).
        Exception: Any other error from litellm is re-raised.

    """
    try:
        result = litellm.get_max_tokens(model)
        return int(result) if result is not None else None
    except _LITELLM_LOOKUP_ERRORS as e:
        logger.debug(f"Could not get max tokens for model {model}: {e}")
        return None
    except Exception as e:
        if _LITELLM_UNMAPPED_MODEL_MSG.search(str(e)):
            logger.debug(f"Litellm model not in registry for max tokens: {model}: {e}")
            raise LitellmModelNotMappedError(model) from e
        raise

merge_prompt_completion_cost_usd(prompt_cost_usd, completion_cost_usd)

Combine litellm prompt and completion cost parts into a single total.

Either part may be None when the model table lacks that component or estimation failed for that side.

Parameters:

Name	Type	Description	Default
prompt_cost_usd	float | None	Estimated USD cost for prompt tokens, or None.	required
completion_cost_usd	float | None	Estimated USD cost for completion tokens, or None.	required

Returns:

Type	Description
float | None	Sum when at least one part is known, otherwise None.

Source code in deet/utils/tokenisation.py

def merge_prompt_completion_cost_usd(
    prompt_cost_usd: float | None,
    completion_cost_usd: float | None,
) -> float | None:
    """
    Combine litellm prompt and completion cost parts into a single total.

    Either part may be None when the model table lacks that component or
    estimation failed for that side.

    Args:
        prompt_cost_usd: Estimated USD cost for prompt tokens, or None.
        completion_cost_usd: Estimated USD cost for completion tokens, or None.

    Returns:
        Sum when at least one part is known, otherwise None.

    """
    if prompt_cost_usd is not None and completion_cost_usd is not None:
        return prompt_cost_usd + completion_cost_usd
    if prompt_cost_usd is not None:
        return prompt_cost_usd
    if completion_cost_usd is not None:
        return completion_cost_usd
    return None

truncate_to_token_limit(text, model, max_tokens)

Truncate text to fit within a token limit.

Encodes text, truncates the token list, decodes back to string. Uses char-based fallback when encode/decode unavailable.

Parameters:

Name	Type	Description	Default
text	str	Text to truncate.	required
model	str	Model identifier for tokenizer.	required
max_tokens	int	Maximum tokens allowed for the truncated text.	required

Returns:

Type	Description
str	Truncated text.

Source code in deet/utils/tokenisation.py

def truncate_to_token_limit(text: str, model: str, max_tokens: int) -> str:
    """
    Truncate text to fit within a token limit.

    Encodes text, truncates the token list, decodes back to string.
    Uses char-based fallback when encode/decode unavailable.

    Args:
        text: Text to truncate.
        model: Model identifier for tokenizer.
        max_tokens: Maximum tokens allowed for the truncated text.

    Returns:
        Truncated text.

    """
    limit = max(1, max_tokens)
    try:
        tokens = litellm.encode(model=model, text=text)
        if len(tokens) <= limit:
            return text
        truncated_tokens = tokens[:limit]
        decoded = litellm.decode(model=model, tokens=truncated_tokens)
        return str(decoded)
    except _LITELLM_TOKENISER_ERRORS as e:
        logger.debug(f"Could not truncate by tokens for model {model}: {e}")
    # Fallback: rough char estimate (~4 chars per token)
    approx_chars = limit * 4
    if len(text) <= approx_chars:
        return text
    return text[:approx_chars]