Skip to content

API reference

data_models

Data models for the data extraction evaluation toolkit.

base

Core data models regarding annotations.

AnnotationType

Bases: StrEnum

Enumeration of annotation types.

Source code in deet/data_models/base.py 
18
19
20
21
22
class AnnotationType(StrEnum):
    """Enumeration of annotation types."""

    HUMAN = auto()
    LLM = auto()

Attribute pydantic-model

Bases: BaseModel

Core attribute definition for data extraction tasks.

Represents a single piece of information to be extracted from documents.

Fields:

Source code in deet/data_models/base.py 
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
class Attribute(BaseModel):
    """
    Core attribute definition for data extraction tasks.

    Represents a single piece of information to be extracted from documents.
    """

    model_config = ConfigDict()

    prompt: str | None = None  # an optional prompt.
    output_data_type: AttributeType  # One of the defined output data types
    attribute_id: int  # unique identifier for the attribute
    attribute_label: str  # human-readable way of identifying the attribute

    def write_to_csv(self, filepath: Path, mode: Literal["a", "w"] = "a") -> None:
        """
        Write an attribute as a line to a csv file - fields represent columns.

        Args:
            filepath (Path): outfile destination.
            mode (Literal["a", "w"], optional): _w_rite or _a_ppend.
            Defaults to "a" (append).

        """
        dictified = self.model_dump()

        filepath.parent.mkdir(parents=True, exist_ok=True)
        file_exists = filepath.exists() and filepath.stat().st_size > 0
        write_header = not file_exists or mode == "w"

        with filepath.open(mode=mode, newline="", encoding="utf-8") as f:
            writer = csv.DictWriter(f, fieldnames=dictified.keys())

            if write_header:
                writer.writeheader()

            writer.writerow(dictified)

        logger.debug(f"Wrote attribute to {filepath}")

    def populate_prompt_from_dict(
        self, input_dict: dict[str, Any], *, overwrite: bool = True
    ) -> None:
        """
        Populate the `prompt` field in an Attribute instance from a dict.

        The dict must contain following fields:
            - attribute_id
            - prompt
        and attribute_id(dict) must match self.attribute_id.

        NOTE: this would typically be used in a loop to populate
        prompts for a list of attributes from a csv file where every
        row represents an attribute.

        Args:
            input_dict (dict[str, Any]): An input dict, typically a line in a csv file.
            overwrite (bool, optional): Overwrite existing val in `self.prompt`.
            Defaults to True.

        """
        for field in ["attribute_id", "prompt"]:
            if field not in input_dict:
                bad_dict = (
                    "input dict must contain at least `attribute_id` and `prompt`"
                    " fields. currently, it only "
                    f"contains: {', '.join(input_dict.keys())}"
                )
                raise ValueError(bad_dict)

        if int(input_dict["attribute_id"]) != self.attribute_id:
            bad_att_id = (
                f"attribute_id mismatch: input: {input_dict['attribute_id']}. "
                f" self: {self.attribute_id}"
            )
            raise ValueError(bad_att_id)

        if overwrite or (not overwrite and self.prompt is None):
            self.prompt = input_dict["prompt"]
            logger.debug("added prompt  [...] to Attribute instance.")
        else:
            logger.info("overwrite is set to False, no overwrite prompts.")

    def print_tabulated(self) -> None:
        """Print tabulated version of the contents of this attribute."""
        dictified = self.model_dump()
        data = [[k, v] for k, v in dictified.items()]

        print(tabulate(data, headers=["Field", "Value"], tablefmt="simple"))

    def enter_custom_prompt(self, max_tries: int = 5) -> None:
        """Use CLI to add a prompt."""
        self.print_tabulated()
        print("")
        print("Do you want to add a new prompt? y/n. Use CTRL+C to cancel.")
        tries = 0
        while True:
            user_input = input().strip().lower()

            if user_input == "n":
                logger.debug("user chose not to write a prompt...")
                return

            if user_input == "y":
                break

            print("Please answer either `y` or `n`...")
            tries += 1
            if tries >= max_tries:
                return

        def sanitize_prompt(prompt: str) -> str:
            # Remove non-printable/control characters
            return "".join(c for c in prompt if c.isprintable())

        while True:
            print(f"Please enter your prompt (max {MAX_PROMPT_LENGTH} characters): ")
            user_prompt = input().strip()
            user_prompt = sanitize_prompt(user_prompt)
            if len(user_prompt) == 0:
                print("Prompt cannot be empty. Please try again.")
                continue
            if len(user_prompt) > MAX_PROMPT_LENGTH:
                print(f"Prompt exceeds max {MAX_PROMPT_LENGTH} chars. Shorten!.")
                continue
            print("\nYour prompt will be stored as:\n")
            print(f'"{user_prompt}"')
            print("Confirm? y/n")
            confirm = input().strip().lower()
            if confirm == "y":
                self.prompt = user_prompt
                logger.debug(f"wrote prompt {self.prompt[:30]} [...] to prompt field.")
                return
            if confirm == "n":
                print("Prompt entry cancelled. Please enter again or CTRL+C to exit.")
                continue
enter_custom_prompt(max_tries=5)

Use CLI to add a prompt.

Source code in deet/data_models/base.py 
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
def enter_custom_prompt(self, max_tries: int = 5) -> None:
    """Use CLI to add a prompt."""
    self.print_tabulated()
    print("")
    print("Do you want to add a new prompt? y/n. Use CTRL+C to cancel.")
    tries = 0
    while True:
        user_input = input().strip().lower()

        if user_input == "n":
            logger.debug("user chose not to write a prompt...")
            return

        if user_input == "y":
            break

        print("Please answer either `y` or `n`...")
        tries += 1
        if tries >= max_tries:
            return

    def sanitize_prompt(prompt: str) -> str:
        # Remove non-printable/control characters
        return "".join(c for c in prompt if c.isprintable())

    while True:
        print(f"Please enter your prompt (max {MAX_PROMPT_LENGTH} characters): ")
        user_prompt = input().strip()
        user_prompt = sanitize_prompt(user_prompt)
        if len(user_prompt) == 0:
            print("Prompt cannot be empty. Please try again.")
            continue
        if len(user_prompt) > MAX_PROMPT_LENGTH:
            print(f"Prompt exceeds max {MAX_PROMPT_LENGTH} chars. Shorten!.")
            continue
        print("\nYour prompt will be stored as:\n")
        print(f'"{user_prompt}"')
        print("Confirm? y/n")
        confirm = input().strip().lower()
        if confirm == "y":
            self.prompt = user_prompt
            logger.debug(f"wrote prompt {self.prompt[:30]} [...] to prompt field.")
            return
        if confirm == "n":
            print("Prompt entry cancelled. Please enter again or CTRL+C to exit.")
            continue
populate_prompt_from_dict(input_dict, *, overwrite=True)

Populate the prompt field in an Attribute instance from a dict.

The dict must contain following fields
  • attribute_id
  • prompt

and attribute_id(dict) must match self.attribute_id.

NOTE: this would typically be used in a loop to populate prompts for a list of attributes from a csv file where every row represents an attribute.

Parameters:

Name Type Description Default
input_dict dict[str, Any]

An input dict, typically a line in a csv file.

 required
overwrite bool

Overwrite existing val in self.prompt.

 True
Source code in deet/data_models/base.py 
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
def populate_prompt_from_dict(
    self, input_dict: dict[str, Any], *, overwrite: bool = True
) -> None:
    """
    Populate the `prompt` field in an Attribute instance from a dict.

    The dict must contain following fields:
        - attribute_id
        - prompt
    and attribute_id(dict) must match self.attribute_id.

    NOTE: this would typically be used in a loop to populate
    prompts for a list of attributes from a csv file where every
    row represents an attribute.

    Args:
        input_dict (dict[str, Any]): An input dict, typically a line in a csv file.
        overwrite (bool, optional): Overwrite existing val in `self.prompt`.
        Defaults to True.

    """
    for field in ["attribute_id", "prompt"]:
        if field not in input_dict:
            bad_dict = (
                "input dict must contain at least `attribute_id` and `prompt`"
                " fields. currently, it only "
                f"contains: {', '.join(input_dict.keys())}"
            )
            raise ValueError(bad_dict)

    if int(input_dict["attribute_id"]) != self.attribute_id:
        bad_att_id = (
            f"attribute_id mismatch: input: {input_dict['attribute_id']}. "
            f" self: {self.attribute_id}"
        )
        raise ValueError(bad_att_id)

    if overwrite or (not overwrite and self.prompt is None):
        self.prompt = input_dict["prompt"]
        logger.debug("added prompt  [...] to Attribute instance.")
    else:
        logger.info("overwrite is set to False, no overwrite prompts.")
print_tabulated()

Print tabulated version of the contents of this attribute.

Source code in deet/data_models/base.py 
193
194
195
196
197
198
def print_tabulated(self) -> None:
    """Print tabulated version of the contents of this attribute."""
    dictified = self.model_dump()
    data = [[k, v] for k, v in dictified.items()]

    print(tabulate(data, headers=["Field", "Value"], tablefmt="simple"))
write_to_csv(filepath, mode='a')

Write an attribute as a line to a csv file - fields represent columns.

Parameters:

Name Type Description Default
filepath Path

outfile destination.

 required
mode Literal['a', 'w']

_w_rite or _a_ppend.

 'a'
Source code in deet/data_models/base.py 
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
def write_to_csv(self, filepath: Path, mode: Literal["a", "w"] = "a") -> None:
    """
    Write an attribute as a line to a csv file - fields represent columns.

    Args:
        filepath (Path): outfile destination.
        mode (Literal["a", "w"], optional): _w_rite or _a_ppend.
        Defaults to "a" (append).

    """
    dictified = self.model_dump()

    filepath.parent.mkdir(parents=True, exist_ok=True)
    file_exists = filepath.exists() and filepath.stat().st_size > 0
    write_header = not file_exists or mode == "w"

    with filepath.open(mode=mode, newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=dictified.keys())

        if write_header:
            writer.writeheader()

        writer.writerow(dictified)

    logger.debug(f"Wrote attribute to {filepath}")

AttributeType

Bases: StrEnum

Enum of permitted attribute data types.

Source code in deet/data_models/base.py 
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
class AttributeType(StrEnum):
    """Enum of permitted attribute data types."""

    STRING = auto()
    INTEGER = auto()
    FLOAT = auto()
    BOOL = auto()
    LIST = auto()
    DICT = auto()

    def missing_annotation_default(
        self,
    ) -> bool | str | int | float | list[Never] | dict[str, Never]:
        """
        Return default ``output_data`` when no gold-standard annotation exists.

        Used when synthesizing a placeholder annotation (e.g. comparing LLM output
        to gold standard where a value was never annotated).

        Returns a fresh ``list`` or ``dict`` for mutable types so callers do not share
        state.

        Raises:
            ValueError: If this member has no defined default.

        Note:
            This is not ``Enum._missing_``; that hook resolves *unrecognised raw
            values* when constructing enum members, not per-type defaults.

        """
        match self:
            case AttributeType.BOOL:
                return False
            case AttributeType.LIST:
                return []
            case AttributeType.STRING:
                return ""
            case AttributeType.INTEGER:
                return 0
            case AttributeType.FLOAT:
                return 0.0
            case AttributeType.DICT:
                return {}
            case _:
                unsupported = (
                    f"No default for missing annotation when attribute type is {self!s}"
                )
                raise ValueError(unsupported)

    def __str__(self) -> str:
        """Return the string value for JSON serialization."""
        return self.value

    def to_python_type(self) -> type:
        """Map AttributeType to actual Python types."""
        mapping = {
            AttributeType.STRING: str,
            AttributeType.INTEGER: int,
            AttributeType.FLOAT: float,
            AttributeType.BOOL: bool,
            AttributeType.LIST: list,
            AttributeType.DICT: dict,
        }
        return mapping[self]

    def to_json_type(self) -> JsonValue:
        """Map AttributeType to JS types for the JSON schema."""
        mapping: JsonDict = {
            AttributeType.STRING: {"type": "string"},
            AttributeType.INTEGER: {"type": "integer"},
            AttributeType.FLOAT: {"type": "number"},
            AttributeType.BOOL: {"type": "boolean"},
            AttributeType.LIST: {
                "type": "array",
                "items": {"type": "object", "additionalProperties": False},
            },
            AttributeType.DICT: {"type": "object", "additionalProperties": False},
        }
        return mapping[self]
__str__()

Return the string value for JSON serialization.

Source code in deet/data_models/base.py 
74
75
76
def __str__(self) -> str:
    """Return the string value for JSON serialization."""
    return self.value
missing_annotation_default()

Return default output_data when no gold-standard annotation exists.

Used when synthesizing a placeholder annotation (e.g. comparing LLM output to gold standard where a value was never annotated).

Returns a fresh list or dict for mutable types so callers do not share state.

Raises:

Type Description
ValueError

If this member has no defined default.
Note

This is not Enum._missing_; that hook resolves unrecognised raw values when constructing enum members, not per-type defaults.

Source code in deet/data_models/base.py 
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
def missing_annotation_default(
    self,
) -> bool | str | int | float | list[Never] | dict[str, Never]:
    """
    Return default ``output_data`` when no gold-standard annotation exists.

    Used when synthesizing a placeholder annotation (e.g. comparing LLM output
    to gold standard where a value was never annotated).

    Returns a fresh ``list`` or ``dict`` for mutable types so callers do not share
    state.

    Raises:
        ValueError: If this member has no defined default.

    Note:
        This is not ``Enum._missing_``; that hook resolves *unrecognised raw
        values* when constructing enum members, not per-type defaults.

    """
    match self:
        case AttributeType.BOOL:
            return False
        case AttributeType.LIST:
            return []
        case AttributeType.STRING:
            return ""
        case AttributeType.INTEGER:
            return 0
        case AttributeType.FLOAT:
            return 0.0
        case AttributeType.DICT:
            return {}
        case _:
            unsupported = (
                f"No default for missing annotation when attribute type is {self!s}"
            )
            raise ValueError(unsupported)
to_json_type()

Map AttributeType to JS types for the JSON schema.

Source code in deet/data_models/base.py 
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
def to_json_type(self) -> JsonValue:
    """Map AttributeType to JS types for the JSON schema."""
    mapping: JsonDict = {
        AttributeType.STRING: {"type": "string"},
        AttributeType.INTEGER: {"type": "integer"},
        AttributeType.FLOAT: {"type": "number"},
        AttributeType.BOOL: {"type": "boolean"},
        AttributeType.LIST: {
            "type": "array",
            "items": {"type": "object", "additionalProperties": False},
        },
        AttributeType.DICT: {"type": "object", "additionalProperties": False},
    }
    return mapping[self]
to_python_type()

Map AttributeType to actual Python types.

Source code in deet/data_models/base.py 
78
79
80
81
82
83
84
85
86
87
88
def to_python_type(self) -> type:
    """Map AttributeType to actual Python types."""
    mapping = {
        AttributeType.STRING: str,
        AttributeType.INTEGER: int,
        AttributeType.FLOAT: float,
        AttributeType.BOOL: bool,
        AttributeType.LIST: list,
        AttributeType.DICT: dict,
    }
    return mapping[self]

GoldStandardAnnotation pydantic-model

Bases: BaseModel

A single gold standard annotation for an attribute.

raw_data stores the data as it comes from source, output_data is computed and coerces raw_data into the correct type. This can change if the AttributeType of the attribute changes.

Fields:

Source code in deet/data_models/base.py 
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
class GoldStandardAnnotation(BaseModel):
    """
    A single gold standard annotation for an attribute.

    `raw_data` stores the data as it comes from source,
    `output_data` is computed and coerces raw_data into the correct type.
    This can change if the `AttributeType` of the `attribute` changes.
    """

    attribute: Attribute
    raw_data: Any = Field(
        description=(
            "The output data exactly as it was first seen"
            " without any coercion to the correct type"
        )
    )
    annotation_type: AnnotationType
    additional_text: str | None = Field(
        description="Notes provided by the annotator - usually the citation "
        " from the paper containing the context window where the attribute is found",
        default=None,
    )
    reasoning: str | None = Field(
        description="Reasoning, taken from LLM response", default=None
    )

    @model_validator(mode="before")
    @classmethod
    def handle_output_data_input(cls, data: SUPPORTED_TYPES) -> SUPPORTED_TYPES:
        """Catch instantations with output_data and send this to raw_data."""
        if isinstance(data, dict) and "output_data" in data and "raw_data" not in data:
            data["raw_data"] = data.pop("output_data")
        return data

    @computed_field  # type: ignore[prop-decorator]
    @property
    def output_data(self) -> SUPPORTED_TYPES | None:
        """Coerce raw data to correct type based on attribute."""
        strategy = ANNOTATION_COERCION_STRATEGIES.get(self.attribute.output_data_type)

        if strategy:
            return strategy(self.raw_data)

        return self.raw_data
additional_text = None pydantic-field

Notes provided by the annotator - usually the citation from the paper containing the context window where the attribute is found

output_data property

Coerce raw data to correct type based on attribute.

raw_data pydantic-field

The output data exactly as it was first seen without any coercion to the correct type

reasoning = None pydantic-field

Reasoning, taken from LLM response

handle_output_data_input(data) classmethod

Catch instantations with output_data and send this to raw_data.

Source code in deet/data_models/base.py 
356
357
358
359
360
361
362
@model_validator(mode="before")
@classmethod
def handle_output_data_input(cls, data: SUPPORTED_TYPES) -> SUPPORTED_TYPES:
    """Catch instantations with output_data and send this to raw_data."""
    if isinstance(data, dict) and "output_data" in data and "raw_data" not in data:
        data["raw_data"] = data.pop("output_data")
    return data

LLMAnnotationResponse pydantic-model

Bases: BaseModel

LLM response model for a single annotation.

This mirrors EppiGoldStandardAnnotation structure but uses attribute_id instead of full EppiAttribute object, as the LLM cannot provide the full attribute object.

Config:

  • extra: forbid

Fields:

Source code in deet/data_models/base.py 
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
class LLMAnnotationResponse(BaseModel):
    """
    LLM response model for a single annotation.

    This mirrors EppiGoldStandardAnnotation structure but uses attribute_id
    instead of full EppiAttribute object, as the LLM cannot provide the full
    attribute object.
    """

    attribute_id: int = Field(
        ..., description="The ID of the EPPI attribute being annotated"
    )
    output_data: Any = Field(
        ...,
        description="The LLM's annotation.",
        json_schema_extra=cast(
            "JsonDict",
            {
                "anyOf": [
                    attribute_type.to_json_type() for attribute_type in AttributeType
                ]
            },
        ),
    )
    additional_text: str | None = Field(
        ...,
        description=(
            "Supporting text from document containing the context window "
            "where the attribute is found"
        ),
    )
    reasoning: str | None = Field(
        ...,
        description="Reasoning or explanation for the annotation decision",
    )

    # Note: arm_id, arm_title, arm_description, item_attribute_full_text_details
    # are not included as they're EPPI-specific metadata the LLM cannot provide

    model_config = ConfigDict(extra="forbid")
additional_text pydantic-field

Supporting text from document containing the context window where the attribute is found

attribute_id pydantic-field

The ID of the EPPI attribute being annotated

output_data pydantic-field

The LLM's annotation.

reasoning pydantic-field

Reasoning or explanation for the annotation decision

LLMInputSchema pydantic-model

Bases: BaseModel

Schema for data going into the LLM.

Config:

  • extra: ignore

Fields:

Validators:

Source code in deet/data_models/base.py 
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
class LLMInputSchema(BaseModel):
    """Schema for data going into the LLM."""

    prompt: str
    attribute_id: int
    output_data_type: AttributeType

    model_config = ConfigDict(extra="ignore")

    @model_validator(mode="before")
    @classmethod
    def fill_prompt(cls, data: dict, fill_from_field: str = "attribute_label") -> dict:
        """
        Fill `prompt` field if empty.

        Args:
            data (dict): the incoming data
            fill_from_field (str, optional): field to use to fill prompt if empty.
                Defaults to "attribute_label".

        Returns:
            dict: the populated data.

        """
        if data["prompt"] is not None:
            return data
        logger.debug(data)
        if fill_from_field not in data:
            no_fill_field = f" '{fill_from_field}' is missing from data"
            raise ValueError(no_fill_field)
        data["prompt"] = data[fill_from_field]
        logger.debug(f"filled `prompt` with {data['prompt']}.")
        return data
fill_prompt(data, fill_from_field='attribute_label') pydantic-validator

Fill prompt field if empty.

Parameters:

Name Type Description Default
data dict

the incoming data

 required
fill_from_field str

field to use to fill prompt if empty. Defaults to "attribute_label".

 'attribute_label'

Returns:

Name Type Description
dict dict

the populated data.
Source code in deet/data_models/base.py 
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
@model_validator(mode="before")
@classmethod
def fill_prompt(cls, data: dict, fill_from_field: str = "attribute_label") -> dict:
    """
    Fill `prompt` field if empty.

    Args:
        data (dict): the incoming data
        fill_from_field (str, optional): field to use to fill prompt if empty.
            Defaults to "attribute_label".

    Returns:
        dict: the populated data.

    """
    if data["prompt"] is not None:
        return data
    logger.debug(data)
    if fill_from_field not in data:
        no_fill_field = f" '{fill_from_field}' is missing from data"
        raise ValueError(no_fill_field)
    data["prompt"] = data[fill_from_field]
    logger.debug(f"filled `prompt` with {data['prompt']}.")
    return data

LLMResponseSchema pydantic-model

Bases: BaseModel

Root schema for LLM annotation extraction response.

This structure matches the expected format that can be converted to list[GoldStandardAnnotation] after attribute resolution.

Config:

  • extra: forbid

Fields:

Source code in deet/data_models/base.py 
459
460
461
462
463
464
465
466
467
468
469
470
471
class LLMResponseSchema(BaseModel):
    """
    Root schema for LLM annotation extraction response.

    This structure matches the expected format that can be converted
    to list[GoldStandardAnnotation] after attribute resolution.
    """

    annotations: list[LLMAnnotationResponse] = Field(
        ..., description="List of annotations extracted from the document"
    )

    model_config = ConfigDict(extra="forbid")
annotations pydantic-field

List of annotations extracted from the document

coerce_annotation_to_bool(val)

Coerce an annotation to a bool.

Source code in deet/data_models/base.py 
256
257
258
259
260
261
262
263
264
265
266
def coerce_annotation_to_bool(val: SUPPORTED_TYPES) -> bool:
    """Coerce an annotation to a bool."""
    if isinstance(val, bool):
        return val
    if isinstance(val, str) and val.lower() in ("false", "0"):
        return False

    if isinstance(val, int | float):
        return bool(val)

    return True

coerce_annotation_to_float(val)

Coerce an annotation to a float.

Source code in deet/data_models/base.py 
283
284
285
286
287
288
289
290
291
292
293
294
def coerce_annotation_to_float(val: SUPPORTED_TYPES) -> float | None:
    """Coerce an annotation to a float."""
    if isinstance(val, float):
        return val
    if isinstance(val, str | bool | int):
        try:
            return float(val)
        except ValueError:
            logger.warning(f"Could not convert {val} to float")

    logger.warning(f"Unsupported type for float conversion: {type(val).__name__}")
    return None

coerce_annotation_to_int(val)

Coerce an annotation to a int.

Source code in deet/data_models/base.py 
269
270
271
272
273
274
275
276
277
278
279
280
def coerce_annotation_to_int(val: SUPPORTED_TYPES) -> int | None:
    """Coerce an annotation to a int."""
    if isinstance(val, int):
        return val
    if isinstance(val, str | float | bool):
        try:
            return int(val)
        except ValueError:
            logger.warning("Could not convert {val} to int")

    logger.warning(f"Unsupported type for int conversion: {type(val).__name__}")
    return None

coerce_annotation_to_list(val)

Coerce an annotation to list.

Source code in deet/data_models/base.py 
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
def coerce_annotation_to_list(val: SUPPORTED_TYPES) -> list:
    """Coerce an annotation to list."""
    if val is None:
        return []
    if isinstance(val, list):
        return val
    if isinstance(val, str):
        list_of_strings = [item.strip() for item in val.split(";;;")]
        try:
            return [int(item) for item in list_of_strings]
        except (ValueError, TypeError):
            logger.debug("Could not convert items to int.")

        try:
            return [float(item) for item in list_of_strings]
        except (ValueError, TypeError):
            logger.debug("Could not convert items to float")

        return list_of_strings
    return [val]

coerce_annotation_to_str(val)

Coerce an annotation to a string.

Source code in deet/data_models/base.py 
251
252
253
def coerce_annotation_to_str(val: SUPPORTED_TYPES) -> str:
    """Coerce an annotation to a string."""
    return str(val) if val else ""

documents

Data models concerning documents and how to represent them in deet.

ContextType

Bases: StrEnum

Types of context that can be provided to the LLM.

Source code in deet/data_models/documents.py 
42
43
44
45
46
47
48
49
class ContextType(StrEnum):
    """Types of context that can be provided to the LLM."""

    EMPTY = auto()
    FULL_DOCUMENT = auto()
    ABSTRACT_ONLY = auto()
    RAG_SNIPPETS = auto()
    CUSTOM = auto()

Document pydantic-model

Bases: BaseModel

Represents a document.

This can be used both for references itemised in a document listing gold standard annotations (e.g. eppi.json) AND for a document coming from a file (e.g. pdf) without linking to a gold standard annotations document with references.

Config:

  • extra: allow
  • validate_assignment: True

Fields:

Validators:

Source code in deet/data_models/documents.py 
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
class Document(BaseModel):
    """
    Represents a document.

    This can be used both for references itemised
    in a document listing gold standard annotations (e.g. eppi.json)
    AND
    for a document coming from a file (e.g. pdf) without
    linking to a gold standard annotations document with references.
    """

    # `extra` allows extra fields, e.g. for EppiDocument.
    model_config = ConfigDict(extra="allow", validate_assignment=True)
    # `validate_assignment` runs model/field validators
    # not only on instantiation, but also when we change values,
    # e.g. when we set is_linked=True, thereby preventing
    # us from saying something is linked if it isn't.

    name: str
    citation: ReferenceFileInput
    context: str | None = None  # new defaults, empty
    context_type: ContextType | None = ContextType.EMPTY
    document_id: int | None = None
    document_identity: DocumentIdentity | None = None

    parsed_document: ParsedOutput | None = None
    original_doc_filepath: Path | None = (
        None  # NOTE -- add S3/blob support when required.
    )

    is_final: bool = False
    is_linked: bool = False

    @property
    def safe_identity(self) -> DocumentIdentity:
        """
        Definitely Return an identity.
        Initialise identity if not set already,
        or raise an error if this is not possible.
        """
        if self.document_identity is None:
            self.init_document_identity()
        if self.document_identity is None:
            no_id = "Failed to initialise document identity"
            raise RuntimeError(no_id)
        return self.document_identity

    @property
    def safe_parsed_document(self) -> ParsedOutput:
        """Return the parsed_document, or raise an error if document is not linked."""
        if self.parsed_document is None:
            unlinked = "Document is not linked, cannot access parsed_document"
            raise RuntimeError(unlinked)
        return self.parsed_document

    @model_validator(mode="after")
    def validate_linking_complete(self) -> Self:
        """Validate linking is completed if `is_linked=True`."""
        base_err_msg = "requirements not met for linking: "
        if self.is_linked:
            if self.context is None:
                no_context_err = base_err_msg + "`context` is empty."
                raise ValueError(no_context_err)
            if (
                self.context_type is None
                or self.context_type != ContextType.FULL_DOCUMENT
            ):
                no_context_type_err = (
                    base_err_msg + "`context_type` is not FULL_DOCUMENT."
                )
                raise ValueError(no_context_type_err)
            if self.document_identity is None:
                no_doc_id_err = (
                    base_err_msg + "`document_identity` is empty.  "
                    "run `init_document_identity() to populate."
                )
                raise ValueError(no_doc_id_err)
            if self.parsed_document is None:
                no_parsed_doc_err = base_err_msg + "`parsed_document` is empty. "
                raise ValueError(no_parsed_doc_err)

        return self

    @model_validator(mode="after")
    def validate_final(self) -> Self:
        """Validate Document is permitted to be `is_final`."""
        base_err_msg = "requirements not met for `Document().is_final`: "
        if self.is_final:
            if self.context is None:
                no_context_err = base_err_msg + "`context` is empty."
                raise ValueError(no_context_err)
            if self.context_type is None or self.context_type == ContextType.EMPTY:
                bad_context_type_err = (
                    base_err_msg + "`context_type` musnt be None or EMPTY."
                )
                raise ValueError(bad_context_type_err)
            if self.document_identity is None:
                no_doc_id_err = (
                    base_err_msg + "`document_identity` is empty.  "
                    "run `init_document_identity() to populate."
                )
                raise ValueError(no_doc_id_err)

        return self

    def init_document_identity(
        self,
        existing_ids: set[int] | None = None,
        *,
        return_id: bool = True,
    ) -> int | None:
        """Initialise document_identity field using available metadata."""
        labs_ref = LabsReference(reference=self.citation)  # convert for easy access
        self.document_identity = DocumentIdentity(
            document_id=self.document_id,
            doi=labs_ref.doi,
            first_author=labs_ref.first_author,
            year=str(labs_ref.publication_year),
        )

        logger.info("populating id & id source...")
        self.document_identity.populate_id(existing_ids=existing_ids)
        if self.document_id is None:
            logger.info(
                "populating Document-level `document_id` field with "
                f"newly populated id {self.document_identity.document_id}... "
            )
            self.document_id = self.document_identity.document_id

        if return_id:
            return self.document_identity.document_id
        return None

    def author_year_from_document_identity(
        self, substring_strategy: Literal["longest", "last"]
    ) -> str:
        """
        Create lower-case `author_year` guess from a Document's
        DocumentIdentity field.
        The idea is to take the last name of the first author.

        NOTE: this can probably improved with more knowledge
        of how destiny encodes the first_author field.

        Returns:
            author_year (str): `author_year`

        """
        if self.document_identity is None:
            logger.debug("document identity is None, initialising...")
            self.init_document_identity()

        if (
            self.document_identity is None
            or self.document_identity.first_author is None
            or self.document_identity.year is None
        ):
            whats_what = f"self.document_identity: {self.document_identity}; "
            if self.document_identity is not None:
                whats_what += (
                    "self.document_identity.first_author: "
                    f"{self.document_identity.first_author}; "
                    f"self.document_identity.year: {self.document_identity.year}."
                )
            logger.warning(whats_what)
            raise ValueError
        author_name = self.document_identity.first_author
        year = self.document_identity.year

        name_components = author_name.split(" ")
        if substring_strategy == "longest":
            name_guess = max(name_components, key=len)
        elif substring_strategy == "last":
            name_guess = name_components[-1]
        else:
            missing_name_guess_method_err = (
                f"method {substring_strategy} is not implemented."
            )
            raise NotImplementedError(missing_name_guess_method_err)

        return f"{name_guess.lower()}_{year}"

    def set_abstract_context(self) -> None:
        """Set the abstract, contained in `citation` field, as context."""
        abstract = LabsReference(reference=self.citation).abstract
        if abstract is not None:
            self.context_type = ContextType.ABSTRACT_ONLY
            self.context = abstract
            logger.info(
                "set context type to ABSTRACT_ONLY; set context to abstract."
                f" snippet: {abstract[:20]}"
            )
            return
        no_abstract = "No abstract found"
        raise NoAbstractError(no_abstract)

    def link_parsed_document(
        self,
        parsed_document: ParsedOutput,
        original_doc_filepath: Path | None = None,
    ) -> None:
        """
        Link parsed document and document metadata/`reference`.

        Args:
            parsed_document (ParsedOutput): the output from the parser
            original_doc_filepath (Path): full filepath to the original doc.

        """
        self.parsed_document = parsed_document
        if original_doc_filepath and (
            not original_doc_filepath.is_file() or not original_doc_filepath.exists()
        ):
            logger.warning(
                "supplied `original_doc_filepath` does not resolve. writing None."
            )
            original_doc_filepath = None
        self.original_doc_filepath = original_doc_filepath

    def set_context_from_parsed(self) -> None:
        """Symlink context to parsed_document.text."""
        if self.parsed_document and self.parsed_document.text:
            self.context = self.parsed_document.text
            self.context_type = ContextType.FULL_DOCUMENT
        else:
            logger.warning("no text in parsed_document!")

    def save(self, path: Path) -> None:
        """Save linked document to .json."""
        data = self.model_dump(by_alias=False)

        # convert images to base64 for json serialization
        # NOTE @all -- we leave ourselves open to
        # malicious stuff being injected here. open to
        # suggestions as to how to validate/circumvent.
        # however, we do control what goes in and out,
        # so the danger here might be overstated.
        if self.parsed_document and self.parsed_document.images:
            images_b64 = {}
            for key, img in self.parsed_document.images.items():
                buffer = BytesIO()
                img.save(buffer, format="PNG")
                images_b64[key] = base64.b64encode(buffer.getvalue()).decode()
            data["parsed_document"]["images"] = images_b64

        path.parent.mkdir(parents=True, exist_ok=True)
        with path.open("w", encoding="utf-8") as f:
            json.dump(data, f, indent=2, default=str)
        logger.info(f"Saved Document fulltext link to {path}")

    @classmethod
    def load(cls, path: Path) -> Self:
        """Load linked document from .json."""
        data = json.loads(path.read_text(encoding="utf-8"))
        logger.debug(data)

        # convert base64 back to PIL img
        if data.get("parsed_document") is not None and data.get(
            "parsed_document", {}
        ).get("images"):
            images = {}
            for key, img_b64 in data["parsed_document"]["images"].items():
                img_bytes = base64.b64decode(img_b64)
                images[key] = Image.open(BytesIO(img_bytes))
            data["parsed_document"]["images"] = images

        return cls(**data)
safe_identity property

Definitely Return an identity. Initialise identity if not set already, or raise an error if this is not possible.

safe_parsed_document property

Return the parsed_document, or raise an error if document is not linked.

author_year_from_document_identity(substring_strategy)

Create lower-case author_year guess from a Document's DocumentIdentity field. The idea is to take the last name of the first author.

NOTE: this can probably improved with more knowledge of how destiny encodes the first_author field.

Returns:

Name Type Description
author_year str

author_year
Source code in deet/data_models/documents.py 
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
def author_year_from_document_identity(
    self, substring_strategy: Literal["longest", "last"]
) -> str:
    """
    Create lower-case `author_year` guess from a Document's
    DocumentIdentity field.
    The idea is to take the last name of the first author.

    NOTE: this can probably improved with more knowledge
    of how destiny encodes the first_author field.

    Returns:
        author_year (str): `author_year`

    """
    if self.document_identity is None:
        logger.debug("document identity is None, initialising...")
        self.init_document_identity()

    if (
        self.document_identity is None
        or self.document_identity.first_author is None
        or self.document_identity.year is None
    ):
        whats_what = f"self.document_identity: {self.document_identity}; "
        if self.document_identity is not None:
            whats_what += (
                "self.document_identity.first_author: "
                f"{self.document_identity.first_author}; "
                f"self.document_identity.year: {self.document_identity.year}."
            )
        logger.warning(whats_what)
        raise ValueError
    author_name = self.document_identity.first_author
    year = self.document_identity.year

    name_components = author_name.split(" ")
    if substring_strategy == "longest":
        name_guess = max(name_components, key=len)
    elif substring_strategy == "last":
        name_guess = name_components[-1]
    else:
        missing_name_guess_method_err = (
            f"method {substring_strategy} is not implemented."
        )
        raise NotImplementedError(missing_name_guess_method_err)

    return f"{name_guess.lower()}_{year}"
init_document_identity(existing_ids=None, *, return_id=True)

Initialise document_identity field using available metadata.

Source code in deet/data_models/documents.py 
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
def init_document_identity(
    self,
    existing_ids: set[int] | None = None,
    *,
    return_id: bool = True,
) -> int | None:
    """Initialise document_identity field using available metadata."""
    labs_ref = LabsReference(reference=self.citation)  # convert for easy access
    self.document_identity = DocumentIdentity(
        document_id=self.document_id,
        doi=labs_ref.doi,
        first_author=labs_ref.first_author,
        year=str(labs_ref.publication_year),
    )

    logger.info("populating id & id source...")
    self.document_identity.populate_id(existing_ids=existing_ids)
    if self.document_id is None:
        logger.info(
            "populating Document-level `document_id` field with "
            f"newly populated id {self.document_identity.document_id}... "
        )
        self.document_id = self.document_identity.document_id

    if return_id:
        return self.document_identity.document_id
    return None

Link parsed document and document metadata/reference.

Parameters:

Name Type Description Default
parsed_document ParsedOutput

the output from the parser

 required
original_doc_filepath Path

full filepath to the original doc.

 None
Source code in deet/data_models/documents.py 
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
def link_parsed_document(
    self,
    parsed_document: ParsedOutput,
    original_doc_filepath: Path | None = None,
) -> None:
    """
    Link parsed document and document metadata/`reference`.

    Args:
        parsed_document (ParsedOutput): the output from the parser
        original_doc_filepath (Path): full filepath to the original doc.

    """
    self.parsed_document = parsed_document
    if original_doc_filepath and (
        not original_doc_filepath.is_file() or not original_doc_filepath.exists()
    ):
        logger.warning(
            "supplied `original_doc_filepath` does not resolve. writing None."
        )
        original_doc_filepath = None
    self.original_doc_filepath = original_doc_filepath
load(path) classmethod

Load linked document from .json.

Source code in deet/data_models/documents.py 
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
@classmethod
def load(cls, path: Path) -> Self:
    """Load linked document from .json."""
    data = json.loads(path.read_text(encoding="utf-8"))
    logger.debug(data)

    # convert base64 back to PIL img
    if data.get("parsed_document") is not None and data.get(
        "parsed_document", {}
    ).get("images"):
        images = {}
        for key, img_b64 in data["parsed_document"]["images"].items():
            img_bytes = base64.b64decode(img_b64)
            images[key] = Image.open(BytesIO(img_bytes))
        data["parsed_document"]["images"] = images

    return cls(**data)
save(path)

Save linked document to .json.

Source code in deet/data_models/documents.py 
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
def save(self, path: Path) -> None:
    """Save linked document to .json."""
    data = self.model_dump(by_alias=False)

    # convert images to base64 for json serialization
    # NOTE @all -- we leave ourselves open to
    # malicious stuff being injected here. open to
    # suggestions as to how to validate/circumvent.
    # however, we do control what goes in and out,
    # so the danger here might be overstated.
    if self.parsed_document and self.parsed_document.images:
        images_b64 = {}
        for key, img in self.parsed_document.images.items():
            buffer = BytesIO()
            img.save(buffer, format="PNG")
            images_b64[key] = base64.b64encode(buffer.getvalue()).decode()
        data["parsed_document"]["images"] = images_b64

    path.parent.mkdir(parents=True, exist_ok=True)
    with path.open("w", encoding="utf-8") as f:
        json.dump(data, f, indent=2, default=str)
    logger.info(f"Saved Document fulltext link to {path}")
set_abstract_context()

Set the abstract, contained in citation field, as context.

Source code in deet/data_models/documents.py 
426
427
428
429
430
431
432
433
434
435
436
437
438
def set_abstract_context(self) -> None:
    """Set the abstract, contained in `citation` field, as context."""
    abstract = LabsReference(reference=self.citation).abstract
    if abstract is not None:
        self.context_type = ContextType.ABSTRACT_ONLY
        self.context = abstract
        logger.info(
            "set context type to ABSTRACT_ONLY; set context to abstract."
            f" snippet: {abstract[:20]}"
        )
        return
    no_abstract = "No abstract found"
    raise NoAbstractError(no_abstract)
set_context_from_parsed()

Symlink context to parsed_document.text.

Source code in deet/data_models/documents.py 
463
464
465
466
467
468
469
def set_context_from_parsed(self) -> None:
    """Symlink context to parsed_document.text."""
    if self.parsed_document and self.parsed_document.text:
        self.context = self.parsed_document.text
        self.context_type = ContextType.FULL_DOCUMENT
    else:
        logger.warning("no text in parsed_document!")
validate_final() pydantic-validator

Validate Document is permitted to be is_final.

Source code in deet/data_models/documents.py 
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
@model_validator(mode="after")
def validate_final(self) -> Self:
    """Validate Document is permitted to be `is_final`."""
    base_err_msg = "requirements not met for `Document().is_final`: "
    if self.is_final:
        if self.context is None:
            no_context_err = base_err_msg + "`context` is empty."
            raise ValueError(no_context_err)
        if self.context_type is None or self.context_type == ContextType.EMPTY:
            bad_context_type_err = (
                base_err_msg + "`context_type` musnt be None or EMPTY."
            )
            raise ValueError(bad_context_type_err)
        if self.document_identity is None:
            no_doc_id_err = (
                base_err_msg + "`document_identity` is empty.  "
                "run `init_document_identity() to populate."
            )
            raise ValueError(no_doc_id_err)

    return self
validate_linking_complete() pydantic-validator

Validate linking is completed if is_linked=True.

Source code in deet/data_models/documents.py 
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
@model_validator(mode="after")
def validate_linking_complete(self) -> Self:
    """Validate linking is completed if `is_linked=True`."""
    base_err_msg = "requirements not met for linking: "
    if self.is_linked:
        if self.context is None:
            no_context_err = base_err_msg + "`context` is empty."
            raise ValueError(no_context_err)
        if (
            self.context_type is None
            or self.context_type != ContextType.FULL_DOCUMENT
        ):
            no_context_type_err = (
                base_err_msg + "`context_type` is not FULL_DOCUMENT."
            )
            raise ValueError(no_context_type_err)
        if self.document_identity is None:
            no_doc_id_err = (
                base_err_msg + "`document_identity` is empty.  "
                "run `init_document_identity() to populate."
            )
            raise ValueError(no_doc_id_err)
        if self.parsed_document is None:
            no_parsed_doc_err = base_err_msg + "`parsed_document` is empty. "
            raise ValueError(no_parsed_doc_err)

    return self

DocumentIDSource

Bases: StrEnum

Sources for a given document_id. Can be e.g. eppi_item_id.

To be extended if e.g. we start working with non-eppi gold standard references.

Source code in deet/data_models/documents.py 
52
53
54
55
56
57
58
59
60
61
62
63
64
class DocumentIDSource(StrEnum):
    """
    Sources for a given document_id. Can be e.g. eppi_item_id.

    To be extended if e.g. we start working with
    non-eppi gold standard references.
    """

    EPPI_ITEM_ID = auto()
    DOI_AUTHOR_YEAR = auto()
    DOI_ID = auto()
    AUTHOR_YEAR_ID = auto()
    RANDINT = auto()

DocumentIdentity pydantic-model

Bases: BaseModel

A unified identity for a document, deriveable from multiple sources.

Fields:

Source code in deet/data_models/documents.py 
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
class DocumentIdentity(BaseModel):
    """A unified identity for a document, deriveable from multiple sources."""

    document_id: int | None = None
    document_id_source: DocumentIDSource | None = None

    # parsed citation info
    doi: str | None
    first_author: str | None
    year: str | None

    def populate_id(
        self,
        existing_ids: set[int] | None = None,
        hierarchy: list[DocumentIDSource] | None = None,
    ) -> None:
        """
        Populate document_id using a hierarchical list of ID creation methods.

        Tries each method in order until a unique ID is generated. If an ID
        conflicts with existing_ids, tries the next method. RANDINT always
        succeeds as fallback.

        NOTE: we will have to implement some sort of matching thing, if we are
        concerned that an id-collision might be becuase we have already
        parsed&linked a document.

        Args:
            existing_ids: List of existing IDs to check for conflicts.
            hierarchy: Ordered list of DocumentIDSource methods to try.
                Defaults to [EPPI_ITEM_ID, DOI_AUTHOR_YEAR, DOI_ID,
                AUTHOR_YEAR_ID, RANDINT].

        Raises:
            BadDocumentIdError: If unable to generate unique ID (should never
                happen as RANDINT is always in hierarchy).

        """
        if existing_ids is None:
            existing_ids = set()

        if hierarchy is None:
            hierarchy = list(DocumentIDSource)  # retain the order from the enum

        if DocumentIDSource.RANDINT not in hierarchy:
            hierarchy.append(
                DocumentIDSource.RANDINT
            )  # always keep random as a fallback

        attempted_sources = []

        for id_source in hierarchy:
            try:
                id_factory = self._create_id_factory(id_source)
                logger.debug(f"created id_factory: {id_factory.__name__}")
                potential_id = id_factory()
                logger.debug(
                    f"created potential id: {potential_id} "
                    f"using factory {id_factory.__name__}"
                )

                # id collisions?
                if potential_id not in existing_ids:
                    self.document_id = potential_id
                    self.document_id_source = id_source
                    logger.debug(
                        f"successfully created document_id {potential_id} "
                        f"using {id_source}"
                    )
                    return

                logger.debug(
                    f"id {potential_id} from {id_source} conflicts with existing IDs"
                )
                attempted_sources.append(id_source)

            except (BadDocumentIdError, MissingCitationElementError) as e:
                logger.debug(f"Failed to create ID using {id_source}: {e}")
                attempted_sources.append(id_source)
                continue

        failed_sources = ", ".join(str(s) for s in attempted_sources)
        err_msg = (
            f"Failed to generate unique document_id after trying: {failed_sources}"
        )

        if len(attempted_sources) == len(hierarchy):
            max_attempts = 10
            attempts = 0
            for _ in range(max_attempts):
                potential_id = self._random_int_id()
                if potential_id not in existing_ids:
                    self.document_id = potential_id
                    self.document_id_source = DocumentIDSource.RANDINT
                    logger.debug(
                        f"successfully created document_id {potential_id} "
                        f"using {id_source}"
                    )
                    return

        err_msg += f" plus {attempts} randint attempts."
        raise BadDocumentIdError(err_msg)

    def _create_id_factory(self, id_source: DocumentIDSource) -> Callable:
        """
        Return an id-creating method given specific value of DocumentIDSource.

        Returns:
            int: the id.

        """
        id_creation_map = {
            DocumentIDSource.EPPI_ITEM_ID: self._eppi_item_id,
            DocumentIDSource.DOI_ID: self._doi_id,
            DocumentIDSource.AUTHOR_YEAR_ID: self._author_year_id,
            DocumentIDSource.DOI_AUTHOR_YEAR: self._doi_author_year_id,
            DocumentIDSource.RANDINT: self._random_int_id,
        }

        return id_creation_map[id_source]

    def _eppi_item_id(self) -> int:
        """Map an existing item_id (parsed as document_id)."""
        # we're going to assume that our `document_id`, received
        # from parsing eppi-json to EppiDocument is always going
        # to be eppi, otherwise this method should be extended to
        # reflect it coming from somewhere else.
        # Either way, it must be a positive integer with a number of digits
        # between MIN_DOCUMENT_ID_DIGITS and MAX_DOCUMENT_ID_DIGITS (inclusive).
        if (
            self.document_id is not None
            and isinstance(self.document_id, int)
            and self.document_id > 0
        ):
            digit_count = len(str(abs(self.document_id)))
            if MIN_DOCUMENT_ID_DIGITS <= digit_count <= MAX_DOCUMENT_ID_DIGITS:
                return self.document_id
        bad_doc_id = f"id {self.document_id} is not a valid eppi item_id."
        raise BadDocumentIdError(bad_doc_id)

    def _citation_id_hasher(self, target_fields: list[str]) -> int:
        """Create an id from _n_ citation fields."""
        if not all(field in self.model_dump() for field in target_fields):
            missing_citation = (
                f"required fields are missing in citation. "
                f"required: {', '.join(target_fields)}"
                f"actual: {','.join(self.model_dump())}"
            )
            raise MissingCitationElementError(missing_citation)
        payload = [self.model_dump()[field] for field in target_fields]

        if "" in payload or None in payload:
            none_or_empty = (
                "some or all of target fields are "
                f"None or empty strings: {','.join(target_fields)} "
            )
            raise MissingCitationElementError(none_or_empty)
        return hash_n_strings_to_document_id(payload)

    def _doi_id(self) -> int:
        """Create an integer id as a function of doi."""
        return self._citation_id_hasher(["doi"])

    def _doi_author_year_id(self) -> int:
        """Create an integer id as a function of doi, author and year."""
        return self._citation_id_hasher(["doi", "first_author", "year"])

    def _author_year_id(self) -> int:
        """Create an 8-digit integer id as a function of author and year."""
        return self._citation_id_hasher(["first_author", "year"])

    @staticmethod
    def _random_int_id() -> int:
        """Create a random integer id with 8 digits."""
        return randint(MIN_DOCUMENT_ID, MAX_DOCUMENT_ID)  # noqa: S311
populate_id(existing_ids=None, hierarchy=None)

Populate document_id using a hierarchical list of ID creation methods.

Tries each method in order until a unique ID is generated. If an ID conflicts with existing_ids, tries the next method. RANDINT always succeeds as fallback.

NOTE: we will have to implement some sort of matching thing, if we are concerned that an id-collision might be becuase we have already parsed&linked a document.

Parameters:

Name Type Description Default
existing_ids set[int] | None

List of existing IDs to check for conflicts.

 None
hierarchy list[DocumentIDSource] | None

Ordered list of DocumentIDSource methods to try. Defaults to [EPPI_ITEM_ID, DOI_AUTHOR_YEAR, DOI_ID, AUTHOR_YEAR_ID, RANDINT].

 None

Raises:

Type Description
BadDocumentIdError

If unable to generate unique ID (should never happen as RANDINT is always in hierarchy).
Source code in deet/data_models/documents.py 
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
def populate_id(
    self,
    existing_ids: set[int] | None = None,
    hierarchy: list[DocumentIDSource] | None = None,
) -> None:
    """
    Populate document_id using a hierarchical list of ID creation methods.

    Tries each method in order until a unique ID is generated. If an ID
    conflicts with existing_ids, tries the next method. RANDINT always
    succeeds as fallback.

    NOTE: we will have to implement some sort of matching thing, if we are
    concerned that an id-collision might be becuase we have already
    parsed&linked a document.

    Args:
        existing_ids: List of existing IDs to check for conflicts.
        hierarchy: Ordered list of DocumentIDSource methods to try.
            Defaults to [EPPI_ITEM_ID, DOI_AUTHOR_YEAR, DOI_ID,
            AUTHOR_YEAR_ID, RANDINT].

    Raises:
        BadDocumentIdError: If unable to generate unique ID (should never
            happen as RANDINT is always in hierarchy).

    """
    if existing_ids is None:
        existing_ids = set()

    if hierarchy is None:
        hierarchy = list(DocumentIDSource)  # retain the order from the enum

    if DocumentIDSource.RANDINT not in hierarchy:
        hierarchy.append(
            DocumentIDSource.RANDINT
        )  # always keep random as a fallback

    attempted_sources = []

    for id_source in hierarchy:
        try:
            id_factory = self._create_id_factory(id_source)
            logger.debug(f"created id_factory: {id_factory.__name__}")
            potential_id = id_factory()
            logger.debug(
                f"created potential id: {potential_id} "
                f"using factory {id_factory.__name__}"
            )

            # id collisions?
            if potential_id not in existing_ids:
                self.document_id = potential_id
                self.document_id_source = id_source
                logger.debug(
                    f"successfully created document_id {potential_id} "
                    f"using {id_source}"
                )
                return

            logger.debug(
                f"id {potential_id} from {id_source} conflicts with existing IDs"
            )
            attempted_sources.append(id_source)

        except (BadDocumentIdError, MissingCitationElementError) as e:
            logger.debug(f"Failed to create ID using {id_source}: {e}")
            attempted_sources.append(id_source)
            continue

    failed_sources = ", ".join(str(s) for s in attempted_sources)
    err_msg = (
        f"Failed to generate unique document_id after trying: {failed_sources}"
    )

    if len(attempted_sources) == len(hierarchy):
        max_attempts = 10
        attempts = 0
        for _ in range(max_attempts):
            potential_id = self._random_int_id()
            if potential_id not in existing_ids:
                self.document_id = potential_id
                self.document_id_source = DocumentIDSource.RANDINT
                logger.debug(
                    f"successfully created document_id {potential_id} "
                    f"using {id_source}"
                )
                return

    err_msg += f" plus {attempts} randint attempts."
    raise BadDocumentIdError(err_msg)

GoldStandardAnnotatedDocument pydantic-model

Bases: BaseModel, Generic[DocumentTypeVar, GoldStandardAnnotationTypeVar]

A document with its gold standard annotations.

Fields:

  • document (DocumentTypeVar)
  • annotations (list[GoldStandardAnnotationTypeVar])
Source code in deet/data_models/documents.py 
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
class GoldStandardAnnotatedDocument(
    BaseModel, Generic[DocumentTypeVar, GoldStandardAnnotationTypeVar]
):
    """A document with its gold standard annotations."""

    document: DocumentTypeVar
    annotations: list[GoldStandardAnnotationTypeVar]

    def get_attribute_annotation(self, attribute: Attribute) -> GoldStandardAnnotation:
        """Get the value of the annotation of the corresponding attribute."""
        result = None
        output_data: Any
        for annotation in self.annotations:
            if annotation.attribute.attribute_id == attribute.attribute_id:
                if result is not None:
                    multiple_matches = (
                        "More than one annotation found for "
                        f"attribute: {attribute.attribute_label}. We don't know how to"
                        "interpret which is the canonical version."
                    )
                    raise DuplicateAnnotationError(multiple_matches)
                result = annotation

        if result is None:
            try:
                output_data = attribute.output_data_type.missing_annotation_default()
            except ValueError as err:
                not_found = (
                    "Attribute not found in annotations."
                    " Don't know how to interpret this when attribute is of type "
                    f"{attribute.output_data_type}"
                )
                raise ValueError(not_found) from err
            return GoldStandardAnnotation(
                attribute=attribute,
                raw_data=output_data,
                annotation_type=AnnotationType.HUMAN,
            )

        return result
get_attribute_annotation(attribute)

Get the value of the annotation of the corresponding attribute.

Source code in deet/data_models/documents.py 
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
def get_attribute_annotation(self, attribute: Attribute) -> GoldStandardAnnotation:
    """Get the value of the annotation of the corresponding attribute."""
    result = None
    output_data: Any
    for annotation in self.annotations:
        if annotation.attribute.attribute_id == attribute.attribute_id:
            if result is not None:
                multiple_matches = (
                    "More than one annotation found for "
                    f"attribute: {attribute.attribute_label}. We don't know how to"
                    "interpret which is the canonical version."
                )
                raise DuplicateAnnotationError(multiple_matches)
            result = annotation

    if result is None:
        try:
            output_data = attribute.output_data_type.missing_annotation_default()
        except ValueError as err:
            not_found = (
                "Attribute not found in annotations."
                " Don't know how to interpret this when attribute is of type "
                f"{attribute.output_data_type}"
            )
            raise ValueError(not_found) from err
        return GoldStandardAnnotation(
            attribute=attribute,
            raw_data=output_data,
            annotation_type=AnnotationType.HUMAN,
        )

    return result

GoldStandardAnnotatedDocumentList pydantic-model

Bases: BaseModel, Generic[GoldStandardAnnotatedDocumentTypeVar]

A list of GoldStandardAnnotatedDocuments (or subclasses thereof). This list is indexed to enable easy retrieval by document_id.

Fields:

  • gold_standard_annotations (Sequence[GoldStandardAnnotatedDocumentTypeVar])
Source code in deet/data_models/documents.py 
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
class GoldStandardAnnotatedDocumentList(
    BaseModel, Generic[GoldStandardAnnotatedDocumentTypeVar]
):
    """
    A list of GoldStandardAnnotatedDocuments (or subclasses thereof).
    This list is indexed to enable easy retrieval by document_id.
    """

    gold_standard_annotations: Sequence[GoldStandardAnnotatedDocumentTypeVar]

    @cached_property
    def annotation_index(self) -> dict[int, GoldStandardAnnotatedDocumentTypeVar]:
        """Cached index to enable retrieving annotated documents by id."""
        return {
            doc.document.safe_identity.document_id: doc
            for doc in self.gold_standard_annotations
        }

    def get_by_id(self, document_id: int) -> GoldStandardAnnotatedDocumentTypeVar:
        """
        Get GoldStandardAnnotatedDocument where document.document_identity
        matches document_identity.
        """
        try:
            return self.annotation_index[document_id]
        except KeyError as err:
            not_found = f"Document with ID {document_id} not found in annotated"
            " doc list"
            raise MissingDocumentError(not_found) from err
annotation_index cached property

Cached index to enable retrieving annotated documents by id.

get_by_id(document_id)

Get GoldStandardAnnotatedDocument where document.document_identity matches document_identity.

Source code in deet/data_models/documents.py 
581
582
583
584
585
586
587
588
589
590
591
def get_by_id(self, document_id: int) -> GoldStandardAnnotatedDocumentTypeVar:
    """
    Get GoldStandardAnnotatedDocument where document.document_identity
    matches document_identity.
    """
    try:
        return self.annotation_index[document_id]
    except KeyError as err:
        not_found = f"Document with ID {document_id} not found in annotated"
        " doc list"
        raise MissingDocumentError(not_found) from err

enums

A store for plain enums, so they can quickly be imported in the CLI. We can use these to set argument types and defaults, without needing large imports, that would slow the CLI down during autocomplete, or when asking for --help.

CustomPromptPopulationMethod

Bases: StrEnum

Methods of populating prompts.

Source code in deet/data_models/enums.py 
11
12
13
14
15
class CustomPromptPopulationMethod(StrEnum):
    """Methods of populating prompts."""

    FILE = auto()
    CLI = auto()

eppi

EPPI-specific data models extending the core models.

AttributeAnswerCoT pydantic-model

Bases: BaseModel

Detailed answer format for a single attribute with reasoning.

Fields:

Source code in deet/data_models/eppi.py 
392
393
394
395
396
397
398
399
400
401
402
class AttributeAnswerCoT(BaseModel):
    """Detailed answer format for a single attribute with reasoning."""

    attribute_name: str = Field(
        description="The name of the attribute being asked about"
    )
    answer: str = Field(description="The answer to the question, 'True' or 'False'")
    reasoning: str = Field(description="The reasoning behind the answer")
    citation: str | None = Field(
        description="The citation from the Research Information to support the answer"
    )
answer pydantic-field

The answer to the question, 'True' or 'False'

attribute_name pydantic-field

The name of the attribute being asked about

citation pydantic-field

The citation from the Research Information to support the answer

reasoning pydantic-field

The reasoning behind the answer

BatchAnswerFormatCoT pydantic-model

Bases: BaseModel

Batch answers for all attributes with reasoning.

Fields:

Source code in deet/data_models/eppi.py 
405
406
407
408
409
410
class BatchAnswerFormatCoT(BaseModel):
    """Batch answers for all attributes with reasoning."""

    answers: list[AttributeAnswerCoT] = Field(
        description="List of answers for each attribute"
    )
answers pydantic-field

List of answers for each attribute

EppiAttribute pydantic-model

Bases: Attribute

EPPI-specific attribute with additional fields.

Extends the core Attribute class with EPPI-specific metadata and hierarchy information.

Uses alias generators to automatically map camelCase EPPI JSON fields to snake_case Python fields.

Config:

  • validate_by_name: True
  • validate_by_alias: True

Fields:

Source code in deet/data_models/eppi.py 
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
class EppiAttribute(Attribute):
    """
    EPPI-specific attribute with additional fields.

    Extends the core Attribute class with EPPI-specific
    metadata and hierarchy information.

    Uses alias generators to automatically map
    camelCase EPPI JSON fields to snake_case Python fields.
    """

    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)  # type: ignore[typeddict-unknown-key]

    attribute_id: int = Field(
        validation_alias=AliasChoices("AttributeId", "attribute_id")
    )
    attribute_selection_type: EppiAttributeSelectionType = Field(
        default=EppiAttributeSelectionType.UNSPECIFIED,
        validation_alias=AliasChoices(
            "AttributeType", "attribute_type", "attribute_selection_type"
        ),
    )
    output_data_type: AttributeType = DEFAULT_ATTRIBUTE_TYPE
    attribute_label: str = Field(alias="AttributeName")

    # EPPI-specific fields - these map automatically from camelCase JSON
    attribute_set_description: str | None = Field(
        description="Description of the attribute set this attribute belongs to",
        default=None,
    )
    hierarchy_path: str | None = Field(
        description="Dot-separated path showing the hierarchical "
        " position of this attribute",
        default=None,
    )
    hierarchy_level: int = Field(
        description="Numeric level indicating depth in "
        " the attribute hierarchy (0 = root level)",
        default=0,
    )
    is_leaf: bool = Field(
        description="Whether this attribute is a leaf node  (has no child attributes)",
        default=True,
    )
    parent_attribute_id: int | None = Field(
        description="ID of the parent attribute in the hierarchy", default=None
    )
    attribute_description: str | None = Field(
        description="Detailed description explaining what this attribute represents",
        default=None,
    )
attribute_description = None pydantic-field

Detailed description explaining what this attribute represents

attribute_set_description = None pydantic-field

Description of the attribute set this attribute belongs to

hierarchy_level = 0 pydantic-field

Numeric level indicating depth in the attribute hierarchy (0 = root level)

hierarchy_path = None pydantic-field

Dot-separated path showing the hierarchical position of this attribute

is_leaf = True pydantic-field

Whether this attribute is a leaf node (has no child attributes)

parent_attribute_id = None pydantic-field

ID of the parent attribute in the hierarchy

EppiAttributeSelectionType

Bases: StrEnum

AttributeType as it appears in eppi json.

Source code in deet/data_models/eppi.py 
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
class EppiAttributeSelectionType(StrEnum):
    """`AttributeType` as it appears in eppi json."""

    SELECTABLE = "Selectable (show checkbox)"
    OUTCOME = "Outcome"
    INTERVENTION = "Intervention"
    NOT_SELECTABLE = "Not Selectable (no checkbox)"
    UNSPECIFIED = "Unspecified"

    @classmethod
    def _missing_(cls, value: object) -> "EppiAttributeSelectionType | None":
        """Handle case-insensitive assignment & lookup."""
        if isinstance(value, str):
            for member in cls:
                if member.value.lower() == value.lower():
                    return member
        return None

EppiCodeSet pydantic-model

Bases: BaseModel

Represents a single CodeSet from EPPI JSON.

CodeSets contain hierarchical attribute definitions used in EPPI-Reviewer.

Fields:

Source code in deet/data_models/eppi.py 
343
344
345
346
347
348
349
350
351
352
353
354
355
356
class EppiCodeSet(BaseModel):
    """
    Represents a single CodeSet from EPPI JSON.

    CodeSets contain hierarchical attribute definitions used in EPPI-Reviewer.
    """

    attributes: dict[str, Any] | None = Field(alias="Attributes", default=None)

    def get_attributes_list(self) -> list[dict[str, Any]]:
        """Extract AttributesList from the CodeSet."""
        if self.attributes and "AttributesList" in self.attributes:
            return self.attributes["AttributesList"]
        return []
get_attributes_list()

Extract AttributesList from the CodeSet.

Source code in deet/data_models/eppi.py 
352
353
354
355
356
def get_attributes_list(self) -> list[dict[str, Any]]:
    """Extract AttributesList from the CodeSet."""
    if self.attributes and "AttributesList" in self.attributes:
        return self.attributes["AttributesList"]
    return []

EppiDocument pydantic-model

Bases: Document

EPPI-specific document.

Uses alias generators to automatically map camelCase EPPI JSON fields to snake_case Python fields.

Config:

  • validate_by_name: True
  • validate_by_alias: True

Fields:

  • citation (ReferenceFileInput)
  • context (str | None)
  • context_type (ContextType | None)
  • document_identity (DocumentIdentity | None)
  • parsed_document (ParsedOutput | None)
  • original_doc_filepath (Path | None)
  • is_final (bool)
  • is_linked (bool)
  • name (str)
  • document_id (int)
  • parent_title (str | None)
  • short_title (str | None)
  • date_created (datetime | None)
  • created_by (str | None)
  • edited_by (str | None)
  • year (int | None)
  • month (str | None)
  • abstract (str | None)
  • authors (str | None)
  • keywords (str | None)
  • doi (str | None)

Validators:

Source code in deet/data_models/eppi.py 
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
class EppiDocument(Document):
    """
    EPPI-specific document.

    Uses alias generators to automatically map
    camelCase EPPI JSON fields to snake_case Python fields.
    """

    name: str = Field(default="", validation_alias=AliasChoices("Title", "name"))
    document_id: int = Field(validation_alias=AliasChoices("ItemId", "document_id"))

    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)  # type: ignore[typeddict-unknown-key]

    parent_title: str | None = Field(
        default=None, validation_alias=AliasChoices("ParentTitle", "parent_title")
    )
    short_title: str | None = Field(
        default=None, validation_alias=AliasChoices("ShortTitle", "short_title")
    )
    date_created: datetime | None = Field(
        default=None, validation_alias=AliasChoices("DateCreated", "date_created")
    )
    created_by: str | None = Field(
        default=None, validation_alias=AliasChoices("CreatedBy", "created_by")
    )
    edited_by: str | None = Field(
        default=None, validation_alias=AliasChoices("EditedBy", "edited_by")
    )
    year: int | None = Field(
        default=None, validation_alias=AliasChoices("Year", "year")
    )
    month: str | None = Field(
        default=None, validation_alias=AliasChoices("Month", "month")
    )
    abstract: str | None = Field(
        default=None, validation_alias=AliasChoices("Abstract", "abstract")
    )
    authors: str | None = Field(
        default=None, validation_alias=AliasChoices("Authors", "authors")
    )
    keywords: str | None = Field(
        default=None, validation_alias=AliasChoices("Keywords", "keywords")
    )
    doi: str | None = Field(default=None, validation_alias=AliasChoices("DOI", "doi"))

    @field_validator("year", mode="before")
    @classmethod
    def empty_year_string_to_none(cls, value: str) -> str | None:
        """Parse an empty string year to None or return as is."""
        if value == "":
            return None
        return value

    @field_validator("date_created", mode="before")
    @classmethod
    def parse_date_string(cls, value: str) -> datetime | None:
        """Parse a string datetime to native datetime."""
        if value is None or value == "":
            return None
        if isinstance(value, datetime):
            return value
        if isinstance(value, str):
            # add as we encounter other formats, if ever relevant
            formats = [
                "%d/%m/%Y",  # OG EPPI
                "%Y-%m-%d %H:%M:%S%z",  # ISO format with timezone,
                # result of dumping is_final EppiDocument to json
                "%Y-%m-%d",  # simple ISO date
            ]

            for fmt in formats:
                try:
                    return datetime.strptime(value, fmt).replace(tzinfo=UTC)
                except ValueError:
                    continue
            no_parsage = "unable to parse date_created."
            raise ValueError(no_parsage)

        return None

    @model_validator(mode="before")
    @classmethod
    def populate_citation_field(cls, data: dict[str, Any]) -> dict:
        """
        Populate the `citation` field with a Destiny
        reference derived from the EPPI data.
        """
        # if not isinstance(data, dict):
        #     return data
        if "citation" in data:
            # we have already created citation,
            # no need to do it again
            return data

        citation = parse_citation_to_destiny(reference=data)
        data["citation"] = citation

        return data
empty_year_string_to_none(value) pydantic-validator

Parse an empty string year to None or return as is.

Source code in deet/data_models/eppi.py 
230
231
232
233
234
235
236
@field_validator("year", mode="before")
@classmethod
def empty_year_string_to_none(cls, value: str) -> str | None:
    """Parse an empty string year to None or return as is."""
    if value == "":
        return None
    return value
parse_date_string(value) pydantic-validator

Parse a string datetime to native datetime.

Source code in deet/data_models/eppi.py 
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
@field_validator("date_created", mode="before")
@classmethod
def parse_date_string(cls, value: str) -> datetime | None:
    """Parse a string datetime to native datetime."""
    if value is None or value == "":
        return None
    if isinstance(value, datetime):
        return value
    if isinstance(value, str):
        # add as we encounter other formats, if ever relevant
        formats = [
            "%d/%m/%Y",  # OG EPPI
            "%Y-%m-%d %H:%M:%S%z",  # ISO format with timezone,
            # result of dumping is_final EppiDocument to json
            "%Y-%m-%d",  # simple ISO date
        ]

        for fmt in formats:
            try:
                return datetime.strptime(value, fmt).replace(tzinfo=UTC)
            except ValueError:
                continue
        no_parsage = "unable to parse date_created."
        raise ValueError(no_parsage)

    return None
populate_citation_field(data) pydantic-validator

Populate the citation field with a Destiny reference derived from the EPPI data.

Source code in deet/data_models/eppi.py 
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
@model_validator(mode="before")
@classmethod
def populate_citation_field(cls, data: dict[str, Any]) -> dict:
    """
    Populate the `citation` field with a Destiny
    reference derived from the EPPI data.
    """
    # if not isinstance(data, dict):
    #     return data
    if "citation" in data:
        # we have already created citation,
        # no need to do it again
        return data

    citation = parse_citation_to_destiny(reference=data)
    data["citation"] = citation

    return data

EppiGoldStandardAnnotatedDocument pydantic-model

Bases: GoldStandardAnnotatedDocument[EppiDocument, EppiGoldStandardAnnotation]

EPPI-specific gold standard annotated document.

Fields:

  • document (DocumentTypeVar)
  • annotations (list[GoldStandardAnnotationTypeVar])
Source code in deet/data_models/eppi.py 
337
338
339
340
class EppiGoldStandardAnnotatedDocument(
    GoldStandardAnnotatedDocument[EppiDocument, EppiGoldStandardAnnotation]
):
    """EPPI-specific gold standard annotated document."""

EppiGoldStandardAnnotation pydantic-model

Bases: GoldStandardAnnotation

EPPI-specific gold standard annotation.

In EPPI-Reviewer context, an "arm" refers to a study group or intervention group within a research study (e.g., "Treatment Group", "Control Group", "Placebo Group"). Each annotation is associated with a specific arm to indicate which study group the extracted information relates to.

Fields:

Source code in deet/data_models/eppi.py 
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
class EppiGoldStandardAnnotation(GoldStandardAnnotation):
    """
    EPPI-specific gold standard annotation.

    In EPPI-Reviewer context, an "arm" refers to a study group or intervention group
    within a research study (e.g., "Treatment Group", "Control Group", "Placebo Group").
    Each annotation is associated with a specific arm to indicate which study group
    the extracted information relates to.
    """

    arm_id: int | None = Field(
        description="ID of the study arm this annotation relates to", default=None
    )
    arm_title: str | None = Field(
        description="Title or name of the study arm", default=None
    )
    arm_description: str | None = Field(
        description="Detailed description of the study arm", default=None
    )
    item_attribute_full_text_details: list[EppiItemAttributeFullTextDetails] | None = (
        Field(
            description="List of detailed text extracts and "
            " arm-specific information for this annotation",
            default=None,
        )
    )
arm_description = None pydantic-field

Detailed description of the study arm

arm_id = None pydantic-field

ID of the study arm this annotation relates to

arm_title = None pydantic-field

Title or name of the study arm

item_attribute_full_text_details = None pydantic-field

List of detailed text extracts and arm-specific information for this annotation

EppiItemAttributeFullTextDetails pydantic-model

Bases: BaseModel

EPPI-specific item attribute full text details.

Arm specific information, exact text keywords for the attribute.

Fields:

  • item_document_id (int | None)
  • text (str | None)
  • item_arm (str | None)

Validators:

Source code in deet/data_models/eppi.py 
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
class EppiItemAttributeFullTextDetails(BaseModel):
    """
    EPPI-specific item attribute full text details.

    Arm specific information, exact text keywords for the attribute.
    """

    item_document_id: int | None = None
    text: str | None = None
    item_arm: str | None = None

    @model_validator(mode="before")
    @classmethod
    def validate_at_least_one_field(cls, data: dict) -> dict:
        """Ensure at least one field is not None."""
        if all(v is None for k, v in data.items()):
            msg = (
                "At least one field must be provided "
                "(item_document_id, text, or item_arm)"
            )
            raise ValueError(msg)
        return data
validate_at_least_one_field(data) pydantic-validator

Ensure at least one field is not None.

Source code in deet/data_models/eppi.py 
296
297
298
299
300
301
302
303
304
305
306
@model_validator(mode="before")
@classmethod
def validate_at_least_one_field(cls, data: dict) -> dict:
    """Ensure at least one field is not None."""
    if all(v is None for k, v in data.items()):
        msg = (
            "At least one field must be provided "
            "(item_document_id, text, or item_arm)"
        )
        raise ValueError(msg)
    return data

EppiRawData pydantic-model

Bases: BaseModel

Represents the complete EPPI JSON structure.

This model validates and structures the raw EPPI JSON data, making it easier to work with and validate.

Fields:

Source code in deet/data_models/eppi.py 
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
class EppiRawData(BaseModel):
    """
    Represents the complete EPPI JSON structure.

    This model validates and structures the raw EPPI JSON data,
    making it easier to work with and validate.
    """

    code_sets: list[EppiCodeSet] = Field(alias="CodeSets", default=[])
    references: list[dict[str, Any]] = Field(alias="References", default=[])

    def extract_all_attributes(
        self, flatten_hierarchy_func: Callable[[list], list]
    ) -> list[dict[str, Any]]:
        """
        Extract and flatten attributes from all CodeSets.

        Args:
            flatten_hierarchy_func: Function to flatten attribute hierarchy

        Returns:
            List of flattened attribute dictionaries

        """
        all_attributes = []
        for codeset in self.code_sets:
            attributes_list = codeset.get_attributes_list()
            if attributes_list:
                flattened = flatten_hierarchy_func(attributes_list)
                all_attributes.extend(flattened)
        return all_attributes
extract_all_attributes(flatten_hierarchy_func)

Extract and flatten attributes from all CodeSets.

Parameters:

Name Type Description Default
flatten_hierarchy_func Callable[[list], list]

Function to flatten attribute hierarchy

 required

Returns:

Type Description
list[dict[str, Any]]

List of flattened attribute dictionaries
Source code in deet/data_models/eppi.py 
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
def extract_all_attributes(
    self, flatten_hierarchy_func: Callable[[list], list]
) -> list[dict[str, Any]]:
    """
    Extract and flatten attributes from all CodeSets.

    Args:
        flatten_hierarchy_func: Function to flatten attribute hierarchy

    Returns:
        List of flattened attribute dictionaries

    """
    all_attributes = []
    for codeset in self.code_sets:
        attributes_list = codeset.get_attributes_list()
        if attributes_list:
            flattened = flatten_hierarchy_func(attributes_list)
            all_attributes.extend(flattened)
    return all_attributes

parse_citation_to_destiny(reference)

Create a ReferenceFileInput object from document data.

NOTE: we are not using the wrapping parser method in repository as it is for the whole document, and if it fails, we wouldn't be able to map a destiny reference.

See https://github.com/destiny-evidence/destiny-repository/issues/458

Parameters:

Name Type Description Default
reference dict[str, Any]

one reference from the eppi json.

 required
Source code in deet/data_models/eppi.py 
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
def parse_citation_to_destiny(reference: dict[str, Any]) -> ReferenceFileInput:
    """
    Create a ReferenceFileInput object from document data.

    NOTE: we are not using the wrapping parser method in
    repository as it is for the whole document, and
    if it fails, we wouldn't be able to map a destiny reference.

    See https://github.com/destiny-evidence/destiny-repository/issues/458

    Args:
        reference: one reference from the eppi json.

    """
    if "DOI" in reference:
        reference["DOI"] = sanitise_doi(reference["DOI"])
    raw_enhancement_content = [
        c
        for c in [
            (
                eppi_destiny_parser._parse_abstract_enhancement(reference),  # noqa: SLF001
                EnhancementType.ABSTRACT,
            ),
            (
                eppi_destiny_parser._parse_bibliographic_enhancement(reference),  # noqa: SLF001
                EnhancementType.BIBLIOGRAPHIC,
            ),
            (
                eppi_destiny_parser._create_annotation_enhancement(),  # noqa: SLF001
                EnhancementType.ANNOTATION,
            ),
        ]
        if c[0] is not None
    ]

    enhancements = [
        EnhancementFileInput(
            source=eppi_destiny_parser.parser_source,
            visibility=Visibility.PUBLIC,
            content=content[0],  # type:ignore[arg-type]
            enhancement_type=content[1],  # type:ignore[call-arg]
        )
        for content in raw_enhancement_content
    ]

    destiny_ids = None
    try:
        destiny_ids = eppi_destiny_parser._parse_identifiers(  # noqa: SLF001
            ref_to_import=reference
        )
    except ExternalIdentifierNotFoundError as e:
        logger.warning(f"no identifier for reference. storing `None`. error: {e}")

    return ReferenceFileInput(
        visibility=Visibility.PUBLIC,
        identifiers=destiny_ids,
        enhancements=enhancements,
    )

sanitise_doi(doi_candidate, *, raise_on_fail=False)

Clean DOI strings in EPPI jsons.

Source code in deet/data_models/eppi.py 
38
39
40
41
42
43
44
45
46
47
48
49
50
def sanitise_doi(doi_candidate: str, *, raise_on_fail: bool = False) -> str:
    """Clean DOI strings in EPPI jsons."""
    doi = DOI_REGEX.search(doi_candidate)
    if doi and isinstance(doi, re.Match):
        return doi[0]
    if raise_on_fail:
        bad_doi = f"doi {doi} is bad."
        raise ValueError(bad_doi)
    logger.debug(
        "not found a valid DOI, returning empty string."
        " to modify this behaviour, set raise_on_fail=True"
    )
    return ""

evaluation

Data models to help with evaluation.

AttributeMetric pydantic-model

Bases: BaseModel

Data structure storing a metric for an attribute for a data extraction run.

Fields:

Source code in deet/data_models/evaluation.py 
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
class AttributeMetric(BaseModel):
    """Data structure storing a metric for an attribute for a data extraction run."""

    attribute: Attribute
    metric_name: str
    value: float | None
    extraction_run_id: str

    def dictify(self) -> dict:
        """
        Return a dictionary representation, unpacking the attribute into ID
            and label.
        """
        return {
            "attribute_id": self.attribute.attribute_id,
            "attribute_label": self.attribute.attribute_label,
            "value": self.value,
            "extraction_run_id": self.extraction_run_id,
        }

    def save_to_csv(self, filepath: Path, mode: Literal["a", "w"] = "a") -> None:
        """
        Write an evaluation metric for an attribute as a line to a csv file.

        Args:
            filepath (Path): outfile destination.
            mode (Literal["a", "w"], optional): _w_rite or _a_ppend.
            Defaults to "a" (append).

        """
        dictified = self.dictify()

        filepath.parent.mkdir(parents=True, exist_ok=True)
        file_exists = filepath.exists() and filepath.stat().st_size > 0
        write_header = not file_exists or mode == "w"

        with filepath.open(mode=mode, newline="", encoding="utf-8") as f:
            writer = csv.DictWriter(f, fieldnames=dictified.keys())

            if write_header:
                writer.writeheader()

            writer.writerow(dictified)

        logger.debug(f"Wrote metric to {filepath}")
dictify()

Return a dictionary representation, unpacking the attribute into ID and label.

Source code in deet/data_models/evaluation.py 
 96
 97
 98
 99
100
101
102
103
104
105
106
def dictify(self) -> dict:
    """
    Return a dictionary representation, unpacking the attribute into ID
        and label.
    """
    return {
        "attribute_id": self.attribute.attribute_id,
        "attribute_label": self.attribute.attribute_label,
        "value": self.value,
        "extraction_run_id": self.extraction_run_id,
    }
save_to_csv(filepath, mode='a')

Write an evaluation metric for an attribute as a line to a csv file.

Parameters:

Name Type Description Default
filepath Path

outfile destination.

 required
mode Literal['a', 'w']

_w_rite or _a_ppend.

 'a'
Source code in deet/data_models/evaluation.py 
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
def save_to_csv(self, filepath: Path, mode: Literal["a", "w"] = "a") -> None:
    """
    Write an evaluation metric for an attribute as a line to a csv file.

    Args:
        filepath (Path): outfile destination.
        mode (Literal["a", "w"], optional): _w_rite or _a_ppend.
        Defaults to "a" (append).

    """
    dictified = self.dictify()

    filepath.parent.mkdir(parents=True, exist_ok=True)
    file_exists = filepath.exists() and filepath.stat().st_size > 0
    write_header = not file_exists or mode == "w"

    with filepath.open(mode=mode, newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=dictified.keys())

        if write_header:
            writer.writeheader()

        writer.writerow(dictified)

    logger.debug(f"Wrote metric to {filepath}")

check_metric_returns_float(metric)

Check whether a metric returns a scalar.

Source code in deet/data_models/evaluation.py 
23
24
25
26
27
28
def check_metric_returns_float(metric: MetricFunction) -> bool:
    """Check whether a metric returns a scalar."""
    y_true = [1, 0, 0, 1]
    y_pred = [1, 0, 0, 0]
    result = metric(y_true, y_pred)
    return isinstance(result, float)

get_metrics_for_attribute_type(attribute_type)

Return the metric set registered for the given attribute data type.

Some types map to an empty dict when no suitable default metrics are implemented yet (float, list, dict); callers may still merge in custom metrics.

Source code in deet/data_models/evaluation.py 
76
77
78
79
80
81
82
83
84
85
def get_metrics_for_attribute_type(
    attribute_type: AttributeType,
) -> dict[str, MetricFunction]:
    """
    Return the metric set registered for the given attribute data type.

    Some types map to an empty dict when no suitable default metrics are
    implemented yet (float, list, dict); callers may still merge in custom metrics.
    """
    return METRICS_BY_ATTRIBUTE_TYPE[attribute_type]

n_labels(y_true, y_pred)

Count the number of positive instances of the class in gold data.

Source code in deet/data_models/evaluation.py 
31
32
33
def n_labels(y_true: list[int], y_pred: list[int]) -> float:  # noqa: ARG001
    """Count the number of positive instances of the class in gold data."""
    return sum(y_true)

extraction

Data models for LLM extraction outputs.

DocumentExtractionResult pydantic-model

Bases: BaseModel

Result of extracting data from a single document via an LLM.

Fields:

Validators:

Source code in deet/data_models/extraction.py 
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
class DocumentExtractionResult(BaseModel):
    """Result of extracting data from a single document via an LLM."""

    annotations: list[GoldStandardAnnotation]
    messages: list[dict[str, Any]]
    input_tokens: int = 0
    output_tokens: int = 0
    model: str | None = None
    total_cost_usd: float | None = None
    timestamp: datetime = Field(default_factory=lambda: datetime.now(tz=UTC))

    @model_validator(mode="after")
    def compute_total_cost_usd(self) -> Self:
        """Populate ``total_cost_usd`` from tokens and model (``estimate_cost_usd``)."""
        if self.model is None:
            self.total_cost_usd = None
            return self
        prompt_c, completion_c = estimate_cost_usd(
            self.model,
            prompt_tokens=self.input_tokens,
            completion_tokens=self.output_tokens,
        )
        merged = merge_prompt_completion_cost_usd(prompt_c, completion_c)
        self.total_cost_usd = round(merged, 6) if merged is not None else None
        return self
compute_total_cost_usd() pydantic-validator

Populate total_cost_usd from tokens and model (estimate_cost_usd).

Source code in deet/data_models/extraction.py 
24
25
26
27
28
29
30
31
32
33
34
35
36
37
@model_validator(mode="after")
def compute_total_cost_usd(self) -> Self:
    """Populate ``total_cost_usd`` from tokens and model (``estimate_cost_usd``)."""
    if self.model is None:
        self.total_cost_usd = None
        return self
    prompt_c, completion_c = estimate_cost_usd(
        self.model,
        prompt_tokens=self.input_tokens,
        completion_tokens=self.output_tokens,
    )
    merged = merge_prompt_completion_cost_usd(prompt_c, completion_c)
    self.total_cost_usd = round(merged, 6) if merged is not None else None
    return self

ExtractionRunMetadata pydantic-model

Bases: BaseModel

Aggregate metadata for a batch extraction run.

Fields:

Source code in deet/data_models/extraction.py 
40
41
42
43
44
45
46
47
48
class ExtractionRunMetadata(BaseModel):
    """Aggregate metadata for a batch extraction run."""

    model: str | None = None
    total_input_tokens: int = 0
    total_output_tokens: int = 0
    total_cost_usd: float | None = None
    per_document_tokens: dict[str, dict[str, int]] = Field(default_factory=dict)
    timestamp: datetime = Field(default_factory=lambda: datetime.now(tz=UTC))

ExtractionRunOutput pydantic-model

Bases: BaseModel

Top-level output from a batch extraction run.

Fields:

Source code in deet/data_models/extraction.py 
51
52
53
54
55
class ExtractionRunOutput(BaseModel):
    """Top-level output from a batch extraction run."""

    annotated_documents: list[GoldStandardAnnotatedDocument]
    metadata: ExtractionRunMetadata

pipeline

Models to employ for implementing DEET jobs in sequential, harmonised pipelines.

BaseExecutor

Bases: ABC

Abstract base class for all executors.

Source code in deet/data_models/pipeline.py 
104
105
106
107
108
109
110
111
112
113
114
class BaseExecutor(ABC):
    """Abstract base class for all executors."""

    @abstractmethod
    def _execute(
        self,
        job: "Job",
        args: list[Any] | None = None,
        kwargs: dict[str, Any] | None = None,
    ) -> Any:  # noqa: ANN401
        pass

CodeExecutor

Bases: BaseExecutor

Executor for Python callable.

Source code in deet/data_models/pipeline.py 
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
class CodeExecutor(BaseExecutor):
    """Executor for Python callable."""

    def _execute(
        self,
        job: "Job",
        args: list[Any] | None = None,
        kwargs: dict[str, Any] | None = None,
    ) -> Any:  # noqa: ANN401
        args = args or []
        kwargs = kwargs or {}
        if isinstance(job.job, Path):
            malspecified_job = "job has to be a callable, not a path to a script."
            raise JobExecutionError(malspecified_job)
        result = job.job(*args, **kwargs)
        if job.capture_output:
            return result
        return None

EgressMethod

Bases: StrEnum

An enum of egree methods for a PipelineStage.

Source code in deet/data_models/pipeline.py 
60
61
62
63
64
class EgressMethod(StrEnum):
    """An enum of egree methods for a PipelineStage."""

    FILE = auto()
    MEMORY = auto()

Executor

A wrapper for all kinds of executors.

Source code in deet/data_models/pipeline.py 
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
class Executor:
    """A wrapper for all kinds of executors."""

    def __init__(self, executor: BaseExecutor) -> None:
        """Init new executor instance."""
        self.executor = executor

    def execute(
        self,
        job: "Job",
        args: list[Any] | None = None,
        kwargs: dict[str, Any] | None = None,
    ) -> Any:  # noqa: ANN401
        """Execute a job."""
        return self.executor._execute(job, args=args, kwargs=kwargs)  # noqa: SLF001
__init__(executor)

Init new executor instance.

Source code in deet/data_models/pipeline.py 
325
326
327
def __init__(self, executor: BaseExecutor) -> None:
    """Init new executor instance."""
    self.executor = executor
execute(job, args=None, kwargs=None)

Execute a job.

Source code in deet/data_models/pipeline.py 
329
330
331
332
333
334
335
336
def execute(
    self,
    job: "Job",
    args: list[Any] | None = None,
    kwargs: dict[str, Any] | None = None,
) -> Any:  # noqa: ANN401
    """Execute a job."""
    return self.executor._execute(job, args=args, kwargs=kwargs)  # noqa: SLF001

IngressMethod

Bases: StrEnum

An enum of ingress methods for a PipelineStage.

Source code in deet/data_models/pipeline.py 
51
52
53
54
55
56
57
class IngressMethod(StrEnum):
    """An enum of ingress methods for a PipelineStage."""

    FILE = auto()
    MEMORY = auto()
    HTTP = auto()  # we may need to download data
    RANDOM = auto()  # there might be jobs & pipeline stages where we start with a seed

Job pydantic-model

Bases: BaseModel

The attributes describing a specific job.

Config:

  • arbitrary_types_allowed: True

Fields:

Source code in deet/data_models/pipeline.py 
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
class Job(BaseModel):
    """The attributes describing a specific job."""

    name: str
    job_format: JobFormat
    job_type: JobType | list[JobType]
    language: Language
    ingress_method: IngressMethod | None = (
        None  # we may have a job that starts with no data
    )
    egress_method: EgressMethod
    job: Callable | Path
    script_args: list[str] | None
    func_args: list[Any] | None = None
    func_kwargs: dict[str, Any] | None = None
    capture_output: bool = True
    executor: Executor

    model_config = ConfigDict(arbitrary_types_allowed=True)

    def run_job(self) -> None | str:
        """Run the job defined in this model instance."""
        logger.debug(f"Running job {self.name}")
        logger.debug(f"func args: {self.func_args}")
        logger.debug(f"func kwargs: {self.func_kwargs}")

        # Use func_args/func_kwargs for CODE jobs, script_args for SCRIPT jobs
        if self.job_format == JobFormat.CODE:
            output = self.executor.execute(
                job=self, args=self.func_args, kwargs=self.func_kwargs
            )
        elif self.job_format == JobFormat.SCRIPT:
            output = self.executor.execute(job=self, args=self.script_args)

        if self.capture_output:
            logger.debug(output)
        return output
run_job()

Run the job defined in this model instance.

Source code in deet/data_models/pipeline.py 
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
def run_job(self) -> None | str:
    """Run the job defined in this model instance."""
    logger.debug(f"Running job {self.name}")
    logger.debug(f"func args: {self.func_args}")
    logger.debug(f"func kwargs: {self.func_kwargs}")

    # Use func_args/func_kwargs for CODE jobs, script_args for SCRIPT jobs
    if self.job_format == JobFormat.CODE:
        output = self.executor.execute(
            job=self, args=self.func_args, kwargs=self.func_kwargs
        )
    elif self.job_format == JobFormat.SCRIPT:
        output = self.executor.execute(job=self, args=self.script_args)

    if self.capture_output:
        logger.debug(output)
    return output

JobExecutionError

Bases: Exception

To raise when a job hits a generic error.

Source code in deet/data_models/pipeline.py 
47
48
class JobExecutionError(Exception):
    """To raise when a job hits a generic error."""

JobFormat

Bases: StrEnum

An enum of job formats.

Jobs are the building blocks of pipeline stages. The job format describes the 'medium' in which the job is provided to the job object.

Source code in deet/data_models/pipeline.py 
67
68
69
70
71
72
73
74
75
76
77
class JobFormat(StrEnum):
    """
    An enum of job formats.

    Jobs are the building blocks of pipeline stages.
    The job format describes the 'medium' in which the job
    is provided to the job object.
    """

    SCRIPT = auto()  # when job is supplied in a file.
    CODE = auto()  # when job is supplied within the pipeline.

JobType

Bases: StrEnum

An enum of job types.

This is a descriptive label of the broad category of what the job is doing.

Source code in deet/data_models/pipeline.py 
80
81
82
83
84
85
86
87
88
89
90
91
class JobType(StrEnum):
    """
    An enum of job types.

    This is a descriptive label of the broad category
    of what the job is doing.
    """

    DATA_PROCESSING = auto()
    DATA_COLLECTION = auto()
    CLASSIFICATION = auto()
    EXTRACTION = auto()

Language

Bases: StrEnum

An enum of permitted languages a job can be specified in.

Source code in deet/data_models/pipeline.py 
 94
 95
 96
 97
 98
 99
100
101
class Language(StrEnum):
    """An enum of permitted languages a job can be specified in."""

    PYTHON = auto()
    R = auto()
    SHELL = auto()
    SQL = auto()
    LLM_PROMPT = auto()

MissingBinaryError

Bases: Exception

To raise when we're missing a binary required to run a script.

Source code in deet/data_models/pipeline.py 
43
44
class MissingBinaryError(Exception):
    """To raise when we're missing a binary required to run a script."""

Pipeline pydantic-model

Bases: BaseModel

A complete pipeline consisting of several PipelineStage objects.

Fields:

Source code in deet/data_models/pipeline.py 
472
473
474
475
476
477
478
479
480
481
482
483
484
class Pipeline(BaseModel):
    """A complete pipeline consisting of several `PipelineStage` objects."""

    model_config = ConfigDict()

    name: str
    stages: list[PipelineStage]

    def run(self) -> None:
        """Run all pipeline stages."""
        logger.info(f"Pipeline {self.name} has {len(self.stages)} stages.")
        for stage in self.stages:
            stage.run_jobs()
run()

Run all pipeline stages.

Source code in deet/data_models/pipeline.py 
480
481
482
483
484
def run(self) -> None:
    """Run all pipeline stages."""
    logger.info(f"Pipeline {self.name} has {len(self.stages)} stages.")
    for stage in self.stages:
        stage.run_jobs()

PipelineStage pydantic-model

Bases: BaseModel

A stage in a DEET pipeline.

Fields:

Validators:

Source code in deet/data_models/pipeline.py 
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
class PipelineStage(BaseModel):
    """A stage in a DEET pipeline."""

    model_config = ConfigDict()

    name: str
    skip_jobs_if_failed: bool = False
    input_file: Path | None = None  # handled in job
    data: Any | None = None  # handled in job
    jobs: Job | list[Job]
    logfile: Path | None = None
    default_func_args: list[Any] | None = None
    default_func_kwargs: dict[str, Any] | None = None

    @field_validator("jobs", mode="before")
    @classmethod
    def convert_jobs_to_list(cls, v: Job | list[Job]) -> list[Job]:
        """Convert jobs to list of jobs if just one job supplied."""
        if isinstance(v, Job):
            v = [v]
        return v

    @staticmethod
    def write_stage_logfile(payload: str, filepath: Path) -> None:
        """Write logfile for a specific stage."""
        filepath.write_text(payload, encoding="utf-8")

    def run_jobs(
        self,
        func_args: list[Any] | None = None,
        func_kwargs: dict[str, Any] | None = None,
    ) -> None:
        """Run all jobs in a pipeline stage."""
        if isinstance(
            self.jobs, Job
        ):  # for mypy -- can we remove this given field_validator?
            self.jobs = [self.jobs]

        args_to_use = func_args or self.default_func_args
        kwargs_to_use = func_kwargs or self.default_func_kwargs

        logger.info(f"Pipeline stage {self.name} has {len(self.jobs)} stages.")
        for i, job in enumerate(self.jobs):
            logger.info(
                f"Running job {i + 1} out of {len(self.jobs)}, name: {job.name}."
            )
            try:
                # Use job's own args if they exist, otherwise fall
                # back to stage/method args
                if job.job_format == JobFormat.CODE:
                    args_to_use = job.func_args or func_args or self.default_func_args
                    kwargs_to_use = (
                        job.func_kwargs or func_kwargs or self.default_func_kwargs
                    )

                    # Only override if the job doesn't have its own args
                    if job.func_args is None:
                        job.func_args = args_to_use
                    if job.func_kwargs is None:
                        job.func_kwargs = kwargs_to_use

                job_output = job.run_job()
                if (
                    job.capture_output
                    and job_output is not None
                    and self.logfile is not None
                ):
                    self.write_stage_logfile(payload=job_output, filepath=self.logfile)
                # should we `yield` the job_output here?
            except Exception as e:
                if not self.skip_jobs_if_failed:
                    raise
                logger.error(
                    f"Encountered error {e} on job {i}, {job.name}"
                    f" in pipeline stage {self.name}, moving to next job..."
                )
                logger.error(f"Error type: {type(e).__name__}")
                logger.error(f"Error message: {e}")
                logger.error("Stack trace:")
                logger.error(traceback.format_exc())
                logger.info("Moving to next job...")

                continue
convert_jobs_to_list(v) pydantic-validator

Convert jobs to list of jobs if just one job supplied.

Source code in deet/data_models/pipeline.py 
395
396
397
398
399
400
401
@field_validator("jobs", mode="before")
@classmethod
def convert_jobs_to_list(cls, v: Job | list[Job]) -> list[Job]:
    """Convert jobs to list of jobs if just one job supplied."""
    if isinstance(v, Job):
        v = [v]
    return v
run_jobs(func_args=None, func_kwargs=None)

Run all jobs in a pipeline stage.

Source code in deet/data_models/pipeline.py 
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
def run_jobs(
    self,
    func_args: list[Any] | None = None,
    func_kwargs: dict[str, Any] | None = None,
) -> None:
    """Run all jobs in a pipeline stage."""
    if isinstance(
        self.jobs, Job
    ):  # for mypy -- can we remove this given field_validator?
        self.jobs = [self.jobs]

    args_to_use = func_args or self.default_func_args
    kwargs_to_use = func_kwargs or self.default_func_kwargs

    logger.info(f"Pipeline stage {self.name} has {len(self.jobs)} stages.")
    for i, job in enumerate(self.jobs):
        logger.info(
            f"Running job {i + 1} out of {len(self.jobs)}, name: {job.name}."
        )
        try:
            # Use job's own args if they exist, otherwise fall
            # back to stage/method args
            if job.job_format == JobFormat.CODE:
                args_to_use = job.func_args or func_args or self.default_func_args
                kwargs_to_use = (
                    job.func_kwargs or func_kwargs or self.default_func_kwargs
                )

                # Only override if the job doesn't have its own args
                if job.func_args is None:
                    job.func_args = args_to_use
                if job.func_kwargs is None:
                    job.func_kwargs = kwargs_to_use

            job_output = job.run_job()
            if (
                job.capture_output
                and job_output is not None
                and self.logfile is not None
            ):
                self.write_stage_logfile(payload=job_output, filepath=self.logfile)
            # should we `yield` the job_output here?
        except Exception as e:
            if not self.skip_jobs_if_failed:
                raise
            logger.error(
                f"Encountered error {e} on job {i}, {job.name}"
                f" in pipeline stage {self.name}, moving to next job..."
            )
            logger.error(f"Error type: {type(e).__name__}")
            logger.error(f"Error message: {e}")
            logger.error("Stack trace:")
            logger.error(traceback.format_exc())
            logger.info("Moving to next job...")

            continue
write_stage_logfile(payload, filepath) staticmethod

Write logfile for a specific stage.

Source code in deet/data_models/pipeline.py 
403
404
405
406
@staticmethod
def write_stage_logfile(payload: str, filepath: Path) -> None:
    """Write logfile for a specific stage."""
    filepath.write_text(payload, encoding="utf-8")

ScriptExecutor

Bases: BaseExecutor

An executor class for different kinds of scripts.

Source code in deet/data_models/pipeline.py 
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
class ScriptExecutor(BaseExecutor):
    """An executor class for different kinds of scripts."""

    def __init__(
        self,
        python_path: Path | None = None,
        r_path: Path | None = None,
        bash_path: Path = Path("/bin/bash"),
    ) -> None:
        """Create ScriptExecutor instance."""
        python_which = shutil.which("python")
        r_which = shutil.which("R")

        self.python_path = (
            python_path
            if python_path
            else (Path(python_which) if python_which else None)
        )
        self.r_path = r_path if r_path else (Path(r_which) if r_which else None)
        self.bash_path = bash_path

        logger.debug(f"python path: {self.python_path}")
        logger.debug(f"r path: {self.r_path}")
        logger.debug(f"bash path: {self.bash_path}")

    def _execute(
        self,
        job: "Job",
        args: list[str] | None = None,
        kwargs: dict[str, Any] | None = None,
    ) -> Any:  # noqa: ANN401
        if not isinstance(job.job, Path):
            malspecified_job = (
                "ScriptExecutor requires job.job to be a Path, not a Callable."
            )
            raise JobExecutionError(malspecified_job)

        if job.language == Language.PYTHON:
            return self.python_executor(
                job.job, args=args, capture_output=job.capture_output
            )
        if job.language == Language.R:
            return self.r_executor(
                job.job, args=args, capture_output=job.capture_output
            )
        if job.language == Language.SHELL:
            return self.bash_executor(
                job.job, args=args, capture_output=job.capture_output
            )
        missing_language = (
            f"Script execution not implemented for language: {job.language}"
        )
        raise NotImplementedError(missing_language)

    @staticmethod
    def verify_filetype(filename: str, filetype: Literal[".py", ".R", ".sh"]) -> bool:
        """
        Verify a given file is of a given filetype via checking the ending.

        Args:
            filename (str): Name of file.
            filetype (Literal[&quot;.py&quot;, &quot;.R&quot;, &quot;.sh&quot;]):
                    file ending.

        Raises:
            WrongFiletypeError: When ending doesnt match the input.

        Returns:
            bool: True if OK.

        """
        if not filename[-len(filetype) :] == filetype:
            raise WrongFiletypeError
        return True

    def python_executor(
        self, script_path: Path, args: list[str] | None, *, capture_output: bool = True
    ) -> None | str:
        """
        Execute a python script.

        Args:
            script_path (Path): file path to script.
            args (list[str]): args to run with script.
            capture_output (bool, optional): Defaults to True.

        Returns:
            None | str: output from stdout or None.

        """
        self.verify_filetype(script_path.name, ".py")
        if self.python_path is None:
            python_missing = "can't find python binary. please find it/install."
            raise MissingBinaryError(python_missing)

        cmd: list[str] = [str(self.python_path), str(script_path)]
        if args:
            cmd.extend(args)

        env = restricted_env.copy()
        env.update({"PYTHONPATH": ""})

        output = subprocess.run(  # noqa: S603
            cmd,
            check=True,
            capture_output=capture_output,
            text=True,
            env=env,
        )
        if capture_output:
            return (
                output.stderr
            )  # loguru writes to stderr; we're using loguru for messages.
        # if you want to capture `print`-ed messages, capture stdout.
        return None

    def r_executor(
        self, script_path: Path, args: list[str] | None, *, capture_output: bool = True
    ) -> None | str:
        """Execute an R script."""
        self.verify_filetype(script_path.name, ".R")
        if self.r_path is None:
            r_missing = "can't find r binary. please find it/install."
            raise MissingBinaryError(r_missing)

        cmd: list[str] = [str(self.r_path), str(script_path)]
        if args:
            cmd.extend(args)

        env = restricted_env.copy()
        env.update(
            {
                "R_LIBS_USER": "",  # prevent loading from user library paths
                "R_PROFILE_USER": "",  # disable user profile scripts
                "R_ENVIRON_USER": "",  # disable user environment files
                "R_HISTFILE": "",  # disable history file
            }
        )

        output = subprocess.run(  # noqa: S603
            cmd,
            check=True,
            capture_output=capture_output,
            text=True,
            env=env,
        )
        if capture_output:
            return output.stdout
        return None

    def bash_executor(
        self, script_path: Path, args: list[str] | None, *, capture_output: bool = True
    ) -> None | str:
        """Execute a bash script."""
        self.verify_filetype(script_path.name, ".sh")
        if self.r_path is None:
            r_missing = "can't find bash binary. please find it/install."
            raise MissingBinaryError(r_missing)

        cmd: list[str] = [str(self.bash_path), str(script_path)]
        if args:
            cmd.extend(args)

        env = restricted_env.copy()
        env.update(
            {
                "SHELL": str(self.bash_path),
                "IFS": " \t\n",  # safe input field separator
                "ENV": "",  # disable shell startup file
                "BASH_ENV": "",  # disable bash startup file
            }
        )

        output = subprocess.run(  # noqa: S603
            cmd,
            check=True,
            capture_output=capture_output,
            text=True,
            env=env,
        )
        if capture_output:
            return output.stdout
        return None
__init__(python_path=None, r_path=None, bash_path=Path('/bin/bash'))

Create ScriptExecutor instance.

Source code in deet/data_models/pipeline.py 
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
def __init__(
    self,
    python_path: Path | None = None,
    r_path: Path | None = None,
    bash_path: Path = Path("/bin/bash"),
) -> None:
    """Create ScriptExecutor instance."""
    python_which = shutil.which("python")
    r_which = shutil.which("R")

    self.python_path = (
        python_path
        if python_path
        else (Path(python_which) if python_which else None)
    )
    self.r_path = r_path if r_path else (Path(r_which) if r_which else None)
    self.bash_path = bash_path

    logger.debug(f"python path: {self.python_path}")
    logger.debug(f"r path: {self.r_path}")
    logger.debug(f"bash path: {self.bash_path}")
bash_executor(script_path, args, *, capture_output=True)

Execute a bash script.

Source code in deet/data_models/pipeline.py 
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
def bash_executor(
    self, script_path: Path, args: list[str] | None, *, capture_output: bool = True
) -> None | str:
    """Execute a bash script."""
    self.verify_filetype(script_path.name, ".sh")
    if self.r_path is None:
        r_missing = "can't find bash binary. please find it/install."
        raise MissingBinaryError(r_missing)

    cmd: list[str] = [str(self.bash_path), str(script_path)]
    if args:
        cmd.extend(args)

    env = restricted_env.copy()
    env.update(
        {
            "SHELL": str(self.bash_path),
            "IFS": " \t\n",  # safe input field separator
            "ENV": "",  # disable shell startup file
            "BASH_ENV": "",  # disable bash startup file
        }
    )

    output = subprocess.run(  # noqa: S603
        cmd,
        check=True,
        capture_output=capture_output,
        text=True,
        env=env,
    )
    if capture_output:
        return output.stdout
    return None
python_executor(script_path, args, *, capture_output=True)

Execute a python script.

Parameters:

Name Type Description Default
script_path Path

file path to script.

 required
args list[str]

args to run with script.

 required
capture_output bool

Defaults to True.

 True

Returns:

Type Description
None | str

None | str: output from stdout or None.
Source code in deet/data_models/pipeline.py 
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
def python_executor(
    self, script_path: Path, args: list[str] | None, *, capture_output: bool = True
) -> None | str:
    """
    Execute a python script.

    Args:
        script_path (Path): file path to script.
        args (list[str]): args to run with script.
        capture_output (bool, optional): Defaults to True.

    Returns:
        None | str: output from stdout or None.

    """
    self.verify_filetype(script_path.name, ".py")
    if self.python_path is None:
        python_missing = "can't find python binary. please find it/install."
        raise MissingBinaryError(python_missing)

    cmd: list[str] = [str(self.python_path), str(script_path)]
    if args:
        cmd.extend(args)

    env = restricted_env.copy()
    env.update({"PYTHONPATH": ""})

    output = subprocess.run(  # noqa: S603
        cmd,
        check=True,
        capture_output=capture_output,
        text=True,
        env=env,
    )
    if capture_output:
        return (
            output.stderr
        )  # loguru writes to stderr; we're using loguru for messages.
    # if you want to capture `print`-ed messages, capture stdout.
    return None
r_executor(script_path, args, *, capture_output=True)

Execute an R script.

Source code in deet/data_models/pipeline.py 
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
def r_executor(
    self, script_path: Path, args: list[str] | None, *, capture_output: bool = True
) -> None | str:
    """Execute an R script."""
    self.verify_filetype(script_path.name, ".R")
    if self.r_path is None:
        r_missing = "can't find r binary. please find it/install."
        raise MissingBinaryError(r_missing)

    cmd: list[str] = [str(self.r_path), str(script_path)]
    if args:
        cmd.extend(args)

    env = restricted_env.copy()
    env.update(
        {
            "R_LIBS_USER": "",  # prevent loading from user library paths
            "R_PROFILE_USER": "",  # disable user profile scripts
            "R_ENVIRON_USER": "",  # disable user environment files
            "R_HISTFILE": "",  # disable history file
        }
    )

    output = subprocess.run(  # noqa: S603
        cmd,
        check=True,
        capture_output=capture_output,
        text=True,
        env=env,
    )
    if capture_output:
        return output.stdout
    return None
verify_filetype(filename, filetype) staticmethod

Verify a given file is of a given filetype via checking the ending.

Parameters:

Name Type Description Default
filename str

Name of file.

 required
filetype Literal[&quot;.py&quot;, &quot;.R&quot;, &quot;.sh&quot;] 
file ending.
required

Raises:

Type Description
WrongFiletypeError

When ending doesnt match the input.

Returns:

Name Type Description
bool bool

True if OK.
Source code in deet/data_models/pipeline.py 
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
@staticmethod
def verify_filetype(filename: str, filetype: Literal[".py", ".R", ".sh"]) -> bool:
    """
    Verify a given file is of a given filetype via checking the ending.

    Args:
        filename (str): Name of file.
        filetype (Literal[&quot;.py&quot;, &quot;.R&quot;, &quot;.sh&quot;]):
                file ending.

    Raises:
        WrongFiletypeError: When ending doesnt match the input.

    Returns:
        bool: True if OK.

    """
    if not filename[-len(filetype) :] == filetype:
        raise WrongFiletypeError
    return True

WrongFiletypeError

Bases: Exception

Raise for wrong filetype.

Source code in deet/data_models/pipeline.py 
30
31
32
33
34
35
36
37
38
39
40
class WrongFiletypeError(Exception):
    """Raise for wrong filetype."""

    def __init__(
        self,
        msg: str = "Supplied filetype is not correct.",
        *args,  # noqa: ANN002
        **kwargs,
    ) -> None:
        """Init the exception with default message."""
        super().__init__(msg, *args, **kwargs)
__init__(msg='Supplied filetype is not correct.', *args, **kwargs)

Init the exception with default message.

Source code in deet/data_models/pipeline.py 
33
34
35
36
37
38
39
40
def __init__(
    self,
    msg: str = "Supplied filetype is not correct.",
    *args,  # noqa: ANN002
    **kwargs,
) -> None:
    """Init the exception with default message."""
    super().__init__(msg, *args, **kwargs)

jobify(name, job_type=JobType.DATA_PROCESSING, func_args=None, func_kwargs=None, *, capture_output=True)

Decorate to wrap a function as a Job instance.

Source code in deet/data_models/pipeline.py 
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
def jobify(
    name: str,
    job_type: JobType | list[JobType] = JobType.DATA_PROCESSING,
    func_args: list[Any] | None = None,
    func_kwargs: dict[str, Any] | None = None,
    *,
    capture_output: bool = True,
) -> Callable[[F], Job]:
    """Decorate to wrap a function as a Job instance."""

    def decorator(func: F) -> Job:
        """Wrap around target callable."""
        return Job(
            name=name,
            job_format=JobFormat.CODE,
            job_type=job_type,
            language=Language.PYTHON,
            ingress_method=None,
            egress_method=EgressMethod.MEMORY,
            job=func,
            script_args=None,
            func_args=func_args,
            func_kwargs=func_kwargs,
            capture_output=capture_output,
            executor=Executor(executor=CodeExecutor()),
        )

    return decorator

stage_from_job(job=None, stage_name=None, input_file=None, logfile=None, *, skip_jobs_if_failed=False) 

stage_from_job(
    job: Job,
    stage_name: str | None = None,
    input_file: Path | None = None,
    logfile: Path | None = None,
    *,
    skip_jobs_if_failed: bool = False,
) -> PipelineStage

stage_from_job(
    job: None = None,
    stage_name: str | None = None,
    input_file: Path | None = None,
    logfile: Path | None = None,
    *,
    skip_jobs_if_failed: bool = False,
) -> Callable[[Job], PipelineStage]

Create a PipelineStage from a single Job.

Can be used as a function or as a decorator.

Parameters:

Name Type Description Default
job Job | None

The Job to wrap in a PipelineStage. If None, returns a decorator.

 None
stage_name str | None

Name for the stage. Defaults to job name if not provided.

 None
input_file Path | None

Optional input file for the stage.

 None
logfile Path | None

Optional logfile for the stage.

 None
skip_jobs_if_failed bool

Whether to skip remaining jobs if one fails.

 False

Returns:

Type Description
PipelineStage | Callable[[Job], PipelineStage]

PipelineStage or decorator function.

Examples:

As a function:

>>> stage = stage_from_job(my_job, stage_name="my_stage")

As a decorator:

>>> @stage_from_job(stage_name="my_stage")
... @jobify(name="my_job")
... def my_function():
...     pass
Source code in deet/data_models/pipeline.py 
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
def stage_from_job(
    job: Job | None = None,
    stage_name: str | None = None,
    input_file: Path | None = None,
    logfile: Path | None = None,
    *,
    skip_jobs_if_failed: bool = False,
) -> PipelineStage | Callable[[Job], PipelineStage]:
    """
    Create a PipelineStage from a single Job.

    Can be used as a function or as a decorator.

    Args:
        job: The Job to wrap in a PipelineStage. If None, returns a decorator.
        stage_name: Name for the stage. Defaults to job name if not provided.
        input_file: Optional input file for the stage.
        logfile: Optional logfile for the stage.
        skip_jobs_if_failed: Whether to skip remaining jobs if one fails.

    Returns:
        PipelineStage or decorator function.

    Examples:
        As a function:
        >>> stage = stage_from_job(my_job, stage_name="my_stage")

        As a decorator:
        >>> @stage_from_job(stage_name="my_stage")
        ... @jobify(name="my_job")
        ... def my_function():
        ...     pass

    """

    def _create_stage(j: Job) -> PipelineStage:
        return PipelineStage(
            name=stage_name or j.name,
            skip_jobs_if_failed=skip_jobs_if_failed,
            input_file=input_file,
            data=None,
            jobs=j,
            logfile=logfile,
        )

    # ff job is provided, return the stage directly
    if job is not None:
        return _create_stage(job)

    # otherwise, return a decorator
    return _create_stage

processed_gold_standard_annotations

Data models for procesed annotation data.

ProcessedAnnotationData pydantic-model

Bases: ProcessedAttributeData, Generic[AttributeTypeVar, DocumentTypeVar, GoldStandardAnnotationTypeVar, GoldStandardAnnotatedDocumentTypeVar]

Structured result from annotation processing.

This model provides a clean, validated structure for all processed annotation data with useful properties and methods.

Fields:

  • attributes (list[AttributeTypeVar])
  • documents (list[DocumentTypeVar])
  • annotations (list[GoldStandardAnnotationTypeVar])
  • annotated_documents (list[GoldStandardAnnotatedDocumentTypeVar])
  • attribute_id_to_label (dict[int, str])
Source code in deet/data_models/processed_gold_standard_annotations.py 
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
class ProcessedAnnotationData(
    ProcessedAttributeData,
    Generic[
        AttributeTypeVar,
        DocumentTypeVar,
        GoldStandardAnnotationTypeVar,
        GoldStandardAnnotatedDocumentTypeVar,
    ],
):
    """
    Structured result from annotation processing.

    This model provides a clean, validated structure for all processed
    annotation data with useful properties and methods.
    """

    documents: list[DocumentTypeVar]
    annotations: list[GoldStandardAnnotationTypeVar]
    annotated_documents: list[GoldStandardAnnotatedDocumentTypeVar]
    attribute_id_to_label: dict[int, str]

    @property
    def total_documents(self) -> int:
        """Total number of documents processed."""
        return len(self.documents)

    @property
    def total_annotations(self) -> int:
        """Total number of annotations processed."""
        return len(self.annotations)

    @property
    def total_annotated_documents(self) -> int:
        """Total number of documents with annotations."""
        return len(self.annotated_documents)

    def get_attributes_by_attribute_type(
        self, attribute_type: AttributeType
    ) -> list[AttributeTypeVar]:
        """Get all attributes of a specific type."""
        return [
            attr for attr in self.attributes if attr.output_data_type == attribute_type
        ]

    def get_documents_with_annotations(self) -> list[DocumentTypeVar]:
        """Get only documents that have annotations."""
        annotated_doc_ids = {
            doc.document.document_id for doc in self.annotated_documents
        }
        return [doc for doc in self.documents if doc.document_id in annotated_doc_ids]

    def get_annotations_by_annotation_type(
        self, annotation_type: AnnotationType
    ) -> list[GoldStandardAnnotationTypeVar]:
        """Get all annotations of a specific type (human/llm)."""
        return [
            ann for ann in self.annotations if ann.annotation_type == annotation_type
        ]

    def get_attribute_by_id(self, attribute_id: int) -> Attribute | None:
        """Get an attribute by its ID."""
        for attr in self.attributes:
            if attr.attribute_id == attribute_id:
                return attr
        return None

    def export_linkage_mapper_csv(self, file_path: Path) -> None:
        """Export a csv mapper to link document IDs and filenames."""
        with file_path.open("w", newline="", encoding="utf-8") as f:
            writer = csv.DictWriter(f, fieldnames=["document_id", "name", "file_path"])
            writer.writeheader()
            for d in self.documents:
                if (
                    d.document_identity is None
                    or d.document_identity.document_id is None
                ):
                    d.init_document_identity()
                if d.document_identity is None:
                    no_doc_id_err = f"document_identity was not set for document {d}"
                    raise ValueError(no_doc_id_err)
                writer.writerow(
                    {
                        "document_id": d.document_identity.document_id,
                        "name": d.name,
                        "file_path": None,
                    }
                )
total_annotated_documents property

Total number of documents with annotations.

total_annotations property

Total number of annotations processed.

total_documents property

Total number of documents processed.

export_linkage_mapper_csv(file_path)

Export a csv mapper to link document IDs and filenames.

Source code in deet/data_models/processed_gold_standard_annotations.py 
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
def export_linkage_mapper_csv(self, file_path: Path) -> None:
    """Export a csv mapper to link document IDs and filenames."""
    with file_path.open("w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=["document_id", "name", "file_path"])
        writer.writeheader()
        for d in self.documents:
            if (
                d.document_identity is None
                or d.document_identity.document_id is None
            ):
                d.init_document_identity()
            if d.document_identity is None:
                no_doc_id_err = f"document_identity was not set for document {d}"
                raise ValueError(no_doc_id_err)
            writer.writerow(
                {
                    "document_id": d.document_identity.document_id,
                    "name": d.name,
                    "file_path": None,
                }
            )
get_annotations_by_annotation_type(annotation_type)

Get all annotations of a specific type (human/llm).

Source code in deet/data_models/processed_gold_standard_annotations.py 
308
309
310
311
312
313
314
def get_annotations_by_annotation_type(
    self, annotation_type: AnnotationType
) -> list[GoldStandardAnnotationTypeVar]:
    """Get all annotations of a specific type (human/llm)."""
    return [
        ann for ann in self.annotations if ann.annotation_type == annotation_type
    ]
get_attribute_by_id(attribute_id)

Get an attribute by its ID.

Source code in deet/data_models/processed_gold_standard_annotations.py 
316
317
318
319
320
321
def get_attribute_by_id(self, attribute_id: int) -> Attribute | None:
    """Get an attribute by its ID."""
    for attr in self.attributes:
        if attr.attribute_id == attribute_id:
            return attr
    return None
get_attributes_by_attribute_type(attribute_type)

Get all attributes of a specific type.

Source code in deet/data_models/processed_gold_standard_annotations.py 
293
294
295
296
297
298
299
def get_attributes_by_attribute_type(
    self, attribute_type: AttributeType
) -> list[AttributeTypeVar]:
    """Get all attributes of a specific type."""
    return [
        attr for attr in self.attributes if attr.output_data_type == attribute_type
    ]
get_documents_with_annotations()

Get only documents that have annotations.

Source code in deet/data_models/processed_gold_standard_annotations.py 
301
302
303
304
305
306
def get_documents_with_annotations(self) -> list[DocumentTypeVar]:
    """Get only documents that have annotations."""
    annotated_doc_ids = {
        doc.document.document_id for doc in self.annotated_documents
    }
    return [doc for doc in self.documents if doc.document_id in annotated_doc_ids]

ProcessedAttributeData pydantic-model

Bases: BaseModel, Generic[AttributeTypeVar]

Structured result from annotation processing.

Contains only attributes, so the ProcessedAnnotationData class can subclass this

Fields:

  • attributes (list[AttributeTypeVar])
Source code in deet/data_models/processed_gold_standard_annotations.py 
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
class ProcessedAttributeData(BaseModel, Generic[AttributeTypeVar]):
    """
    Structured result from annotation processing.

    Contains only attributes, so the ProcessedAnnotationData class can
    subclass this
    """

    attributes: list[AttributeTypeVar]

    def _custom_prompts_cli(self) -> None:
        """
        Use an interactive CLI to have the user enter custom prompts.

        Args:
            attribute (Attribute): a single (Eppi)Attribute

        """
        for attribute in self.attributes:
            attribute.enter_custom_prompt()

    def export_attributes_csv_file(self, filepath: Path) -> None:
        """
        Write a csv file containing all attributes for prompt population.

        Args:
            filepath (Path): outfile path.


        """
        if filepath.suffix != ".csv":
            bad_filetype = "file ending must be .csv"
            raise ValueError(bad_filetype)
        filepath.unlink(missing_ok=True)
        for attribute in self.attributes:
            attribute.write_to_csv(filepath=filepath)

        logger.info(f"wrote attributes to file {filepath}.")

    @staticmethod
    def _validate_csv_file(filepath: Path) -> None:
        """Validate csv file exists and has correct extension."""
        if not filepath.exists():
            no_file = f"CSV file not found: {filepath}"
            raise FileNotFoundError(no_file)

        if filepath.suffix != ".csv":
            bad_suffix = "File must have .csv extension"
            raise ValueError(bad_suffix)

    @staticmethod
    def _validate_csv_headers(fieldnames: Sequence[str] | None) -> None:
        """Validate csv has required headers."""
        if fieldnames is None:
            empty_csv = "csv file is empty or has no headers"
            raise ValueError(empty_csv)

        required_fields = ["attribute_id", "prompt"]
        for field in required_fields:
            if field not in fieldnames:
                csv_missing_fields = (
                    f"csv must contain '{field}' column. "
                    f"Found columns: {', '.join(fieldnames)}"
                )
                raise ValueError(csv_missing_fields)

    def _process_csv_row(
        self,
        row: dict[str, Any],
        csv_attribute_ids_with_prompts: set[int],
        *,
        overwrite: bool = True,
    ) -> bool:
        """
        Process a single csv row and update the matching attribute.

        Returns
            bool: True if row was processed successfully, False otherwise

        """
        try:
            attribute_id = int(row.get("attribute_id"))  # type:ignore[arg-type]
        except ValueError as e:
            logger.warning(e)
            return False

        if (row.get("prompt") == "") or (row.get("prompt") is None):
            logger.debug(
                "prompt field is empty, "
                f"so we don't want this attribute {attribute_id}."
            )
            return False

        # Track attribute IDs that have non-empty prompts
        csv_attribute_ids_with_prompts.add(attribute_id)

        matching_attribute = None
        for attribute in self.attributes:
            if attribute.attribute_id == attribute_id:
                matching_attribute = attribute
                break

        if matching_attribute is None:
            logger.warning(f"No attribute found with ID {attribute_id}, skipping row")
            return False

        # Update attribute with prompt and data type
        try:
            matching_attribute.populate_prompt_from_dict(row, overwrite=overwrite)
            csv_attr_type = AttributeType(row.get("output_data_type"))  # type:ignore[arg-type]
            matching_attribute.output_data_type = csv_attr_type

        except ValueError as e:
            logger.error(
                f"Error processing row for attribute {attribute_id}: {e}. "
                "Setting attribute type to bool."
            )
            matching_attribute.output_data_type = DEFAULT_ATTRIBUTE_TYPE
            return False
        else:
            return True

    def _filter_attributes_by_csv(
        self,
        csv_attribute_ids_with_prompts: set[int],
        *,
        retain_only_csv_attributes: bool = True,
    ) -> None:
        """Filter attributes based on CSV content and retention policy."""
        if retain_only_csv_attributes:
            original_count = len(self.attributes)
            # Only keep attributes that are in CSV AND have non-empty prompts
            self.attributes = [
                attr
                for attr in self.attributes
                if attr.attribute_id in csv_attribute_ids_with_prompts
            ]
            logger.info(
                f"filtered attributes from {original_count} to {len(self.attributes)} "
                f"(retained only those in CSV with non-empty prompts)"
            )

    def _import_prompts_csv_file(
        self,
        filepath: Path,
        *,
        retain_only_csv_attributes: bool = True,
        overwrite: bool = True,
    ) -> None:
        """
        Import prompts from a csv file.

        Args:
            filepath (Path): attribute/prompt input file.
            retain_only_csv_attributes (bool, optional): if True, filter self.attributes
                to only include attributes with ids & a non-null prompt found in csv.
                Defaults to True.
            overwrite (bool, optional): Overwrite existing prompts. Defaults to True.

        """
        self._validate_csv_file(filepath)

        csv_attribute_ids_with_prompts: set[int] = set()
        rows_processed = 0

        with filepath.open(mode="r", newline="", encoding="utf-8") as f:
            reader = csv.DictReader(f)
            if reader.fieldnames is not None:
                reader.fieldnames = [
                    name.lstrip("\ufeff").strip() if name else name
                    for name in reader.fieldnames
                ]

            self._validate_csv_headers(reader.fieldnames)

            for row in reader:
                if self._process_csv_row(
                    row=row,
                    csv_attribute_ids_with_prompts=csv_attribute_ids_with_prompts,
                    overwrite=overwrite,
                ):
                    rows_processed += 1

            logger.info(f"Processed {rows_processed} prompts from {filepath}")

        self._filter_attributes_by_csv(
            csv_attribute_ids_with_prompts=csv_attribute_ids_with_prompts,
            retain_only_csv_attributes=retain_only_csv_attributes,
        )

    def populate_custom_prompts(
        self,
        method: CustomPromptPopulationMethod,
        filepath: Path | None = None,
        **kwargs,
    ) -> None:
        """
        Populate custom prompts.

        Args:
            method (CustomPromptPopulationMethod)
            filepath (Path | None): infile path.

        Raises:
            FileNotFoundError: if method is file and there's no filepath.

        """
        if method == "cli":
            self._custom_prompts_cli()
        elif method == "file":
            if filepath is None:
                missing_filepath = "please specify a filepath!"
                raise FileNotFoundError(missing_filepath)
            self._import_prompts_csv_file(filepath=filepath, **kwargs)
        else:
            not_impl = f"method {method} is not implemented. use cli or file."
            raise NotImplementedError(not_impl)

    @property
    def total_attributes(self) -> int:
        """Total number of attributes processed."""
        return len(self.attributes)
total_attributes property

Total number of attributes processed.

export_attributes_csv_file(filepath)

Write a csv file containing all attributes for prompt population.

Parameters:

Name Type Description Default
filepath Path

outfile path.

 required
Source code in deet/data_models/processed_gold_standard_annotations.py 
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
def export_attributes_csv_file(self, filepath: Path) -> None:
    """
    Write a csv file containing all attributes for prompt population.

    Args:
        filepath (Path): outfile path.


    """
    if filepath.suffix != ".csv":
        bad_filetype = "file ending must be .csv"
        raise ValueError(bad_filetype)
    filepath.unlink(missing_ok=True)
    for attribute in self.attributes:
        attribute.write_to_csv(filepath=filepath)

    logger.info(f"wrote attributes to file {filepath}.")
populate_custom_prompts(method, filepath=None, **kwargs)

Populate custom prompts.

Parameters:

Name Type Description Default
filepath Path | None

infile path.

 None

Raises:

Type Description
FileNotFoundError

if method is file and there's no filepath.
Source code in deet/data_models/processed_gold_standard_annotations.py 
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
def populate_custom_prompts(
    self,
    method: CustomPromptPopulationMethod,
    filepath: Path | None = None,
    **kwargs,
) -> None:
    """
    Populate custom prompts.

    Args:
        method (CustomPromptPopulationMethod)
        filepath (Path | None): infile path.

    Raises:
        FileNotFoundError: if method is file and there's no filepath.

    """
    if method == "cli":
        self._custom_prompts_cli()
    elif method == "file":
        if filepath is None:
            missing_filepath = "please specify a filepath!"
            raise FileNotFoundError(missing_filepath)
        self._import_prompts_csv_file(filepath=filepath, **kwargs)
    else:
        not_impl = f"method {method} is not implemented. use cli or file."
        raise NotImplementedError(not_impl)

ProcessedEppiAnnotationData pydantic-model

Bases: ProcessedAnnotationData[EppiAttribute, EppiDocument, EppiGoldStandardAnnotation, EppiGoldStandardAnnotatedDocument]

Structured result from EPPI annotation processing.

This differs from Base ProcessedAnnotationData by specifying raw_data as an EppiRawData object

Fields:

  • attributes (list[AttributeTypeVar])
  • documents (list[DocumentTypeVar])
  • annotations (list[GoldStandardAnnotationTypeVar])
  • annotated_documents (list[GoldStandardAnnotatedDocumentTypeVar])
  • attribute_id_to_label (dict[int, str])
  • raw_data (EppiRawData | None)
Source code in deet/data_models/processed_gold_standard_annotations.py 
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
class ProcessedEppiAnnotationData(
    ProcessedAnnotationData[
        EppiAttribute,
        EppiDocument,
        EppiGoldStandardAnnotation,
        EppiGoldStandardAnnotatedDocument,
    ]
):
    """
    Structured result from EPPI annotation processing.

    This differs from Base ProcessedAnnotationData by specifying raw_data as an
    EppiRawData object
    """

    raw_data: EppiRawData | None = None

    def populate_custom_prompts(
        self,
        method: CustomPromptPopulationMethod,
        filepath: Path | None = None,
        **kwargs: Any,  # noqa: ANN401
    ) -> None:
        """
        Populate custom prompts, then recompute each annotation's ``raw_data``.

        An EPPI ``Report.json`` (or similar) only carries EPPI-Reviewer field names;
        it does not set each attribute's ``output_data_type`` in DEET. Ingested
        attributes default to ``bool`` from the codeset. The custom prompt
        definition file (e.g. a CSV with ``output_data_type`` such as
        ``string`` or ``bool``) is the source of those types, not the JSON. After
        the base method updates prompts and types, we re-apply
        ``eppi_output_data_from_eppi_fields`` so ``raw_data`` matches
        ``AdditionalText`` and the updated type (e.g. ``"32"`` for a string count,
        not boolean ``True`` from initial ingest only).
        """
        super().populate_custom_prompts(method, filepath=filepath, **kwargs)
        self._recompute_eppi_raw_data_from_additional_text()

    def _recompute_eppi_raw_data_from_additional_text(self) -> None:
        """Re-derive each gold ``raw_data`` from types and ``AdditionalText``."""
        # Import inside the method: ``eppi_annotation_converter`` already imports
        # ``ProcessedEppiAnnotationData`` from this module, so a top-level import
        # would risk a circular import.
        from deet.processors.eppi_annotation_converter import (
            eppi_output_data_from_eppi_fields,
        )

        for ann in self.annotations:
            ann.raw_data = eppi_output_data_from_eppi_fields(
                ann.attribute.output_data_type,
                additional_text=ann.additional_text or "",
            )
populate_custom_prompts(method, filepath=None, **kwargs)

Populate custom prompts, then recompute each annotation's raw_data.

An EPPI Report.json (or similar) only carries EPPI-Reviewer field names; it does not set each attribute's output_data_type in DEET. Ingested attributes default to bool from the codeset. The custom prompt definition file (e.g. a CSV with output_data_type such as string or bool) is the source of those types, not the JSON. After the base method updates prompts and types, we re-apply eppi_output_data_from_eppi_fields so raw_data matches AdditionalText and the updated type (e.g. "32" for a string count, not boolean True from initial ingest only).

Source code in deet/data_models/processed_gold_standard_annotations.py 
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
def populate_custom_prompts(
    self,
    method: CustomPromptPopulationMethod,
    filepath: Path | None = None,
    **kwargs: Any,  # noqa: ANN401
) -> None:
    """
    Populate custom prompts, then recompute each annotation's ``raw_data``.

    An EPPI ``Report.json`` (or similar) only carries EPPI-Reviewer field names;
    it does not set each attribute's ``output_data_type`` in DEET. Ingested
    attributes default to ``bool`` from the codeset. The custom prompt
    definition file (e.g. a CSV with ``output_data_type`` such as
    ``string`` or ``bool``) is the source of those types, not the JSON. After
    the base method updates prompts and types, we re-apply
    ``eppi_output_data_from_eppi_fields`` so ``raw_data`` matches
    ``AdditionalText`` and the updated type (e.g. ``"32"`` for a string count,
    not boolean ``True`` from initial ingest only).
    """
    super().populate_custom_prompts(method, filepath=filepath, **kwargs)
    self._recompute_eppi_raw_data_from_additional_text()

evaluators

gold_standard_llm_evaluator

Generalisable evaluation module for comparing data extracted by LLMs with data extracted by hand.

GoldStandardLLMEvaluator

A class to manage the evaluation of LLM-extracted data against "gold-standard" ground truth data.

Source code in deet/evaluators/gold_standard_llm_evaluator.py 
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
class GoldStandardLLMEvaluator:
    """
    A class to manage the evaluation of LLM-extracted data against
    "gold-standard" ground truth data.
    """

    def __init__(
        self,
        gold_standard_annotated_documents: Sequence[
            GoldStandardAnnotatedDocumentTypeVar
        ],
        llm_annotated_documents: Sequence[GoldStandardAnnotatedDocumentTypeVar],
        attributes: Sequence[AttributeTypeVar],
        extraction_run_id: str,
        custom_metrics: list[str] | None = None,
    ) -> None:
        """
        Initialise GoldStandardLLMEvaluator with a list of ground truth and
        LLM-generated data to compare, along with the attributes you want to
        compare.
        """
        self.gold_standard_annotated_documents = gold_standard_annotated_documents
        self.llm_annotated_documents = GoldStandardAnnotatedDocumentList(
            gold_standard_annotations=llm_annotated_documents
        )
        self.attributes = attributes
        self.extraction_run_id = extraction_run_id
        self.metrics_config: dict[str, MetricFunction] = METRICS
        self.custom_metrics: dict[str, MetricFunction] = {}
        self.calculated_metrics: list[AttributeMetric] = []
        if custom_metrics is not None:
            self.add_custom_metrics(custom_metrics)

    def add_custom_metrics(self, custom_metrics: list[str]) -> None:
        """Add custom metrics. These must be valid metrics from sklearn.metrics."""
        for custom_metric_name in custom_metrics:
            custom_metric = getattr(sklearn.metrics, custom_metric_name, None)
            if custom_metric is not None:
                if check_metric_returns_float(custom_metric):
                    self.metrics_config[custom_metric_name] = custom_metric
                    self.custom_metrics[custom_metric_name] = custom_metric
                else:
                    logger.warning(
                        f"Tried to add {custom_metric_name} to"
                        " evaluation metrics, but it does not return a float."
                    )
            else:
                logger.warning(
                    f"Tried to add {custom_metric_name} to"
                    " evaluation metrics, but it does not exist"
                )

    def evaluate_llm_annotations(
        self,
    ) -> None:
        """
        Compare a list of human annotations to those generated by llms.
        Return a list of AttributeMetric objects.
        """
        if self.calculated_metrics:
            logger.warning("Already calculated metrics, deleting and overwriting.")
            self.calculated_metrics = []
        for attribute in self.attributes:
            logger.debug(
                f"Calculating metric for attribute: {attribute.attribute_label}"
            )
            y_true = []
            y_pred: list[Any] = []
            for document in self.gold_standard_annotated_documents:
                doc_id = document.document.safe_identity.document_id
                logger.debug(
                    f"Extracting gold standard and LLM prediction for doc {doc_id}"
                )
                gs_val = document.get_attribute_annotation(attribute).output_data
                y_true.append(gs_val)

                try:
                    llm_doc = self.llm_annotated_documents.get_by_id(doc_id)
                except MissingDocumentError:
                    y_pred.append(None)
                    logger.warning(f"LLM annotated doc not found - ID: {doc_id}")
                    continue
                try:
                    llm_val = llm_doc.get_attribute_annotation(attribute).output_data
                except DuplicateAnnotationError:
                    llm_val = None
                    logger.warning(
                        f"LLM produced multiple annotations for a single"
                        f" attribute with doc: {doc_id}"
                    )
                y_pred.append(llm_val)

            applicable_metrics = get_metrics_for_attribute_type(
                attribute.output_data_type
            )
            combined_metrics = {**applicable_metrics, **self.custom_metrics}

            for metric_name, metric_fn in combined_metrics.items():
                try:
                    value = float(metric_fn(y_true, y_pred))
                except (ValueError, TypeError) as e:
                    logger.warning(
                        f"Metric '{metric_name}' not applicable for "
                        f"attribute '{attribute.attribute_label}' "
                        f"(type={attribute.output_data_type}): {e}"
                    )
                    value = None
                self.calculated_metrics.append(
                    AttributeMetric(
                        attribute=attribute,
                        metric_name=metric_name,
                        value=value,
                        extraction_run_id=self.extraction_run_id,
                    )
                )

    def display_metrics(self) -> None:
        """Print metrics in a nice table to the command line."""
        console = Console()

        table = Table(title="Evaluation Results")

        metric_names = sorted({str(m.metric_name) for m in self.calculated_metrics})

        # Create table with metrics as columns
        table = Table(title="Evaluation Metrics")
        table.add_column("Attribute")
        for name in metric_names:
            table.add_column(name, justify="right")

        # Group metrics by attribute
        metrics_sorted = sorted(
            self.calculated_metrics, key=lambda m: m.attribute.attribute_label
        )
        for attribute, group in groupby(
            metrics_sorted, key=lambda m: m.attribute.attribute_label
        ):
            row = [attribute]
            group_metrics = {str(m.metric_name): m.value for m in group}
            # fill cells in order of metric_names
            row += [
                f"{group_metrics[name]:.4f}"
                if group_metrics.get(name) is not None
                else "N/A"
                for name in metric_names
            ]
            table.add_row(*row)

        console.print(table)

    def write_metrics_to_csv(self, filepath: Path) -> None:
        """Save metrics to csv."""
        if filepath.suffix != ".csv":
            bad_filetype = "file ending must be .csv"
            raise ValueError(bad_filetype)
        for metric in self.calculated_metrics:
            metric.save_to_csv(filepath=filepath)

    def export_llm_comparison(
        self,
        filepath: Path,
    ) -> None:
        """
        Export a comparison CSV for gold vs LLM per document and attribute.

        Columns include identifiers, EPPI-oriented fields, extractions, LLM verbatim,
        fuzzy grounding scores (against the LLM annotated document's ``context``),
        and run id.

        Column semantics:

        - ``attribute_presence``: Whether the gold annotation is present.
        - ``human_additional_text`` / ``item_attribute_full_text_details``: Taken from
          the eppi json file when present; empty when absent.
        - ``human_extraction``: Actual ground truth to be extracted.
        - ``human_verbatim_fuzzy_match_pct``: Grounding of ``human_additional_text``
          against the LLM annotated document's ``context``.
        - ``llm_verbatim_text`` / ``llm_verbatim_fuzzy_match_pct``: LLM
          ``additional_text`` and its grounding against the same ``context``.

        Example row (illustrative types): ``attribute_presence`` is the string
        ``"True"`` or ``"False"``; ``human_verbatim_fuzzy_match_pct`` and
        ``llm_verbatim_fuzzy_match_pct`` are decimal strings (e.g. ``"100.00"``,
        ``"87.50"``); ``human_extraction`` / ``llm_extraction`` serialize according to
        the attribute's coerced value (e.g. bool, int, or str) as written by
        :class:`csv.DictWriter`.
        """
        with filepath.open("w", newline="", encoding="utf-8") as f:
            writer = csv.DictWriter(
                f,
                fieldnames=[
                    "document_id",
                    "document_name",
                    "attribute_id",
                    "attribute_label",
                    "attribute_presence",
                    "human_additional_text",
                    "item_attribute_full_text_details",
                    "human_extraction",
                    "llm_extraction",
                    "llm_reasoning",
                    "llm_verbatim_text",
                    "human_verbatim_fuzzy_match_pct",
                    "llm_verbatim_fuzzy_match_pct",
                    "extraction_run_id",
                ],
            )
            writer.writeheader()
            for doc in self.gold_standard_annotated_documents:
                try:
                    llm_annotated_doc = self.llm_annotated_documents.get_by_id(
                        doc.document.safe_identity.document_id
                    )
                except MissingDocumentError:
                    llm_annotated_doc = None

                context: str | None = (
                    None
                    if llm_annotated_doc is None
                    else (str(t) if (t := llm_annotated_doc.document.context) else None)
                )

                for attribute in self.attributes:
                    human_ann = doc.get_attribute_annotation(attribute)
                    gold_real = next(
                        (
                            ann
                            for ann in doc.annotations
                            if ann.attribute.attribute_id == attribute.attribute_id
                        ),
                        None,
                    )
                    if gold_real is not None:
                        human_additional_text: str = gold_real.additional_text or ""
                        item_attr_full: str = _eppi_full_text_details_colon_separated(
                            gold_real
                        )
                    else:
                        human_additional_text = ""
                        item_attr_full = ""
                    present = gold_real is not None
                    human_fuzzy = _verbatim_fuzzy_match_pct(
                        human_additional_text, context
                    )

                    llm_extraction: Any = None
                    llm_reasoning: str | None = None
                    llm_verbatim: str | None = None
                    llm_fuzzy = 0.0

                    if llm_annotated_doc is None:
                        llm_reasoning = (
                            "LLM did not produce an output for this document."
                            " Check the logs carefully to find out why"
                        )
                    else:
                        try:
                            llm_annotation = llm_annotated_doc.get_attribute_annotation(
                                attribute
                            )
                            llm_extraction = llm_annotation.output_data
                            llm_reasoning = llm_annotation.reasoning
                            llm_verbatim = llm_annotation.additional_text
                            llm_fuzzy = _verbatim_fuzzy_match_pct(llm_verbatim, context)
                        except DuplicateAnnotationError:
                            llm_reasoning = (
                                "The LLM produced multiple annotations"
                                "for this single attribute"
                            )

                    writer.writerow(
                        {
                            "document_id": doc.document.safe_identity.document_id,
                            "document_name": doc.document.name,
                            "attribute_id": attribute.attribute_id,
                            "attribute_label": attribute.attribute_label,
                            "attribute_presence": str(present),
                            "human_additional_text": human_additional_text,
                            "item_attribute_full_text_details": item_attr_full,
                            "human_extraction": human_ann.output_data,
                            "llm_extraction": llm_extraction,
                            "llm_reasoning": llm_reasoning,
                            "llm_verbatim_text": llm_verbatim,
                            "human_verbatim_fuzzy_match_pct": f"{human_fuzzy:.2f}",
                            "llm_verbatim_fuzzy_match_pct": f"{llm_fuzzy:.2f}",
                            "extraction_run_id": self.extraction_run_id,
                        }
                    )
__init__(gold_standard_annotated_documents, llm_annotated_documents, attributes, extraction_run_id, custom_metrics=None)

Initialise GoldStandardLLMEvaluator with a list of ground truth and LLM-generated data to compare, along with the attributes you want to compare.

Source code in deet/evaluators/gold_standard_llm_evaluator.py 
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
def __init__(
    self,
    gold_standard_annotated_documents: Sequence[
        GoldStandardAnnotatedDocumentTypeVar
    ],
    llm_annotated_documents: Sequence[GoldStandardAnnotatedDocumentTypeVar],
    attributes: Sequence[AttributeTypeVar],
    extraction_run_id: str,
    custom_metrics: list[str] | None = None,
) -> None:
    """
    Initialise GoldStandardLLMEvaluator with a list of ground truth and
    LLM-generated data to compare, along with the attributes you want to
    compare.
    """
    self.gold_standard_annotated_documents = gold_standard_annotated_documents
    self.llm_annotated_documents = GoldStandardAnnotatedDocumentList(
        gold_standard_annotations=llm_annotated_documents
    )
    self.attributes = attributes
    self.extraction_run_id = extraction_run_id
    self.metrics_config: dict[str, MetricFunction] = METRICS
    self.custom_metrics: dict[str, MetricFunction] = {}
    self.calculated_metrics: list[AttributeMetric] = []
    if custom_metrics is not None:
        self.add_custom_metrics(custom_metrics)
add_custom_metrics(custom_metrics)

Add custom metrics. These must be valid metrics from sklearn.metrics.

Source code in deet/evaluators/gold_standard_llm_evaluator.py 
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
def add_custom_metrics(self, custom_metrics: list[str]) -> None:
    """Add custom metrics. These must be valid metrics from sklearn.metrics."""
    for custom_metric_name in custom_metrics:
        custom_metric = getattr(sklearn.metrics, custom_metric_name, None)
        if custom_metric is not None:
            if check_metric_returns_float(custom_metric):
                self.metrics_config[custom_metric_name] = custom_metric
                self.custom_metrics[custom_metric_name] = custom_metric
            else:
                logger.warning(
                    f"Tried to add {custom_metric_name} to"
                    " evaluation metrics, but it does not return a float."
                )
        else:
            logger.warning(
                f"Tried to add {custom_metric_name} to"
                " evaluation metrics, but it does not exist"
            )
display_metrics()

Print metrics in a nice table to the command line.

Source code in deet/evaluators/gold_standard_llm_evaluator.py 
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
def display_metrics(self) -> None:
    """Print metrics in a nice table to the command line."""
    console = Console()

    table = Table(title="Evaluation Results")

    metric_names = sorted({str(m.metric_name) for m in self.calculated_metrics})

    # Create table with metrics as columns
    table = Table(title="Evaluation Metrics")
    table.add_column("Attribute")
    for name in metric_names:
        table.add_column(name, justify="right")

    # Group metrics by attribute
    metrics_sorted = sorted(
        self.calculated_metrics, key=lambda m: m.attribute.attribute_label
    )
    for attribute, group in groupby(
        metrics_sorted, key=lambda m: m.attribute.attribute_label
    ):
        row = [attribute]
        group_metrics = {str(m.metric_name): m.value for m in group}
        # fill cells in order of metric_names
        row += [
            f"{group_metrics[name]:.4f}"
            if group_metrics.get(name) is not None
            else "N/A"
            for name in metric_names
        ]
        table.add_row(*row)

    console.print(table)
evaluate_llm_annotations()

Compare a list of human annotations to those generated by llms. Return a list of AttributeMetric objects.

Source code in deet/evaluators/gold_standard_llm_evaluator.py 
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
def evaluate_llm_annotations(
    self,
) -> None:
    """
    Compare a list of human annotations to those generated by llms.
    Return a list of AttributeMetric objects.
    """
    if self.calculated_metrics:
        logger.warning("Already calculated metrics, deleting and overwriting.")
        self.calculated_metrics = []
    for attribute in self.attributes:
        logger.debug(
            f"Calculating metric for attribute: {attribute.attribute_label}"
        )
        y_true = []
        y_pred: list[Any] = []
        for document in self.gold_standard_annotated_documents:
            doc_id = document.document.safe_identity.document_id
            logger.debug(
                f"Extracting gold standard and LLM prediction for doc {doc_id}"
            )
            gs_val = document.get_attribute_annotation(attribute).output_data
            y_true.append(gs_val)

            try:
                llm_doc = self.llm_annotated_documents.get_by_id(doc_id)
            except MissingDocumentError:
                y_pred.append(None)
                logger.warning(f"LLM annotated doc not found - ID: {doc_id}")
                continue
            try:
                llm_val = llm_doc.get_attribute_annotation(attribute).output_data
            except DuplicateAnnotationError:
                llm_val = None
                logger.warning(
                    f"LLM produced multiple annotations for a single"
                    f" attribute with doc: {doc_id}"
                )
            y_pred.append(llm_val)

        applicable_metrics = get_metrics_for_attribute_type(
            attribute.output_data_type
        )
        combined_metrics = {**applicable_metrics, **self.custom_metrics}

        for metric_name, metric_fn in combined_metrics.items():
            try:
                value = float(metric_fn(y_true, y_pred))
            except (ValueError, TypeError) as e:
                logger.warning(
                    f"Metric '{metric_name}' not applicable for "
                    f"attribute '{attribute.attribute_label}' "
                    f"(type={attribute.output_data_type}): {e}"
                )
                value = None
            self.calculated_metrics.append(
                AttributeMetric(
                    attribute=attribute,
                    metric_name=metric_name,
                    value=value,
                    extraction_run_id=self.extraction_run_id,
                )
            )
export_llm_comparison(filepath)

Export a comparison CSV for gold vs LLM per document and attribute.

Columns include identifiers, EPPI-oriented fields, extractions, LLM verbatim, fuzzy grounding scores (against the LLM annotated document's context), and run id.

Column semantics:

  • attribute_presence: Whether the gold annotation is present.
  • human_additional_text / item_attribute_full_text_details: Taken from the eppi json file when present; empty when absent.
  • human_extraction: Actual ground truth to be extracted.
  • human_verbatim_fuzzy_match_pct: Grounding of human_additional_text against the LLM annotated document's context.
  • llm_verbatim_text / llm_verbatim_fuzzy_match_pct: LLM additional_text and its grounding against the same context.

Example row (illustrative types): attribute_presence is the string "True" or "False"; human_verbatim_fuzzy_match_pct and llm_verbatim_fuzzy_match_pct are decimal strings (e.g. "100.00", "87.50"); human_extraction / llm_extraction serialize according to the attribute's coerced value (e.g. bool, int, or str) as written by :class:csv.DictWriter.

Source code in deet/evaluators/gold_standard_llm_evaluator.py 
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
def export_llm_comparison(
    self,
    filepath: Path,
) -> None:
    """
    Export a comparison CSV for gold vs LLM per document and attribute.

    Columns include identifiers, EPPI-oriented fields, extractions, LLM verbatim,
    fuzzy grounding scores (against the LLM annotated document's ``context``),
    and run id.

    Column semantics:

    - ``attribute_presence``: Whether the gold annotation is present.
    - ``human_additional_text`` / ``item_attribute_full_text_details``: Taken from
      the eppi json file when present; empty when absent.
    - ``human_extraction``: Actual ground truth to be extracted.
    - ``human_verbatim_fuzzy_match_pct``: Grounding of ``human_additional_text``
      against the LLM annotated document's ``context``.
    - ``llm_verbatim_text`` / ``llm_verbatim_fuzzy_match_pct``: LLM
      ``additional_text`` and its grounding against the same ``context``.

    Example row (illustrative types): ``attribute_presence`` is the string
    ``"True"`` or ``"False"``; ``human_verbatim_fuzzy_match_pct`` and
    ``llm_verbatim_fuzzy_match_pct`` are decimal strings (e.g. ``"100.00"``,
    ``"87.50"``); ``human_extraction`` / ``llm_extraction`` serialize according to
    the attribute's coerced value (e.g. bool, int, or str) as written by
    :class:`csv.DictWriter`.
    """
    with filepath.open("w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(
            f,
            fieldnames=[
                "document_id",
                "document_name",
                "attribute_id",
                "attribute_label",
                "attribute_presence",
                "human_additional_text",
                "item_attribute_full_text_details",
                "human_extraction",
                "llm_extraction",
                "llm_reasoning",
                "llm_verbatim_text",
                "human_verbatim_fuzzy_match_pct",
                "llm_verbatim_fuzzy_match_pct",
                "extraction_run_id",
            ],
        )
        writer.writeheader()
        for doc in self.gold_standard_annotated_documents:
            try:
                llm_annotated_doc = self.llm_annotated_documents.get_by_id(
                    doc.document.safe_identity.document_id
                )
            except MissingDocumentError:
                llm_annotated_doc = None

            context: str | None = (
                None
                if llm_annotated_doc is None
                else (str(t) if (t := llm_annotated_doc.document.context) else None)
            )

            for attribute in self.attributes:
                human_ann = doc.get_attribute_annotation(attribute)
                gold_real = next(
                    (
                        ann
                        for ann in doc.annotations
                        if ann.attribute.attribute_id == attribute.attribute_id
                    ),
                    None,
                )
                if gold_real is not None:
                    human_additional_text: str = gold_real.additional_text or ""
                    item_attr_full: str = _eppi_full_text_details_colon_separated(
                        gold_real
                    )
                else:
                    human_additional_text = ""
                    item_attr_full = ""
                present = gold_real is not None
                human_fuzzy = _verbatim_fuzzy_match_pct(
                    human_additional_text, context
                )

                llm_extraction: Any = None
                llm_reasoning: str | None = None
                llm_verbatim: str | None = None
                llm_fuzzy = 0.0

                if llm_annotated_doc is None:
                    llm_reasoning = (
                        "LLM did not produce an output for this document."
                        " Check the logs carefully to find out why"
                    )
                else:
                    try:
                        llm_annotation = llm_annotated_doc.get_attribute_annotation(
                            attribute
                        )
                        llm_extraction = llm_annotation.output_data
                        llm_reasoning = llm_annotation.reasoning
                        llm_verbatim = llm_annotation.additional_text
                        llm_fuzzy = _verbatim_fuzzy_match_pct(llm_verbatim, context)
                    except DuplicateAnnotationError:
                        llm_reasoning = (
                            "The LLM produced multiple annotations"
                            "for this single attribute"
                        )

                writer.writerow(
                    {
                        "document_id": doc.document.safe_identity.document_id,
                        "document_name": doc.document.name,
                        "attribute_id": attribute.attribute_id,
                        "attribute_label": attribute.attribute_label,
                        "attribute_presence": str(present),
                        "human_additional_text": human_additional_text,
                        "item_attribute_full_text_details": item_attr_full,
                        "human_extraction": human_ann.output_data,
                        "llm_extraction": llm_extraction,
                        "llm_reasoning": llm_reasoning,
                        "llm_verbatim_text": llm_verbatim,
                        "human_verbatim_fuzzy_match_pct": f"{human_fuzzy:.2f}",
                        "llm_verbatim_fuzzy_match_pct": f"{llm_fuzzy:.2f}",
                        "extraction_run_id": self.extraction_run_id,
                    }
                )
write_metrics_to_csv(filepath)

Save metrics to csv.

Source code in deet/evaluators/gold_standard_llm_evaluator.py 
266
267
268
269
270
271
272
def write_metrics_to_csv(self, filepath: Path) -> None:
    """Save metrics to csv."""
    if filepath.suffix != ".csv":
        bad_filetype = "file ending must be .csv"
        raise ValueError(bad_filetype)
    for metric in self.calculated_metrics:
        metric.save_to_csv(filepath=filepath)

exceptions

Custom exceptions.

BadDocumentIdError

Bases: Exception

Raise when our Document.document_id field doesn't satisfy our criteria.

Parameters:

Name Type Description Default
Exception _type_

description

 required
Source code in deet/exceptions.py 
101
102
103
104
105
106
107
108
109
class BadDocumentIdError(Exception):
    """
    Raise when our `Document.document_id` field
    doesn't satisfy our criteria.

    Args:
        Exception (_type_): _description_

    """

DuplicateAnnotationError

Bases: Exception

Raise when multiple annotations for a single attribute are present.

Parameters:

Name Type Description Default
Exception _type_
required
Source code in deet/exceptions.py 
16
17
18
19
20
21
22
23
class DuplicateAnnotationError(Exception):
    """
    Raise when multiple annotations for a single attribute are present.

    Args:
        Exception (_type_):

    """

EmptyPdfExtractionError

Bases: Exception

Raise when PDF parsing yields no extractable text.

Occurs when the PDF has no mappable text (e.g. image-only) or when text is represented in a way pdfminer cannot decode.

Source code in deet/exceptions.py 
76
77
78
79
80
81
82
83
84
85
86
87
88
class EmptyPdfExtractionError(Exception):
    """
    Raise when PDF parsing yields no extractable text.

    Occurs when the PDF has no mappable text (e.g. image-only) or when text
    is represented in a way pdfminer cannot decode.

    """

    DEFAULT_MESSAGE = (
        "PDF contained no extractable text (e.g. image-only or text in "
        "unsupported encoding)."
    )

FileParserMismatchError

Bases: Exception

Raise when we have an input-file <> parser mismatch.

Parameters:

Name Type Description Default
Exception _type_
required
Source code in deet/exceptions.py 
56
57
58
59
60
61
62
63
class FileParserMismatchError(Exception):
    """
    Raise when we have an input-file <> parser mismatch.

    Args:
        Exception (_type_):

    """

InvalidFileTypeError

Bases: Exception

Raise when user supplies a not permitted file.

Parameters:

Name Type Description Default
Exception _type_
required
Source code in deet/exceptions.py 
46
47
48
49
50
51
52
53
class InvalidFileTypeError(Exception):
    """
    Raise when user supplies a not permitted file.

    Args:
        Exception (_type_):

    """

InvalidInputFileTypeError

Bases: Exception

Raise when user supplies a not permitted input file type.

Parameters:

Name Type Description Default
Exception _type_
required
Source code in deet/exceptions.py 
26
27
28
29
30
31
32
33
class InvalidInputFileTypeError(Exception):
    """
    Raise when user supplies a not permitted input file type.

    Args:
        Exception (_type_):

    """

InvalidOutputFileTypeError

Bases: Exception

Raise when user supplies a not permitted output file.

Parameters:

Name Type Description Default
Exception _type_
required
Source code in deet/exceptions.py 
36
37
38
39
40
41
42
43
class InvalidOutputFileTypeError(Exception):
    """
    Raise when user supplies a not permitted output file.

    Args:
        Exception (_type_):

    """

JsonStyleError

Bases: Exception

Raise when a json containing document-reference-linkages is incorrectly formatted.

Parameters:

Name Type Description Default
Exception _type_

description

 required
Source code in deet/exceptions.py 
112
113
114
115
116
117
118
119
120
class JsonStyleError(Exception):
    """
    Raise when a json containing document-reference-linkages
    is incorrectly formatted.

    Args:
        Exception (_type_): _description_

    """

LitellmModelNotMappedError

Bases: Exception

Raised when litellm reports the model is missing from its registry.

litellm.get_max_tokens can raise a bare Exception with a characteristic message; we translate that to this type so callers can handle it without a broad except Exception.

Source code in deet/exceptions.py 
161
162
163
164
165
166
167
168
class LitellmModelNotMappedError(Exception):
    """
    Raised when litellm reports the model is missing from its registry.

    ``litellm.get_max_tokens`` can raise a bare ``Exception`` with a
    characteristic message; we translate that to this type so callers can
    handle it without a broad ``except Exception``.
    """

MalformedLanguageError

Bases: Exception

Raise when language checker fails.

Parameters:

Name Type Description Default
Exception _type_

description

 required
Source code in deet/exceptions.py 
66
67
68
69
70
71
72
73
class MalformedLanguageError(Exception):
    """
    Raise when language checker fails.

    Args:
        Exception (_type_): _description_

    """

MissingCitationElementError

Bases: Exception

Raise when required element of citation is missing.

Parameters:

Name Type Description Default
Exception _type_

description

 required
Source code in deet/exceptions.py 
91
92
93
94
95
96
97
98
class MissingCitationElementError(Exception):
    """
    Raise when required element of citation is missing.

    Args:
        Exception (_type_): _description_

    """

MissingDocumentError

Bases: Exception

Raise when looking up a document by id fails.

Parameters:

Name Type Description Default
Exception _type_
required
Source code in deet/exceptions.py 
 6
 7
 8
 9
10
11
12
13
class MissingDocumentError(Exception):
    """
    Raise when looking up a document by id fails.

    Args:
        Exception (_type_):

    """

NoAbstractError

Bases: Exception

Raise when we can't find an abstract in our citation info.

Parameters:

Name Type Description Default
Exception _type_

description

 required
Source code in deet/exceptions.py 
123
124
125
126
127
128
129
130
class NoAbstractError(Exception):
    """
    Raise when we can't find an abstract in our citation info.

    Args:
        Exception (_type_): _description_

    """

UnsupportedEppiAttributeTypeError

Bases: ValueError

Raised when EPPI AdditionalText / Codes mapping cannot handle an attribute type.

Subclasses :class:ValueError so existing except ValueError call sites remain valid while allowing targeted handling via UnsupportedEppiAttributeTypeError.

Parameters:

Name Type Description Default
output_data_type AttributeType

The :class:~deet.data_models.base.AttributeType that has no mapping in the EPPI raw_data pipeline (see deet.processors.eppi_annotation_converter.eppi_output_data_from_eppi_fields).

 required
Source code in deet/exceptions.py 
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
class UnsupportedEppiAttributeTypeError(ValueError):
    """
    Raised when EPPI ``AdditionalText`` / Codes mapping cannot handle an attribute type.

    Subclasses :class:`ValueError` so existing ``except ValueError`` call sites remain
    valid while allowing targeted handling via ``UnsupportedEppiAttributeTypeError``.

    Args:
        output_data_type: The :class:`~deet.data_models.base.AttributeType` that has
            no mapping in the EPPI ``raw_data`` pipeline (see
            ``deet.processors.eppi_annotation_converter.eppi_output_data_from_eppi_fields``).

    """

    def __init__(
        self,
        output_data_type: AttributeType,
        *,
        message: str | None = None,
    ) -> None:
        """Store ``output_data_type`` and build a default message when omitted."""
        self.output_data_type = output_data_type
        text = message or (
            f"Unsupported AttributeType for EPPI mapping: {output_data_type!s}"
        )
        super().__init__(text)

__init__(output_data_type, *, message=None)

Store output_data_type and build a default message when omitted.

Source code in deet/exceptions.py 
147
148
149
150
151
152
153
154
155
156
157
158
def __init__(
    self,
    output_data_type: AttributeType,
    *,
    message: str | None = None,
) -> None:
    """Store ``output_data_type`` and build a default message when omitted."""
    self.output_data_type = output_data_type
    text = message or (
        f"Unsupported AttributeType for EPPI mapping: {output_data_type!s}"
    )
    super().__init__(text)

extractors

cli_helpers

Helper functions to run extraction via the CLI.

init_extraction_run(out_dir, run_name)

Set up ID, folder and logging for data extraction run.

Source code in deet/extractors/cli_helpers.py 
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
def init_extraction_run(
    out_dir: Path,
    run_name: str,
) -> tuple[str, Path]:
    """Set up ID, folder and logging for data extraction run."""
    extraction_run_id = (
        datetime.datetime.now(tz=datetime.UTC).strftime("%Y-%m-%d_%H-%M-%S")
        + f"_{run_name}"
    )

    experiment_out_dir = out_dir / extraction_run_id
    experiment_out_dir.mkdir(parents=True)

    logger.add(experiment_out_dir / "deet.log", level="DEBUG")

    return extraction_run_id, experiment_out_dir

load_or_init_config(config_path)

Load config, or initialise default config.

Source code in deet/extractors/cli_helpers.py 
16
17
18
19
20
21
22
23
24
25
26
def load_or_init_config(config_path: Path) -> DataExtractionConfig:
    """Load config, or initialise default config."""
    if config_path.exists():
        config = DataExtractionConfig(**yaml.safe_load(config_path.read_text()))
    else:
        echo_and_log(
            f"Config file: {config_path} does not exist."
            " Initialising config with default settings."
        )
        config = DataExtractionConfig()
    return config

prepare_documents(documents, config, linked_document_path, pdf_dir, link_map_path)

Load documents depending on the context type we want.

NOTE: while there are no arg-defaults defined here, when used in cli.py, we populate defaults via typer arg defaults.

If fulltext, try to load linked documents, or create them if not.

Source code in deet/extractors/cli_helpers.py 
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
def prepare_documents(
    documents: Sequence[Document],
    config: DataExtractionConfig,
    linked_document_path: Path,
    pdf_dir: Path,
    link_map_path: Path | None,
) -> Sequence[Document]:
    """
    Load documents depending on the context type we want.

    NOTE: while there are no arg-defaults defined here,
    when used in cli.py, we populate defaults via
    typer arg defaults.

    If fulltext, try to load linked documents, or create them if not.
    """
    if config.default_context_type == ContextType.ABSTRACT_ONLY:
        return documents
    if config.default_context_type == ContextType.FULL_DOCUMENT:
        if link_map_path is not None:
            echo_and_log(f"Linking documents using link map: {link_map_path}")
            linker = DocumentReferenceLinker(
                references=documents,
                document_base_dir=pdf_dir,
                document_reference_mapping=link_map_path,
                linking_strategies=[LinkingStrategy.MAPPING_FILE],
            )
            documents = linker.link_many_references_parsed_documents()
            for linked_document in documents:
                file_path = (
                    linked_document_path
                    / f"{linked_document.safe_identity.document_id}.json"
                )
                linked_document.save(file_path)

            if not documents:
                no_links = (
                    f"context type {config.default_context_type} selected"
                    " but no linked documents could be found or created"
                )
                fail_with_message(no_links)

            return documents
        if linked_document_path.exists():
            echo_and_log(f"Loading linked documents from {linked_document_path}")
            documents = [Document.load(f) for f in linked_document_path.glob("*.json")]
            if documents:
                return documents
            fail_with_message(
                "Linked document path does not contain any linked documents."
                " Please use a link map"
                " to create linked documents "
                "(`deet init-linkage-mapping-file`)"
            )
        else:
            fail_with_message(
                "Linked document path does not exist. Please use a link map"
                " to create linked documents "
                "(`deet init-linkage-mapping-file`)"
            )
    else:
        message = f"context type {config.default_context_type} not supported"
        fail_with_message(message)

    return None

llm_data_extractor

Generalisable data extraction module for LLM-based document analysis.

DataExtractionConfig pydantic-model

Bases: BaseModel

Configuration for data extraction tasks.

Fields:

Validators:

Source code in deet/extractors/llm_data_extractor.py 
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
class DataExtractionConfig(BaseModel):
    """Configuration for data extraction tasks."""

    model_config = ConfigDict()

    # LLM
    model: str = settings.llm_model
    provider: LLMProvider = settings.llm_provider
    temperature: float = settings.llm_temperature
    max_tokens: int | None = settings.llm_max_tokens

    # Context
    default_context_type: ContextType = Field(
        default=ContextType.FULL_DOCUMENT, description="Type of context to provide"
    )
    max_context_tokens: int | None = Field(
        default_factory=lambda: settings.llm_max_context_tokens,
        description=(
            "Maximum context length in tokens (total payload: system + attributes + "
            "context). None = infer from model. Override to manage costs."
        ),
    )
    truncate_on_overflow: bool = Field(
        default=False,
        description=(
            "When True, automatically truncate context that exceeds "
            "max_context_tokens. When False (default), raise ValueError."
        ),
    )

    # Prompt
    prompt_config: PromptConfig = Field(
        default_factory=PromptConfig, description="Prompt configuration"
    )

    # Output
    include_reasoning: bool = Field(
        default=True, description="Include reasoning in output"
    )
    include_additional_text: bool = Field(
        default=True, description="Include additional text/citations in output"
    )

    @model_validator(mode="after")
    def populate_max_context_tokens_from_model(self) -> "DataExtractionConfig":
        """Populate max_context_tokens from model when not set."""
        if self.max_context_tokens is not None:
            return self
        model_str = _model_string_for_tokenisation(self.provider, self.model)
        try:
            inferred = get_model_max_tokens(model_str)
        except LitellmModelNotMappedError:
            inferred = None
        if inferred is not None:
            self.max_context_tokens = inferred
        else:
            # Use shared fallback when model max tokens cannot be inferred.
            self.max_context_tokens = DEFAULT_LLM_MAX_CONTEXT_TOKENS_FALLBACK
        return self
default_context_type = ContextType.FULL_DOCUMENT pydantic-field

Type of context to provide

include_additional_text = True pydantic-field

Include additional text/citations in output

include_reasoning = True pydantic-field

Include reasoning in output

max_context_tokens pydantic-field

Maximum context length in tokens (total payload: system + attributes + context). None = infer from model. Override to manage costs.

prompt_config pydantic-field

Prompt configuration

truncate_on_overflow = False pydantic-field

When True, automatically truncate context that exceeds max_context_tokens. When False (default), raise ValueError.

populate_max_context_tokens_from_model() pydantic-validator

Populate max_context_tokens from model when not set.

Source code in deet/extractors/llm_data_extractor.py 
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
@model_validator(mode="after")
def populate_max_context_tokens_from_model(self) -> "DataExtractionConfig":
    """Populate max_context_tokens from model when not set."""
    if self.max_context_tokens is not None:
        return self
    model_str = _model_string_for_tokenisation(self.provider, self.model)
    try:
        inferred = get_model_max_tokens(model_str)
    except LitellmModelNotMappedError:
        inferred = None
    if inferred is not None:
        self.max_context_tokens = inferred
    else:
        # Use shared fallback when model max tokens cannot be inferred.
        self.max_context_tokens = DEFAULT_LLM_MAX_CONTEXT_TOKENS_FALLBACK
    return self

LLMDataExtractor

Generalisable module for LLM-based data extraction from documents.

This module provides a flexible interface for extracting structured data from documents using LLMs, with support for different context types and customizable prompts.

Source code in deet/extractors/llm_data_extractor.py 
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
class LLMDataExtractor:
    """
    Generalisable module for LLM-based data extraction from documents.

    This module provides a flexible interface for extracting structured data
    from documents using LLMs, with support for different context types and
    customizable prompts.
    """

    def __init__(
        self,
        config: DataExtractionConfig,
        custom_system_prompt_file: Path | None = None,
        *,
        show_litellm_debug_messages: bool = False,
    ) -> None:
        """
        Initialise the data extraction module.

        Args:
            config (DataExtractionConfig): config obj for data extraction run
            custom_system_prompt_file (Path | None, optional): path to non-defualt
            sys prompt file. Defaults to None.
            show_litellm_debug_messages (bool, optional): show verbose litellm logs.
            Defaults to False.

        """
        self.config = config
        self.custom_system_prompt_file = custom_system_prompt_file
        if self.config.provider == LLMProvider.AZURE:
            self.model = f"azure/{self.config.model}"
            self.llm_api_key = settings.azure_api_key.get_secret_value()  # type: ignore[union-attr]
            self.api_base = settings.azure_api_base.get_secret_value()  # type: ignore[union-attr]
        elif self.config.provider == LLMProvider.OLLAMA:
            self.model = f"ollama/{self.config.model}"
            self.llm_api_key = None
            self.api_base = None
        else:
            error_message = f"Unsupported LLM provider: {self.config.provider}"
            raise ValueError(error_message)

        logger.info(f"Using {self.config.provider} with model: {self.model}")
        if self.config.max_tokens is not None:
            logger.info(f"max_tokens={self.config.max_tokens}")

        if show_litellm_debug_messages:
            litellm._turn_on_debug()  # noqa: SLF001

        if (
            self.custom_system_prompt_file
            and self.custom_system_prompt_file
            != self.config.prompt_config.system_prompt
        ):
            logger.debug("found custom sys prompt. loading...")
            self.config.prompt_config.system_prompt = (
                self.custom_system_prompt_file.read_text()
            )

    def extract_from_document(
        self,
        attributes: list[Attribute],
        filter_attribute_ids: list[int] | None = None,
        *,
        payload: str | None = None,
        md_path: Path | None = None,
        context_type: ContextType | None = None,
    ) -> DocumentExtractionResult:
        """
        Extract data from a single document.

        Call with either payload (document text) or md_path (path to markdown file).
        If md_path is provided, the file is read and used as the payload.
        Prompt payloads are not written here; the batch entry point
        extract_from_documents writes them to prompt_outfile when provided.

        Args:
            attributes: List of attributes to extract.
            payload: Document text to extract from. Required if md_path not set.
            md_path: Path to a markdown file to read as payload.
                Required if payload not set.
            context_type: Override config context type; if None, use config default.

        Returns:
            DocumentExtractionResult with annotations, messages, token counts,
            cost, model name, and timestamp.

        Raises:
            ValueError: If no attributes are selected for extraction after filtering.
            ValueError: If neither payload nor md_path provided, or both provided.

        """
        if (payload is None and md_path is None) or (
            payload is not None and md_path is not None
        ):
            msg = "Exactly one of payload or md_path must be provided"
            raise ValueError(msg)
        if md_path is not None:
            if not md_path.exists():
                msg = f"Markdown file not found: {md_path}"
                raise FileNotFoundError(msg)
            payload = md_path.read_text(encoding="utf-8")
        payload = cast("str", payload)

        selected_attributes = attributes
        if filter_attribute_ids and len(filter_attribute_ids) > 0:
            try:
                selected_attributes = self._filter_attributes(
                    selected_attributes, filter_ids=filter_attribute_ids
                )
            except (ValueError, TypeError):
                logger.warning(
                    f"Invalid attribute IDs in config: "
                    f"{filter_attribute_ids}. "
                    "No attributes will be selected."
                )

        if not selected_attributes:
            msg = "No attributes selected for extraction"
            logger.warning(msg)
            raise ValueError(msg)

        context = self._prepare_context(payload=payload, context_type=context_type)
        prompt = self._generate_user_message_json(
            payload=context, attributes=selected_attributes
        )
        llm_response, messages, output_tokens, input_tokens = self._call_llm(
            prompt=prompt
        )
        annotations = self._parse_llm_response(
            response_content=llm_response, attributes=selected_attributes
        )

        return DocumentExtractionResult(
            annotations=annotations,
            messages=messages,
            input_tokens=input_tokens,
            output_tokens=output_tokens,
            model=self.model,
        )

    def extract_from_documents(  # noqa: PLR0913
        self,
        attributes: list[Attribute],
        documents: Sequence[Document],
        filter_attribute_ids: list[int] | None = None,
        output_file: Path | None = None,
        context_type: ContextType | None = None,
        prompt_outfile: Path | None = None,
        *,
        show_progress: bool = False,
    ) -> ExtractionRunOutput:
        """
        Extract data from all documents.

        Loops over documents and extracts data using list of attributes.

        Args:
            attributes: List of attributes to extract.
            documents: Sequence of Document instances (required).
            filter_attribute_ids: Optional list of attribute IDs to filter by.
            output_file: Optional path to save combined results JSON.
            context_type: Override config context type; if None, use config default.
            prompt_outfile: Optional path to write a single JSON object:
                keys are document IDs, values are prompt payload (messages).
            show_progress: Whether to show a progress bar.

        Returns:
            ExtractionRunOutput containing annotated documents and run metadata.

        """
        if context_type is None:
            context_type = self.config.default_context_type

        prompt_payloads: dict[str, Any] = {}
        per_doc_tokens: dict[str, dict[str, int]] = {}

        llm_annotated_docs: list[GoldStandardAnnotatedDocument] = []
        total_input_tokens = 0
        total_output_tokens = 0
        total_cost: float | None = None

        with optional_progress(
            documents, show_progress=show_progress
        ) as iterable_documents:
            for document in iterable_documents:
                logger.info(f"Processing document: {document.name}")

                if context_type == ContextType.ABSTRACT_ONLY:
                    document.set_abstract_context()
                elif context_type == ContextType.FULL_DOCUMENT:
                    document.context = document.safe_parsed_document.text

                try:
                    result = self.extract_from_document(
                        attributes=attributes,
                        filter_attribute_ids=filter_attribute_ids,
                        payload=document.context,
                        context_type=context_type,
                    )

                    llm_annotated_docs.append(
                        GoldStandardAnnotatedDocument(
                            document=document, annotations=result.annotations
                        )
                    )
                    doc_id_str = str(document.safe_identity.document_id)
                    prompt_payloads[doc_id_str] = result.messages
                    per_doc_tokens[doc_id_str] = {
                        "input_tokens": result.input_tokens,
                        "output_tokens": result.output_tokens,
                    }
                    total_input_tokens += result.input_tokens
                    total_output_tokens += result.output_tokens
                    if result.total_cost_usd is not None:
                        total_cost = (total_cost or 0.0) + result.total_cost_usd

                except Exception as e:  # noqa: BLE001
                    logger.error(f"Failed to process {document.name}: {e}")
                    logger.debug("Error details", exc_info=True)

        run_metadata = ExtractionRunMetadata(
            model=self.model,
            total_input_tokens=total_input_tokens,
            total_output_tokens=total_output_tokens,
            total_cost_usd=round(total_cost, 6) if total_cost is not None else None,
            per_document_tokens=per_doc_tokens,
        )
        run_output = ExtractionRunOutput(
            annotated_documents=llm_annotated_docs,
            metadata=run_metadata,
        )

        if output_file is not None:
            self._save_results(run_output, output_file)
            logger.info(f"Combined LLM classifications written to: {output_file}")

        if prompt_outfile is not None:
            prompt_outfile.write_text(
                json.dumps(prompt_payloads, indent=2), encoding="utf-8"
            )
            logger.info(f"Prompt payloads saved to: {prompt_outfile}")

        return run_output

    def _write_json_if_path(
        self, data: dict[str, Any] | list[Any], path: Path | None
    ) -> None:
        """
        Write data as JSON to path if path is not None; otherwise no-op.

        Args:
            data: Dict or list to serialize as JSON.
            path: Optional file path; when None, nothing is written.

        """
        if path is not None:
            path.write_text(json.dumps(data, indent=2), encoding="utf-8")

    def _filter_attributes(
        self, attributes: list[Attribute], filter_ids: list[int] | None = None
    ) -> list[Attribute]:
        """
        Filter attributes using provided attribute IDs.

        Args:
            attributes: List of attributes to filter
            filter_ids: Optional list of attribute IDs (ints) to filter by.
                        If None, returns all attributes.
                        If empty list, returns empty list.

        Returns:
            Filtered list of attributes matching the provided IDs, or all attributes
            if filter_ids is None, or empty list if filter_ids is empty.

        """
        if filter_ids is None:
            logger.debug(
                f"No attribute filtering applied, "
                f"using all {len(attributes)} attributes"
            )
            return attributes

        filtered = [attr for attr in attributes if attr.attribute_id in filter_ids]
        logger.debug(
            f"Filtered {len(attributes)} attributes to {len(filtered)} "
            f"using filter_ids: {filter_ids}"
        )
        return filtered

    def _prepare_context(
        self,
        payload: str,
        context_type: ContextType | None = None,
    ) -> str:
        """Prepare context based on context type."""
        ctx = (
            context_type
            if context_type is not None
            else self.config.default_context_type
        )
        logger.debug(f"Using context type: {ctx}")
        if ctx == ContextType.FULL_DOCUMENT:
            context = payload
            logger.debug(f"Using full document context (length: {len(str(context))})")
        elif ctx == ContextType.ABSTRACT_ONLY:
            context = payload
            logger.debug(f"Using abstract context (length: {len(str(context))})")
        elif ctx == ContextType.RAG_SNIPPETS:
            rag_not_impl = "rag-snippets context type is not implemented."
            raise NotImplementedError(rag_not_impl)
        elif ctx == ContextType.CUSTOM:
            custom_not_impl = "custom context type is not implemented."
            raise NotImplementedError(custom_not_impl)
        else:
            other_not_allowed = f"{ctx} context type is not allowed."
            raise ValueError(other_not_allowed)

        if isinstance(context, list):
            logger.debug(f"Converting list context to string (items: {len(context)})")
            context = " ".join(context)

        return context

    def _generate_user_message_json(
        self,
        payload: str,
        attributes: list[Attribute],
    ) -> str:
        """
        Generate structured JSON input for the LLM user message.

        The payload contains the prepared document context and an array
        `LLMInputSchema` objects, containing the attribute id, prompt and
        target output data type.

        NOTE: If `prompt` field is not populated in incoming data,
        LLMInputSchema will populate from `attribute_label`
        field, or fail.

        Args:
            payload: Prepared document context string.
            attributes: List of Attribute objects to extract.

        Returns:
            JSON string containing `context` and `attributes`.

        """
        logger.debug(f"Generating prompt for {len(attributes)} attributes")
        attributes_payload = []
        for attr in attributes:
            # validate schema & fill prompt if not yet filled
            # Use exclude_none=False to ensure prompt field is included even if None
            attr_dict = attr.model_dump(exclude_none=False)
            llm_input_attr = LLMInputSchema(**attr_dict)
            attributes_payload.append(llm_input_attr.model_dump())

        unserialised_prompt = {
            "context": payload,
            "attributes": attributes_payload,
        }

        logger.debug(f"attributes payload: {attributes_payload}")
        prompt_json = json.dumps(unserialised_prompt, ensure_ascii=False)
        logger.debug(f"Generated prompt JSON ({len(prompt_json)} characters)")

        return prompt_json

    def _enforce_context_limit(
        self,
        messages: list[dict[str, Any]],
        prompt: str,
        system_prompt: str,
    ) -> None:
        """
        Enforce max_context_tokens on the messages payload.

        When the total token count exceeds max_context_tokens:
        - If ``truncate_on_overflow`` is False (default), raises ValueError.
        - If ``truncate_on_overflow`` is True, truncates the "context" field
          inside the user prompt JSON to fit. Mutates messages in place.

        Args:
            messages: List of message dicts with "role" and "content";
                messages[1] is the user message (prompt JSON).
            prompt: Current user prompt JSON string.
            system_prompt: System prompt text (for token counting).

        Raises:
            ValueError: When payload exceeds max_context_tokens and
                truncate_on_overflow is False, or when truncate_on_overflow is
                True but the prompt JSON cannot be parsed or messages are not
                in the expected shape so truncation cannot be applied.

        """
        max_ctx = self.config.max_context_tokens
        if max_ctx is None:
            return
        messages_text = " ".join(str(m.get("content", "")) for m in messages)
        total_tokens = count_tokens(self.model, messages_text)
        if total_tokens <= max_ctx:
            return

        if not self.config.truncate_on_overflow:
            msg = (
                f"Payload ({total_tokens} tokens) exceeds "
                f"max_context_tokens ({max_ctx} tokens). "
                "Set truncate_on_overflow=True in your config to "
                "automatically truncate, or increase max_context_tokens."
            )
            raise ValueError(msg)

        try:
            prompt_data = json.loads(prompt)
            context = prompt_data.get("context", "")
            attributes_payload = prompt_data.get("attributes", [])
            attributes_part = json.dumps(
                {"context": "", "attributes": attributes_payload},
                ensure_ascii=False,
            )
            system_tokens = count_tokens(self.model, str(system_prompt))
            attributes_tokens = count_tokens(self.model, attributes_part)
            # Buffer for token-count discrepancies or extra tokens from
            # serialization/whitespace that LLM APIs may add.
            buffer = 50
            context_limit = max_ctx - system_tokens - attributes_tokens - buffer
            if context_limit > 0:
                context = truncate_to_token_limit(context, self.model, context_limit)
                prompt_data["context"] = context
                truncated_prompt = json.dumps(prompt_data, ensure_ascii=False)
                messages[1]["content"] = truncated_prompt
                logger.warning(
                    f"Truncated context to fit {max_ctx} "
                    "tokens. Edit `max_context_tokens` in your config."
                )
            else:
                logger.warning(
                    "System prompt and attributes exceed "
                    "max_context_tokens; context will be empty."
                )
                prompt_data["context"] = ""
                messages[1]["content"] = json.dumps(prompt_data, ensure_ascii=False)
        except (json.JSONDecodeError, KeyError, IndexError) as e:
            logger.debug(f"Could not truncate by tokens: {e}")
            logger.warning(
                "Could not enforce max_context_tokens by truncating the prompt JSON; "
                "prompt appears to be invalid or not in the expected shape."
            )
            msg = (
                "Failed to enforce max_context_tokens: invalid or unexpected prompt "
                "JSON structure while truncate_on_overflow=True."
            )
            raise ValueError(msg) from e

    def _call_llm(self, prompt: str) -> tuple[str, list[dict[str, Any]], int, int]:
        """
        Call the LLM with the given prompt.

        Args:
            prompt: The user prompt (with context and attributes).

        Returns:
            Tuple of (LLM response text, messages list, output token count,
            input/prompt token count from the response usage).

        """
        system_prompt = self.config.prompt_config.system_prompt
        messages: list[dict[str, Any]] = [
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": prompt},
        ]

        self._enforce_context_limit(messages, prompt, str(system_prompt))

        logger.debug(f"Model: {self.model}")
        logger.debug(f"Temperature: {self.config.temperature}")
        logger.debug(f" sys message: {messages[0]['content'][:1000]}")
        logger.debug(f" user message: {messages[1]['content'][:1000]}")

        messages_text = " ".join(str(m.get("content", "")) for m in messages)
        input_tokens = count_tokens(self.model, messages_text)
        prompt_cost, _ = estimate_cost_usd(
            self.model,
            prompt_tokens=input_tokens,
            completion_tokens=0,
        )
        if prompt_cost is not None:
            logger.info(
                f"Estimated input cost: ${prompt_cost:.6f} USD "
                f"({input_tokens} tokens)"
            )

        response = litellm.completion(
            model=self.model,
            api_key=self.llm_api_key,
            api_base=self.api_base,
            messages=messages,
            temperature=self.config.temperature,
            response_format={
                "type": "json_schema",
                "json_schema": {
                    "name": "llm_annotation_response",
                    "schema": LLMResponseSchema.model_json_schema(),
                    "strict": True,
                },
            },
            max_tokens=self.config.max_tokens,
        )

        response_content = response.choices[0].message.content
        logger.debug(f"raw response: {response_content}")

        input_tokens = 0
        output_tokens = 0
        if response.usage is not None:
            if hasattr(response.usage, "prompt_tokens"):
                input_tokens = response.usage.prompt_tokens or 0
            if hasattr(response.usage, "completion_tokens"):
                output_tokens = response.usage.completion_tokens or 0

        return response_content, messages, output_tokens, input_tokens

    def _parse_llm_response(
        self,
        response_content: str,
        attributes: list[Attribute],
    ) -> list[GoldStandardAnnotation]:
        """
        Parse and validate LLM response against GoldStandardAnnotation structure.

        Args:
            response_content: Raw JSON string response from LLM
            attributes: List of attributes to match against

        Returns:
            List of GoldStandardAnnotation objects

        Raises:
            ValidationError: If response fails schema validation.
            ValueError: If JSON parsing fails.

        """
        try:
            validated_response = LLMResponseSchema.model_validate_json(response_content)
        except ValidationError as ve:
            logger.error(f"LLM response failed schema validation: {ve}")
            logger.debug(f"Response content: {response_content}")
            raise
        except json.JSONDecodeError as je:
            error_msg = f"Invalid JSON in LLM response: {je}"
            logger.error(f"Failed to parse LLM response as JSON: {je}")
            raise ValueError(error_msg) from je

        annotations = []
        logger.debug(
            f"Parsing LLM response with {len(validated_response.annotations)} "
            f"annotations"
        )
        for llm_annotation in validated_response.annotations:
            # Resolve attribute_id to full Attribute
            attribute = next(
                (
                    attr
                    for attr in attributes
                    if attr.attribute_id == llm_annotation.attribute_id
                ),
                None,
            )

            if not attribute:
                logger.warning(
                    f"No attribute found for ID: {llm_annotation.attribute_id}"
                )
                continue

            additional_text = (
                llm_annotation.additional_text
                if self.config.include_additional_text
                else None
            )
            reasoning = (
                llm_annotation.reasoning if self.config.include_reasoning else None
            )
            # Convert to full EppiGoldStandardAnnotation
            annotation = GoldStandardAnnotation(
                attribute=attribute,
                raw_data=llm_annotation.output_data,
                annotation_type=AnnotationType.LLM,
                additional_text=additional_text,
                reasoning=reasoning,
            )
            annotations.append(annotation)
            logger.debug(
                f"Created annotation for attribute {attribute.attribute_id}: "
                f"output_data={llm_annotation.output_data}"
            )

        logger.debug(f"Successfully parsed {len(annotations)} annotations")
        return annotations

    def _save_results(
        self,
        run_output: ExtractionRunOutput,
        output_file: Path,
    ) -> None:
        """
        Serialize an ExtractionRunOutput to JSON and write it to disk.

        Args:
            run_output: The complete extraction run output to persist.
            output_file: Destination file path.

        """
        output_file.write_text(
            run_output.model_dump_json(indent=2),
            encoding="utf-8",
        )
        logger.info(f"Results saved to: {output_file}")
__init__(config, custom_system_prompt_file=None, *, show_litellm_debug_messages=False)

Initialise the data extraction module.

Parameters:

Name Type Description Default
config DataExtractionConfig

config obj for data extraction run

 required
custom_system_prompt_file Path | None

path to non-defualt

 None
show_litellm_debug_messages bool

show verbose litellm logs.

 False
Source code in deet/extractors/llm_data_extractor.py 
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
def __init__(
    self,
    config: DataExtractionConfig,
    custom_system_prompt_file: Path | None = None,
    *,
    show_litellm_debug_messages: bool = False,
) -> None:
    """
    Initialise the data extraction module.

    Args:
        config (DataExtractionConfig): config obj for data extraction run
        custom_system_prompt_file (Path | None, optional): path to non-defualt
        sys prompt file. Defaults to None.
        show_litellm_debug_messages (bool, optional): show verbose litellm logs.
        Defaults to False.

    """
    self.config = config
    self.custom_system_prompt_file = custom_system_prompt_file
    if self.config.provider == LLMProvider.AZURE:
        self.model = f"azure/{self.config.model}"
        self.llm_api_key = settings.azure_api_key.get_secret_value()  # type: ignore[union-attr]
        self.api_base = settings.azure_api_base.get_secret_value()  # type: ignore[union-attr]
    elif self.config.provider == LLMProvider.OLLAMA:
        self.model = f"ollama/{self.config.model}"
        self.llm_api_key = None
        self.api_base = None
    else:
        error_message = f"Unsupported LLM provider: {self.config.provider}"
        raise ValueError(error_message)

    logger.info(f"Using {self.config.provider} with model: {self.model}")
    if self.config.max_tokens is not None:
        logger.info(f"max_tokens={self.config.max_tokens}")

    if show_litellm_debug_messages:
        litellm._turn_on_debug()  # noqa: SLF001

    if (
        self.custom_system_prompt_file
        and self.custom_system_prompt_file
        != self.config.prompt_config.system_prompt
    ):
        logger.debug("found custom sys prompt. loading...")
        self.config.prompt_config.system_prompt = (
            self.custom_system_prompt_file.read_text()
        )
extract_from_document(attributes, filter_attribute_ids=None, *, payload=None, md_path=None, context_type=None)

Extract data from a single document.

Call with either payload (document text) or md_path (path to markdown file). If md_path is provided, the file is read and used as the payload. Prompt payloads are not written here; the batch entry point extract_from_documents writes them to prompt_outfile when provided.

Parameters:

Name Type Description Default
attributes list[Attribute]

List of attributes to extract.

 required
payload str | None

Document text to extract from. Required if md_path not set.

 None
md_path Path | None

Path to a markdown file to read as payload. Required if payload not set.

 None
context_type ContextType | None

Override config context type; if None, use config default.

 None

Returns:

Type Description
DocumentExtractionResult

DocumentExtractionResult with annotations, messages, token counts,
DocumentExtractionResult

cost, model name, and timestamp.

Raises:

Type Description
ValueError

If no attributes are selected for extraction after filtering.
ValueError

If neither payload nor md_path provided, or both provided.
Source code in deet/extractors/llm_data_extractor.py 
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
def extract_from_document(
    self,
    attributes: list[Attribute],
    filter_attribute_ids: list[int] | None = None,
    *,
    payload: str | None = None,
    md_path: Path | None = None,
    context_type: ContextType | None = None,
) -> DocumentExtractionResult:
    """
    Extract data from a single document.

    Call with either payload (document text) or md_path (path to markdown file).
    If md_path is provided, the file is read and used as the payload.
    Prompt payloads are not written here; the batch entry point
    extract_from_documents writes them to prompt_outfile when provided.

    Args:
        attributes: List of attributes to extract.
        payload: Document text to extract from. Required if md_path not set.
        md_path: Path to a markdown file to read as payload.
            Required if payload not set.
        context_type: Override config context type; if None, use config default.

    Returns:
        DocumentExtractionResult with annotations, messages, token counts,
        cost, model name, and timestamp.

    Raises:
        ValueError: If no attributes are selected for extraction after filtering.
        ValueError: If neither payload nor md_path provided, or both provided.

    """
    if (payload is None and md_path is None) or (
        payload is not None and md_path is not None
    ):
        msg = "Exactly one of payload or md_path must be provided"
        raise ValueError(msg)
    if md_path is not None:
        if not md_path.exists():
            msg = f"Markdown file not found: {md_path}"
            raise FileNotFoundError(msg)
        payload = md_path.read_text(encoding="utf-8")
    payload = cast("str", payload)

    selected_attributes = attributes
    if filter_attribute_ids and len(filter_attribute_ids) > 0:
        try:
            selected_attributes = self._filter_attributes(
                selected_attributes, filter_ids=filter_attribute_ids
            )
        except (ValueError, TypeError):
            logger.warning(
                f"Invalid attribute IDs in config: "
                f"{filter_attribute_ids}. "
                "No attributes will be selected."
            )

    if not selected_attributes:
        msg = "No attributes selected for extraction"
        logger.warning(msg)
        raise ValueError(msg)

    context = self._prepare_context(payload=payload, context_type=context_type)
    prompt = self._generate_user_message_json(
        payload=context, attributes=selected_attributes
    )
    llm_response, messages, output_tokens, input_tokens = self._call_llm(
        prompt=prompt
    )
    annotations = self._parse_llm_response(
        response_content=llm_response, attributes=selected_attributes
    )

    return DocumentExtractionResult(
        annotations=annotations,
        messages=messages,
        input_tokens=input_tokens,
        output_tokens=output_tokens,
        model=self.model,
    )
extract_from_documents(attributes, documents, filter_attribute_ids=None, output_file=None, context_type=None, prompt_outfile=None, *, show_progress=False)

Extract data from all documents.

Loops over documents and extracts data using list of attributes.

Parameters:

Name Type Description Default
attributes list[Attribute]

List of attributes to extract.

 required
documents Sequence[Document]

Sequence of Document instances (required).

 required
filter_attribute_ids list[int] | None

Optional list of attribute IDs to filter by.

 None
output_file Path | None

Optional path to save combined results JSON.

 None
context_type ContextType | None

Override config context type; if None, use config default.

 None
prompt_outfile Path | None

Optional path to write a single JSON object: keys are document IDs, values are prompt payload (messages).

 None
show_progress bool

Whether to show a progress bar.

 False

Returns:

Type Description
ExtractionRunOutput

ExtractionRunOutput containing annotated documents and run metadata.
Source code in deet/extractors/llm_data_extractor.py 
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
def extract_from_documents(  # noqa: PLR0913
    self,
    attributes: list[Attribute],
    documents: Sequence[Document],
    filter_attribute_ids: list[int] | None = None,
    output_file: Path | None = None,
    context_type: ContextType | None = None,
    prompt_outfile: Path | None = None,
    *,
    show_progress: bool = False,
) -> ExtractionRunOutput:
    """
    Extract data from all documents.

    Loops over documents and extracts data using list of attributes.

    Args:
        attributes: List of attributes to extract.
        documents: Sequence of Document instances (required).
        filter_attribute_ids: Optional list of attribute IDs to filter by.
        output_file: Optional path to save combined results JSON.
        context_type: Override config context type; if None, use config default.
        prompt_outfile: Optional path to write a single JSON object:
            keys are document IDs, values are prompt payload (messages).
        show_progress: Whether to show a progress bar.

    Returns:
        ExtractionRunOutput containing annotated documents and run metadata.

    """
    if context_type is None:
        context_type = self.config.default_context_type

    prompt_payloads: dict[str, Any] = {}
    per_doc_tokens: dict[str, dict[str, int]] = {}

    llm_annotated_docs: list[GoldStandardAnnotatedDocument] = []
    total_input_tokens = 0
    total_output_tokens = 0
    total_cost: float | None = None

    with optional_progress(
        documents, show_progress=show_progress
    ) as iterable_documents:
        for document in iterable_documents:
            logger.info(f"Processing document: {document.name}")

            if context_type == ContextType.ABSTRACT_ONLY:
                document.set_abstract_context()
            elif context_type == ContextType.FULL_DOCUMENT:
                document.context = document.safe_parsed_document.text

            try:
                result = self.extract_from_document(
                    attributes=attributes,
                    filter_attribute_ids=filter_attribute_ids,
                    payload=document.context,
                    context_type=context_type,
                )

                llm_annotated_docs.append(
                    GoldStandardAnnotatedDocument(
                        document=document, annotations=result.annotations
                    )
                )
                doc_id_str = str(document.safe_identity.document_id)
                prompt_payloads[doc_id_str] = result.messages
                per_doc_tokens[doc_id_str] = {
                    "input_tokens": result.input_tokens,
                    "output_tokens": result.output_tokens,
                }
                total_input_tokens += result.input_tokens
                total_output_tokens += result.output_tokens
                if result.total_cost_usd is not None:
                    total_cost = (total_cost or 0.0) + result.total_cost_usd

            except Exception as e:  # noqa: BLE001
                logger.error(f"Failed to process {document.name}: {e}")
                logger.debug("Error details", exc_info=True)

    run_metadata = ExtractionRunMetadata(
        model=self.model,
        total_input_tokens=total_input_tokens,
        total_output_tokens=total_output_tokens,
        total_cost_usd=round(total_cost, 6) if total_cost is not None else None,
        per_document_tokens=per_doc_tokens,
    )
    run_output = ExtractionRunOutput(
        annotated_documents=llm_annotated_docs,
        metadata=run_metadata,
    )

    if output_file is not None:
        self._save_results(run_output, output_file)
        logger.info(f"Combined LLM classifications written to: {output_file}")

    if prompt_outfile is not None:
        prompt_outfile.write_text(
            json.dumps(prompt_payloads, indent=2), encoding="utf-8"
        )
        logger.info(f"Prompt payloads saved to: {prompt_outfile}")

    return run_output

PromptConfig pydantic-model

Bases: BaseModel

Configuration for prompts used in data extraction.

Fields:

Validators:

Source code in deet/extractors/llm_data_extractor.py 
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
class PromptConfig(BaseModel):
    """Configuration for prompts used in data extraction."""

    model_config = ConfigDict()

    system_prompt: str | Path = Field(
        description="System prompt that defines the task and role",
        default_factory=default_system_prompt,
    )

    @model_validator(mode="after")
    def load_system_prompt_file(self) -> "PromptConfig":
        """Load system prompt from file if Path provided."""
        if isinstance(self.system_prompt, Path):
            if not self.system_prompt.exists():
                sys_prompt_missing = f"sys prompt {self.system_prompt} not found."
                raise ValueError(sys_prompt_missing)
            logger.debug(f"Reading system prompt from {self.system_prompt}")
            self.system_prompt = self.system_prompt.read_text()

        return self
system_prompt pydantic-field

System prompt that defines the task and role

load_system_prompt_file() pydantic-validator

Load system prompt from file if Path provided.

Source code in deet/extractors/llm_data_extractor.py 
68
69
70
71
72
73
74
75
76
77
78
@model_validator(mode="after")
def load_system_prompt_file(self) -> "PromptConfig":
    """Load system prompt from file if Path provided."""
    if isinstance(self.system_prompt, Path):
        if not self.system_prompt.exists():
            sys_prompt_missing = f"sys prompt {self.system_prompt} not found."
            raise ValueError(sys_prompt_missing)
        logger.debug(f"Reading system prompt from {self.system_prompt}")
        self.system_prompt = self.system_prompt.read_text()

    return self

default_system_prompt()

Get default system prompt included in the package.

Source code in deet/extractors/llm_data_extractor.py 
53
54
55
def default_system_prompt() -> str:
    """Get default system prompt included in the package."""
    return (files("deet.prompts") / "system_prompt.txt").read_text()

logger

Customisations on the loguru logger.

processors

base_converter

Generic classes and functions for converters.

AnnotationConverter

Bases: Generic[DocumentTypeVar, AttributeTypeVar, GoldStandardAnnotationTypeVar, GoldStandardAnnotatedDocumentTypeVar], ABC

Abstract base class to define expected behaviour of an annotationconverter.

OUTFILE_LOADERS maps the outfiles to be read/written to a filename, and a TypeAdapter defining the type of Pydantic Model to read back in.

Source code in deet/processors/base_converter.py 
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
class AnnotationConverter(
    Generic[
        DocumentTypeVar,
        AttributeTypeVar,
        GoldStandardAnnotationTypeVar,
        GoldStandardAnnotatedDocumentTypeVar,
    ],
    ABC,
):
    """
    Abstract base class to define expected behaviour of an annotationconverter.

    OUTFILE_LOADERS maps the outfiles to be read/written to a filename, and a
    TypeAdapter defining the type of Pydantic Model to read back in.
    """

    OUTFILE_LOADERS: dict[Outfiles, tuple[str, TypeAdapter]]

    @property
    def outfile_names(self) -> dict[Outfiles, str]:
        """Return a dictionary of outfiles to names, using outfile_loaders."""
        return {k: v[0] for k, v in self.OUTFILE_LOADERS.items()}

    def __init__(
        self,
        base_output_dir: str | Path | None = DEFAULT_BASE_OUTPUT_DIR,
    ) -> None:
        """
        Initialise the converter with configurable output paths.

        Args:
            output_dir: Base directory for saving processed files
            attributes_filename: Filename for attributes output
            documents_filename: Filename for documents output
            annotated_documents_filename: Filename for annotated documents output
            attribute_mapping_filename: Filename for attribute ID to label mapping

        """
        if base_output_dir is None:
            logger.debug(
                "`base_output_dir` set to None; "
                "converting to empty string for compatibility."
            )
            base_output_dir = ""
        self.base_output_dir = Path(base_output_dir)

    @property
    @abstractmethod
    def processed_data_type(
        self,
    ) -> type[
        ProcessedAnnotationData[
            AttributeTypeVar,
            DocumentTypeVar,
            GoldStandardAnnotationTypeVar,
            GoldStandardAnnotatedDocumentTypeVar,
        ]
    ]:
        """Return the class to use when instantiating processed data."""
        ...

    @abstractmethod
    def process_annotation_file(
        self,
        file_path: str | Path,
        set_attribute_type: str | AttributeType | None = None,
    ) -> ProcessedAnnotationData:
        """Parse a raw input format into processed data."""
        ...

    def write_processed_data_to_file(
        self,
        processed_data: ProcessedAnnotationData,
        output_dir: str | Path,
        outfiles_to_write: list[Outfiles] | None = None,
    ) -> dict[str, str]:
        """
        Save processed data to structured files using Pydantic model serialisation.

        Args:
            processed_data: The processed data from process_annotation_file
            output_dir: Write all output (json) files from conversion to this
            directory. NOTE: we output files will live in a sub-directory
            `self.base_output_dir`, which by default is `DEFAULT_BASE_OUTPUT_DIR`.
            so, if you want output files to go straight to `output_dir`, set
            `self.base_output_dir` to '' or None.

        Returns:
            Dictionary mapping data types to saved file paths

        """
        file_mappings = {
            Outfiles.ATTRIBUTES: processed_data.attributes,
            Outfiles.DOCUMENTS: processed_data.documents,
            Outfiles.ANNOTATED_DOCUMENTS: processed_data.annotated_documents,
            Outfiles.ATTRIBUTE_LABEL_MAPPING: processed_data.attribute_id_to_label,
        }
        # setting here to avoid mutable default.
        if outfiles_to_write is None:
            outfiles_to_write = [Outfiles.ATTRIBUTES, Outfiles.DOCUMENTS]

        file_mappings = {
            k: v for k, v in file_mappings.items() if k in outfiles_to_write
        }
        files_to_write_message = {",".join(k.value for k in file_mappings)}
        logger.info(f"writing {files_to_write_message} out...")

        user_dir = Path(output_dir)
        target_dir = user_dir / self.base_output_dir
        target_dir.mkdir(parents=True, exist_ok=True)
        logger.info(f"writing files to dir: {target_dir}")

        saved_files = {}

        for file_type, data_list in file_mappings.items():
            file_path = target_dir / self.outfile_names[file_type]
            logger.debug(f"writing file {file_type} to {file_path}")
            if file_type == Outfiles.ATTRIBUTE_LABEL_MAPPING:
                file_path.write_text(json.dumps(data_list), encoding="utf-8")
            else:
                file_path.write_text(
                    json.dumps(
                        [item.model_dump(mode="json") for item in data_list],  # type: ignore[attr-defined]
                        indent=2,
                    ),
                    encoding="utf-8",
                )
            saved_files[file_type.value] = str(file_path)

        return saved_files

    def reload_output(self, file_path: Path) -> ProcessedAnnotationData:
        """
        Read data back in, using OUTFILE_LOADERS and the
        subclass's processed_data_type.
        """
        loaded_data = {}
        for key, (filename, adapter) in self.OUTFILE_LOADERS.items():
            path = file_path / self.base_output_dir / filename
            if not path.exists():
                message = f"Expected {key.value} at {path}"
                raise FileNotFoundError(message)
            loaded_data[key] = adapter.validate_json(path.read_text())

        return self.processed_data_type(
            attributes=loaded_data[Outfiles.ATTRIBUTES],
            documents=loaded_data[Outfiles.DOCUMENTS],
            annotations=[],  # or derive from loaded_data if applicable
            annotated_documents=loaded_data[Outfiles.ANNOTATED_DOCUMENTS],
            attribute_id_to_label=loaded_data[Outfiles.ATTRIBUTE_LABEL_MAPPING],
        )
outfile_names property

Return a dictionary of outfiles to names, using outfile_loaders.

processed_data_type abstractmethod property

Return the class to use when instantiating processed data.

__init__(base_output_dir=DEFAULT_BASE_OUTPUT_DIR)

Initialise the converter with configurable output paths.

Parameters:

Name Type Description Default
output_dir

Base directory for saving processed files

 required
attributes_filename

Filename for attributes output

 required
documents_filename

Filename for documents output

 required
annotated_documents_filename

Filename for annotated documents output

 required
attribute_mapping_filename

Filename for attribute ID to label mapping

 required
Source code in deet/processors/base_converter.py 
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
def __init__(
    self,
    base_output_dir: str | Path | None = DEFAULT_BASE_OUTPUT_DIR,
) -> None:
    """
    Initialise the converter with configurable output paths.

    Args:
        output_dir: Base directory for saving processed files
        attributes_filename: Filename for attributes output
        documents_filename: Filename for documents output
        annotated_documents_filename: Filename for annotated documents output
        attribute_mapping_filename: Filename for attribute ID to label mapping

    """
    if base_output_dir is None:
        logger.debug(
            "`base_output_dir` set to None; "
            "converting to empty string for compatibility."
        )
        base_output_dir = ""
    self.base_output_dir = Path(base_output_dir)
process_annotation_file(file_path, set_attribute_type=None) abstractmethod

Parse a raw input format into processed data.

Source code in deet/processors/base_converter.py 
 97
 98
 99
100
101
102
103
104
@abstractmethod
def process_annotation_file(
    self,
    file_path: str | Path,
    set_attribute_type: str | AttributeType | None = None,
) -> ProcessedAnnotationData:
    """Parse a raw input format into processed data."""
    ...
reload_output(file_path)

Read data back in, using OUTFILE_LOADERS and the subclass's processed_data_type.

Source code in deet/processors/base_converter.py 
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
def reload_output(self, file_path: Path) -> ProcessedAnnotationData:
    """
    Read data back in, using OUTFILE_LOADERS and the
    subclass's processed_data_type.
    """
    loaded_data = {}
    for key, (filename, adapter) in self.OUTFILE_LOADERS.items():
        path = file_path / self.base_output_dir / filename
        if not path.exists():
            message = f"Expected {key.value} at {path}"
            raise FileNotFoundError(message)
        loaded_data[key] = adapter.validate_json(path.read_text())

    return self.processed_data_type(
        attributes=loaded_data[Outfiles.ATTRIBUTES],
        documents=loaded_data[Outfiles.DOCUMENTS],
        annotations=[],  # or derive from loaded_data if applicable
        annotated_documents=loaded_data[Outfiles.ANNOTATED_DOCUMENTS],
        attribute_id_to_label=loaded_data[Outfiles.ATTRIBUTE_LABEL_MAPPING],
    )
write_processed_data_to_file(processed_data, output_dir, outfiles_to_write=None)

Save processed data to structured files using Pydantic model serialisation.

Parameters:

Name Type Description Default
processed_data ProcessedAnnotationData

The processed data from process_annotation_file

 required
output_dir str | Path

Write all output (json) files from conversion to this

 required
directory. NOTE

we output files will live in a sub-directory

 required

Returns:

Type Description
dict[str, str]

Dictionary mapping data types to saved file paths
Source code in deet/processors/base_converter.py 
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
def write_processed_data_to_file(
    self,
    processed_data: ProcessedAnnotationData,
    output_dir: str | Path,
    outfiles_to_write: list[Outfiles] | None = None,
) -> dict[str, str]:
    """
    Save processed data to structured files using Pydantic model serialisation.

    Args:
        processed_data: The processed data from process_annotation_file
        output_dir: Write all output (json) files from conversion to this
        directory. NOTE: we output files will live in a sub-directory
        `self.base_output_dir`, which by default is `DEFAULT_BASE_OUTPUT_DIR`.
        so, if you want output files to go straight to `output_dir`, set
        `self.base_output_dir` to '' or None.

    Returns:
        Dictionary mapping data types to saved file paths

    """
    file_mappings = {
        Outfiles.ATTRIBUTES: processed_data.attributes,
        Outfiles.DOCUMENTS: processed_data.documents,
        Outfiles.ANNOTATED_DOCUMENTS: processed_data.annotated_documents,
        Outfiles.ATTRIBUTE_LABEL_MAPPING: processed_data.attribute_id_to_label,
    }
    # setting here to avoid mutable default.
    if outfiles_to_write is None:
        outfiles_to_write = [Outfiles.ATTRIBUTES, Outfiles.DOCUMENTS]

    file_mappings = {
        k: v for k, v in file_mappings.items() if k in outfiles_to_write
    }
    files_to_write_message = {",".join(k.value for k in file_mappings)}
    logger.info(f"writing {files_to_write_message} out...")

    user_dir = Path(output_dir)
    target_dir = user_dir / self.base_output_dir
    target_dir.mkdir(parents=True, exist_ok=True)
    logger.info(f"writing files to dir: {target_dir}")

    saved_files = {}

    for file_type, data_list in file_mappings.items():
        file_path = target_dir / self.outfile_names[file_type]
        logger.debug(f"writing file {file_type} to {file_path}")
        if file_type == Outfiles.ATTRIBUTE_LABEL_MAPPING:
            file_path.write_text(json.dumps(data_list), encoding="utf-8")
        else:
            file_path.write_text(
                json.dumps(
                    [item.model_dump(mode="json") for item in data_list],  # type: ignore[attr-defined]
                    indent=2,
                ),
                encoding="utf-8",
            )
        saved_files[file_type.value] = str(file_path)

    return saved_files

Outfiles

Bases: StrEnum

Enum of all outfiles producable by this module. Extend as required.

Source code in deet/processors/base_converter.py 
27
28
29
30
31
32
33
class Outfiles(StrEnum):
    """Enum of all outfiles producable by this module. Extend as required."""

    ATTRIBUTES = auto()
    DOCUMENTS = auto()
    ANNOTATED_DOCUMENTS = auto()
    ATTRIBUTE_LABEL_MAPPING = auto()

converter_register

A register of supported supported annotation formats and a map to their converters.

SupportedImportFormat

Bases: StrEnum

Supported formats to import gold standard annotation data from.

Source code in deet/processors/converter_register.py 
13
14
15
16
17
18
19
20
21
class SupportedImportFormat(StrEnum):
    """Supported formats to import gold standard annotation data from."""

    EPPI_JSON = auto()
    GENERIC_CSV = auto()

    def get_annotation_converter(self) -> AnnotationConverter:
        """Return an instance of the parser for the given data type."""
        return CONVERTER_REGISTRY[self]()
get_annotation_converter()

Return an instance of the parser for the given data type.

Source code in deet/processors/converter_register.py 
19
20
21
def get_annotation_converter(self) -> AnnotationConverter:
    """Return an instance of the parser for the given data type."""
    return CONVERTER_REGISTRY[self]()

csv_annotation_converter

Convert annotation CSV files to Pydantic models.

CSVAnnotationConverter

Bases: AnnotationConverter

A class to convert raw CSV (e.g. Covidence) annotations into structured Pydantic models.

This converter operates on flat CSV columns, infers field/column types, and produces attributes, documents, and annotated document records.

Source code in deet/processors/csv_annotation_converter.py 
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
class CSVAnnotationConverter(AnnotationConverter):
    """
    A class to convert raw CSV (e.g. Covidence) annotations
    into structured Pydantic models.

     This converter operates on flat CSV columns, infers field/column types,
    and produces attributes, documents, and annotated document records.
    """

    def __init__(  # noqa: PLR0913
        self,
        base_output_dir: str | Path | None = DEFAULT_BASE_OUTPUT_DIR,
        attributes_filename: str = DEFAULT_ATTRIBUTES_FILENAME,
        documents_filename: str = DEFAULT_DOCUMENTS_FILENAME,
        annotated_documents_filename: str = DEFAULT_ANNOTATED_DOCUMENTS_FILENAME,
        attribute_mapping_filename: str = DEFAULT_ATTRIBUTE_MAPPING_FILENAME,
        config: CSVParserConfig | None = None,
    ) -> None:
        """
        Initialize the converter output configurations (base directory + filenames) to
        save the multiple files created during csv processing.

        Args:
            base_output_dir: Base directory for saving processed files
            attributes_filename: Filename for attributes output
            documents_filename: Filename for documents output
            annotated_documents_filename: Filename for annotated documents output
            attribute_mapping_filename: Filename for attribute ID to label mapping

        """
        self.config = config or CSVParserConfig()
        # If no directory given, write everything relative to current working directory
        if base_output_dir is None:
            logger.debug(
                "`base_output_dir` set to None; "
                "converting to empty string for compatibility."
            )
            base_output_dir = ""
        self.base_output_dir = Path(base_output_dir)

        # Output registry:
        # For each output type, provides the filename and how to serialize/validate it.
        # Used when writing and reloading the results

        self.OUTFILE_LOADERS: dict[Outfiles, tuple[str, TypeAdapter]] = {
            Outfiles.ATTRIBUTES: (
                attributes_filename,
                TypeAdapter(list[Attribute]),
            ),
            Outfiles.DOCUMENTS: (
                documents_filename,
                TypeAdapter(list[Document]),
            ),
            Outfiles.ANNOTATED_DOCUMENTS: (
                annotated_documents_filename,
                TypeAdapter(list[GoldStandardAnnotatedDocument]),
            ),
            Outfiles.ATTRIBUTE_LABEL_MAPPING: (
                attribute_mapping_filename,
                TypeAdapter(dict[int, str]),
            ),
        }

    @property
    def processed_data_type(self) -> type[ProcessedAnnotationData]:
        """Return ProcessedAnnotationData."""
        return ProcessedAnnotationData

    @staticmethod
    def _find_duplicate_column_names(column_names: list[str]) -> list[dict]:
        """
        Find duplicate column names and return their positions.

        Given a list of field/column names, returns a list of dictionaries
        containing the duplicated column name and the indices where it appears.

        Example:
            column_names = ["id", "name", "title", "name", "title"] ->
            {"name": [1, 3], "title": [2, 4]}

        """
        positions = defaultdict(list)
        for idx, name in enumerate(column_names):
            positions[name].append(idx)

        return [{name: ids} for name, ids in positions.items() if len(ids) > 1]

    @staticmethod
    def _infer_type(value: str) -> type | None:
        """
        Given a string value infer its type.
        Returns bool, int, float, or str depending on the value.
        Empty or null-like values return None.
        """
        if value is None or str(value).strip() == "":
            return None

        v = str(value).strip().lower()

        if v in {"true", "false", "t", "f"}:
            return bool
        try:
            float(v)
            if v.lstrip("-+").isdigit():
                return int
            return float  # noqa: TRY300
        except ValueError:
            return str

    # TODO: Extend support for list and dict AttributeType values.
    # Discussed use case: 'tags' associated with a document ([USA, Germnay, India])

    @staticmethod
    def _resolve_types(types: list) -> AttributeType:
        """
        Given a list of types, resolve to a single type and return an
        AttributeType object.

        Examples:
        [int, int, None, int, None, int] -> int -> AttributeType.INTEGER
        [int, int, None, float, None, int] -> float -> AttributeType.FLOAT

        """
        unique = set(types) - {None}
        if len(unique) == 0:
            msg = "Type not inferred. Sample is null."
            raise ValueError(msg)
        if unique == {str}:
            return AttributeType.STRING
        if unique == {bool}:
            return AttributeType.BOOL
        if unique == {int}:
            return AttributeType.INTEGER
        if unique.issubset({float, int}) and bool not in unique:
            return AttributeType.FLOAT

        msg = f"Multiple incompatible types found: {unique}"
        raise ColumnTypeInferenceError(msg)

    def _infer_column_types(
        self,
        rows: list[dict],
        column_names: list[str],
        sample_size: int | None = None,
    ) -> dict[str, AttributeType]:
        """
        Infer the data type of each column using up to `sample_size` non-null samples.

        By default, `sample_size` is the total number of rows. Specifying a smaller
        sample can speed up inference but may bias results if columns are sparse.

        Args:
            rows: List of dictionaries representing CSV rows.
            column_names: List of column names for which to infer data types.
            sample_size: Maximum number of non-null samples to use per column.

        Returns:
        - Dictionary mapping each column name to its inferred `AttributeType`.

        Example:
            rows = [
                {"id": "1", "age": "42", "gender": "F"},
                {"id": "2", "age": "", "gender": "M"},
                {"id": "3", "age": "36", "gender": ""},
            ]
            fieldnames = ["id", "age", "gender"]

            converter = CovidenceAnnotationConverter()
            converter._infer_column_types(rows, fieldnames, sample_size=2)
            # Output: {"id": int, "age": int, "gender": str}

        """
        n_rows = len(rows)

        # Collect observed value types per column (skipping missing values).
        # Used as evidence for downstream column type inference.
        # Example: column_type_observations = {age: [int, int], gender: [str, str]}
        column_type_evidence: defaultdict[str, list] = defaultdict(list)

        if sample_size is None:
            sample_size = n_rows

        for col_name in column_names:
            row_num = 0

            while (
                len(column_type_evidence[col_name]) < sample_size and row_num < n_rows
            ):
                att_type = self._infer_type(rows[row_num][col_name])

                if att_type is not None:
                    column_type_evidence[col_name].append(att_type)

                row_num += 1

        resolved_column_types: dict[str, AttributeType] = {}
        for col_name, col_types in column_type_evidence.items():
            resolved_column_types[col_name] = self._resolve_types(col_types)

        return resolved_column_types

    @staticmethod
    def _build_destiny_reference_dict_from_row(
        mapping: dict[str, str],
        row: dict[str, Any],
    ) -> dict[str, Any]:
        """
        Build a nested dictionary of reference data from a CSV row.

        Maps flat CSV columns to the nested structure expected by
        destiny_sdk.enhancements models, The resulting dictionary is used to create
        `EnhancementFileInput` and `ReferenceFileInput` objects.

        Args:
            mapping: Dictionary mapping nested keys(dot-separated strings) to CSV
            column names.
            row: Dictionary representing a CSV row.

        Returns:
            Nested dictionary representing the reference data. Example:

        -----------------------------------------------
        USER GUIDE
        -----------
        The `mapping` argument defines how CSV columns map to Destiny SDK fields.

        Format:
            mapping = {
                "<destiny_field>": "<csv_column_name>",
            }
        Rules:
            - Keys are dot-separated Destiny SDK field paths.
            - Values are CSV column names.

        SUPPORTED FIELDS
        ----------------
        - abstract
        - authorship
        - cited_by_count
        - created_date
        - updated_date
        - publication_date
        - publication_year
        - publisher
        - title
        - pagination.volume
        - pagination.issue
        - pagination.first_page
        - pagination.last_page
        - publication_venue.display_name
        - publication_venue.issn
        - publication_venue.venue_type
        - publication_venue.issn_l
        - publication_venue.host_organization_name

        -----------------------------------------------

        Example:
        mapping = {"publication_venue.display_name": "journal", "abstract": "abstract"}
        row = {"journal": "Nature Climate Change", "abstract": "ABCDEFXXXXXX"}
        # Output
            {
            "publication_venue": {"display_name": "Nature Climate Change"},
            "abstract": "ABCDEFXXXXXX"
            }


        """
        result: dict[str, Any] = {}

        for key, column_name in mapping.items():
            value = row[column_name].strip()
            if value == "":
                value = None

            parts = key.split(".")
            d = result

            for part in parts[:-1]:
                d = d.setdefault(part, {})

            d[parts[-1]] = value

        return result

    # TODO: Allow users to provide seperator
    def _build_destiny_authorship_list(self, authors: str) -> list[Authorship]:
        """
        Build a list of a `Authorship` objects  (as defined in destiny_sdk.enhancements)
        from a string. The returned list is in the format expected by
        `BibliographicMetadataEnhancement` for authorship.

        Args:
            authors: A semicolon-separated string of author names. eg:("Alice; Bob; Mo")

        Returns:
            List of `Authorship` objects representing each author with their position.

        """
        # Split on semicolons, strip whitespace, and remove any empty entries
        sep: str = self.config.author_separator
        clean_authors = [
            author.strip() for author in authors.split(sep) if author.strip()
        ]

        authorship: list[Authorship] = []
        if not clean_authors:
            return authorship

        for i, author in enumerate(clean_authors):
            position = AuthorPosition.MIDDLE
            if i == 0:
                position = AuthorPosition.FIRST
            elif i == len(clean_authors) - 1:
                position = AuthorPosition.LAST
            authorship.append(Authorship(display_name=author, position=position))
        return authorship

    def build_destiny_reference(
        self,
        row: dict,
        mapping: dict[str, Any],
        source: str = "deet CSV converter",
    ) -> ReferenceFileInput:
        """

        Convert a CSV row into a `ReferenceFileInput` for destiny_sdk.reference.

        This method extracts bibliographic metadata and abstract content from the
        given row using the provided reference field mapping. It constructs
        `BibliographicMetadataEnhancement` and `AbstractContentEnhancement`
        objects, then wraps them in `EnhancementFileInput` objects, and finally
        returns a `ReferenceFileInput`.

        Args:
            row: Dictionary representing a single CSV row.
            mapping: Dictionary mapping nested keys (dot-separated strings) to CSV
                 column names.
            source: Optional string indicating the source of the data; defaults to
                "deet CSV converter"
        Returns:
            ReferenceFileInput object containing all extracted enhancements

        """
        reference_dict = self._build_destiny_reference_dict_from_row(mapping, row)

        abstract = (
            AbstractContentEnhancement(
                abstract=reference_dict["abstract"],
                process=AbstractProcessType.UNINVERTED,
            )
            if "abstract" in reference_dict
            else None
        )

        reference_dict["authorship"] = (
            self._build_destiny_authorship_list(reference_dict["authorship"])
            if "authorship" in reference_dict
            and isinstance(reference_dict["authorship"], str)
            else None
        )

        bibliographic_data = BibliographicMetadataEnhancement(**reference_dict)

        enhancements = [
            EnhancementFileInput(
                source=source,
                visibility=Visibility.PUBLIC,
                content=content,
            )
            for content in [abstract, bibliographic_data]
            if content is not None
        ]

        return ReferenceFileInput(
            visibility=Visibility.PUBLIC,
            enhancements=enhancements,
        )

    def load_csv(
        self,
        file_path: Path,
        attribute_fields: list[str] | None = None,
        reference_fields: dict | None = None,
    ) -> tuple[list[str], dict[str, str], list[str], list[dict[str, Any]]]:
        """
        Load a CSV, normalize headers, and return all column names, attribute names,
        reference names, and rows.
        """
        path = Path(file_path)
        with path.open(newline="", encoding="utf-8-sig") as f:
            csv_reader = csv.DictReader(f)

            # normalize headers BEFORE reading rows
            raw_headers = csv_reader.fieldnames or []
            colnames: list[str] = [h.strip().lower() for h in raw_headers]
            csv_reader.fieldnames = colnames

            # --- validate duplicates ---
            dup_fields = self._find_duplicate_column_names(colnames)
            if dup_fields:
                msg = f"{len(dup_fields)} Duplicate fieldnames found: {dup_fields}"
                raise ValueError(msg)

            # --- validate required fields ---
            meta_fields = {"name", "document_id"}
            missing = meta_fields - set(colnames)
            if missing:
                msg = f"Required columns missing: {missing}"
                raise ValueError(msg)

            # --- validate reference fields ---
            # If reference_fields is not povided....
            if reference_fields is None:
                if self.config.auto_assign_reference_fields:
                    logger.info("Auto assigning reference fields")
                    reference_fields = {
                        ref_field: ref_field
                        for ref_field in ALLOWED_REFERENCE_MAPPING_KEYS
                        if ref_field in colnames
                    }
                else:
                    logger.info("No reference fields provided and auto assign is False")
                    reference_fields = {}

            # If reference_fields is povided....
            if reference_fields:
                invalid_keys = set(reference_fields) - ALLOWED_REFERENCE_MAPPING_KEYS
                if invalid_keys:
                    msg = f"Invalid mapping keys: {invalid_keys}"
                    raise ValueError(msg)

                # normalize reference fields
                reference_fields = {
                    k: v.strip().lower() for k, v in reference_fields.items()
                }

                unknown = set(reference_fields.values()) - set(colnames)
                if unknown:
                    msg = f"Reference fields not found in CSV: {unknown}"
                    raise ValueError(msg)

            # --- validate attribute fields ---
            # normalize and validate provided attribute fields
            if attribute_fields is None:
                logger.info("No attribute fields provided")
                excluded_fields = meta_fields | set(reference_fields.values())
                resolved_attribute_fields = [
                    h for h in colnames if h not in excluded_fields
                ]
            else:
                resolved_attribute_fields = [
                    field.strip().lower() for field in attribute_fields
                ]
                unknown_attributes = set(resolved_attribute_fields) - set(colnames)
                if unknown_attributes:
                    msg = f"Attribute fields not found in CSV: {unknown_attributes}"
                    raise ValueError(msg)

            # Read rows into mem once
            rows = list(csv_reader)

        return colnames, reference_fields, resolved_attribute_fields, rows

    def build_attributes(
        self,
        attribute_fields: list[str],
        rows: list[dict],
    ) -> list[Attribute]:
        """
        Build a list of `Attribute` objects from CSV rows and specified attribute
        columns.

        Infers the `AttributeType` for each attribute field and assigns a unique ID
        to each `Attribute`.
        """
        attribute_types = self._infer_column_types(rows, attribute_fields)
        attributes = []
        for idx, (k, v) in enumerate(attribute_types.items()):
            attribute = Attribute(
                output_data_type=v,
                attribute_id=idx,
                attribute_label=k,
            )
            attributes.append(attribute)

        return attributes

    def build_documents_and_annotations(
        self,
        attributes: list[Attribute],
        reference_fields: dict,
        rows: list[dict],
    ) -> tuple[list[Document], list[GoldStandardAnnotatedDocument]]:
        """
        Build `Document` objects and their corresponding GoldStandardAnnotatedDocument`s
        from CSV rows.

        Args:
            attributes:
            reference_fields: Dictionary mapping reference field labels (as defined in
                destiny_sdk.enhancements) to CSV column names.

            rows: List of dictionaries representing all CSV rows.

        Returns:
            A tuple containing: Document and List[GoldStandardAnnotatedDocument]

        """
        annotated_documents: list[GoldStandardAnnotatedDocument] = []
        documents: list[Document] = []

        # --- To acces attributes by their names ---
        attr_by_label = {a.attribute_label: a for a in attributes}

        for row in rows:
            # --- Build destiny Reference given row ---
            row_reference = self.build_destiny_reference(row, reference_fields)
            # --- Build Document ---
            document = Document(
                name=row["name"],
                citation=row_reference,
                document_id=row["document_id"],
            )
            document.init_document_identity()
            documents.append(document)

            # --- Build Document Annotations ---
            annotations = []
            for label, attr in attr_by_label.items():
                raw_value = row[label].strip()

                annotation = GoldStandardAnnotation(
                    attribute=attr,
                    raw_data=raw_value,
                    annotation_type=AnnotationType.HUMAN,
                )

                annotations.append(annotation)

            # --- Build Annotated Documents = Attach annotations to document ---
            annotated_doc = GoldStandardAnnotatedDocument(
                document=document,
                annotations=annotations,
            )

            annotated_documents.append(annotated_doc)

        return documents, annotated_documents

    def process_annotation_file(
        self,
        file_path: str | Path,
        set_attribute_type: str | AttributeType | None = None,
        attribute_fields: list | None = None,
        reference_fields: dict | None = None,
    ) -> ProcessedAnnotationData:
        """
        Process a complete CSV annotation file and return structured data.

        Each row is assumed to represent a document, and columns correspond to
        different types of fields (metadata, reference information, and attributes).

        Args:
            file_path: Path to the CSV annotation file.
            attribute_fields: List of column names to be treated as document attributes.
            reference_fields: Dictionary mapping reference field labels(as defined in
            destiny_sdk.enhancements) to corresponding CSV column names.


        Returns:
            ProcessedAnnotationData containing all processed data.

        """
        file_path = Path(file_path)

        if set_attribute_type is not None:
            msg = (
                "CsvAnnotationConverter does not support set_attribute_type; "
                "use attribute_fields instead."
            )
            raise NotImplementedError(msg)
        logger.info(f"Processing annotation file: {file_path}")

        colnames, reference_fields, attribute_fields, rows = self.load_csv(
            file_path, attribute_fields, reference_fields
        )

        attributes = self.build_attributes(attribute_fields, rows)

        documents, annotated_documents = self.build_documents_and_annotations(
            attributes, reference_fields, rows
        )
        attribute_id_to_label = {
            attr.attribute_id: attr.attribute_label for attr in attributes
        }

        logger.info(
            f"Processed {len(attributes)} attributes, {len(documents)} documents, "
            f" {len(annotated_documents)} annotated documents"
        )

        return ProcessedAnnotationData(
            attributes=attributes,
            documents=documents,
            annotated_documents=annotated_documents,
            attribute_id_to_label=attribute_id_to_label,
            annotations=[],  # keep type compatible
        )
processed_data_type property

Return ProcessedAnnotationData.

__init__(base_output_dir=DEFAULT_BASE_OUTPUT_DIR, attributes_filename=DEFAULT_ATTRIBUTES_FILENAME, documents_filename=DEFAULT_DOCUMENTS_FILENAME, annotated_documents_filename=DEFAULT_ANNOTATED_DOCUMENTS_FILENAME, attribute_mapping_filename=DEFAULT_ATTRIBUTE_MAPPING_FILENAME, config=None)

Initialize the converter output configurations (base directory + filenames) to save the multiple files created during csv processing.

Parameters:

Name Type Description Default
base_output_dir str | Path | None

Base directory for saving processed files

 DEFAULT_BASE_OUTPUT_DIR
attributes_filename str

Filename for attributes output

 DEFAULT_ATTRIBUTES_FILENAME
documents_filename str

Filename for documents output

 DEFAULT_DOCUMENTS_FILENAME
annotated_documents_filename str

Filename for annotated documents output

 DEFAULT_ANNOTATED_DOCUMENTS_FILENAME
attribute_mapping_filename str

Filename for attribute ID to label mapping

 DEFAULT_ATTRIBUTE_MAPPING_FILENAME
Source code in deet/processors/csv_annotation_converter.py 
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
def __init__(  # noqa: PLR0913
    self,
    base_output_dir: str | Path | None = DEFAULT_BASE_OUTPUT_DIR,
    attributes_filename: str = DEFAULT_ATTRIBUTES_FILENAME,
    documents_filename: str = DEFAULT_DOCUMENTS_FILENAME,
    annotated_documents_filename: str = DEFAULT_ANNOTATED_DOCUMENTS_FILENAME,
    attribute_mapping_filename: str = DEFAULT_ATTRIBUTE_MAPPING_FILENAME,
    config: CSVParserConfig | None = None,
) -> None:
    """
    Initialize the converter output configurations (base directory + filenames) to
    save the multiple files created during csv processing.

    Args:
        base_output_dir: Base directory for saving processed files
        attributes_filename: Filename for attributes output
        documents_filename: Filename for documents output
        annotated_documents_filename: Filename for annotated documents output
        attribute_mapping_filename: Filename for attribute ID to label mapping

    """
    self.config = config or CSVParserConfig()
    # If no directory given, write everything relative to current working directory
    if base_output_dir is None:
        logger.debug(
            "`base_output_dir` set to None; "
            "converting to empty string for compatibility."
        )
        base_output_dir = ""
    self.base_output_dir = Path(base_output_dir)

    # Output registry:
    # For each output type, provides the filename and how to serialize/validate it.
    # Used when writing and reloading the results

    self.OUTFILE_LOADERS: dict[Outfiles, tuple[str, TypeAdapter]] = {
        Outfiles.ATTRIBUTES: (
            attributes_filename,
            TypeAdapter(list[Attribute]),
        ),
        Outfiles.DOCUMENTS: (
            documents_filename,
            TypeAdapter(list[Document]),
        ),
        Outfiles.ANNOTATED_DOCUMENTS: (
            annotated_documents_filename,
            TypeAdapter(list[GoldStandardAnnotatedDocument]),
        ),
        Outfiles.ATTRIBUTE_LABEL_MAPPING: (
            attribute_mapping_filename,
            TypeAdapter(dict[int, str]),
        ),
    }
build_attributes(attribute_fields, rows)

Build a list of Attribute objects from CSV rows and specified attribute columns.

Infers the AttributeType for each attribute field and assigns a unique ID to each Attribute.

Source code in deet/processors/csv_annotation_converter.py 
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
def build_attributes(
    self,
    attribute_fields: list[str],
    rows: list[dict],
) -> list[Attribute]:
    """
    Build a list of `Attribute` objects from CSV rows and specified attribute
    columns.

    Infers the `AttributeType` for each attribute field and assigns a unique ID
    to each `Attribute`.
    """
    attribute_types = self._infer_column_types(rows, attribute_fields)
    attributes = []
    for idx, (k, v) in enumerate(attribute_types.items()):
        attribute = Attribute(
            output_data_type=v,
            attribute_id=idx,
            attribute_label=k,
        )
        attributes.append(attribute)

    return attributes
build_destiny_reference(row, mapping, source='deet CSV converter')

Convert a CSV row into a ReferenceFileInput for destiny_sdk.reference.

This method extracts bibliographic metadata and abstract content from the given row using the provided reference field mapping. It constructs BibliographicMetadataEnhancement and AbstractContentEnhancement objects, then wraps them in EnhancementFileInput objects, and finally returns a ReferenceFileInput.

Parameters:

Name Type Description Default
row dict

Dictionary representing a single CSV row.

 required
mapping dict[str, Any]

Dictionary mapping nested keys (dot-separated strings) to CSV column names.

 required
source str

Optional string indicating the source of the data; defaults to "deet CSV converter"

 'deet CSV converter'

Returns: ReferenceFileInput object containing all extracted enhancements

Source code in deet/processors/csv_annotation_converter.py 
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
def build_destiny_reference(
    self,
    row: dict,
    mapping: dict[str, Any],
    source: str = "deet CSV converter",
) -> ReferenceFileInput:
    """

    Convert a CSV row into a `ReferenceFileInput` for destiny_sdk.reference.

    This method extracts bibliographic metadata and abstract content from the
    given row using the provided reference field mapping. It constructs
    `BibliographicMetadataEnhancement` and `AbstractContentEnhancement`
    objects, then wraps them in `EnhancementFileInput` objects, and finally
    returns a `ReferenceFileInput`.

    Args:
        row: Dictionary representing a single CSV row.
        mapping: Dictionary mapping nested keys (dot-separated strings) to CSV
             column names.
        source: Optional string indicating the source of the data; defaults to
            "deet CSV converter"
    Returns:
        ReferenceFileInput object containing all extracted enhancements

    """
    reference_dict = self._build_destiny_reference_dict_from_row(mapping, row)

    abstract = (
        AbstractContentEnhancement(
            abstract=reference_dict["abstract"],
            process=AbstractProcessType.UNINVERTED,
        )
        if "abstract" in reference_dict
        else None
    )

    reference_dict["authorship"] = (
        self._build_destiny_authorship_list(reference_dict["authorship"])
        if "authorship" in reference_dict
        and isinstance(reference_dict["authorship"], str)
        else None
    )

    bibliographic_data = BibliographicMetadataEnhancement(**reference_dict)

    enhancements = [
        EnhancementFileInput(
            source=source,
            visibility=Visibility.PUBLIC,
            content=content,
        )
        for content in [abstract, bibliographic_data]
        if content is not None
    ]

    return ReferenceFileInput(
        visibility=Visibility.PUBLIC,
        enhancements=enhancements,
    )
build_documents_and_annotations(attributes, reference_fields, rows)

Build Document objects and their corresponding GoldStandardAnnotatedDocument`s from CSV rows.

Parameters:

Name Type Description Default
attributes list[Attribute]
required
reference_fields dict

Dictionary mapping reference field labels (as defined in destiny_sdk.enhancements) to CSV column names.

 required
rows list[dict]

List of dictionaries representing all CSV rows.

 required

Returns:

Type Description
tuple[list[Document], list[GoldStandardAnnotatedDocument]]

A tuple containing: Document and List[GoldStandardAnnotatedDocument]
Source code in deet/processors/csv_annotation_converter.py 
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
def build_documents_and_annotations(
    self,
    attributes: list[Attribute],
    reference_fields: dict,
    rows: list[dict],
) -> tuple[list[Document], list[GoldStandardAnnotatedDocument]]:
    """
    Build `Document` objects and their corresponding GoldStandardAnnotatedDocument`s
    from CSV rows.

    Args:
        attributes:
        reference_fields: Dictionary mapping reference field labels (as defined in
            destiny_sdk.enhancements) to CSV column names.

        rows: List of dictionaries representing all CSV rows.

    Returns:
        A tuple containing: Document and List[GoldStandardAnnotatedDocument]

    """
    annotated_documents: list[GoldStandardAnnotatedDocument] = []
    documents: list[Document] = []

    # --- To acces attributes by their names ---
    attr_by_label = {a.attribute_label: a for a in attributes}

    for row in rows:
        # --- Build destiny Reference given row ---
        row_reference = self.build_destiny_reference(row, reference_fields)
        # --- Build Document ---
        document = Document(
            name=row["name"],
            citation=row_reference,
            document_id=row["document_id"],
        )
        document.init_document_identity()
        documents.append(document)

        # --- Build Document Annotations ---
        annotations = []
        for label, attr in attr_by_label.items():
            raw_value = row[label].strip()

            annotation = GoldStandardAnnotation(
                attribute=attr,
                raw_data=raw_value,
                annotation_type=AnnotationType.HUMAN,
            )

            annotations.append(annotation)

        # --- Build Annotated Documents = Attach annotations to document ---
        annotated_doc = GoldStandardAnnotatedDocument(
            document=document,
            annotations=annotations,
        )

        annotated_documents.append(annotated_doc)

    return documents, annotated_documents
load_csv(file_path, attribute_fields=None, reference_fields=None)

Load a CSV, normalize headers, and return all column names, attribute names, reference names, and rows.

Source code in deet/processors/csv_annotation_converter.py 
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
def load_csv(
    self,
    file_path: Path,
    attribute_fields: list[str] | None = None,
    reference_fields: dict | None = None,
) -> tuple[list[str], dict[str, str], list[str], list[dict[str, Any]]]:
    """
    Load a CSV, normalize headers, and return all column names, attribute names,
    reference names, and rows.
    """
    path = Path(file_path)
    with path.open(newline="", encoding="utf-8-sig") as f:
        csv_reader = csv.DictReader(f)

        # normalize headers BEFORE reading rows
        raw_headers = csv_reader.fieldnames or []
        colnames: list[str] = [h.strip().lower() for h in raw_headers]
        csv_reader.fieldnames = colnames

        # --- validate duplicates ---
        dup_fields = self._find_duplicate_column_names(colnames)
        if dup_fields:
            msg = f"{len(dup_fields)} Duplicate fieldnames found: {dup_fields}"
            raise ValueError(msg)

        # --- validate required fields ---
        meta_fields = {"name", "document_id"}
        missing = meta_fields - set(colnames)
        if missing:
            msg = f"Required columns missing: {missing}"
            raise ValueError(msg)

        # --- validate reference fields ---
        # If reference_fields is not povided....
        if reference_fields is None:
            if self.config.auto_assign_reference_fields:
                logger.info("Auto assigning reference fields")
                reference_fields = {
                    ref_field: ref_field
                    for ref_field in ALLOWED_REFERENCE_MAPPING_KEYS
                    if ref_field in colnames
                }
            else:
                logger.info("No reference fields provided and auto assign is False")
                reference_fields = {}

        # If reference_fields is povided....
        if reference_fields:
            invalid_keys = set(reference_fields) - ALLOWED_REFERENCE_MAPPING_KEYS
            if invalid_keys:
                msg = f"Invalid mapping keys: {invalid_keys}"
                raise ValueError(msg)

            # normalize reference fields
            reference_fields = {
                k: v.strip().lower() for k, v in reference_fields.items()
            }

            unknown = set(reference_fields.values()) - set(colnames)
            if unknown:
                msg = f"Reference fields not found in CSV: {unknown}"
                raise ValueError(msg)

        # --- validate attribute fields ---
        # normalize and validate provided attribute fields
        if attribute_fields is None:
            logger.info("No attribute fields provided")
            excluded_fields = meta_fields | set(reference_fields.values())
            resolved_attribute_fields = [
                h for h in colnames if h not in excluded_fields
            ]
        else:
            resolved_attribute_fields = [
                field.strip().lower() for field in attribute_fields
            ]
            unknown_attributes = set(resolved_attribute_fields) - set(colnames)
            if unknown_attributes:
                msg = f"Attribute fields not found in CSV: {unknown_attributes}"
                raise ValueError(msg)

        # Read rows into mem once
        rows = list(csv_reader)

    return colnames, reference_fields, resolved_attribute_fields, rows
process_annotation_file(file_path, set_attribute_type=None, attribute_fields=None, reference_fields=None)

Process a complete CSV annotation file and return structured data.

Each row is assumed to represent a document, and columns correspond to different types of fields (metadata, reference information, and attributes).

Parameters:

Name Type Description Default
file_path str | Path

Path to the CSV annotation file.

 required
attribute_fields list | None

List of column names to be treated as document attributes.

 None
reference_fields dict | None

Dictionary mapping reference field labels(as defined in

 None

Returns:

Type Description
ProcessedAnnotationData

ProcessedAnnotationData containing all processed data.
Source code in deet/processors/csv_annotation_converter.py 
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
def process_annotation_file(
    self,
    file_path: str | Path,
    set_attribute_type: str | AttributeType | None = None,
    attribute_fields: list | None = None,
    reference_fields: dict | None = None,
) -> ProcessedAnnotationData:
    """
    Process a complete CSV annotation file and return structured data.

    Each row is assumed to represent a document, and columns correspond to
    different types of fields (metadata, reference information, and attributes).

    Args:
        file_path: Path to the CSV annotation file.
        attribute_fields: List of column names to be treated as document attributes.
        reference_fields: Dictionary mapping reference field labels(as defined in
        destiny_sdk.enhancements) to corresponding CSV column names.


    Returns:
        ProcessedAnnotationData containing all processed data.

    """
    file_path = Path(file_path)

    if set_attribute_type is not None:
        msg = (
            "CsvAnnotationConverter does not support set_attribute_type; "
            "use attribute_fields instead."
        )
        raise NotImplementedError(msg)
    logger.info(f"Processing annotation file: {file_path}")

    colnames, reference_fields, attribute_fields, rows = self.load_csv(
        file_path, attribute_fields, reference_fields
    )

    attributes = self.build_attributes(attribute_fields, rows)

    documents, annotated_documents = self.build_documents_and_annotations(
        attributes, reference_fields, rows
    )
    attribute_id_to_label = {
        attr.attribute_id: attr.attribute_label for attr in attributes
    }

    logger.info(
        f"Processed {len(attributes)} attributes, {len(documents)} documents, "
        f" {len(annotated_documents)} annotated documents"
    )

    return ProcessedAnnotationData(
        attributes=attributes,
        documents=documents,
        annotated_documents=annotated_documents,
        attribute_id_to_label=attribute_id_to_label,
        annotations=[],  # keep type compatible
    )

CSVParserConfig pydantic-model

Bases: BaseModel

Configuration Seetings for parsing CSV.

Fields:

  • author_separator (str)
  • auto_assign_reference_fields (Boolean)
Source code in deet/processors/csv_annotation_converter.py 
71
72
73
74
75
class CSVParserConfig(BaseModel):
    """Configuration Seetings for parsing CSV."""

    author_separator: str = ";"
    auto_assign_reference_fields: Boolean = False

ColumnTypeInferenceError

Bases: Exception

Raised when column type inference fails due to incompatible types.

Source code in deet/processors/csv_annotation_converter.py 
67
68
class ColumnTypeInferenceError(Exception):
    """Raised when column type inference fails due to incompatible types."""

eppi_annotation_converter

Convert annotation JSON files to Pydantic models.

EppiAnnotationConverter

Bases: AnnotationConverter

A class to convert raw EPPI-Reviewer JSON annotations into structured Pydantic models.

This converter handles the complex hierarchical structure of EPPI attributes by flattening them while preserving parent-child relationships through path information.

Source code in deet/processors/eppi_annotation_converter.py 
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
class EppiAnnotationConverter(AnnotationConverter):
    """
    A class to convert raw EPPI-Reviewer JSON annotations
    into structured Pydantic models.

    This converter handles the complex hierarchical
    structure of EPPI attributes by flattening
    them while preserving parent-child relationships
    through path information.
    """

    def __init__(
        self,
        base_output_dir: str | Path | None = DEFAULT_BASE_OUTPUT_DIR,
        attributes_filename: str = DEFAULT_ATTRIBUTES_FILENAME,
        documents_filename: str = DEFAULT_DOCUMENTS_FILENAME,
        annotated_documents_filename: str = DEFAULT_ANNOTATED_DOCUMENTS_FILENAME,
        attribute_mapping_filename: str = DEFAULT_ATTRIBUTE_MAPPING_FILENAME,
    ) -> None:
        """
        Initialise the converter with configurable output paths.
        Set self.OUTFILE_LOADERS mapping the outfiles to be read/written to a
        filename, and a TypeAdapter defining the type of Pydantic Model to
        read back in when deserialising.

        Args:
            output_dir: Base directory for saving processed files
            attributes_filename: Filename for attributes output
            documents_filename: Filename for documents output
            annotated_documents_filename: Filename for annotated documents output
            attribute_mapping_filename: Filename for attribute ID to label mapping

        """
        if base_output_dir is None:
            logger.debug(
                "`base_output_dir` set to None; "
                "converting to empty string for compatibility."
            )
            base_output_dir = ""
        self.base_output_dir = Path(base_output_dir)

        # extend below if adding more output files in `Outfiles`.
        self.OUTFILE_LOADERS: dict[Outfiles, tuple[str, TypeAdapter]] = {
            Outfiles.ATTRIBUTES: (
                attributes_filename,
                TypeAdapter(list[EppiAttribute]),
            ),
            Outfiles.DOCUMENTS: (
                documents_filename,
                TypeAdapter(list[EppiDocument]),
            ),
            Outfiles.ANNOTATED_DOCUMENTS: (
                annotated_documents_filename,
                TypeAdapter(list[EppiGoldStandardAnnotatedDocument]),
            ),
            Outfiles.ATTRIBUTE_LABEL_MAPPING: (
                attribute_mapping_filename,
                TypeAdapter(dict[int, str]),
            ),
        }

    @property
    def processed_data_type(self) -> type[ProcessedEppiAnnotationData]:
        """Return ProcessedEppiAnnotationData."""
        return ProcessedEppiAnnotationData

    def flatten_attributes_hierarchy(
        self, attributes_list: list[dict[str, Any]], parent_path: str = ""
    ) -> list[dict[str, Any]]:
        """
        Recursively flatten the hierarchical attributes structure.

        Args:
            attributes_list: List of attribute dictionaries from the JSON
            parent_path: Path to the parent attribute (for hierarchy tracking)

        Returns:
            List of flattened attribute dictionaries with hierarchy information

        """
        flattened = []

        for attr in attributes_list:
            # extract children before modifying  dict
            child_attributes = attr.get("Attributes", {}).get("AttributesList", [])

            attr["hierarchy_path"] = parent_path
            attr["hierarchy_level"] = (
                len(parent_path.split(" > ")) if parent_path else 0
            )
            attr["is_leaf"] = not bool(child_attributes)

            flattened.append(attr)

            # recursive extension
            if child_attributes:
                current_path = (
                    f"{parent_path} > {attr.get('AttributeName', '')}"
                    if parent_path
                    else attr.get("AttributeName", "")
                )
                flattened.extend(
                    self.flatten_attributes_hierarchy(child_attributes, current_path)
                )

        return flattened

    def _extract_attributes_from_codesets(
        self, raw_data: EppiRawData
    ) -> list[dict[str, Any]]:
        """Extract and flatten attributes from CodeSets using structured models."""
        return raw_data.extract_all_attributes(self.flatten_attributes_hierarchy)

    def convert_to_eppi_attributes(
        self,
        flattened_attributes: list[dict[str, Any]],
        set_attribute_type: AttributeType | None = None,
    ) -> list[EppiAttribute]:
        """
        Convert flattened attribute data to EppiAttribute models.

        Args:
            flattened_attributes: List of flattened attribute dictionaries

        Returns:
            List of EppiAttribute models

        """
        out = []
        for att_dict in flattened_attributes:
            if "AttributeId" not in att_dict:
                att_dict["AttributeId"] = 0
            new_attribute = EppiAttribute(**att_dict)
            if set_attribute_type:
                logger.debug(
                    f"setting custom attribute type {set_attribute_type.value} "
                    f"for attribute {new_attribute.attribute_id}"
                )
                new_attribute.output_data_type = set_attribute_type
            out.append(new_attribute)
        return out

    def _process_text_details(
        self, text_details: list[dict[str, Any]]
    ) -> tuple[list[str], list[EppiItemAttributeFullTextDetails]]:
        """
        Process ItemAttributeFullTextDetails to extract texts and create detail objects.

        Args:
            text_details: List of text detail dictionaries from EPPI JSON

        Returns:
            Tuple of (extracted_texts, item_attribute_details)

        """
        extracted_texts = []
        item_attribute_details = []

        for text_detail in text_details:
            text = text_detail.get("Text", "")
            if text:
                extracted_texts.append(text)

            detail = EppiItemAttributeFullTextDetails(
                item_document_id=text_detail.get("ItemDocumentId"),
                text=text,
                item_arm=text_detail.get("ItemArm", ""),
            )
            item_attribute_details.append(detail)

        return extracted_texts, item_attribute_details

    def _convert_single_annotation(
        self,
        annotation: dict[str, Any],
        attributes_lookup: dict[int, EppiAttribute],
        attribute_id_to_label: dict[int, str] | None = None,
    ) -> EppiGoldStandardAnnotation:
        """
        Convert a single annotation dictionary to EppiGoldStandardAnnotation.

        Args:
            annotation: Single annotation dictionary from EPPI JSON
            attributes_lookup: Lookup dictionary for attributes
            attribute_id_to_label: Mapping from attribute ID to label

        Returns:
            EppiGoldStandardAnnotation model

        Note:
            If attribute is not found in lookup, creates a basic attribute using
            the attribute_id_to_label mapping. All annotations from JSON are
            marked as HUMAN type. ``raw_data`` follows
            :func:`eppi_output_data_from_eppi_fields` (booleans from code presence;
            other types from ``AdditionalText`` only).

        """
        text_details = annotation.get("ItemAttributeFullTextDetails", [])
        _extracted_texts, item_attribute_details = self._process_text_details(
            text_details
        )

        # Look up the attribute from the attributes list
        if (attribute_id := annotation.get("AttributeId")) is None:
            missing_attr_id_msg = (
                "Annotation is missing required field 'AttributeId'. "
                "All annotations must have an AttributeId."
            )
            raise ValueError(missing_attr_id_msg)

        # find attribute in attributes_lookup
        if (attribute := attributes_lookup.get(attribute_id)) is None:
            attr_not_found_msg = (
                f"Attribute with ID {attribute_id} not found in attributes list. "
                "All annotations must reference a valid attribute from the CodeSets."
            )
            raise ValueError(attr_not_found_msg)

        # ensure the attribute has the correct label from the mapping if available
        if attribute_id_to_label is not None and attribute_id in attribute_id_to_label:
            attribute.attribute_label = attribute_id_to_label[attribute_id]

        additional_text = str(annotation.get("AdditionalText", "") or "")
        typed_raw_data: bool | str | int | float | list[Any] | dict[str, Any] = (
            eppi_output_data_from_eppi_fields(
                attribute.output_data_type, additional_text=additional_text
            )
        )

        return EppiGoldStandardAnnotation(
            attribute=attribute,
            additional_text=annotation.get("AdditionalText", ""),
            arm_id=annotation.get("ArmId"),
            arm_title=annotation.get("ArmTitle", ""),
            arm_description=annotation.get("ArmDescription", ""),
            raw_data=typed_raw_data,
            annotation_type=AnnotationType.HUMAN,
            item_attribute_full_text_details=item_attribute_details,
        )

    def _merge_raw_values(
        self, existing: SUPPORTED_TYPES, new: SUPPORTED_TYPES
    ) -> SUPPORTED_TYPES:
        """Merge the raw_data of duplicated annotations."""
        if existing is None:
            return new
        if new is None:
            return existing
        if isinstance(existing, list) and isinstance(new, list):
            return existing + new
        if isinstance(existing, dict) and isinstance(new, dict):
            return {**existing, **new}

        return f"{existing};;; {new}"

    def convert_to_eppi_annotations(
        self,
        annotations_data: list[dict[str, Any]],
        attributes_lookup: dict[int, EppiAttribute],
        attribute_id_to_label: dict[int, str] | None = None,
    ) -> list[EppiGoldStandardAnnotation]:
        """
        Convert several dicts to a list of EppiGoldStandardAnnotations.

        Args:
            annotations_data: List of human, gold standard
                annotation dicts from EPPI JSON
            document: The document these annotations belong to
            attributes_lookup: Lookup dictionary for attributes
            attribute_id_to_label: Mapping from attribute ID to label

        Returns:
            List of EppiGoldStandardAnnotation models

        """
        results = []
        for annotation in annotations_data:
            try:
                converted = self._convert_single_annotation(
                    annotation, attributes_lookup, attribute_id_to_label
                )
                results.append(converted)
            except ValueError as e:
                logger.warning(f"Skipping annotation due to error: {e}")
                continue
        return results

    def dedup_annotations(
        self, annotations: list[EppiGoldStandardAnnotation]
    ) -> list[EppiGoldStandardAnnotation]:
        """Merge annotations with the same attribute id."""
        merged: dict[int, EppiGoldStandardAnnotation] = {}

        for ann in annotations:
            attr_id = ann.attribute.attribute_id

            if attr_id not in merged:
                merged[attr_id] = ann
                continue

            target = merged[attr_id]

            target.raw_data = self._merge_raw_values(target.raw_data, ann.raw_data)

            if ann.additional_text:
                existing_text = target.additional_text or ""
                target.additional_text = f"{existing_text};;; {ann.additional_text}"

        return list(merged.values())

    def process_annotation_file(
        self,
        file_path: str | Path,
        set_attribute_type: str | AttributeType | None = None,
    ) -> ProcessedEppiAnnotationData:
        """
        Process a complete annotation file and return structured data.

        Args:
            file_path: Path to the JSON annotation file
            set_attribute_type: custom AttributeType to set for incoming annotations.

        Returns:
            ProcessedAnnotationData containing all processed data

        """
        logger.info(f"Processing annotation file: {file_path}")

        with Path(file_path).open("r", encoding="utf-8") as f:
            data: dict = json.load(f)

        raw_data = EppiRawData.model_validate(data)

        all_attributes_raw = self._extract_attributes_from_codesets(raw_data)

        if isinstance(set_attribute_type, str):
            set_attribute_type = AttributeType(set_attribute_type)
        attributes = self.convert_to_eppi_attributes(
            flattened_attributes=all_attributes_raw,
            set_attribute_type=set_attribute_type,
        )

        attributes_lookup: dict[int, EppiAttribute] = {
            attr.attribute_id: attr for attr in attributes
        }

        attribute_id_to_label: dict[int, str] = {
            attr.attribute_id: attr.attribute_label for attr in attributes
        }

        annotated_documents = []
        all_annotations = []
        documents_by_item_id: dict[int, EppiDocument] = {}

        # Process each reference with its annotations directly
        # Annotations are already nested within their parent reference
        for reference in data.get("References", []):
            item_id = reference.get("ItemId")
            if item_id is None:
                continue

            # Create or retrieve document using ItemId as unique identifier
            if item_id not in documents_by_item_id:
                document = EppiDocument.model_validate(reference)
                documents_by_item_id[item_id] = document
            else:
                document = documents_by_item_id[item_id]

            # Get annotations directly from this reference's Codes array
            reference_codes = reference.get("Codes", [])
            if reference_codes:
                annotations = self.convert_to_eppi_annotations(
                    reference_codes,
                    attributes_lookup,
                    attribute_id_to_label,
                )

                annotations = self.dedup_annotations(annotations)

                annotated_doc = EppiGoldStandardAnnotatedDocument(
                    document=document, annotations=annotations
                )

                annotated_documents.append(annotated_doc)
                all_annotations.extend(annotations)

        logger.info(
            f"Processed {len(attributes)} attributes,"
            f" {len(documents_by_item_id)} documents, "
            f"{len(all_annotations)} annotations,"
            f" {len(annotated_documents)} annotated documents"
        )

        return ProcessedEppiAnnotationData(
            attributes=attributes,
            documents=list(documents_by_item_id.values()),
            annotations=all_annotations,
            annotated_documents=annotated_documents,
            attribute_id_to_label=attribute_id_to_label,
            raw_data=raw_data,
        )
processed_data_type property

Return ProcessedEppiAnnotationData.

__init__(base_output_dir=DEFAULT_BASE_OUTPUT_DIR, attributes_filename=DEFAULT_ATTRIBUTES_FILENAME, documents_filename=DEFAULT_DOCUMENTS_FILENAME, annotated_documents_filename=DEFAULT_ANNOTATED_DOCUMENTS_FILENAME, attribute_mapping_filename=DEFAULT_ATTRIBUTE_MAPPING_FILENAME)

Initialise the converter with configurable output paths. Set self.OUTFILE_LOADERS mapping the outfiles to be read/written to a filename, and a TypeAdapter defining the type of Pydantic Model to read back in when deserialising.

Parameters:

Name Type Description Default
output_dir

Base directory for saving processed files

 required
attributes_filename str

Filename for attributes output

 DEFAULT_ATTRIBUTES_FILENAME
documents_filename str

Filename for documents output

 DEFAULT_DOCUMENTS_FILENAME
annotated_documents_filename str

Filename for annotated documents output

 DEFAULT_ANNOTATED_DOCUMENTS_FILENAME
attribute_mapping_filename str

Filename for attribute ID to label mapping

 DEFAULT_ATTRIBUTE_MAPPING_FILENAME
Source code in deet/processors/eppi_annotation_converter.py 
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
def __init__(
    self,
    base_output_dir: str | Path | None = DEFAULT_BASE_OUTPUT_DIR,
    attributes_filename: str = DEFAULT_ATTRIBUTES_FILENAME,
    documents_filename: str = DEFAULT_DOCUMENTS_FILENAME,
    annotated_documents_filename: str = DEFAULT_ANNOTATED_DOCUMENTS_FILENAME,
    attribute_mapping_filename: str = DEFAULT_ATTRIBUTE_MAPPING_FILENAME,
) -> None:
    """
    Initialise the converter with configurable output paths.
    Set self.OUTFILE_LOADERS mapping the outfiles to be read/written to a
    filename, and a TypeAdapter defining the type of Pydantic Model to
    read back in when deserialising.

    Args:
        output_dir: Base directory for saving processed files
        attributes_filename: Filename for attributes output
        documents_filename: Filename for documents output
        annotated_documents_filename: Filename for annotated documents output
        attribute_mapping_filename: Filename for attribute ID to label mapping

    """
    if base_output_dir is None:
        logger.debug(
            "`base_output_dir` set to None; "
            "converting to empty string for compatibility."
        )
        base_output_dir = ""
    self.base_output_dir = Path(base_output_dir)

    # extend below if adding more output files in `Outfiles`.
    self.OUTFILE_LOADERS: dict[Outfiles, tuple[str, TypeAdapter]] = {
        Outfiles.ATTRIBUTES: (
            attributes_filename,
            TypeAdapter(list[EppiAttribute]),
        ),
        Outfiles.DOCUMENTS: (
            documents_filename,
            TypeAdapter(list[EppiDocument]),
        ),
        Outfiles.ANNOTATED_DOCUMENTS: (
            annotated_documents_filename,
            TypeAdapter(list[EppiGoldStandardAnnotatedDocument]),
        ),
        Outfiles.ATTRIBUTE_LABEL_MAPPING: (
            attribute_mapping_filename,
            TypeAdapter(dict[int, str]),
        ),
    }
convert_to_eppi_annotations(annotations_data, attributes_lookup, attribute_id_to_label=None)

Convert several dicts to a list of EppiGoldStandardAnnotations.

Parameters:

Name Type Description Default
annotations_data list[dict[str, Any]]

List of human, gold standard annotation dicts from EPPI JSON

 required
document

The document these annotations belong to

 required
attributes_lookup dict[int, EppiAttribute]

Lookup dictionary for attributes

 required
attribute_id_to_label dict[int, str] | None

Mapping from attribute ID to label

 None

Returns:

Type Description
list[EppiGoldStandardAnnotation]

List of EppiGoldStandardAnnotation models
Source code in deet/processors/eppi_annotation_converter.py 
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
def convert_to_eppi_annotations(
    self,
    annotations_data: list[dict[str, Any]],
    attributes_lookup: dict[int, EppiAttribute],
    attribute_id_to_label: dict[int, str] | None = None,
) -> list[EppiGoldStandardAnnotation]:
    """
    Convert several dicts to a list of EppiGoldStandardAnnotations.

    Args:
        annotations_data: List of human, gold standard
            annotation dicts from EPPI JSON
        document: The document these annotations belong to
        attributes_lookup: Lookup dictionary for attributes
        attribute_id_to_label: Mapping from attribute ID to label

    Returns:
        List of EppiGoldStandardAnnotation models

    """
    results = []
    for annotation in annotations_data:
        try:
            converted = self._convert_single_annotation(
                annotation, attributes_lookup, attribute_id_to_label
            )
            results.append(converted)
        except ValueError as e:
            logger.warning(f"Skipping annotation due to error: {e}")
            continue
    return results
convert_to_eppi_attributes(flattened_attributes, set_attribute_type=None)

Convert flattened attribute data to EppiAttribute models.

Parameters:

Name Type Description Default
flattened_attributes list[dict[str, Any]]

List of flattened attribute dictionaries

 required

Returns:

Type Description
list[EppiAttribute]

List of EppiAttribute models
Source code in deet/processors/eppi_annotation_converter.py 
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
def convert_to_eppi_attributes(
    self,
    flattened_attributes: list[dict[str, Any]],
    set_attribute_type: AttributeType | None = None,
) -> list[EppiAttribute]:
    """
    Convert flattened attribute data to EppiAttribute models.

    Args:
        flattened_attributes: List of flattened attribute dictionaries

    Returns:
        List of EppiAttribute models

    """
    out = []
    for att_dict in flattened_attributes:
        if "AttributeId" not in att_dict:
            att_dict["AttributeId"] = 0
        new_attribute = EppiAttribute(**att_dict)
        if set_attribute_type:
            logger.debug(
                f"setting custom attribute type {set_attribute_type.value} "
                f"for attribute {new_attribute.attribute_id}"
            )
            new_attribute.output_data_type = set_attribute_type
        out.append(new_attribute)
    return out
dedup_annotations(annotations)

Merge annotations with the same attribute id.

Source code in deet/processors/eppi_annotation_converter.py 
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
def dedup_annotations(
    self, annotations: list[EppiGoldStandardAnnotation]
) -> list[EppiGoldStandardAnnotation]:
    """Merge annotations with the same attribute id."""
    merged: dict[int, EppiGoldStandardAnnotation] = {}

    for ann in annotations:
        attr_id = ann.attribute.attribute_id

        if attr_id not in merged:
            merged[attr_id] = ann
            continue

        target = merged[attr_id]

        target.raw_data = self._merge_raw_values(target.raw_data, ann.raw_data)

        if ann.additional_text:
            existing_text = target.additional_text or ""
            target.additional_text = f"{existing_text};;; {ann.additional_text}"

    return list(merged.values())
flatten_attributes_hierarchy(attributes_list, parent_path='')

Recursively flatten the hierarchical attributes structure.

Parameters:

Name Type Description Default
attributes_list list[dict[str, Any]]

List of attribute dictionaries from the JSON

 required
parent_path str

Path to the parent attribute (for hierarchy tracking)

 ''

Returns:

Type Description
list[dict[str, Any]]

List of flattened attribute dictionaries with hierarchy information
Source code in deet/processors/eppi_annotation_converter.py 
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
def flatten_attributes_hierarchy(
    self, attributes_list: list[dict[str, Any]], parent_path: str = ""
) -> list[dict[str, Any]]:
    """
    Recursively flatten the hierarchical attributes structure.

    Args:
        attributes_list: List of attribute dictionaries from the JSON
        parent_path: Path to the parent attribute (for hierarchy tracking)

    Returns:
        List of flattened attribute dictionaries with hierarchy information

    """
    flattened = []

    for attr in attributes_list:
        # extract children before modifying  dict
        child_attributes = attr.get("Attributes", {}).get("AttributesList", [])

        attr["hierarchy_path"] = parent_path
        attr["hierarchy_level"] = (
            len(parent_path.split(" > ")) if parent_path else 0
        )
        attr["is_leaf"] = not bool(child_attributes)

        flattened.append(attr)

        # recursive extension
        if child_attributes:
            current_path = (
                f"{parent_path} > {attr.get('AttributeName', '')}"
                if parent_path
                else attr.get("AttributeName", "")
            )
            flattened.extend(
                self.flatten_attributes_hierarchy(child_attributes, current_path)
            )

    return flattened
process_annotation_file(file_path, set_attribute_type=None)

Process a complete annotation file and return structured data.

Parameters:

Name Type Description Default
file_path str | Path

Path to the JSON annotation file

 required
set_attribute_type str | AttributeType | None

custom AttributeType to set for incoming annotations.

 None

Returns:

Type Description
ProcessedEppiAnnotationData

ProcessedAnnotationData containing all processed data
Source code in deet/processors/eppi_annotation_converter.py 
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
def process_annotation_file(
    self,
    file_path: str | Path,
    set_attribute_type: str | AttributeType | None = None,
) -> ProcessedEppiAnnotationData:
    """
    Process a complete annotation file and return structured data.

    Args:
        file_path: Path to the JSON annotation file
        set_attribute_type: custom AttributeType to set for incoming annotations.

    Returns:
        ProcessedAnnotationData containing all processed data

    """
    logger.info(f"Processing annotation file: {file_path}")

    with Path(file_path).open("r", encoding="utf-8") as f:
        data: dict = json.load(f)

    raw_data = EppiRawData.model_validate(data)

    all_attributes_raw = self._extract_attributes_from_codesets(raw_data)

    if isinstance(set_attribute_type, str):
        set_attribute_type = AttributeType(set_attribute_type)
    attributes = self.convert_to_eppi_attributes(
        flattened_attributes=all_attributes_raw,
        set_attribute_type=set_attribute_type,
    )

    attributes_lookup: dict[int, EppiAttribute] = {
        attr.attribute_id: attr for attr in attributes
    }

    attribute_id_to_label: dict[int, str] = {
        attr.attribute_id: attr.attribute_label for attr in attributes
    }

    annotated_documents = []
    all_annotations = []
    documents_by_item_id: dict[int, EppiDocument] = {}

    # Process each reference with its annotations directly
    # Annotations are already nested within their parent reference
    for reference in data.get("References", []):
        item_id = reference.get("ItemId")
        if item_id is None:
            continue

        # Create or retrieve document using ItemId as unique identifier
        if item_id not in documents_by_item_id:
            document = EppiDocument.model_validate(reference)
            documents_by_item_id[item_id] = document
        else:
            document = documents_by_item_id[item_id]

        # Get annotations directly from this reference's Codes array
        reference_codes = reference.get("Codes", [])
        if reference_codes:
            annotations = self.convert_to_eppi_annotations(
                reference_codes,
                attributes_lookup,
                attribute_id_to_label,
            )

            annotations = self.dedup_annotations(annotations)

            annotated_doc = EppiGoldStandardAnnotatedDocument(
                document=document, annotations=annotations
            )

            annotated_documents.append(annotated_doc)
            all_annotations.extend(annotations)

    logger.info(
        f"Processed {len(attributes)} attributes,"
        f" {len(documents_by_item_id)} documents, "
        f"{len(all_annotations)} annotations,"
        f" {len(annotated_documents)} annotated documents"
    )

    return ProcessedEppiAnnotationData(
        attributes=attributes,
        documents=list(documents_by_item_id.values()),
        annotations=all_annotations,
        annotated_documents=annotated_documents,
        attribute_id_to_label=attribute_id_to_label,
        raw_data=raw_data,
    )

eppi_output_data_from_eppi_fields(output_data_type, *, additional_text)

Map EPPI evidence onto typed raw_data for coerced output_data.

Glossary

  • Codes: Rows under References[].Codes in EPPI export JSON. Each row means the reviewer applied that code for the reference (e.g. ticked a box).
  • raw_data: The value stored on GoldStandardAnnotation before / during coercion to the Python type implied by the attribute.
  • output_data: The coerced, typed value used in evaluation (derived from raw_data). For EPPI ingest, booleans reflect code presence; other types come from the AdditionalText field.

A Code row exists means the attribute was applied. For boolean attributes that is True even when AdditionalText is empty (the checkbox alone carries the positive annotation).

For every non-boolean type, only the info-box AdditionalText is used. ItemAttributeFullTextDetails is not used for the stored value (it may still be attached to the model for other uses).

Parameters:

Name Type Description Default
output_data_type AttributeType

Target attribute type (from codeset or prompt CSV).

 required
additional_text str

EPPI AdditionalText / info-box value.

 required

Returns:

Type Description
EppiRawDataValue

Value to store in GoldStandardAnnotation.raw_data (then coerced via
EppiRawDataValue

output_data). Never None; see module-level note on
EppiRawDataValue

EppiRawDataValue.
Source code in deet/processors/eppi_annotation_converter.py 
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
def eppi_output_data_from_eppi_fields(
    output_data_type: AttributeType,
    *,
    additional_text: str,
) -> EppiRawDataValue:
    """
    Map EPPI evidence onto typed ``raw_data`` for coerced ``output_data``.

    **Glossary**

    - **Codes:** Rows under ``References[].Codes`` in EPPI export JSON. Each row
      means the reviewer applied that code for the reference (e.g. ticked a box).
    - **raw_data:** The value stored on ``GoldStandardAnnotation`` before / during
      coercion to the Python type implied by the attribute.
    - **output_data:** The coerced, typed value used in evaluation (derived from
      ``raw_data``). For EPPI ingest, booleans reflect **code presence**; other types
      come from the ``AdditionalText`` field.

    A Code row exists means the attribute was applied. For boolean attributes that is
    ``True`` even when ``AdditionalText`` is empty (the checkbox alone carries the
    positive annotation).

    For every non-boolean type, only the info-box ``AdditionalText`` is used.
    ``ItemAttributeFullTextDetails`` is not used for the stored value (it may still be
    attached to the model for other uses).

    Args:
        output_data_type: Target attribute type (from codeset or prompt CSV).
        additional_text: EPPI ``AdditionalText`` / info-box value.

    Returns:
        Value to store in ``GoldStandardAnnotation.raw_data`` (then coerced via
        ``output_data``). Never ``None``; see module-level note on
        ``EppiRawDataValue``.

    """
    additional = (additional_text or "").strip()

    if output_data_type == AttributeType.BOOL:
        return True
    if output_data_type == AttributeType.STRING:
        return additional
    if output_data_type == AttributeType.INTEGER:
        return _parse_eppi_integer(additional, output_data_type)
    if output_data_type == AttributeType.FLOAT:
        return _parse_eppi_float(additional, output_data_type)
    if output_data_type in (AttributeType.LIST, AttributeType.DICT):
        return _parse_eppi_list_or_dict(additional, output_data_type)

    raise UnsupportedEppiAttributeTypeError(output_data_type)

linker

Tools for linking references/citations with parsed documents.

DocumentReferenceLinker

Core class for linking references/citations with parsed document text.

Source code in deet/processors/linker.py 
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
class DocumentReferenceLinker:
    """Core class for linking references/citations with parsed document text."""

    LINKING_STRATEGY_HIERARCHY = [
        LinkingStrategy.MAPPING_FILE,
        LinkingStrategy.FILENAME_ID,
    ]

    def __init__(
        self,
        references: Sequence[Document],
        document_reference_mapping: list[DocumentReferenceMapping] | Path | None = None,
        document_base_dir: Path | None = None,
        parser: DocumentParser = parser,
        linking_strategies: list[LinkingStrategy] | None = None,
    ) -> None:
        """Initialise DocumentReferenceLinker class."""
        # deep copies to ensure lookup tables don't modify validators
        # on our existing reference docuemnts.
        tmp_refs = [doc.model_copy(deep=True) for doc in references]
        self.documents_references = references

        # lookup dics for O(1) id & author_year based lookup
        self._references_by_id: dict[int, Document] = {
            doc.document_identity.document_id: doc  # type:ignore[union-attr]
            for doc in tmp_refs
            if doc.document_identity.document_id is not None  # type:ignore[union-attr]
        }
        logger.debug(self._references_by_id.keys())
        try:
            self._references_by_author_year_longest: dict[str, Document] = {
                doc.author_year_from_document_identity(
                    substring_strategy="longest"
                ): doc
                for doc in tmp_refs
            }
        except ValueError:
            logger.warning("author or year missing, returning empty dict.")
            self._references_by_author_year_longest = {}

        try:
            self._references_by_author_year_last: dict[str, Document] = {
                doc.author_year_from_document_identity(substring_strategy="last"): doc
                for doc in tmp_refs
            }
        except ValueError:
            logger.warning("author or year missing, returning empty dict.")
            self._references_by_author_year_last = {}

        self.document_base_dir = document_base_dir
        if isinstance(document_reference_mapping, Path):
            logger.debug(
                f"document_reference_mapping type: {type(document_reference_mapping)}"
            )
            document_reference_mapping = MappingImporter(
                mapping_file_path=document_reference_mapping,
                document_base_dir=document_base_dir,
            ).import_mapping()
        self.document_reference_mapping = document_reference_mapping
        self.parser = parser

        self.linking_strategies = linking_strategies or self.LINKING_STRATEGY_HIERARCHY

    def _create_linking_factory(
        self, linking_strategy: LinkingStrategy
    ) -> Callable[[], Generator[LinkedInterimPayload, None, None]]:
        """
        Create a match function contingent on linking strategy.


        Args:
            linking_strategy (LinkingStrategy): see the enum for available ones.


        Returns:
            function: the function to run to retrieve objects to be linked,
            which will always yield a LinkedInterimPayload.


        """
        linking_strategy_map: dict[LinkingStrategy, Callable] = {  # extend as needed.
            LinkingStrategy.MAPPING_FILE: self._get_linkages_mapping_file,
            LinkingStrategy.FILENAME_AUTHOR_YEAR_LONGEST: partial(
                self._get_linkages_filename_author_year, "longest"
            ),
            LinkingStrategy.FILENAME_AUTHOR_YEAR_LAST: partial(
                self._get_linkages_filename_author_year, "last"
            ),
            LinkingStrategy.FILENAME_ID: self._get_linkages_filename_id,
        }

        return linking_strategy_map[linking_strategy]

    def _get_linkages_mapping_file(self) -> Generator[LinkedInterimPayload]:
        """
        Yield linkages between files (pdf/md) and document_ids.


        This is contingent on
        - successful reading of a mapping file (csv/json) and
          storing in self.document_reference_mapping
        - there being at least 1 document that matches the ids
          therein.


        Yields:
             Generator[LinkedInterimPayload].

        """
        if (
            self.document_reference_mapping is not None
            and False
            in [
                isinstance(x, DocumentReferenceMapping)
                for x in self.document_reference_mapping
            ]
        ) or self.document_reference_mapping is None:
            bad_mapping = (
                "self.document_reference_mapping needs to be a list of "
                " DocumentReferenceMapping objects."
                f"actual: {type(self.document_reference_mapping)}"
            )
            raise TypeError(bad_mapping)

        for mapping in self.document_reference_mapping:
            # get the right unlinked doc (reference)
            unlinked_doc = self._references_by_id.get(mapping.document_id)
            if unlinked_doc is None:
                logger.debug(
                    f"no reference document found for id {mapping.document_id}. next!"
                )
                continue
            interim_payload_dict = {
                **mapping.model_dump(),
                "unlinked_document": unlinked_doc,
            }
            yield LinkedInterimPayload(**interim_payload_dict)

    def _get_linkages_filename_author_year(
        self, substring_strategy: Literal["longest", "last"]
    ) -> Generator[LinkedInterimPayload]:
        """
        Yield linkages between files and reference-documents based on
        best guess at `author_year.pdf` filename structure.


        Yields:
            Generator[LinkedInterimPayload]:


        """
        # we could type-check or existence-check self.document_reference_mapping
        # here; but there might be an instance in which the user chooses this
        # regardless of other situation, so really this should be handled elsewhere.
        # necessary condition for this method to atempt is only availability of
        # files.
        if not self.document_base_dir or not self.document_base_dir.is_dir():
            bad_base_dir = "self.document_base_dir needs to be a valid directory."
            raise ValueError(bad_base_dir)

        lookup_dict: dict[str, Document]
        if substring_strategy == "longest":
            lookup_dict = self._references_by_author_year_longest
        elif substring_strategy == "last":
            lookup_dict = self._references_by_author_year_last
        else:
            bad_strat_err = "unimnplemented substring strategy for finding author"
            raise NotImplementedError(bad_strat_err)
        logger.debug(f"last name strategy is {substring_strategy}")

        for file in self.document_base_dir.iterdir():
            if file.suffix not in [".md", ".pdf"]:
                logger.warning(f"file {file} is not pdf/md. next!")
                continue
            author_year_guess = file.name.split(".")[0].lower()
            unlinked_doc = lookup_dict.get(author_year_guess)
            if unlinked_doc is None:
                logger.debug(
                    f"no reference document found for id {author_year_guess}. next!"
                )
                continue
            if unlinked_doc.document_identity is None:
                unlinked_doc.init_document_identity()

            yield LinkedInterimPayload(
                document_id=unlinked_doc.document_identity.document_id,  # type:ignore[union-attr, arg-type]
                file_path=file,
                format=cast("Literal['md', 'pdf']", file.suffix[1:]),
                unlinked_document=unlinked_doc,
            )

    def _get_linkages_filename_id(self) -> Generator[LinkedInterimPayload]:
        """
        Yield linkages between files and reference-documents based on
        assumption that files are named `id.pdf`, e.g `12345678.pdf`.


        Yields:
            Generator[LinkedInterimPayload]:


        """
        if not self.document_base_dir or not self.document_base_dir.is_dir():
            bad_base_dir = "self.document_base_dir needs to be a valid directory."
            raise ValueError(bad_base_dir)

        for file in self.document_base_dir.iterdir():
            if file.suffix not in [".md", ".pdf"]:
                logger.warning(f"file {file} is not pdf/md. next!")
                continue
            id_guess = int(file.name.split(".")[0])
            unlinked_doc = self._references_by_id.get(id_guess)
            if unlinked_doc is None:
                logger.debug(f"no reference document found for id {id_guess}. next!")
                continue
            if unlinked_doc.document_identity is None:
                unlinked_doc.init_document_identity()
            yield LinkedInterimPayload(
                document_id=unlinked_doc.document_identity.document_id,  # type:ignore[union-attr, arg-type]
                file_path=file,
                format=cast("Literal['md', 'pdf']", file.suffix[1:]),
                unlinked_document=unlinked_doc,
            )

    @staticmethod
    def _parse_pdf(
        path_to_pdf: Path, *, return_images: bool = False, return_metadata: bool = False
    ) -> ParsedOutput:
        """
        Parse a pdf to ParsedOutput.


        NOTE: wrapper around DocumentParser.parse().


        Args:
            path_to_pdf (Path): where to find the file
            return_images (bool, optional): store images or not. Defaults to False.
            return_metadata (bool, optional): store metadata or not. Defaults to False.


        Raises:
            TypeError: when it's not a pdf file.


        Returns:
            ParsedOutput: a container for markdown, images and metadata.


        """
        if path_to_pdf.suffix != ".pdf":
            not_pdf = "need a valid pdf file."
            raise TypeError(not_pdf)
        return parser(
            input_=path_to_pdf,
            return_images=return_images,
            return_metadata=return_metadata,
        )

    @staticmethod
    def link_reference_parsed_document(
        reference: Document,
        parsed_output: ParsedOutput,
        original_filepath: Path | None = None,
    ) -> Document:
        """
        Link a reference, in `Document` format with a parsed document,
        in ParsedOutput format.


        Args:
            reference (Document): the reference, e.g. 'document' from eppi json.
            parsed_output (ParsedOutput): parser output.
            original_filepath (Path | None, optional): Defaults to None.


        Returns:
            Document: a linked Document with
            all required fields populated, and is_linked==True,
            and required fields' presence validated.


        """
        if not reference.document_identity:
            logger.debug(
                "initialising document identity for "
                f"reference with id {reference.document_id}."
            )

            reference.init_document_identity()

        logger.debug(
            f"adding parsed_output to reference with id {reference.document_id}."
        )
        reference.link_parsed_document(
            parsed_document=parsed_output,
            original_doc_filepath=original_filepath,
        )

        # set context & context-type
        reference.set_context_from_parsed()
        # reference.context_type = ContextType.FULL_DOCUMENT

        # set is_linked -- this will raise a ValidationError
        # if min requirements are not met
        reference.is_final = True
        reference.is_linked = True

        return reference

    def link_many_references_parsed_documents(
        self,
        *,
        return_images: bool = False,
        return_metadata: bool = False,
    ) -> list[Document]:
        """
        Link multiple references to parsed
        documents using available LinkingStrategy(s).


        Iterates over linking strategies in
        hierarchical order, attempting to link
        each reference-doc to its corresponding file.


        If required, parses files.
        Creates Document objects where is_linked=True.


        Args:
            return_images: Whether to include images in parsed output
            return_metadata: Whether to include metadata in parsed output


        Returns:
            List of Document objects successfully linked and parsed


        """
        linked_documents: list[Document] = []
        processed_doc_ids = set()
        n_docs_to_link = len(self.documents_references)

        for strategy in self.linking_strategies:
            if len(linked_documents) == n_docs_to_link:
                logger.info("all linking jobs completed.")
                break
            logger.info(f"Attempting linking strategy: {strategy}")

            try:
                linking_function = self._create_linking_factory(strategy)  # type:ignore[arg-type] #mypy is wrong here...

                for interim_payload in linking_function():
                    if interim_payload.document_id in processed_doc_ids:
                        logger.debug(
                            f"document {interim_payload.document_id} "
                            " already linked, next"
                        )
                        continue

                    # parse only if pdf.
                    if interim_payload.format == "pdf":
                        parsed_output = self._parse_pdf(
                            interim_payload.file_path,
                            return_images=return_images,
                            return_metadata=return_metadata,
                        )
                    elif interim_payload.format == "md":
                        parsed_output = ParsedOutput(
                            text=interim_payload.file_path.read_text(encoding="utf-8"),
                            parser_library="unknown",
                        )
                    else:
                        logger.warning(
                            f"unsupported format {interim_payload.format} for "
                            f"document {interim_payload.document_id}, skipping"
                        )
                        continue

                    # link!
                    linked_doc = self.link_reference_parsed_document(
                        reference=interim_payload.unlinked_document,
                        parsed_output=parsed_output,
                        original_filepath=interim_payload.file_path,
                    )

                    linked_documents.append(linked_doc)
                    processed_doc_ids.add(interim_payload.document_id)

                    logger.info(
                        f"successfully linked document {interim_payload.document_id} "
                        f"with file {interim_payload.file_path} "
                        f"using {strategy}"
                    )

            except (TypeError, ValueError) as e:
                # if i + 1 < len(self.linking_strategies):
                logger.error(f"Error with linking strategy {strategy}: {e}")
                continue
                # raise

        total_refs = len(self.documents_references)
        linked_count = len(linked_documents)
        logger.info(
            f"linking complete: {linked_count}/{total_refs} "
            "references successfully linked"
        )

        if linked_count < total_refs:
            unlinked_ids = [
                doc.document_identity.document_id  # type:ignore[union-attr]
                for doc in self.documents_references
                if doc.document_identity.document_id not in processed_doc_ids  # type:ignore[operator, union-attr]
            ]
            logger.warning(f"Unlinked document IDs: {unlinked_ids}")

        return linked_documents
__init__(references, document_reference_mapping=None, document_base_dir=None, parser=parser, linking_strategies=None)

Initialise DocumentReferenceLinker class.

Source code in deet/processors/linker.py 
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
def __init__(
    self,
    references: Sequence[Document],
    document_reference_mapping: list[DocumentReferenceMapping] | Path | None = None,
    document_base_dir: Path | None = None,
    parser: DocumentParser = parser,
    linking_strategies: list[LinkingStrategy] | None = None,
) -> None:
    """Initialise DocumentReferenceLinker class."""
    # deep copies to ensure lookup tables don't modify validators
    # on our existing reference docuemnts.
    tmp_refs = [doc.model_copy(deep=True) for doc in references]
    self.documents_references = references

    # lookup dics for O(1) id & author_year based lookup
    self._references_by_id: dict[int, Document] = {
        doc.document_identity.document_id: doc  # type:ignore[union-attr]
        for doc in tmp_refs
        if doc.document_identity.document_id is not None  # type:ignore[union-attr]
    }
    logger.debug(self._references_by_id.keys())
    try:
        self._references_by_author_year_longest: dict[str, Document] = {
            doc.author_year_from_document_identity(
                substring_strategy="longest"
            ): doc
            for doc in tmp_refs
        }
    except ValueError:
        logger.warning("author or year missing, returning empty dict.")
        self._references_by_author_year_longest = {}

    try:
        self._references_by_author_year_last: dict[str, Document] = {
            doc.author_year_from_document_identity(substring_strategy="last"): doc
            for doc in tmp_refs
        }
    except ValueError:
        logger.warning("author or year missing, returning empty dict.")
        self._references_by_author_year_last = {}

    self.document_base_dir = document_base_dir
    if isinstance(document_reference_mapping, Path):
        logger.debug(
            f"document_reference_mapping type: {type(document_reference_mapping)}"
        )
        document_reference_mapping = MappingImporter(
            mapping_file_path=document_reference_mapping,
            document_base_dir=document_base_dir,
        ).import_mapping()
    self.document_reference_mapping = document_reference_mapping
    self.parser = parser

    self.linking_strategies = linking_strategies or self.LINKING_STRATEGY_HIERARCHY

Link multiple references to parsed documents using available LinkingStrategy(s).

Iterates over linking strategies in hierarchical order, attempting to link each reference-doc to its corresponding file.

If required, parses files. Creates Document objects where is_linked=True.

Parameters:

Name Type Description Default
return_images bool

Whether to include images in parsed output

 False
return_metadata bool

Whether to include metadata in parsed output

 False

Returns:

Type Description
list[Document]

List of Document objects successfully linked and parsed
Source code in deet/processors/linker.py 
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
def link_many_references_parsed_documents(
    self,
    *,
    return_images: bool = False,
    return_metadata: bool = False,
) -> list[Document]:
    """
    Link multiple references to parsed
    documents using available LinkingStrategy(s).


    Iterates over linking strategies in
    hierarchical order, attempting to link
    each reference-doc to its corresponding file.


    If required, parses files.
    Creates Document objects where is_linked=True.


    Args:
        return_images: Whether to include images in parsed output
        return_metadata: Whether to include metadata in parsed output


    Returns:
        List of Document objects successfully linked and parsed


    """
    linked_documents: list[Document] = []
    processed_doc_ids = set()
    n_docs_to_link = len(self.documents_references)

    for strategy in self.linking_strategies:
        if len(linked_documents) == n_docs_to_link:
            logger.info("all linking jobs completed.")
            break
        logger.info(f"Attempting linking strategy: {strategy}")

        try:
            linking_function = self._create_linking_factory(strategy)  # type:ignore[arg-type] #mypy is wrong here...

            for interim_payload in linking_function():
                if interim_payload.document_id in processed_doc_ids:
                    logger.debug(
                        f"document {interim_payload.document_id} "
                        " already linked, next"
                    )
                    continue

                # parse only if pdf.
                if interim_payload.format == "pdf":
                    parsed_output = self._parse_pdf(
                        interim_payload.file_path,
                        return_images=return_images,
                        return_metadata=return_metadata,
                    )
                elif interim_payload.format == "md":
                    parsed_output = ParsedOutput(
                        text=interim_payload.file_path.read_text(encoding="utf-8"),
                        parser_library="unknown",
                    )
                else:
                    logger.warning(
                        f"unsupported format {interim_payload.format} for "
                        f"document {interim_payload.document_id}, skipping"
                    )
                    continue

                # link!
                linked_doc = self.link_reference_parsed_document(
                    reference=interim_payload.unlinked_document,
                    parsed_output=parsed_output,
                    original_filepath=interim_payload.file_path,
                )

                linked_documents.append(linked_doc)
                processed_doc_ids.add(interim_payload.document_id)

                logger.info(
                    f"successfully linked document {interim_payload.document_id} "
                    f"with file {interim_payload.file_path} "
                    f"using {strategy}"
                )

        except (TypeError, ValueError) as e:
            # if i + 1 < len(self.linking_strategies):
            logger.error(f"Error with linking strategy {strategy}: {e}")
            continue
            # raise

    total_refs = len(self.documents_references)
    linked_count = len(linked_documents)
    logger.info(
        f"linking complete: {linked_count}/{total_refs} "
        "references successfully linked"
    )

    if linked_count < total_refs:
        unlinked_ids = [
            doc.document_identity.document_id  # type:ignore[union-attr]
            for doc in self.documents_references
            if doc.document_identity.document_id not in processed_doc_ids  # type:ignore[operator, union-attr]
        ]
        logger.warning(f"Unlinked document IDs: {unlinked_ids}")

    return linked_documents

Link a reference, in Document format with a parsed document, in ParsedOutput format.

Parameters:

Name Type Description Default
reference Document

the reference, e.g. 'document' from eppi json.

 required
parsed_output ParsedOutput

parser output.

 required
original_filepath Path | None

Defaults to None.

 None

Returns:

Name Type Description
Document Document

a linked Document with
Document

all required fields populated, and is_linked==True,
Document

and required fields' presence validated.
Source code in deet/processors/linker.py 
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
@staticmethod
def link_reference_parsed_document(
    reference: Document,
    parsed_output: ParsedOutput,
    original_filepath: Path | None = None,
) -> Document:
    """
    Link a reference, in `Document` format with a parsed document,
    in ParsedOutput format.


    Args:
        reference (Document): the reference, e.g. 'document' from eppi json.
        parsed_output (ParsedOutput): parser output.
        original_filepath (Path | None, optional): Defaults to None.


    Returns:
        Document: a linked Document with
        all required fields populated, and is_linked==True,
        and required fields' presence validated.


    """
    if not reference.document_identity:
        logger.debug(
            "initialising document identity for "
            f"reference with id {reference.document_id}."
        )

        reference.init_document_identity()

    logger.debug(
        f"adding parsed_output to reference with id {reference.document_id}."
    )
    reference.link_parsed_document(
        parsed_document=parsed_output,
        original_doc_filepath=original_filepath,
    )

    # set context & context-type
    reference.set_context_from_parsed()
    # reference.context_type = ContextType.FULL_DOCUMENT

    # set is_linked -- this will raise a ValidationError
    # if min requirements are not met
    reference.is_final = True
    reference.is_linked = True

    return reference

DocumentReferenceMapping pydantic-model

Bases: BaseModel

Data model for incoming, manual mappings of references (via integer ids) to documents, via filename.

Fields:

  • document_id (int)
  • file_path (Path)
  • format (Literal['md', 'pdf'] | None)

Validators:

Source code in deet/processors/linker.py 
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
class DocumentReferenceMapping(BaseModel):
    """
    Data model for incoming, manual mappings
    of references (via integer ids) to
    documents, via filename.
    """

    document_id: int
    file_path: Path
    # NOTE: may want to inherit this from supported
    # formats in parser.py
    format: Literal["md", "pdf"] | None = None

    @field_validator("document_id", mode="before")
    @classmethod
    def ensure_valid_doc_id(cls, value: int) -> int:
        """Ensure supplied document_id has a valid number of digits."""
        int_value = int(value)
        if int_value <= 0:
            invalid_int_id_error = (
                f"`document_id` must be a positive integer. Supplied: {int_value}"
            )
            raise ValueError(invalid_int_id_error)
        num_digits = len(str(abs(int_value)))
        if not (MIN_DOCUMENT_ID_DIGITS <= num_digits <= MAX_DOCUMENT_ID_DIGITS):
            val_err = (
                f"`document_id` must be between {MIN_DOCUMENT_ID_DIGITS} "
                f"and {MAX_DOCUMENT_ID_DIGITS} digits. Supplied: {int_value}"
            )
            raise ValueError(val_err)
        return int_value

    @model_validator(mode="after")
    def ensure_file_exists(self) -> Self:
        """
        Ensure either md_path or pdf_path are populated,
        and the associated file exists.
        """
        # check path resolves
        if not self.file_path.is_file():
            not_a_file = f"{self.file_path} is not a file."
            raise ValueError(not_a_file)
        # check suffix
        if self.file_path.suffix not in [".md", ".pdf"]:
            unsupported = (
                f"{self.file_path.suffix} is not supported. either .md or .pdf"
            )
            raise ValueError(unsupported)
        self.format = self.file_path.suffix[1:]  # type:ignore[assignment] # omit .

        return self
ensure_file_exists() pydantic-validator

Ensure either md_path or pdf_path are populated, and the associated file exists.

Source code in deet/processors/linker.py 
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
@model_validator(mode="after")
def ensure_file_exists(self) -> Self:
    """
    Ensure either md_path or pdf_path are populated,
    and the associated file exists.
    """
    # check path resolves
    if not self.file_path.is_file():
        not_a_file = f"{self.file_path} is not a file."
        raise ValueError(not_a_file)
    # check suffix
    if self.file_path.suffix not in [".md", ".pdf"]:
        unsupported = (
            f"{self.file_path.suffix} is not supported. either .md or .pdf"
        )
        raise ValueError(unsupported)
    self.format = self.file_path.suffix[1:]  # type:ignore[assignment] # omit .

    return self
ensure_valid_doc_id(value) pydantic-validator

Ensure supplied document_id has a valid number of digits.

Source code in deet/processors/linker.py 
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
@field_validator("document_id", mode="before")
@classmethod
def ensure_valid_doc_id(cls, value: int) -> int:
    """Ensure supplied document_id has a valid number of digits."""
    int_value = int(value)
    if int_value <= 0:
        invalid_int_id_error = (
            f"`document_id` must be a positive integer. Supplied: {int_value}"
        )
        raise ValueError(invalid_int_id_error)
    num_digits = len(str(abs(int_value)))
    if not (MIN_DOCUMENT_ID_DIGITS <= num_digits <= MAX_DOCUMENT_ID_DIGITS):
        val_err = (
            f"`document_id` must be between {MIN_DOCUMENT_ID_DIGITS} "
            f"and {MAX_DOCUMENT_ID_DIGITS} digits. Supplied: {int_value}"
        )
        raise ValueError(val_err)
    return int_value

LinkedInterimPayload pydantic-model

Bases: DocumentReferenceMapping

Interim output from a linking factory method; extending DocumentReferenceMapping.

Interim as the document may a) still need to be parsed, and b) still needs to be coerced into ParsedOutput.

Fields:

Validators:

Source code in deet/processors/linker.py 
86
87
88
89
90
91
92
93
94
95
96
97
class LinkedInterimPayload(DocumentReferenceMapping):
    """
    Interim output from a linking factory method; extending
    DocumentReferenceMapping.


    Interim as the document may
    a) still need to be parsed, and
    b) still needs to be coerced into ParsedOutput.
    """

    unlinked_document: Document

LinkingStrategy

Bases: StrEnum

Enum of permitted/implemented ref<>parsed_doc linking strategies.

Source code in deet/processors/linker.py 
22
23
24
25
26
27
28
class LinkingStrategy(StrEnum):
    """Enum of permitted/implemented ref<>parsed_doc linking strategies."""

    MAPPING_FILE = auto()
    FILENAME_AUTHOR_YEAR_LONGEST = auto()
    FILENAME_AUTHOR_YEAR_LAST = auto()
    FILENAME_ID = auto()

MappingImporter

Tool for importing manual mappings from csv/json to list[DocumentReferenceMapping].

Source code in deet/processors/linker.py 
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
class MappingImporter:
    """
    Tool for importing manual mappings from csv/json
    to list[DocumentReferenceMapping].
    """

    def __init__(
        self, mapping_file_path: Path, document_base_dir: Path | None = None
    ) -> None:
        """
        Initialise MappingImporter instance.


        Args:
            mapping_file_path (Path): Path to csv/json file containing mappings.
            document_base_dir (Path | None, optional): Optional directory path
            where documents (pdf/md) live. Defaults to None.


        """
        if mapping_file_path.suffix not in [".csv", ".json"]:
            bad_file = "mapping file needs to be supplied in csv or json format."
            raise ValueError(bad_file)

        if document_base_dir and not document_base_dir.is_dir():
            bad_dir = "document_base_dir is optional, but must be a dir if supplied."
            raise ValueError(bad_dir)

        self.mapping_file_path = mapping_file_path
        self.mapping_file_type = mapping_file_path.suffix[1:]
        self.document_base_dir = document_base_dir

    def import_mapping(self) -> list[DocumentReferenceMapping]:
        """Parse a csv/json file to a list od DocumentReferenceMapping objects."""
        if self.mapping_file_type == "json":
            logger.debug("importing mapping from json")
            payload = self._load_json()
        elif self.mapping_file_type == "csv":
            logger.debug("importing mapping from csv")
            payload = self._load_csv()
        else:
            bad_file = "only json or csv files are supported."
            raise ValueError(bad_file)

        return [
            DocumentReferenceMapping(document_id=doc_id, file_path=file_path)
            for doc_id, file_path in payload.items()
        ]

    def _load_json(self) -> dict[int, Path]:
        """
        Load json file with mappings to dict.


        NOTE: json can be in...
        - array style:
        [
        {"document_id": 12345678, "file_path": "path/to/file.pdf"},
        {"document_id": 87654321, "file_path": "path/to/other.md"}
        ]


        - dict style:
        {
        "12345678": "path/to/file.pdf",
        "87654321": "path/to/other.md"
        }


        Raises:
            JsonStyleError: if bad json style.


        Returns:
            dict[int, Path]: the pre-validation object.


        """
        with self.mapping_file_path.open() as f:
            data = json.load(f)

        result = {}

        # list[dict] (array) style
        if isinstance(data, list):
            logger.debug("json format is array.")
            for item in data:
                doc_id = int(item["document_id"])
                file_path = self._resolve_file_path(item["file_path"])
                if file_path:
                    logger.debug(f"file path {file_path} resolved, adding dict entry.")
                    result[doc_id] = Path(file_path)
                logger.debug(
                    f"file path {file_path} not resoved resolved, adding dict entry."
                )

        # dict style
        elif isinstance(data, dict):
            logger.debug("json format is dict.")
            for doc_id_str, file_path in data.items():
                doc_id = int(doc_id_str)
                file_path_out = self._resolve_file_path(file_path)
                if file_path_out:
                    logger.debug(
                        f"file path {file_path_out} resolved, adding dict entry."
                    )
                    result[doc_id] = Path(file_path_out)
                logger.debug(
                    f"file path {file_path_out} not resolved, not adding dict entry."
                )

        else:
            bad_json = "json must be either a list(array) or dict format."
            raise JsonStyleError(bad_json)

        return result

    def _load_csv(self) -> dict[int, Path]:
        result = {}

        with self.mapping_file_path.open(newline="", encoding="utf-8") as f:
            reader = csv.DictReader(f)

            # ensure required columns exist
            if (
                reader.fieldnames is None
                or "document_id" not in reader.fieldnames
                or "file_path" not in reader.fieldnames
            ):
                required_cols = "csv must contain 'document_id' and 'file_path' columns"
                raise ValueError(required_cols)

            for row in reader:
                doc_id = int(row["document_id"])
                file_path = self._resolve_file_path(row["file_path"])
                if file_path:
                    logger.debug(f"file path {file_path} resolved, adding dict entry.")
                    result[doc_id] = Path(file_path)
                logger.debug(
                    f"file path not {file_path} resolved, not adding dict entry."
                )

        return result

    @staticmethod
    def merge_partial_paths(parts_a: list[str], parts_b: list[str]) -> list[str]:
        """
        Merge partial file path components.

        Implements the Knuth-Morris-Pratt (KMP) algorithm. Nice!
        https://en.wikipedia.org/wiki/Knuth%E2%80%93Morris%E2%80%93Pratt_algorithm

        Args:
            parts_a (list[str]): the longer of the path parts
            parts_b (list[str]): the shorter of the path parts.

        Returns:
            list[str]: the merged combined path

        """
        # build combined list with sentinel
        sentinel_combined = [*parts_b, None, *parts_a]

        # prefix function (KMP)
        pi = [0] * len(sentinel_combined)  # this is the nomenclature KMP uses
        for i in range(1, len(sentinel_combined)):
            j = pi[i - 1]
            while j > 0 and sentinel_combined[i] != sentinel_combined[j]:
                j = pi[j - 1]
            if sentinel_combined[i] == sentinel_combined[j]:
                j += 1
            pi[i] = j

        overlap = pi[-1]  # length of prefix of b matching suffix of a

        return parts_a[: len(parts_a) - overlap] + parts_b

    def _resolve_file_path(self, file_path: str | None | Path) -> Path | None:
        """
        Resolve file path, handling absolute
        paths and document_base_dir.


        Args:
            file_path: File path from mapping file
            (can be relative or absolute)


        Returns:
            Resolved Path object


        Raises:
            FileNotFoundError: If resolved path doesn't exist


        """
        if file_path == "" or file_path is None:
            return None
        file_path = Path(file_path)

        # if already findable, use as-is
        if file_path.exists():
            logger.debug(f"Using absolute path: {file_path.absolute()!s}")
            return file_path.absolute()

        # if document_base_dir provided, prepend it -- if appropriate
        if not self.document_base_dir or not self.document_base_dir.is_dir():
            dir_missing = (
                f"self.document_base_dir {self.document_base_dir!s}"
                " does not exist or is not defined."
            )
            raise FileNotFoundError(dir_missing)

        # try finding a file that simply appends to base path and exists
        if (self.document_base_dir / file_path).exists():
            return (self.document_base_dir / file_path).absolute()

        # now we're facing a situation where we try to resolve it all...
        base_dir_parts = list(self.document_base_dir.absolute().parts)
        file_path_parts = list(file_path.parts)
        merged_partials = self.merge_partial_paths(base_dir_parts, file_path_parts)
        merged_partial_path = Path("/".join(merged_partials))
        if merged_partial_path.exists():
            return merged_partial_path
        cant_resolve = f"merged file path {merged_partial_path!s} cant be resolved."
        raise FileNotFoundError(cant_resolve)
__init__(mapping_file_path, document_base_dir=None)

Initialise MappingImporter instance.

Parameters:

Name Type Description Default
mapping_file_path Path

Path to csv/json file containing mappings.

 required
document_base_dir Path | None

Optional directory path

 None
Source code in deet/processors/linker.py 
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
def __init__(
    self, mapping_file_path: Path, document_base_dir: Path | None = None
) -> None:
    """
    Initialise MappingImporter instance.


    Args:
        mapping_file_path (Path): Path to csv/json file containing mappings.
        document_base_dir (Path | None, optional): Optional directory path
        where documents (pdf/md) live. Defaults to None.


    """
    if mapping_file_path.suffix not in [".csv", ".json"]:
        bad_file = "mapping file needs to be supplied in csv or json format."
        raise ValueError(bad_file)

    if document_base_dir and not document_base_dir.is_dir():
        bad_dir = "document_base_dir is optional, but must be a dir if supplied."
        raise ValueError(bad_dir)

    self.mapping_file_path = mapping_file_path
    self.mapping_file_type = mapping_file_path.suffix[1:]
    self.document_base_dir = document_base_dir
import_mapping()

Parse a csv/json file to a list od DocumentReferenceMapping objects.

Source code in deet/processors/linker.py 
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
def import_mapping(self) -> list[DocumentReferenceMapping]:
    """Parse a csv/json file to a list od DocumentReferenceMapping objects."""
    if self.mapping_file_type == "json":
        logger.debug("importing mapping from json")
        payload = self._load_json()
    elif self.mapping_file_type == "csv":
        logger.debug("importing mapping from csv")
        payload = self._load_csv()
    else:
        bad_file = "only json or csv files are supported."
        raise ValueError(bad_file)

    return [
        DocumentReferenceMapping(document_id=doc_id, file_path=file_path)
        for doc_id, file_path in payload.items()
    ]
merge_partial_paths(parts_a, parts_b) staticmethod

Merge partial file path components.

Implements the Knuth-Morris-Pratt (KMP) algorithm. Nice! https://en.wikipedia.org/wiki/Knuth%E2%80%93Morris%E2%80%93Pratt_algorithm

Parameters:

Name Type Description Default
parts_a list[str]

the longer of the path parts

 required
parts_b list[str]

the shorter of the path parts.

 required

Returns:

Type Description
list[str]

list[str]: the merged combined path
Source code in deet/processors/linker.py 
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
@staticmethod
def merge_partial_paths(parts_a: list[str], parts_b: list[str]) -> list[str]:
    """
    Merge partial file path components.

    Implements the Knuth-Morris-Pratt (KMP) algorithm. Nice!
    https://en.wikipedia.org/wiki/Knuth%E2%80%93Morris%E2%80%93Pratt_algorithm

    Args:
        parts_a (list[str]): the longer of the path parts
        parts_b (list[str]): the shorter of the path parts.

    Returns:
        list[str]: the merged combined path

    """
    # build combined list with sentinel
    sentinel_combined = [*parts_b, None, *parts_a]

    # prefix function (KMP)
    pi = [0] * len(sentinel_combined)  # this is the nomenclature KMP uses
    for i in range(1, len(sentinel_combined)):
        j = pi[i - 1]
        while j > 0 and sentinel_combined[i] != sentinel_combined[j]:
            j = pi[j - 1]
        if sentinel_combined[i] == sentinel_combined[j]:
            j += 1
        pi[i] = j

    overlap = pi[-1]  # length of prefix of b matching suffix of a

    return parts_a[: len(parts_a) - overlap] + parts_b

parser

Utilities for parsing input files (e.g. pdf) into output files (e.g. md).

DocumentParser

Parse documents from target format to other target format.

Source code in deet/processors/parser.py 
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
class DocumentParser:
    """Parse documents from target format to other target format."""

    DEFAULT_PARSERS: dict[str, type[ParserLibrary]] = {
        "pdf": PdfminerParser,
        "epub": PandocParser,
        "html": PandocParser,
        "xml": PandocParser,
    }

    def __init__(
        self, parsers: dict[str, type[ParserLibrary]] = DEFAULT_PARSERS
    ) -> None:
        """
        Initialise instance of DocumentParser with default parsers.
        Default parsers are in dict.

        """
        self.parsers = parsers

        if self.parsers is not None and isinstance(self.parsers, dict):
            for parser_name, parser in self.parsers.items():
                logger.debug(f"default {parser_name} parser: {parser.name}")

    def __call__(  # noqa: PLR0913 - img/meta needs to be explicit (non-kwargs) here.
        self,
        input_: str | PathLike,
        out_path: str | PathLike | None = None,
        parser: type[ParserLibrary] | None = None,
        input_type: InputFileType | str | None = None,
        *,
        return_images: bool = False,
        return_metadata: bool = False,
        **kwargs,
    ) -> ParsedOutput:
        """
        Run the parser on one input_.

        Args:
            input_ (str | PathLike): File(path) or str of input_.
            out_path (str | PathLike | None): If None, return parsed content as str.
            parser (ParserLibrary | None, optional): Defaults to None.
                If None, uses the default parser.
            input_type (InputFileType | None, optional): Defaults to None.
                If None, infers file type using `detect_filetype`.
            return_images (bool): Defaults to False. Whether to write
                parsed images (JPEG) to file, or not. `out_path`. can't be None.
            return_metadata (bool): Defaults to None. Whether to write
                parsed metadata (json).

        Returns:
            str: ParsedOutput object.

        """
        logger.debug(f"kwargs: {kwargs}")
        if input_type is None:
            logger.debug(
                "no input file type provided. using `detect_filetype` to infer."
            )
            try:
                input_type = InputFileType(
                    self.detect_filetype(
                        file=input_,
                        permitted_file_enum_list=list(InputFileType),
                    )
                )
            except ValueError as ve:
                raise InvalidInputFileTypeError(ve) from ve
        logger.debug(f"input file type: {input_type}.")

        if parser is not None and (
            (not isinstance(parser, type)) or (not issubclass(parser, ParserLibrary))
        ):
            bad_parser_err = f"parser {parser} is not a valid ParserLibrary."
            raise FileParserMismatchError(bad_parser_err)
        if parser is None and input_type is not None:
            logger.debug("parser not supplied. selecting default parser for file_type.")
            if isinstance(input_type, str):
                try:
                    input_type = InputFileType(input_type)
                except ValueError as ve:
                    if "is not a valid InputFileType" in str(ve):
                        invalid_input_ft = f"{input_type} is not a valid InputFileType"
                        raise InvalidInputFileTypeError(invalid_input_ft) from ve
            if (
                self.parsers is None
                or (isinstance(input_type, str) and input_type not in self.parsers)
                or (
                    isinstance(input_type, InputFileType)
                    and input_type.value not in self.parsers
                )
            ):
                missing_parser = "no parser supplied."
                raise ValueError(missing_parser)
            if isinstance(input_type, InputFileType):
                parser = self.parsers[input_type.value]
            elif isinstance(input_type, str):
                parser = self.parsers[input_type]
            else:
                missing_parser = "no parser supplied."
                raise ValueError(missing_parser)
        logger.debug(f"parser: {parser}.")
        kwargs["input_type"] = input_type

        parsed = self.parse(
            input_=input_,
            parser=parser,
            return_images=return_images,
            return_metadata=return_metadata,
            **kwargs,
        )

        if out_path:
            self.write_files(
                out_path=out_path,
                parser=parser,
                write_metadata=return_metadata,
                write_images=return_images,
                text=parsed.text,
                metadata=parsed.metadata,
                images=parsed.images,
            )

        return parsed

    def parse(
        self,
        input_: str | PathLike,
        parser: type[ParserLibrary],
        *,
        return_metadata: bool = False,
        return_images: bool = False,
        **kwargs,
    ) -> ParsedOutput:
        """
        Parse target file.
        Wraps around specific parser methods.

        Args:
            input_ (str | PathLike):
            input_type (InputFileType):
            parser (ParserLibrary):
            parse_method (Callable[[str  |  PathLike, ParserLibrary], str]):

        Returns:
            str: _description_

        """
        logger.debug(f"kwargs: {kwargs}")
        if return_metadata and OutputFileType.JSON not in parser.output_file_types:
            metadata_not_allowed = (
                f"metadata out not permitted for parser {parser.name}."
            )
            raise InvalidOutputFileTypeError(metadata_not_allowed)
        if return_images and OutputFileType.JPEG not in parser.output_file_types:
            images_not_allowed = f"images out not permitted for parser {parser.name}."
            raise InvalidOutputFileTypeError(images_not_allowed)

        return parser.parse(
            input_=input_,
            return_metadata=return_metadata,
            return_images=return_images,
            **kwargs,
        )

    @staticmethod
    def detect_filetype(
        file: str | PathLike,
        permitted_file_enum_list: list[InputFileType]
        | list[OutputFileType]
        | list[str]
        | None = None,
    ) -> str:
        """
        Detect file type from a file_path.

        Args:
            file (str | PathLike): _description_

        Raises:
            InvalidInputFileTypeError: If file extension isn't permitted.

        Returns:
            InputFileType: _description_

        """
        if permitted_file_enum_list is None:
            permitted_file_enum_list = list(InputFileType) + list(OutputFileType)
        permitted_extensions_str = {
            x.value
            for x in permitted_file_enum_list
            if isinstance(x, (InputFileType | OutputFileType))
        }

        extension = str(file).split(".")[-1]
        if extension not in permitted_extensions_str:
            has_input = any(
                isinstance(ft, InputFileType) for ft in permitted_file_enum_list
            )
            has_output = any(
                isinstance(ft, OutputFileType) for ft in permitted_file_enum_list
            )
            target_error: type[Exception]
            if has_input and not has_output:
                target_error = InvalidInputFileTypeError
            elif not has_input and has_output:
                target_error = InvalidOutputFileTypeError
            else:
                target_error = InvalidFileTypeError

            forbidden_file_type = (
                f"file type {extension} is not permitted. "
                f" Use one of {permitted_extensions_str}."
            )
            raise target_error(forbidden_file_type)

        logger.debug(f"filetype is: {extension}.")
        return extension

    @staticmethod
    def write_files(  # noqa: PLR0913
        out_path: str | PathLike,
        parser: type[ParserLibrary],
        *,
        write_metadata: bool,
        write_images: bool,
        text: str,
        metadata: dict | None = None,
        images: dict[str, Image] | None = None,
    ) -> None:
        """
        Write parsed content to file(s).

        NOTE: we are taking existence of `out_path` as an intention to
        write all requested objects to file. out_path can be a file or a dir.
        if out_path is a file, we write remaining files to parent dir.

        Args:
            out_path (str | PathLike): _description_
            write_metadata (bool): _description_
            write_images (bool): _description_
            text (str): _description_
            metadata (dict | None, optional): _description_. Defaults to None.
            images (dict[str, Image] | None, optional): _description_. Defaults to None.

        """
        extension = (
            DocumentParser.detect_filetype(  # should raise error if not permitted
                out_path, permitted_file_enum_list=parser.output_file_types
            )
        )

        required_outfiles = ["md"]
        if write_images:
            required_outfiles.append("jpeg")
            if images is None:  # or raise something?
                logger.warning(
                    "`write_images` set to True, but no images obj supplied."
                )
        if write_metadata:
            required_outfiles.append("json")
            if metadata is None:  # or raise something?
                logger.warning(
                    "`write_metadata` set to True, but no metadata obj supplied."
                )
        logger.debug(f"required outfiles: {required_outfiles}")
        if False in [ft in parser.output_file_types for ft in required_outfiles]:
            raise InvalidOutputFileTypeError

        Path(out_path).parent.mkdir(parents=True, exist_ok=True)

        if Path(out_path).is_file() or (extension in required_outfiles):
            logger.debug(f"`out_path` {out_path} points to a file.")

            dir_base = Path(out_path).parent
            filename_base = "".join(Path(out_path).name.split(".")[0:-1])

        if Path(out_path).is_dir():
            logger.debug(f"`out_path` {out_path} points to a dir.")
            # we now have to get our filename base from somewhere...
            dir_base = Path(out_path)
            filename_base = text.split("\n")[0][:15].replace(" ", "_").lower()

        for ext in required_outfiles:
            out = dir_base / (filename_base + "." + ext)
            logger.debug(f"writing out {ext} to {out}.")
            if ext == "md":
                out.write_text(text, encoding="utf-8")
            if ext == "json" and metadata is not None:
                out.write_text(json.dumps(metadata), encoding="utf-8")
            if ext == "jpeg" and images is not None:
                for img_name, img in images.items():
                    img_out = dir_base / (filename_base + "_" + img_name)
                    img.save(img_out, ext)
__call__(input_, out_path=None, parser=None, input_type=None, *, return_images=False, return_metadata=False, **kwargs)

Run the parser on one input_.

Parameters:

Name Type Description Default
input_ str | PathLike

File(path) or str of input_.

 required
out_path str | PathLike | None

If None, return parsed content as str.

 None
parser ParserLibrary | None

Defaults to None. If None, uses the default parser.

 None
input_type InputFileType | None

Defaults to None. If None, infers file type using detect_filetype.

 None
return_images bool

Defaults to False. Whether to write parsed images (JPEG) to file, or not. out_path. can't be None.

 False
return_metadata bool

Defaults to None. Whether to write parsed metadata (json).

 False

Returns:

Name Type Description
str ParsedOutput

ParsedOutput object.
Source code in deet/processors/parser.py 
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
def __call__(  # noqa: PLR0913 - img/meta needs to be explicit (non-kwargs) here.
    self,
    input_: str | PathLike,
    out_path: str | PathLike | None = None,
    parser: type[ParserLibrary] | None = None,
    input_type: InputFileType | str | None = None,
    *,
    return_images: bool = False,
    return_metadata: bool = False,
    **kwargs,
) -> ParsedOutput:
    """
    Run the parser on one input_.

    Args:
        input_ (str | PathLike): File(path) or str of input_.
        out_path (str | PathLike | None): If None, return parsed content as str.
        parser (ParserLibrary | None, optional): Defaults to None.
            If None, uses the default parser.
        input_type (InputFileType | None, optional): Defaults to None.
            If None, infers file type using `detect_filetype`.
        return_images (bool): Defaults to False. Whether to write
            parsed images (JPEG) to file, or not. `out_path`. can't be None.
        return_metadata (bool): Defaults to None. Whether to write
            parsed metadata (json).

    Returns:
        str: ParsedOutput object.

    """
    logger.debug(f"kwargs: {kwargs}")
    if input_type is None:
        logger.debug(
            "no input file type provided. using `detect_filetype` to infer."
        )
        try:
            input_type = InputFileType(
                self.detect_filetype(
                    file=input_,
                    permitted_file_enum_list=list(InputFileType),
                )
            )
        except ValueError as ve:
            raise InvalidInputFileTypeError(ve) from ve
    logger.debug(f"input file type: {input_type}.")

    if parser is not None and (
        (not isinstance(parser, type)) or (not issubclass(parser, ParserLibrary))
    ):
        bad_parser_err = f"parser {parser} is not a valid ParserLibrary."
        raise FileParserMismatchError(bad_parser_err)
    if parser is None and input_type is not None:
        logger.debug("parser not supplied. selecting default parser for file_type.")
        if isinstance(input_type, str):
            try:
                input_type = InputFileType(input_type)
            except ValueError as ve:
                if "is not a valid InputFileType" in str(ve):
                    invalid_input_ft = f"{input_type} is not a valid InputFileType"
                    raise InvalidInputFileTypeError(invalid_input_ft) from ve
        if (
            self.parsers is None
            or (isinstance(input_type, str) and input_type not in self.parsers)
            or (
                isinstance(input_type, InputFileType)
                and input_type.value not in self.parsers
            )
        ):
            missing_parser = "no parser supplied."
            raise ValueError(missing_parser)
        if isinstance(input_type, InputFileType):
            parser = self.parsers[input_type.value]
        elif isinstance(input_type, str):
            parser = self.parsers[input_type]
        else:
            missing_parser = "no parser supplied."
            raise ValueError(missing_parser)
    logger.debug(f"parser: {parser}.")
    kwargs["input_type"] = input_type

    parsed = self.parse(
        input_=input_,
        parser=parser,
        return_images=return_images,
        return_metadata=return_metadata,
        **kwargs,
    )

    if out_path:
        self.write_files(
            out_path=out_path,
            parser=parser,
            write_metadata=return_metadata,
            write_images=return_images,
            text=parsed.text,
            metadata=parsed.metadata,
            images=parsed.images,
        )

    return parsed
__init__(parsers=DEFAULT_PARSERS)

Initialise instance of DocumentParser with default parsers. Default parsers are in dict.

Source code in deet/processors/parser.py 
291
292
293
294
295
296
297
298
299
300
301
302
303
def __init__(
    self, parsers: dict[str, type[ParserLibrary]] = DEFAULT_PARSERS
) -> None:
    """
    Initialise instance of DocumentParser with default parsers.
    Default parsers are in dict.

    """
    self.parsers = parsers

    if self.parsers is not None and isinstance(self.parsers, dict):
        for parser_name, parser in self.parsers.items():
            logger.debug(f"default {parser_name} parser: {parser.name}")
detect_filetype(file, permitted_file_enum_list=None) staticmethod

Detect file type from a file_path.

Parameters:

Name Type Description Default
file str | PathLike

description

 required

Raises:

Type Description
InvalidInputFileTypeError

If file extension isn't permitted.

Returns:

Name Type Description
InputFileType str

description
Source code in deet/processors/parser.py 
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
@staticmethod
def detect_filetype(
    file: str | PathLike,
    permitted_file_enum_list: list[InputFileType]
    | list[OutputFileType]
    | list[str]
    | None = None,
) -> str:
    """
    Detect file type from a file_path.

    Args:
        file (str | PathLike): _description_

    Raises:
        InvalidInputFileTypeError: If file extension isn't permitted.

    Returns:
        InputFileType: _description_

    """
    if permitted_file_enum_list is None:
        permitted_file_enum_list = list(InputFileType) + list(OutputFileType)
    permitted_extensions_str = {
        x.value
        for x in permitted_file_enum_list
        if isinstance(x, (InputFileType | OutputFileType))
    }

    extension = str(file).split(".")[-1]
    if extension not in permitted_extensions_str:
        has_input = any(
            isinstance(ft, InputFileType) for ft in permitted_file_enum_list
        )
        has_output = any(
            isinstance(ft, OutputFileType) for ft in permitted_file_enum_list
        )
        target_error: type[Exception]
        if has_input and not has_output:
            target_error = InvalidInputFileTypeError
        elif not has_input and has_output:
            target_error = InvalidOutputFileTypeError
        else:
            target_error = InvalidFileTypeError

        forbidden_file_type = (
            f"file type {extension} is not permitted. "
            f" Use one of {permitted_extensions_str}."
        )
        raise target_error(forbidden_file_type)

    logger.debug(f"filetype is: {extension}.")
    return extension
parse(input_, parser, *, return_metadata=False, return_images=False, **kwargs)

Parse target file. Wraps around specific parser methods.

Parameters:

Name Type Description Default
input_ str | PathLike
required
input_type InputFileType
required
parser ParserLibrary
required
parse_method Callable[[str | PathLike, ParserLibrary], str]
required

Returns:

Name Type Description
str ParsedOutput

description
Source code in deet/processors/parser.py 
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
def parse(
    self,
    input_: str | PathLike,
    parser: type[ParserLibrary],
    *,
    return_metadata: bool = False,
    return_images: bool = False,
    **kwargs,
) -> ParsedOutput:
    """
    Parse target file.
    Wraps around specific parser methods.

    Args:
        input_ (str | PathLike):
        input_type (InputFileType):
        parser (ParserLibrary):
        parse_method (Callable[[str  |  PathLike, ParserLibrary], str]):

    Returns:
        str: _description_

    """
    logger.debug(f"kwargs: {kwargs}")
    if return_metadata and OutputFileType.JSON not in parser.output_file_types:
        metadata_not_allowed = (
            f"metadata out not permitted for parser {parser.name}."
        )
        raise InvalidOutputFileTypeError(metadata_not_allowed)
    if return_images and OutputFileType.JPEG not in parser.output_file_types:
        images_not_allowed = f"images out not permitted for parser {parser.name}."
        raise InvalidOutputFileTypeError(images_not_allowed)

    return parser.parse(
        input_=input_,
        return_metadata=return_metadata,
        return_images=return_images,
        **kwargs,
    )
write_files(out_path, parser, *, write_metadata, write_images, text, metadata=None, images=None) staticmethod

Write parsed content to file(s).

NOTE: we are taking existence of out_path as an intention to write all requested objects to file. out_path can be a file or a dir. if out_path is a file, we write remaining files to parent dir.

Parameters:

Name Type Description Default
out_path str | PathLike

description

 required
write_metadata bool

description

 required
write_images bool

description

 required
text str

description

 required
metadata dict | None

description. Defaults to None.

 None
images dict[str, Image] | None

description. Defaults to None.

 None
Source code in deet/processors/parser.py 
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
@staticmethod
def write_files(  # noqa: PLR0913
    out_path: str | PathLike,
    parser: type[ParserLibrary],
    *,
    write_metadata: bool,
    write_images: bool,
    text: str,
    metadata: dict | None = None,
    images: dict[str, Image] | None = None,
) -> None:
    """
    Write parsed content to file(s).

    NOTE: we are taking existence of `out_path` as an intention to
    write all requested objects to file. out_path can be a file or a dir.
    if out_path is a file, we write remaining files to parent dir.

    Args:
        out_path (str | PathLike): _description_
        write_metadata (bool): _description_
        write_images (bool): _description_
        text (str): _description_
        metadata (dict | None, optional): _description_. Defaults to None.
        images (dict[str, Image] | None, optional): _description_. Defaults to None.

    """
    extension = (
        DocumentParser.detect_filetype(  # should raise error if not permitted
            out_path, permitted_file_enum_list=parser.output_file_types
        )
    )

    required_outfiles = ["md"]
    if write_images:
        required_outfiles.append("jpeg")
        if images is None:  # or raise something?
            logger.warning(
                "`write_images` set to True, but no images obj supplied."
            )
    if write_metadata:
        required_outfiles.append("json")
        if metadata is None:  # or raise something?
            logger.warning(
                "`write_metadata` set to True, but no metadata obj supplied."
            )
    logger.debug(f"required outfiles: {required_outfiles}")
    if False in [ft in parser.output_file_types for ft in required_outfiles]:
        raise InvalidOutputFileTypeError

    Path(out_path).parent.mkdir(parents=True, exist_ok=True)

    if Path(out_path).is_file() or (extension in required_outfiles):
        logger.debug(f"`out_path` {out_path} points to a file.")

        dir_base = Path(out_path).parent
        filename_base = "".join(Path(out_path).name.split(".")[0:-1])

    if Path(out_path).is_dir():
        logger.debug(f"`out_path` {out_path} points to a dir.")
        # we now have to get our filename base from somewhere...
        dir_base = Path(out_path)
        filename_base = text.split("\n")[0][:15].replace(" ", "_").lower()

    for ext in required_outfiles:
        out = dir_base / (filename_base + "." + ext)
        logger.debug(f"writing out {ext} to {out}.")
        if ext == "md":
            out.write_text(text, encoding="utf-8")
        if ext == "json" and metadata is not None:
            out.write_text(json.dumps(metadata), encoding="utf-8")
        if ext == "jpeg" and images is not None:
            for img_name, img in images.items():
                img_out = dir_base / (filename_base + "_" + img_name)
                img.save(img_out, ext)

InputFileType

Bases: StrEnum

Enumeration of permitted input file types.

Parameters:

Name Type Description Default
StrEnum _type_
required
Source code in deet/processors/parser.py 
38
39
40
41
42
43
44
45
46
47
48
49
50
class InputFileType(StrEnum):
    """
    Enumeration of permitted input file types.

    Args:
        StrEnum (_type_):

    """

    PDF = auto()
    EPUB = auto()
    HTML = auto()
    XML = auto()  # NOTE - this only covers JATS xml.

MarkerParser

Bases: ParserLibrary

Parser with marker backend.

Source code in deet/processors/parser.py 
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
class MarkerParser(ParserLibrary):
    """Parser with `marker` backend."""

    name: Literal["marker"] = "marker"
    input_types = [InputFileType.PDF]
    output_file_types = [OutputFileType.MD, OutputFileType.JPEG, OutputFileType.JSON]

    @classmethod
    @parser_cache.memoize(typed=True, expire=None, tag="marker-converter")
    def _get_converter(cls):  # noqa: ANN206 no return type hint as we don't know PdfConverter yet
        """Lazy initialization of marker converter with disk caching."""
        logger.debug("Initializing marker converter...")
        from marker.converters.pdf import PdfConverter
        from marker.models import create_model_dict

        artifact_dict = create_model_dict()
        return PdfConverter(artifact_dict=artifact_dict)

    @classmethod
    def parse(
        cls,
        input_: str | PathLike,
        *,
        return_metadata: bool = False,
        return_images: bool = False,
        **kwargs,  # noqa: ARG003
    ) -> ParsedOutput:
        """Parse file using marker."""
        from marker.output import text_from_rendered

        converter = cls._get_converter()
        rendered = converter(str(input_))
        text, extension, images = text_from_rendered(rendered)
        out = {"text": text}
        if return_metadata:
            out["metadata"] = rendered.metadata
        if return_images:
            out["images"] = images
        return ParsedOutput(**out, parser_library=cls.name)

    @classmethod
    def clear_cache(cls) -> None:
        """Clear the cached converter."""
        parser_cache.clear()
        logger.info("Marker converter cache cleared")
clear_cache() classmethod

Clear the cached converter.

Source code in deet/processors/parser.py 
190
191
192
193
194
@classmethod
def clear_cache(cls) -> None:
    """Clear the cached converter."""
    parser_cache.clear()
    logger.info("Marker converter cache cleared")
parse(input_, *, return_metadata=False, return_images=False, **kwargs) classmethod

Parse file using marker.

Source code in deet/processors/parser.py 
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
@classmethod
def parse(
    cls,
    input_: str | PathLike,
    *,
    return_metadata: bool = False,
    return_images: bool = False,
    **kwargs,  # noqa: ARG003
) -> ParsedOutput:
    """Parse file using marker."""
    from marker.output import text_from_rendered

    converter = cls._get_converter()
    rendered = converter(str(input_))
    text, extension, images = text_from_rendered(rendered)
    out = {"text": text}
    if return_metadata:
        out["metadata"] = rendered.metadata
    if return_images:
        out["images"] = images
    return ParsedOutput(**out, parser_library=cls.name)

OutputFileType

Bases: StrEnum

Enumeration of permitted output file types.

Parameters:

Name Type Description Default
StrEnum _type_
required
Source code in deet/processors/parser.py 
53
54
55
56
57
58
59
60
61
62
63
64
class OutputFileType(StrEnum):
    """
    Enumeration of permitted output file types.

    Args:
        StrEnum (_type_):

    """

    MD = auto()
    JPEG = auto()
    JSON = auto()

PandocParser

Bases: ParserLibrary

Parser with pandoc backend.

Source code in deet/processors/parser.py 
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
class PandocParser(ParserLibrary):
    """Parser with `pandoc` backend."""

    name: Literal["pandoc"] = "pandoc"
    input_types = [InputFileType.EPUB, InputFileType.HTML, InputFileType.XML]
    output_file_types = [OutputFileType.MD]

    @classmethod
    def parse(
        cls,
        input_: str | PathLike,
        input_type: InputFileType | str | None = None,
        *,
        input_is_string: bool = False,
        return_metadata: bool = False,
        return_images: bool = False,
        **kwargs,  # noqa: ARG003
    ) -> ParsedOutput:
        """Parse file using pandoc."""
        if True in [return_images, return_metadata]:
            image_meta_erro = "PandocParser can't produce images or metadata."
            raise InvalidOutputFileTypeError(image_meta_erro)
        if input_is_string and not input_type:
            missing_filetype = (
                "if input is str in memory, provide format as `input_type`."
            )
            raise InvalidInputFileTypeError(missing_filetype)
        if not input_type:
            input_type = DocumentParser.detect_filetype(input_, cls.input_types)
        if isinstance(input_type, InputFileType):
            input_type = input_type.value
        if input_type == "xml":
            input_type = "jats"

        if input_is_string:
            parse_method = pypandoc.convert_text
        else:
            parse_method = pypandoc.convert_file

        out = {
            "text": parse_method(
                input_,
                to="md",
                format=input_type,
            )
        }
        return ParsedOutput(**out, parser_library=cls.name)
parse(input_, input_type=None, *, input_is_string=False, return_metadata=False, return_images=False, **kwargs) classmethod

Parse file using pandoc.

Source code in deet/processors/parser.py 
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
@classmethod
def parse(
    cls,
    input_: str | PathLike,
    input_type: InputFileType | str | None = None,
    *,
    input_is_string: bool = False,
    return_metadata: bool = False,
    return_images: bool = False,
    **kwargs,  # noqa: ARG003
) -> ParsedOutput:
    """Parse file using pandoc."""
    if True in [return_images, return_metadata]:
        image_meta_erro = "PandocParser can't produce images or metadata."
        raise InvalidOutputFileTypeError(image_meta_erro)
    if input_is_string and not input_type:
        missing_filetype = (
            "if input is str in memory, provide format as `input_type`."
        )
        raise InvalidInputFileTypeError(missing_filetype)
    if not input_type:
        input_type = DocumentParser.detect_filetype(input_, cls.input_types)
    if isinstance(input_type, InputFileType):
        input_type = input_type.value
    if input_type == "xml":
        input_type = "jats"

    if input_is_string:
        parse_method = pypandoc.convert_text
    else:
        parse_method = pypandoc.convert_file

    out = {
        "text": parse_method(
            input_,
            to="md",
            format=input_type,
        )
    }
    return ParsedOutput(**out, parser_library=cls.name)

ParsedOutput pydantic-model

Bases: BaseModel

Output returned from the parser() method of subclasses of ParserLibrary.

Contains

text, str: md-formatted parsed text (required) images, pillow.img: pillow-formatted image(s) (optional) metadata, dict: metadata json (optional) timestamp: datetime: auto-populates with now parser_library: str: name of the ParserLibrary implementation used

Config:

  • arbitrary_types_allowed: True

Fields:

  • text (str)
  • images (dict[str, Image] | None)
  • metadata (dict | None)
  • timestamp (datetime)
  • parser_library (Literal['pandoc', 'marker', 'pdfminer', 'unknown'])

Validators:

Source code in deet/processors/parser.py 
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
class ParsedOutput(BaseModel):
    """
    Output returned from the `parser()` method of subclasses of ParserLibrary.

    Contains:
        text, str: md-formatted parsed text (required)
        images, pillow.img: pillow-formatted image(s) (optional)
        metadata, dict: metadata json (optional)
        timestamp: datetime: auto-populates with _now_
        parser_library: str: name of the ParserLibrary implementation used
    """

    text: str
    images: dict[str, Image] | None = None
    metadata: dict | None = None
    timestamp: datetime = Field(default_factory=lambda: datetime.now(tz=UTC))
    parser_library: Literal[
        "pandoc", "marker", "pdfminer", "unknown"
    ]  # extend when adding new parsers

    model_config = ConfigDict(
        arbitrary_types_allowed=True
    )  # this is to allow our Executor class as a type.

    @field_validator("text", mode="after")
    @classmethod
    def assess_language_quality(cls, value: str) -> str:
        """
        Assess language quality.

        Args:
            text (str): Parsed text.

        Raises:
            MalformedLanguageError: If threshold not met.

        Returns:
            str: parsed text.

        """
        if not check_language(value):
            logger.debug("check lang failed")
            bad_language = "Supplied text didn't pass quality check."
            raise ValueError(bad_language)
        return value
assess_language_quality(value) pydantic-validator

Assess language quality.

Parameters:

Name Type Description Default
text str

Parsed text.

 required

Raises:

Type Description
MalformedLanguageError

If threshold not met.

Returns:

Name Type Description
str str

parsed text.
Source code in deet/processors/parser.py 
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
@field_validator("text", mode="after")
@classmethod
def assess_language_quality(cls, value: str) -> str:
    """
    Assess language quality.

    Args:
        text (str): Parsed text.

    Raises:
        MalformedLanguageError: If threshold not met.

    Returns:
        str: parsed text.

    """
    if not check_language(value):
        logger.debug("check lang failed")
        bad_language = "Supplied text didn't pass quality check."
        raise ValueError(bad_language)
    return value

ParserLibrary

Bases: ABC

Base parser class.

Source code in deet/processors/parser.py 
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
class ParserLibrary(ABC):
    """Base parser class."""

    name: str
    input_types: list[InputFileType]
    output_file_types: list[OutputFileType]

    @classmethod
    @abstractmethod
    def parse(
        cls,
        input_: str | PathLike,
        *,
        return_metadata: bool = False,
        return_images: bool = False,
        **kwargs,
    ) -> ParsedOutput:
        """
        Parse a document.
        Intentionelly left blank as this should be populated in sub-classes.

        Args:
            input_ (str | PathLike): Path to input file or string of input string.
            return_metadata (bool, optional): Return json metadata. Defaults to False.
            return_images (bool, optional): Return images in doc. Defaults to False.

        Raises:
            NotImplementedError: The default, should never actually come.

        Returns:
            str | tuple[str, Any, Any]: There will always be str, but sometimes more.

        """
        raise NotImplementedError
parse(input_, *, return_metadata=False, return_images=False, **kwargs) abstractmethod classmethod

Parse a document. Intentionelly left blank as this should be populated in sub-classes.

Parameters:

Name Type Description Default
input_ str | PathLike

Path to input file or string of input string.

 required
return_metadata bool

Return json metadata. Defaults to False.

 False
return_images bool

Return images in doc. Defaults to False.

 False

Raises:

Type Description
NotImplementedError

The default, should never actually come.

Returns:

Type Description
ParsedOutput

str | tuple[str, Any, Any]: There will always be str, but sometimes more.
Source code in deet/processors/parser.py 
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
@classmethod
@abstractmethod
def parse(
    cls,
    input_: str | PathLike,
    *,
    return_metadata: bool = False,
    return_images: bool = False,
    **kwargs,
) -> ParsedOutput:
    """
    Parse a document.
    Intentionelly left blank as this should be populated in sub-classes.

    Args:
        input_ (str | PathLike): Path to input file or string of input string.
        return_metadata (bool, optional): Return json metadata. Defaults to False.
        return_images (bool, optional): Return images in doc. Defaults to False.

    Raises:
        NotImplementedError: The default, should never actually come.

    Returns:
        str | tuple[str, Any, Any]: There will always be str, but sometimes more.

    """
    raise NotImplementedError

PdfminerParser

Bases: ParserLibrary

Parser with pdfminer.six backend. Fast text extraction, no images or metadata.

Source code in deet/processors/parser.py 
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
class PdfminerParser(ParserLibrary):
    """Parser with pdfminer.six backend. Fast text extraction, no images or metadata."""

    name: Literal["pdfminer"] = "pdfminer"
    input_types = [InputFileType.PDF]
    output_file_types = [OutputFileType.MD]
    _LAPARAMS = LAParams()

    @classmethod
    def parse(
        cls,
        input_: str | PathLike,
        *,
        return_metadata: bool = False,
        return_images: bool = False,
        **kwargs,  # noqa: ARG003
    ) -> ParsedOutput:
        """Parse file using pdfminer.six (no OCR)."""
        if return_metadata or return_images:
            msg = "PdfminerParser can't produce images or metadata."
            raise InvalidOutputFileTypeError(msg)
        rsrcmgr = PDFResourceManager()
        out = StringIO()
        device = TextConverter(rsrcmgr, out, laparams=cls._LAPARAMS)
        with Path(input_).open("rb") as f:
            interpreter = PDFPageInterpreter(rsrcmgr, device)
            for page in PDFPage.get_pages(f):
                interpreter.process_page(page)
        device.close()
        text = out.getvalue() or ""
        if not text.strip():
            raise EmptyPdfExtractionError(EmptyPdfExtractionError.DEFAULT_MESSAGE)
        return ParsedOutput(text=text, parser_library=cls.name)
parse(input_, *, return_metadata=False, return_images=False, **kwargs) classmethod

Parse file using pdfminer.six (no OCR).

Source code in deet/processors/parser.py 
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
@classmethod
def parse(
    cls,
    input_: str | PathLike,
    *,
    return_metadata: bool = False,
    return_images: bool = False,
    **kwargs,  # noqa: ARG003
) -> ParsedOutput:
    """Parse file using pdfminer.six (no OCR)."""
    if return_metadata or return_images:
        msg = "PdfminerParser can't produce images or metadata."
        raise InvalidOutputFileTypeError(msg)
    rsrcmgr = PDFResourceManager()
    out = StringIO()
    device = TextConverter(rsrcmgr, out, laparams=cls._LAPARAMS)
    with Path(input_).open("rb") as f:
        interpreter = PDFPageInterpreter(rsrcmgr, device)
        for page in PDFPage.get_pages(f):
            interpreter.process_page(page)
    device.close()
    text = out.getvalue() or ""
    if not text.strip():
        raise EmptyPdfExtractionError(EmptyPdfExtractionError.DEFAULT_MESSAGE)
    return ParsedOutput(text=text, parser_library=cls.name)

scripts

Scripts for data extraction evaluation toolkit.

cli

A CLI app to run deet pipelines.

export_config_template(output_path=DEFAULT_CONFIG_PATH)

Export the default DataExtractionConfig to a YAML file.

Source code in deet/scripts/cli.py 
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
@app.command()
def export_config_template(
    output_path: Annotated[
        Path,
        typer.Option(help="The output path where your config file will be written"),
    ] = DEFAULT_CONFIG_PATH,
) -> None:
    """Export the default DataExtractionConfig to a YAML file."""
    import yaml  # type:ignore[import-untyped]

    from deet.extractors.llm_data_extractor import DataExtractionConfig

    config = DataExtractionConfig()
    if output_path.exists():
        message = (
            "Config template exists. Proceeding will "
            "overwrite this and you may lose work if you have edited this."
            " Do you want to continue?"
        )
        proceed = typer.confirm(message)
        if proceed:
            echo_and_log("Proceeding to overwrite config template")
            output_path.unlink()
        else:
            raise typer.Abort()  # noqa: RSE102
    output_path.write_text(
        yaml.safe_dump(config.model_dump(mode="json"), sort_keys=False),
        encoding="utf-8",
    )
    echo_and_log(f"✅ Default config exported to {output_path}", fg=typer.colors.GREEN)
    echo_and_log(
        "✏️  Edit this file to adjust options for data extraction.", fg=typer.colors.BLUE
    )

extract_data(gs_data_path, config_path=DEFAULT_CONFIG_PATH, gs_data_format=DEFAULT_IMPORT_FORMAT, prompt_population=None, csv_path=None, linked_document_path=DEFAULT_LINKED_DOCUMENTS_PATH, link_map_path=None, pdf_dir=DEFAULT_PDF_PATH, out_dir=DEFAULT_EXPERIMENT_OUT_DIR, run_name='', custom_evaluation_metrics=None)

Extract data from documents and evaluate.

Load gold standard annotation data, and use an LLM to extract data from the documents in your dataset. Evaluate by comparing the results to the gold standard data.

Source code in deet/scripts/cli.py 
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
@app.command()
def extract_data(  # noqa: PLR0913
    gs_data_path: GS_DATA_PATH,
    config_path: Annotated[
        Path,
        typer.Option(
            help="A path to a config file containing options for data "
            "extraction config. A template can be generated by running "
            "`deet export-config-template."
        ),
    ] = DEFAULT_CONFIG_PATH,
    gs_data_format: GS_DATA_FORMAT = DEFAULT_IMPORT_FORMAT,
    prompt_population: Annotated[
        CustomPromptPopulationMethod | None,
        typer.Option(
            help="A method to define custom prompts for your attributes to be "
            "extracted. Leave blank to use the prompts in your gold standard "
            "data. Set to `file` to provide a file of prompt definitions "
            "(make sure this is supplied below). Set to `cli` to define prompts"
            " interactively in the CLI. With `file`, only attributes that appear "
            "in the CSV with a non-empty `prompt` are kept for extraction and "
            "evaluation (see also `--csv-path`)."
        ),
    ] = None,
    csv_path: Annotated[
        Path | None,
        typer.Option(
            help="A path to read custom prompt definitions from."
            " This must be set if using prompt population from file."
            " Rows with blank `prompt` are ignored; attribute IDs not listed are "
            "dropped from the run."
        ),
    ] = None,
    linked_document_path: Annotated[
        Path,
        typer.Option(
            help="A path to a directory containing documents that have been "
            "linked to their fulltexts. This directory can be populated by "
            "running `deet link-documents-fulltexts`."
        ),
    ] = DEFAULT_LINKED_DOCUMENTS_PATH,
    link_map_path: Annotated[
        Path | None,
        typer.Option(
            help="A path to a link map (create this by running "
            "`deet init-linkage-mapping-file`). You must specify"
            "either a link map or a directory containing linked"
            "documents"
        ),
    ] = None,
    pdf_dir: Annotated[
        Path,
        typer.Option(
            help="Path to a directory containing pdfs. We will attempt to link"
            " these by document ID."
        ),
    ] = DEFAULT_PDF_PATH,
    out_dir: Annotated[
        Path,
        typer.Option(
            help="A path to a directory where you want to store the results of"
            " this, and further instances of extract-data for this project."
        ),
    ] = DEFAULT_EXPERIMENT_OUT_DIR,
    run_name: Annotated[
        str,
        typer.Option(
            help="A name for the run (which will appended to a timestamp) "
            "to help you identify this run later"
        ),
    ] = "",
    custom_evaluation_metrics: Annotated[
        list[str] | None,
        typer.Option(
            help="A list of additional sklearn metrics that you wish to "
            " calculate. Use this option for each additional metric you "
            " would like to add, e.g. `deet extract-data "
            "--custom-evaluation-metrics brier_score_loss "
            "--custom-evaluation-metrics cohen_kappa_score`"
        ),
    ] = None,
) -> None:
    """
    Extract data from documents and evaluate.

    Load gold standard annotation data, and use an LLM to extract data from the
    documents in your dataset. Evaluate by comparing the results to the gold
    standard data.
    """
    import yaml

    from deet.evaluators.gold_standard_llm_evaluator import GoldStandardLLMEvaluator
    from deet.extractors.cli_helpers import (
        init_extraction_run,
        load_or_init_config,
        prepare_documents,
    )
    from deet.extractors.llm_data_extractor import LLMDataExtractor

    config = load_or_init_config(config_path=config_path)

    extraction_run_id, experiment_out_dir = init_extraction_run(out_dir, run_name)

    converter = gs_data_format.get_annotation_converter()
    processed_annotation_data = converter.process_annotation_file(gs_data_path)

    if prompt_population == CustomPromptPopulationMethod.FILE and not csv_path:
        message = "CSV prompt population selected without specifying csv_path"
        fail_with_message(message)
    if prompt_population is not None:
        processed_annotation_data.populate_custom_prompts(
            method=prompt_population, filepath=csv_path
        )

    data_extractor = LLMDataExtractor(config=config)

    documents = prepare_documents(
        processed_annotation_data.documents,
        config,
        linked_document_path=linked_document_path,
        pdf_dir=pdf_dir,
        link_map_path=link_map_path,
    )

    run_output = data_extractor.extract_from_documents(
        attributes=processed_annotation_data.attributes,
        documents=documents,
        context_type=data_extractor.config.default_context_type,
        output_file=experiment_out_dir / "annotated_docs.json",
        show_progress=True,
    )

    config_out = experiment_out_dir / "config.yaml"
    config_out.write_text(
        yaml.safe_dump(data_extractor.config.model_dump(mode="json"), sort_keys=False),
        encoding="utf-8",
    )

    evaluator = GoldStandardLLMEvaluator(
        gold_standard_annotated_documents=processed_annotation_data.annotated_documents,
        llm_annotated_documents=run_output.annotated_documents,
        attributes=processed_annotation_data.attributes,
        custom_metrics=custom_evaluation_metrics,
        extraction_run_id=extraction_run_id,
    )
    evaluator.evaluate_llm_annotations()
    evaluator.write_metrics_to_csv(experiment_out_dir / DEFAULT_METRICS_CSV)
    evaluator.export_llm_comparison(experiment_out_dir / DEFAULT_OUTPUT_COMPARISON_CSV)
    evaluator.display_metrics()

global_options(*, verbose=typer.Option(default=False, help='Display verbose logs.'))

Set global options for all deet commands.

Source code in deet/scripts/cli.py 
402
403
404
405
406
407
408
409
410
411
412
413
414
415
@app.callback()
def global_options(
    *, verbose: bool = typer.Option(default=False, help="Display verbose logs.")
) -> None:
    """Set global options for all deet commands."""
    log_level = "DEBUG" if verbose else "INFO"
    logger.add(
        typer.echo,
        colorize=True,
        level=log_level,
        filter=lambda record: "is_echo" not in record["extra"],
    )
    if not verbose:
        warnings.filterwarnings("ignore", message=".*is ill-defined.*")

init_linkage_mapping_file(gs_data_path, gs_data_format=DEFAULT_IMPORT_FORMAT, link_map_path=DEFAULT_LINK_MAP)

Create a mapping to link documents and their full texts.

Source code in deet/scripts/cli.py 
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
@app.command()
def init_linkage_mapping_file(
    gs_data_path: GS_DATA_PATH,
    gs_data_format: GS_DATA_FORMAT = DEFAULT_IMPORT_FORMAT,
    link_map_path: LINK_MAP_PATH = DEFAULT_LINK_MAP,
) -> None:
    """Create a mapping to link documents and their full texts."""
    if link_map_path.exists():
        message = (
            f"mapping already exists at {link_map_path}. Overwriting"
            " may cause you to lose work. Do you want to continue?"
        )
        proceed = typer.confirm(message)
        if proceed:
            echo_and_log("Proceeding to overwrite config template")
            link_map_path.unlink()
        else:
            raise typer.Abort()  # noqa: RSE102

    converter = gs_data_format.get_annotation_converter()
    processed_annotation_data = converter.process_annotation_file(gs_data_path)
    processed_annotation_data.export_linkage_mapper_csv(link_map_path)

init_prompt_csv(gs_data_path, gs_data_format=DEFAULT_IMPORT_FORMAT, csv_path=DEFAULT_PROMPT_DEFINITION_PATH)

Write a csv to define prompts for your dataset with.

This writes a row for each attribute in your dataset. Edit the prompt column to edit the prompt to be used for that attribute. Attributes without values in the prompt column will not be extracted.

Source code in deet/scripts/cli.py 
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
@app.command()
def init_prompt_csv(
    gs_data_path: GS_DATA_PATH,
    gs_data_format: GS_DATA_FORMAT = DEFAULT_IMPORT_FORMAT,
    csv_path: Annotated[
        Path, typer.Option(help="A path to a file to write your prompt definitions to.")
    ] = DEFAULT_PROMPT_DEFINITION_PATH,
) -> None:
    """
    Write a csv to define prompts for your dataset with.

    This writes a row for each attribute in your dataset. Edit the prompt
    column to edit the prompt to be used for that attribute. Attributes
    without values in the prompt column will not be extracted.
    """
    converter = gs_data_format.get_annotation_converter()
    processed_annotation_data = converter.process_annotation_file(gs_data_path)
    if csv_path.exists():
        message = (
            "Prompt definition csv already exists. Proceeding will "
            "overwrite this and you may lose work. Do you want to continue?"
        )
        proceed = typer.confirm(message)
        if proceed:
            echo_and_log("Proceeding to overwrite prompt definition csv")
            csv_path.unlink()
        else:
            raise typer.Abort()  # noqa: RSE102
    processed_annotation_data.export_attributes_csv_file(filepath=csv_path)

Link documents to their fulltexts.

This creates a document containing the parsed output of its corresponding fulltext in the folder defined in output_path. Linking will be attempted using a mapping file, if provided, then by matching the filename with author and year, then by matching by document id. See deet.processors.linker for more details.

Source code in deet/scripts/cli.py 
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
@app.command()
def link_documents_fulltexts(
    gs_data_path: GS_DATA_PATH,
    link_map_path: LINK_MAP_PATH_READ = DEFAULT_LINK_MAP,
    gs_data_format: GS_DATA_FORMAT = DEFAULT_IMPORT_FORMAT,
    pdf_dir: Annotated[
        Path, typer.Option(help="Path to a directory containing pdfs.")
    ] = DEFAULT_PDF_PATH,
    output_path: Annotated[
        Path,
        typer.Option(help="A path to a directory to write the linked documents to."),
    ] = DEFAULT_LINKED_DOCUMENTS_PATH,
) -> None:
    """
    Link documents to their fulltexts.

    This creates a document containing the parsed output of its corresponding
    fulltext in the folder defined in `output_path`. Linking will be
    attempted using a mapping file, if provided, then by matching the
    filename with author and year, then by matching by document id. See
    `deet.processors.linker` for more details.

    """
    from deet.processors.linker import DocumentReferenceLinker, LinkingStrategy

    converter = gs_data_format.get_annotation_converter()
    processed_annotation_data = converter.process_annotation_file(gs_data_path)

    linker = DocumentReferenceLinker(
        references=processed_annotation_data.documents,
        document_base_dir=pdf_dir,
        document_reference_mapping=link_map_path,
        linking_strategies=[LinkingStrategy.MAPPING_FILE],
    )
    linked_documents = linker.link_many_references_parsed_documents()

    if not output_path.exists():
        output_path.mkdir()

    if len(linked_documents) == 0:
        fail_with_message("Error. Could not link any documents!")

    for linked_document in linked_documents:
        file_path = output_path / f"{linked_document.safe_identity.document_id}.json"
        linked_document.save(file_path)

main()

Run CLI app.

Source code in deet/scripts/cli.py 
418
419
420
def main() -> None:
    """Run CLI app."""
    app()

test_llm_config(config_path=DEFAULT_CONFIG_PATH)

Test llm config.

Source code in deet/scripts/cli.py 
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
@app.command()
def test_llm_config(
    config_path: Annotated[
        Path,
        typer.Option(
            help="A path to a config file containing options for data "
            "extraction config. A template can be generated by running "
            "`deet export-config-template."
        ),
    ] = DEFAULT_CONFIG_PATH,
) -> None:
    """Test llm config."""
    from deet.data_models.base import Attribute, AttributeType
    from deet.extractors.cli_helpers import (
        load_or_init_config,
    )
    from deet.extractors.llm_data_extractor import (
        LLMDataExtractor,
    )

    config = load_or_init_config(config_path=config_path)
    data_extractor = LLMDataExtractor(config=config)
    attr = Attribute(
        output_data_type=AttributeType.BOOL,
        attribute_id=1234,
        attribute_label="Test Attribute",
        prompt="Is the document about climate and health? Return a BOOL",
    )
    context = (
        "This is document, extract data from me please. I am about climate and health"
    )
    response = data_extractor.extract_from_document(
        attributes=[attr],
        payload=context,
        context_type=None,
    )
    echo_and_log(response)

settings

App settings using pydantic-settings.

DataExtractionSettings

Bases: BaseSettings

Settings model for data extraction behavior and provider credentials.

All fields are fully typed and documented. Unknown environment variables are forbidden to help catch configuration drift early.

Source code in deet/settings.py 
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
class DataExtractionSettings(BaseSettings):
    """
    Settings model for data extraction behavior and provider credentials.

    All fields are fully typed and documented. Unknown environment variables
    are forbidden to help catch configuration drift early.
    """

    model_config = SettingsConfigDict(
        env_file=".env", env_file_encoding="utf-8", extra="ignore"
    )

    # General
    log_level: str = Field(default="DEBUG", description="log level for the app logger.")

    runtime: Runtime = Field(
        default=Runtime.LOCAL,
        description="Runtime environment.",
    )

    llm_provider: LLMProvider = Field(
        default=LLMProvider.AZURE, description="LLM Provider"
    )

    # LLM configuration
    llm_model: str = Field(
        default="gpt-4o-mini",
        description="LLM model identifier used for completions.",
    )
    llm_temperature: float = Field(
        default=0.1,
        description="Sampling temperature for the LLM.",
        ge=0.0,
    )
    llm_max_tokens: int | None = Field(
        default=None,
        description=(
            "Maximum number of tokens to generate (None means provider default)."
        ),
    )
    llm_max_context_tokens: int | None = Field(
        default=None,
        description=(
            "Maximum input context length in tokens (system + attributes + "
            "document). None = infer from model (litellm registry), else "
            f"{DEFAULT_LLM_MAX_CONTEXT_TOKENS_FALLBACK} via "
            "DEFAULT_LLM_MAX_CONTEXT_TOKENS_FALLBACK. Override to manage costs."
        ),
    )

    # Provider credentials / settings (secrets redacted)
    azure_api_key: SecretStr | None = Field(
        default=None,
        description="Azure OpenAI API key if using Azure provider.",
    )
    azure_api_base: SecretStr | None = Field(
        default=None, description="Base URL for azure openAI."
    )

    # disk cache folder
    base_disk_cache_dir: Path = Field(
        default=(Path.home() / ".deet_cache"),
        description="the base directory for disk-based caches.",
    )

LLMProvider

Bases: StrEnum

Supported LLM Providers.

Source code in deet/settings.py 
21
22
23
24
25
class LLMProvider(StrEnum):
    """Supported LLM Providers."""

    AZURE = auto()
    OLLAMA = auto()

Runtime

Bases: StrEnum

Permitted runtime environments for DEET.

Source code in deet/settings.py 
11
12
13
14
15
16
17
18
class Runtime(StrEnum):
    """Permitted runtime environments for DEET."""

    LOCAL = auto()
    DEVELOPMENT = auto()
    STAGING = auto()
    TEST = auto()
    PRODUCTION = auto()

get_settings() cached

Return a cached settings instance for reuse across the process.

Source code in deet/settings.py 
101
102
103
104
@lru_cache(maxsize=1)
def get_settings() -> DataExtractionSettings:
    """Return a cached settings instance for reuse across the process."""
    return DataExtractionSettings()

utils

assess_text_quality

Tools for text quality assessments.

EmptyTextError

Bases: Exception

Raise when our text passed to is_english is empty.

Parameters:

Name Type Description Default
Exception _type_
required
Source code in deet/utils/assess_text_quality.py 
41
42
43
44
45
46
47
48
49
50
51
52
class EmptyTextError(Exception):
    """
    Raise when our text passed to is_english is empty.

    Args:
        Exception (_type_):

    """

    def __init__(self, msg: str = "Supplied text is empty.", *args, **kwargs) -> None:  # noqa: ANN002
        """Init the exception with default message."""
        super().__init__(msg, *args, **kwargs)
__init__(msg='Supplied text is empty.', *args, **kwargs)

Init the exception with default message.

Source code in deet/utils/assess_text_quality.py 
50
51
52
def __init__(self, msg: str = "Supplied text is empty.", *args, **kwargs) -> None:  # noqa: ANN002
    """Init the exception with default message."""
    super().__init__(msg, *args, **kwargs)

Language

Bases: Enum

Enum of languages for quality-checking.

Source code in deet/utils/assess_text_quality.py 
16
17
18
19
class Language(Enum):
    """Enum of languages for quality-checking."""

    ENGLISH = "en"

check_language(text, lang=Language.ENGLISH, threshold=0.2)

Assess if text is in the specified language.

Parameters:

Name Type Description Default
text str

Text to assess.

 required
lang Language

Language to check against.

 ENGLISH
threshold float

Threshold for word overlap.

 0.2

Raises:

Type Description
EmptyTextError

If text is empty.

Returns:

Name Type Description
bool bool

True if text matches language, False otherwise.
Source code in deet/utils/assess_text_quality.py 
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
def check_language(
    text: str, lang: Language = Language.ENGLISH, threshold: float = 0.2
) -> bool:
    """
    Assess if text is in the specified language.

    Args:
        text (str): Text to assess.
        lang (Language): Language to check against.
        threshold (float): Threshold for word overlap.

    Raises:
        EmptyTextError: If text is empty.

    Returns:
        bool: True if text matches language, False otherwise.

    """
    if text is None or text == "":
        raise EmptyTextError
    tok = set(re.findall(r"\w+", text.lower()))

    if len(tok) <= 0:
        raise EmptyTextError

    if isinstance(lang, str):
        lang = Language(lang)

    if lang == Language.ENGLISH:
        return len(tok & bc_brown) / len(tok) > threshold
    missing_lang = f"Language '{lang.value}' not supported yet."
    raise NotImplementedError(missing_lang)

get_bc_brown_words()

Get bc_brown words, cached.

Source code in deet/utils/assess_text_quality.py 
22
23
24
25
26
27
28
29
30
31
32
33
34
35
@nltk_cache.memoize(typed=True, expire=None, tag="nltk-brown-words")
def get_bc_brown_words() -> set:
    """Get bc_brown words, cached."""
    from nltk.corpus import brown
    from nltk.data import find

    try:
        find("corpora/brown")
    except LookupError:
        from nltk import download

        download("brown")

    return {x.lower() for x in brown.words()}

is_english(text, threshold=0.2)

Check if text meets minimum English quality.

Parameters:

Name Type Description Default
text str

the text to check.

 required
threshold float

Defaults to 0.2.

 0.2

Returns:

Name Type Description
bool bool

True if 'English', false if not.
Source code in deet/utils/assess_text_quality.py 
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
def is_english(text: str, threshold: float = 0.2) -> bool:
    """
    Check if text meets minimum English quality.

    Args:
        text (str): the text to check.
        threshold (float, optional): Defaults to 0.2.

    Returns:
        bool: True if 'English', false if not.

    """
    return check_language(text, lang=Language.ENGLISH, threshold=threshold)

cli_utils

Utilities to make help integrate cli with rest of the codebase.

echo_and_log(message, **kwargs)

Echo (in typer) and log (via logger) a message simultaenously.

NOTE: pass typer-style stuff via kwargs.

Source code in deet/utils/cli_utils.py 
27
28
29
30
31
32
33
34
def echo_and_log(message: Any, **kwargs) -> None:  # noqa: ANN401
    """
    Echo (in typer) and log (via logger) a message simultaenously.

    NOTE: pass typer-style stuff via kwargs.
    """
    typer.secho(message, **kwargs)
    logger.bind(is_echo=True).info(f"typer .secho: {message}")

fail_with_message(message)

Print message and exit CLI.

Source code in deet/utils/cli_utils.py 
37
38
39
40
def fail_with_message(message: str) -> NoReturn:
    """Print message and exit CLI."""
    echo_and_log(message, fg=typer.colors.RED, err=True)
    raise typer.Exit(code=1)

optional_progress(iterable, *, show_progress=False, label='Processing')

Context manager that yields an iterable. If show_progress is True, wraps it in a Typer progress bar. Otherwise, yields it unchanged.

Source code in deet/utils/cli_utils.py 
11
12
13
14
15
16
17
18
19
20
21
22
23
24
@contextmanager
def optional_progress(
    iterable: Iterable, *, show_progress: bool = False, label: str = "Processing"
) -> Generator[Iterable, None, None]:
    """
    Context manager that yields an iterable.
    If show_progress is True, wraps it in a Typer progress bar.
    Otherwise, yields it unchanged.
    """
    if show_progress:
        with typer.progressbar(iterable, label=label) as progress:
            yield progress
    else:
        yield iterable

destiny_utils

Utilities for working with destiny data types.

ReferencePresenter

A class to present references in a Streamlit application.

Source code in deet/utils/destiny_utils.py 
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
class ReferencePresenter:
    """A class to present references in a Streamlit application."""

    def __init__(self, reference: Reference | ReferenceFileInput) -> None:
        """Init the ReferencePresenter class."""
        self.reference = reference

    @property
    def bibliographic_metadata(self) -> BibliographicMetadataEnhancement | None:
        """Returns the bibliographic metadata enhancement of the reference."""
        for e in self.reference.enhancements or []:
            if e.content.enhancement_type == EnhancementType.BIBLIOGRAPHIC:
                return e.content
        return None

    @property
    def location(self) -> LocationEnhancement | None:
        """Returns the location enhancement of the reference."""
        for e in self.reference.enhancements or []:
            if e.content.enhancement_type == EnhancementType.LOCATION:
                return e.content
        return None

    @property
    def title(self) -> str:
        """Returns the title of the reference."""
        if (
            self.bibliographic_metadata is not None
            and self.bibliographic_metadata.title
        ):
            return fix_text(self.bibliographic_metadata.title)
        return ""

    @property
    def authors(self) -> str:
        """Returns the authors of the reference as a semi-colon-separated string."""
        if self.bibliographic_metadata and self.bibliographic_metadata.authorship:
            return "; ".join(
                [a.display_name for a in self.bibliographic_metadata.authorship]
            )
        return ""

    @property
    def orcids(self) -> str:
        """
        Returns all available ORCIDs for each author of
        the reference as a semi-colon-separated string.
        """
        if self.bibliographic_metadata and self.bibliographic_metadata.authorship:
            return "; ".join(
                [str(a.orcid) for a in self.bibliographic_metadata.authorship]
            )
        return ""

    @property
    def first_author(self) -> Authorship | None:
        """Returns the first author of the reference."""
        if self.bibliographic_metadata and self.bibliographic_metadata.authorship:
            for author in self.bibliographic_metadata.authorship:
                if author.position == AuthorPosition.FIRST:
                    return author
        return None

    @property
    def publisher(self) -> str:
        """Returns the publisher of the reference."""
        return (
            self.bibliographic_metadata.publisher or ""
            if self.bibliographic_metadata
            else ""
        )

    @property
    def publication_date(self) -> str:
        """Returns the publication date of the reference in 'dd mmm yyyy' format."""
        if self.bibliographic_metadata:
            pub_date = self.bibliographic_metadata.publication_date
            if pub_date:
                return pub_date.strftime("%d %b %Y") or ""
        return ""

    @property
    def year(self) -> str:
        """Returns the publication year of the reference."""
        return (
            str(
                self.bibliographic_metadata.publication_year
                or (
                    self.bibliographic_metadata.publication_date.year
                    if self.bibliographic_metadata.publication_date
                    else ""
                )
            )
            if self.bibliographic_metadata
            else ""
        )

    @property
    def publication_locations(self) -> str:
        """
        Returns the publication locations of the
        reference as a semi-colon-separated string.
        """
        if self.location:
            return (
                "; ".join(
                    [
                        str(loc.extra["display_name"]) or ""
                        for loc in self.location.locations
                        if loc.extra
                    ]
                )
                or ""
                if self.location.locations
                else ""
            )
        return ""

    @property
    def publication_links(self) -> str:
        """
        Returns the publication location links of the
        reference as a semi-colon-separated string.
        """
        if self.location:
            return (
                "; ".join(
                    [str(loc.landing_page_url) or "" for loc in self.location.locations]
                )
                or ""
                if self.location.locations
                else ""
            )
        return ""

    @property
    def publication_types(self) -> str:
        """
        Returns the publication types of the
        reference as a semi-colon-separated string.
        """
        if self.location:
            return (
                "; ".join(
                    [
                        str(loc.extra["type"]) or ""
                        for loc in self.location.locations
                        if loc.extra
                    ]
                )
                or ""
                if self.location.locations
                else ""
            )
        return ""

    @property
    def doi(self) -> str | None:
        """Returns the DOI of the reference."""
        for i in self.reference.identifiers or []:
            if i.identifier_type == "doi":
                return f"https://doi.org/{i.identifier}"
        return None

    @property
    def abstract(self) -> str:
        """Returns the abstract of the reference."""
        for e in self.reference.enhancements or []:
            if e.content.enhancement_type == EnhancementType.ABSTRACT:
                return fix_text(e.content.abstract)
        return ""

    @property
    def topics(self) -> str:
        """
        Returns the topic labels of the reference and their id links
        as a semi-colon-separated string of tuples.
        """
        a = ""
        for e in self.reference.enhancements or []:
            if e.content.enhancement_type == EnhancementType.ANNOTATION:
                for x in e.content.annotations or []:
                    if x.scheme == "openalex:topic":
                        a += str(((x.label.title()), x.data["id"] if x.data else ""))
                        a += ";"
        return a.strip(";")

    @property
    def topic_domains(self) -> str:
        """
        Returns the topic domain labels of the reference and their id links
        as a semi-colon-separated string of tuples.
        """
        a = ""
        for e in self.reference.enhancements or []:
            if e.content.enhancement_type == EnhancementType.ANNOTATION:
                for x in e.content.annotations or []:
                    if x.scheme == "openalex:topic" and x.data:
                        a += str(
                            (
                                str(x.data["domain"]["display_name"]).title(),
                                x.data["domain"]["id"],
                            )
                        )
                        a += ";"
        return a.strip(";")

    @property
    def topic_fields(self) -> str:
        """
        Returns the topic field labels of the reference and their id links
        as a semi-colon-separated string of tuples.
        """
        a = ""
        for e in self.reference.enhancements or []:
            if e.content.enhancement_type == EnhancementType.ANNOTATION:
                for x in e.content.annotations or []:
                    if x.scheme == "openalex:topic" and x.data:
                        a += str(
                            (
                                str(x.data["field"]["display_name"]).title(),
                                x.data["field"]["id"],
                            )
                        )
                        a += ";"
        return a.strip(";")

    @property
    def topic_sub_fields(self) -> str:
        """
        Returns the topic sub-field labels of the reference and their id links
        as a semi-colon-separated string of tuples.
        """
        a = ""
        for e in self.reference.enhancements or []:
            if e.content.enhancement_type == EnhancementType.ANNOTATION:
                for x in e.content.annotations or []:
                    if x.scheme == "openalex:topic" and x.data:
                        a += str(
                            (
                                str(x.data["subfield"]["display_name"]).title(),
                                x.data["subfield"]["id"],
                            )
                        )
                        a += ";"
        return a.strip(";")

    @property
    def taxonomy(self) -> str:
        """
        Returns the taxonomy labels of the reference as
        a semi-colon-separated string of tuples.
        """
        a = ""
        for e in self.reference.enhancements or []:
            if e.content.enhancement_type == EnhancementType.ANNOTATION:
                for x in e.content.annotations or []:
                    if x.scheme.startswith("classifier:taxonomy") and x.value:  # type:ignore[union-attr]
                        a += str((x.scheme.split(":")[-1], x.label)).title()
                        a += ";"
        return a.strip(";")

    def to_dict(self) -> dict:
        """
        Convert the reference to a dictionary suitable
        for DataFrame representation.
        """
        return {
            "title": self.title,
            "authors": self.authors,
            "orcids": self.orcids,
            "publisher": self.publisher,
            "publication_date": self.publication_date,
            "year": self.year,
            "publication_locations": self.publication_locations,
            "publication_links": self.publication_links,
            "publication_types": self.publication_types,
            "doi": self.doi,
            "abstract": self.abstract,
            "topics": self.topics,
            "topic_domains": self.topic_domains,
            "topic_fields": self.topic_fields,
            "topic_sub_fields": self.topic_sub_fields,
            "taxonomy": self.taxonomy,
        }
abstract property

Returns the abstract of the reference.

authors property

Returns the authors of the reference as a semi-colon-separated string.

bibliographic_metadata property

Returns the bibliographic metadata enhancement of the reference.

doi property

Returns the DOI of the reference.

first_author property

Returns the first author of the reference.

location property

Returns the location enhancement of the reference.

orcids property

Returns all available ORCIDs for each author of the reference as a semi-colon-separated string.

publication_date property

Returns the publication date of the reference in 'dd mmm yyyy' format.

Returns the publication location links of the reference as a semi-colon-separated string.

publication_locations property

Returns the publication locations of the reference as a semi-colon-separated string.

publication_types property

Returns the publication types of the reference as a semi-colon-separated string.

publisher property

Returns the publisher of the reference.

taxonomy property

Returns the taxonomy labels of the reference as a semi-colon-separated string of tuples.

title property

Returns the title of the reference.

topic_domains property

Returns the topic domain labels of the reference and their id links as a semi-colon-separated string of tuples.

topic_fields property

Returns the topic field labels of the reference and their id links as a semi-colon-separated string of tuples.

topic_sub_fields property

Returns the topic sub-field labels of the reference and their id links as a semi-colon-separated string of tuples.

topics property

Returns the topic labels of the reference and their id links as a semi-colon-separated string of tuples.

year property

Returns the publication year of the reference.

__init__(reference)

Init the ReferencePresenter class.

Source code in deet/utils/destiny_utils.py 
17
18
19
def __init__(self, reference: Reference | ReferenceFileInput) -> None:
    """Init the ReferencePresenter class."""
    self.reference = reference
to_dict()

Convert the reference to a dictionary suitable for DataFrame representation.

Source code in deet/utils/destiny_utils.py 
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
def to_dict(self) -> dict:
    """
    Convert the reference to a dictionary suitable
    for DataFrame representation.
    """
    return {
        "title": self.title,
        "authors": self.authors,
        "orcids": self.orcids,
        "publisher": self.publisher,
        "publication_date": self.publication_date,
        "year": self.year,
        "publication_locations": self.publication_locations,
        "publication_links": self.publication_links,
        "publication_types": self.publication_types,
        "doi": self.doi,
        "abstract": self.abstract,
        "topics": self.topics,
        "topic_domains": self.topic_domains,
        "topic_fields": self.topic_fields,
        "topic_sub_fields": self.topic_sub_fields,
        "taxonomy": self.taxonomy,
    }

identifier_utils

Utility functions for creating, validating and manipulating identifiers.

check_if_id_exists(new_id, id_list)

Check if target id (int) is in a list of ids (list[int]).

Source code in deet/utils/identifier_utils.py 
55
56
57
def check_if_id_exists(new_id: int, id_list: list[int]) -> bool:
    """Check if target id (int) is in a list of ids (list[int])."""
    return new_id in id_list

hash_n_strings_to_document_id(string_list)

Convert n strings into an integer with MIN_DOCUMENT_ID-MAX_DOCUMENT_ID digits (4-10) using hash-based combination.

Parameters:

Name Type Description Default
string_list list[str]

a list of strings to hash.

 required

Returns:

Type Description
int

An integer with between MIN_DOCUMENT_ID-MAX_DOCUMENT_ID digits
int

(from 1000 to 9999999999).
Source code in deet/utils/identifier_utils.py 
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
def hash_n_strings_to_document_id(string_list: list[str]) -> int:
    """
    Convert n strings into an integer with MIN_DOCUMENT_ID-MAX_DOCUMENT_ID
    digits (4-10) using hash-based combination.

    Args:
        string_list: a list of strings to hash.

    Returns:
        An integer with between MIN_DOCUMENT_ID-MAX_DOCUMENT_ID digits
        (from 1000 to 9999999999).

    """
    combined = "|".join(string_list)
    hash_object = hashlib.sha256(combined.encode())
    hash_hex = hash_object.hexdigest()

    # Use first 8 hex chars to select digit count (4-10)
    hash_int_1 = int(hash_hex[:8], 16)
    n_digits = (hash_int_1 % 7) + 4  # 7 possible values (4-10 inclusive)

    # Use next 8 hex chars to select value within that digit range
    hash_int_2 = int(hash_hex[8:16], 16)

    min_for_digits = 10 ** (n_digits - 1)
    max_for_digits = (10**n_digits) - 1
    range_size = max_for_digits - min_for_digits + 1

    id_ = (hash_int_2 % range_size) + min_for_digits

    # Sanity check
    id_len = len(str(id_))
    if not (MIN_DOCUMENT_ID_DIGITS <= id_len <= MAX_DOCUMENT_ID_DIGITS):
        # NOTE: potentially fix ID at source if this error is ever raised,
        # otherwise remove error.
        bad_id = (
            f"id {id_} is bad, it should have between "
            f"{MIN_DOCUMENT_ID_DIGITS} and {MAX_DOCUMENT_ID_DIGITS} digits!"
        )
        raise ValueError(bad_id)

    return id_

tokenisation

Token counting and truncation utilities for LLM context management.

count_tokens(model, text)

Count the number of tokens in text for a given model.

Uses litellm.token_counter with messages format. Falls back to a rough character-based estimate (len/4) for unknown models.

Parameters:

Name Type Description Default
model str

Model identifier for tokenizer selection.

 required
text str

Text to count tokens for.

 required

Returns:

Type Description
int

Number of tokens.
Source code in deet/utils/tokenisation.py 
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
def count_tokens(model: str, text: str) -> int:
    """
    Count the number of tokens in text for a given model.

    Uses litellm.token_counter with messages format. Falls back to
    a rough character-based estimate (len/4) for unknown models.

    Args:
        model: Model identifier for tokenizer selection.
        text: Text to count tokens for.

    Returns:
        Number of tokens.

    """
    try:
        return litellm.token_counter(
            model=model,
            messages=[{"role": "user", "content": text}],
        )
    except _LITELLM_TOKENISER_ERRORS as e:
        logger.warning(
            f"Could not count tokens for model {model}, using char estimate: {e}"
        )
    except Exception as e:  # noqa: BLE001
        # litellm token_counter may raise bare Exception (registry / provider).
        if _LITELLM_UNMAPPED_MODEL_MSG.search(str(e)):
            logger.debug(
                f"Could not count tokens for model {model} (unmapped in registry): {e}"
            )
        else:
            logger.warning(
                f"Could not count tokens for model {model}, using char estimate: {e}"
            )
    return max(1, len(text) // 4)

estimate_cost_usd(model, prompt_tokens=0, completion_tokens=0)

Estimate cost in USD for prompt and completion tokens.

Uses litellm's cost_per_token. Returns (prompt_cost_usd, completion_cost_usd). Either or both may be None if the model is unknown or the call fails. Any other Exception from litellm (e.g. provider HTTP errors) is logged at debug and also yields (None, None) so cost estimation stays best-effort for callers.

Parameters:

Name Type Description Default
model str

Model identifier (e.g. "gpt-4o-mini", "azure/gpt-4o-mini").

 required
prompt_tokens int

Number of input/prompt tokens.

 0
completion_tokens int

Number of output/completion tokens.

 0

Returns:

Type Description
tuple[float | None, float | None]

Tuple of (prompt_cost_usd, completion_cost_usd). Either can be None.
Source code in deet/utils/tokenisation.py 
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
def estimate_cost_usd(
    model: str,
    prompt_tokens: int = 0,
    completion_tokens: int = 0,
) -> tuple[float | None, float | None]:
    """
    Estimate cost in USD for prompt and completion tokens.

    Uses litellm's cost_per_token. Returns (prompt_cost_usd, completion_cost_usd).
    Either or both may be None if the model is unknown or the call fails.
    Any other ``Exception`` from litellm (e.g. provider HTTP errors) is logged
    at debug and also yields ``(None, None)`` so cost estimation stays
    best-effort for callers.

    Args:
        model: Model identifier (e.g. "gpt-4o-mini", "azure/gpt-4o-mini").
        prompt_tokens: Number of input/prompt tokens.
        completion_tokens: Number of output/completion tokens.

    Returns:
        Tuple of (prompt_cost_usd, completion_cost_usd). Either can be None.

    """
    try:
        prompt_cost, completion_cost = litellm.cost_per_token(
            model=model,
            prompt_tokens=prompt_tokens,
            completion_tokens=completion_tokens,
        )
        return (
            float(prompt_cost) if prompt_cost is not None else None,
            float(completion_cost) if completion_cost is not None else None,
        )
    except _LITELLM_LOOKUP_ERRORS as e:
        logger.debug(f"Could not estimate cost for model {model}: {e}")
        return (None, None)
    except Exception as e:  # noqa: BLE001
        # litellm cost_per_token may raise bare Exception (registry / provider).
        if _LITELLM_UNMAPPED_MODEL_MSG.search(str(e)):
            logger.debug(
                f"Could not estimate cost for model {model} (unmapped in registry): {e}"
            )
        else:
            logger.debug(f"Could not estimate cost for model {model}: {e}")
        return (None, None)

get_model_max_tokens(model)

Get the maximum input tokens allowed for a model.

Uses litellm's model registry. Returns None if the model is unknown.

Parameters:

Name Type Description Default
model str

Model identifier (e.g. "gpt-4o-mini", "azure/gpt-4o-mini").

 required

Returns:

Type Description
int | None

Maximum input tokens, or None if model not found.

Raises:

Type Description
LitellmModelNotMappedError

When litellm raises a bare Exception whose message indicates an unmapped model (registry message).
Exception

Any other error from litellm is re-raised.
Source code in deet/utils/tokenisation.py 
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
def get_model_max_tokens(model: str) -> int | None:
    """
    Get the maximum input tokens allowed for a model.

    Uses litellm's model registry. Returns None if the model is unknown.

    Args:
        model: Model identifier (e.g. "gpt-4o-mini", "azure/gpt-4o-mini").

    Returns:
        Maximum input tokens, or None if model not found.

    Raises:
        LitellmModelNotMappedError: When litellm raises a bare ``Exception``
            whose message indicates an unmapped model (registry message).
        Exception: Any other error from litellm is re-raised.

    """
    try:
        result = litellm.get_max_tokens(model)
        return int(result) if result is not None else None
    except _LITELLM_LOOKUP_ERRORS as e:
        logger.debug(f"Could not get max tokens for model {model}: {e}")
        return None
    except Exception as e:
        if _LITELLM_UNMAPPED_MODEL_MSG.search(str(e)):
            logger.debug(f"Litellm model not in registry for max tokens: {model}: {e}")
            raise LitellmModelNotMappedError(model) from e
        raise

merge_prompt_completion_cost_usd(prompt_cost_usd, completion_cost_usd)

Combine litellm prompt and completion cost parts into a single total.

Either part may be None when the model table lacks that component or estimation failed for that side.

Parameters:

Name Type Description Default
prompt_cost_usd float | None

Estimated USD cost for prompt tokens, or None.

 required
completion_cost_usd float | None

Estimated USD cost for completion tokens, or None.

 required

Returns:

Type Description
float | None

Sum when at least one part is known, otherwise None.
Source code in deet/utils/tokenisation.py 
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
def merge_prompt_completion_cost_usd(
    prompt_cost_usd: float | None,
    completion_cost_usd: float | None,
) -> float | None:
    """
    Combine litellm prompt and completion cost parts into a single total.

    Either part may be None when the model table lacks that component or
    estimation failed for that side.

    Args:
        prompt_cost_usd: Estimated USD cost for prompt tokens, or None.
        completion_cost_usd: Estimated USD cost for completion tokens, or None.

    Returns:
        Sum when at least one part is known, otherwise None.

    """
    if prompt_cost_usd is not None and completion_cost_usd is not None:
        return prompt_cost_usd + completion_cost_usd
    if prompt_cost_usd is not None:
        return prompt_cost_usd
    if completion_cost_usd is not None:
        return completion_cost_usd
    return None

truncate_to_token_limit(text, model, max_tokens)

Truncate text to fit within a token limit.

Encodes text, truncates the token list, decodes back to string. Uses char-based fallback when encode/decode unavailable.

Parameters:

Name Type Description Default
text str

Text to truncate.

 required
model str

Model identifier for tokenizer.

 required
max_tokens int

Maximum tokens allowed for the truncated text.

 required

Returns:

Type Description
str

Truncated text.
Source code in deet/utils/tokenisation.py 
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
def truncate_to_token_limit(text: str, model: str, max_tokens: int) -> str:
    """
    Truncate text to fit within a token limit.

    Encodes text, truncates the token list, decodes back to string.
    Uses char-based fallback when encode/decode unavailable.

    Args:
        text: Text to truncate.
        model: Model identifier for tokenizer.
        max_tokens: Maximum tokens allowed for the truncated text.

    Returns:
        Truncated text.

    """
    limit = max(1, max_tokens)
    try:
        tokens = litellm.encode(model=model, text=text)
        if len(tokens) <= limit:
            return text
        truncated_tokens = tokens[:limit]
        decoded = litellm.decode(model=model, tokens=truncated_tokens)
        return str(decoded)
    except _LITELLM_TOKENISER_ERRORS as e:
        logger.debug(f"Could not truncate by tokens for model {model}: {e}")
    # Fallback: rough char estimate (~4 chars per token)
    approx_chars = limit * 4
    if len(text) <= approx_chars:
        return text
    return text[:approx_chars]