Source code for flexeval.schema.config_schema

from pathlib import Path

from pydantic import BaseModel, Field

from flexeval.schema import schema_utils




class Config(BaseModel):


    class Config:
        extra = "ignore"
        # validate_assignment will ensure that fields never take an invalid value e.g. if copied from an eval
        validate_assignment = True


    logs_path: Path | None = Field(
        None,
        description="Log directory path.",
    )
    env_filepath: Path | None = Field(
        None,
        description="A .env file to be processed by python-dotenv before running evals with this config.",
    )
    env: schema_utils.OptionalDict = Field(
        default_factory=dict, description="Any additional environment variables."
    )
    clear_tables: bool = Field(
        False,
        description="Clear any existing tables, if the output SQLite database already exists.",
    )
    max_workers: int = Field(
        1,
        description="Max worker count. Multiple threads will be used if set to > 1. This may have usage limit implications if you are calling APIs.",
    )
    random_seed_conversation_sampling: int = 42
    max_n_conversation_threads: int = (
        50  # TODO allow setting this to None, and set it to None by default
    )
    nb_evaluations_per_thread: int = 1

    raise_on_completion_error: bool = Field(
        False,
        description="If False (default), metrics will be run even if one or more completions fails.",
    )
    raise_on_metric_error: bool = Field(
        False,
        description="If False (default), no exception will be thrown if a metric function raises an exception.",
    )
    raise_on_duplicate_dataset_name: bool = Field(
        False,
        description="If True, throw an exception if two datasets would be created with the same name. Ignored when reuse_dataset_by_name is True.",
    )
    raise_on_unnamed_dataset: bool = Field(
        False,
        description="If True, throw an exception if any dataset is unnamed.",
    )
    reuse_dataset_by_name: bool = Field(
        True,
        description="If True (default), reuse a previously loaded dataset with the same name instead of creating a new one. This avoids redundant data loading and prevents iterator-based data sources from being consumed twice.",
    )