Index A | B | C | D | E | F | G | I | J | K | L | M | N | O | P | R | S | T | V A add_all_metrics_to_objects() (in module flexeval.compute_metrics) add_default_functions (flexeval.schema.evalrun_schema.EvalRun attribute) add_turns() (in module flexeval.data_loader) apply_defaults() (in module flexeval.dependency_graph) B BaseModel (class in flexeval.classes.base) bind_to_database() (in module flexeval.db_utils) build_datasets() (in module flexeval.run_utils) build_eval_set_run() (in module flexeval.run_utils) C check_non_empty() (flexeval.schema.rubric_schema.Rubric class method) choice_scores (flexeval.schema.rubric_schema.Rubric attribute) clear_tables (flexeval.schema.config_schema.Config attribute) completion_llm (flexeval.schema.eval_schema.Eval attribute) CompletionLlm.Config (class in flexeval.schema.eval_schema) compute_metrics() (in module flexeval.compute_metrics) config (flexeval.schema.evalrun_schema.EvalRun attribute) Config.Config (class in flexeval.schema.config_schema) constant() (in module flexeval.configuration.function_metrics) contents (flexeval.schema.evalrun_schema.IterableDataSource attribute) convert_none_or_empty_string_to_dict() (in module flexeval.schema.schema_utils) count_dict_values() (in module flexeval.metrics.access) count_emojis() (in module flexeval.configuration.function_metrics) count_errors() (in module flexeval.configuration.function_metrics) count_llm_models() (in module flexeval.configuration.function_metrics) count_messages() (in module flexeval.configuration.function_metrics) count_messages_per_role() (in module flexeval.configuration.function_metrics) count_numeric_tool_call_params_by_name() (in module flexeval.configuration.function_metrics) count_of_parts_matching_regex() (in module flexeval.configuration.function_metrics) count_rubric_metrics() (in module flexeval.compute_metrics) count_tokens() (in module flexeval.configuration.function_metrics) count_tool_calls() (in module flexeval.configuration.function_metrics) count_tool_calls_by_name() (in module flexeval.configuration.function_metrics) count_turns() (in module flexeval.configuration.function_metrics) create_metrics_graph() (in module flexeval.dependency_graph) create_sqlite_database() (in module flexeval.classes.base) D data_sources (flexeval.schema.evalrun_schema.EvalRun attribute) database (flexeval.classes.eval_runner.EvalRunner attribute) database_path (flexeval.schema.evalrun_schema.EvalRun attribute) Dataset (class in flexeval.classes.dataset) depends_on (flexeval.schema.eval_schema.MetricItem attribute) DependsOnItem.Config (class in flexeval.schema.eval_schema) do_completion (flexeval.schema.eval_schema.Eval attribute) E echo_completion() (in module flexeval.configuration.completion_functions) env (flexeval.schema.config_schema.Config attribute) env_filepath (flexeval.schema.config_schema.Config attribute) eval (flexeval.schema.evalrun_schema.EvalRun attribute) Eval.Config (class in flexeval.schema.eval_schema) EvalRunner (class in flexeval.classes.eval_runner) EvalSetRun (class in flexeval.classes.eval_set_run) extra (flexeval.schema.config_schema.Config.Config attribute) (flexeval.schema.eval_schema.CompletionLlm.Config attribute) (flexeval.schema.eval_schema.DependsOnItem.Config attribute) (flexeval.schema.eval_schema.Eval.Config attribute) (flexeval.schema.eval_schema.GraderLlm.Config attribute) F flesch_kincaid_grade() (in module flexeval.configuration.function_metrics) flesch_reading_ease() (in module flexeval.configuration.function_metrics) flexeval module flexeval.classes module flexeval.classes.base module flexeval.classes.dataset module flexeval.classes.eval_runner module flexeval.classes.eval_set_run module flexeval.classes.message module flexeval.classes.metric module flexeval.classes.thread module flexeval.classes.tool_call module flexeval.classes.turn module flexeval.cli module flexeval.completions module flexeval.compute_metrics module flexeval.configuration module flexeval.configuration.completion_functions module flexeval.configuration.function_metrics module flexeval.data_loader module flexeval.db_utils module flexeval.dependency_graph module flexeval.function_types module flexeval.helpers module flexeval.io module flexeval.log_utils module flexeval.metrics module flexeval.metrics.access module flexeval.metrics.save module flexeval.rubric module flexeval.run_utils module flexeval.runner module flexeval.schema module flexeval.schema.config_schema module flexeval.schema.eval_schema module flexeval.schema.evalrun_schema module flexeval.schema.rubric_schema module flexeval.schema.schema_utils module format (flexeval.schema.evalrun_schema.FileDataSource attribute) function (flexeval.schema.eval_schema.Metrics attribute) function_modules (flexeval.schema.evalrun_schema.EvalRun attribute) function_name (flexeval.schema.eval_schema.CompletionLlm attribute) (flexeval.schema.eval_schema.GraderLlm attribute) functions (flexeval.schema.evalrun_schema.FunctionsCollection attribute) G generate_hash() (in module flexeval.helpers) generic_rest_api_completion() (in module flexeval.configuration.completion_functions) get_acceptable_arg_types() (in module flexeval.function_types) get_all_metrics() (in module flexeval.metrics.access) get_completion() (in module flexeval.completions) get_completion_function() (in module flexeval.completions) get_completions() (in module flexeval.completions) get_default_function_metrics() (in module flexeval.schema.evalrun_schema) get_default_rubric_collection() (in module flexeval.rubric) get_default_rubrics() (in module flexeval.schema.evalrun_schema) get_first_parameter_types() (in module flexeval.function_types) get_function_input() (in module flexeval.function_types) get_metric_info() (in module flexeval.dependency_graph) get_parent_metrics() (in module flexeval.dependency_graph) get_turns() (in module flexeval.data_loader) get_valid_levels_for_callable() (in module flexeval.function_types) global_callback() (in module flexeval.cli) gpt_3p5_turbo() (in module flexeval.configuration.completion_functions) grader_llm (flexeval.schema.eval_schema.Eval attribute) GraderLlm.Config (class in flexeval.schema.eval_schema) I identity() (in module flexeval.configuration.function_metrics) include_system_prompt (flexeval.schema.eval_schema.CompletionLlm attribute) index_in_thread() (in module flexeval.configuration.function_metrics) initialize_database() (in module flexeval.db_utils) is_callable_valid_for_metric_level() (in module flexeval.function_types) is_langgraph_type() (in module flexeval.configuration.function_metrics) is_last_turn_in_thread() (in module flexeval.configuration.function_metrics) is_role() (in module flexeval.configuration.function_metrics) is_rubric_prompt_valid() (flexeval.schema.rubric_schema.Rubric class method) is_sqlite_file() (in module flexeval.classes.dataset) J jan_completion() (in module flexeval.configuration.completion_functions) join_all_contents_to_string() (in module flexeval.function_types) K kwargs (flexeval.schema.eval_schema.CompletionLlm attribute) (flexeval.schema.eval_schema.DependsOnItem attribute) (flexeval.schema.eval_schema.FunctionItem attribute) (flexeval.schema.eval_schema.GraderLlm attribute) (flexeval.schema.eval_schema.RubricItem attribute) L latency() (in module flexeval.configuration.function_metrics) litellm_completion() (in module flexeval.configuration.completion_functions) lm_studio_completion() (in module flexeval.configuration.completion_functions) load_jsonl() (in module flexeval.data_loader) load_langgraph_sqlite() (in module flexeval.data_loader) load_rubric_metrics() (in module flexeval.rubric) load_rubrics_from_yaml_file() (in module flexeval.rubric) load_rubrics_from_yaml_stream() (in module flexeval.rubric) LogLevel (class in flexeval.log_utils) logs_path (flexeval.schema.config_schema.Config attribute) M main() (in module flexeval.cli) max_n_conversation_threads (flexeval.schema.config_schema.Config attribute) max_workers (flexeval.schema.config_schema.Config attribute) Message (class in flexeval.classes.message) message_matches_regex() (in module flexeval.configuration.function_metrics) Metric (class in flexeval.classes.metric) metric_level (flexeval.schema.eval_schema.DependsOnItem attribute) (flexeval.schema.eval_schema.MetricItem attribute) metric_max_value (flexeval.schema.eval_schema.DependsOnItem attribute) metric_min_value (flexeval.schema.eval_schema.DependsOnItem attribute) metric_name (flexeval.schema.eval_schema.DependsOnItem attribute) MetricComputer (class in flexeval.compute_metrics) MetricGraphBuilder (class in flexeval.compute_metrics) metrics (flexeval.schema.eval_schema.Eval attribute) module flexeval flexeval.classes flexeval.classes.base flexeval.classes.dataset flexeval.classes.eval_runner flexeval.classes.eval_set_run flexeval.classes.message flexeval.classes.metric flexeval.classes.thread flexeval.classes.tool_call flexeval.classes.turn flexeval.cli flexeval.completions flexeval.compute_metrics flexeval.configuration flexeval.configuration.completion_functions flexeval.configuration.function_metrics flexeval.data_loader flexeval.db_utils flexeval.dependency_graph flexeval.function_types flexeval.helpers flexeval.io flexeval.log_utils flexeval.metrics flexeval.metrics.access flexeval.metrics.save flexeval.rubric flexeval.run_utils flexeval.runner flexeval.schema flexeval.schema.config_schema flexeval.schema.eval_schema flexeval.schema.evalrun_schema flexeval.schema.rubric_schema flexeval.schema.schema_utils N name (flexeval.classes.eval_set_run.EvalSetRun attribute) (flexeval.schema.eval_schema.DependsOnItem attribute) (flexeval.schema.eval_schema.Eval attribute) (flexeval.schema.eval_schema.MetricItem attribute) (flexeval.schema.evalrun_schema.DataSource attribute) (flexeval.schema.rubric_schema.Rubric attribute) nb_evaluations_per_thread (flexeval.schema.config_schema.Config attribute) no_completion_fn() (in module flexeval.configuration.completion_functions) notes (flexeval.schema.eval_schema.Eval attribute) (flexeval.schema.evalrun_schema.DataSource attribute) (flexeval.schema.rubric_schema.Rubric attribute) O ObjectMetric (class in flexeval.compute_metrics) open_ai_completion() (in module flexeval.configuration.completion_functions) open_ai_completion_async() (in module flexeval.configuration.completion_functions) openai_moderation_api() (in module flexeval.configuration.function_metrics) P path (flexeval.schema.evalrun_schema.FileDataSource attribute) placeholder_completion() (in module flexeval.configuration.completion_functions) process_conversation() (in module flexeval.configuration.function_metrics) process_single_message() (in module flexeval.configuration.function_metrics) prompt (flexeval.schema.rubric_schema.Rubric attribute) R raise_on_completion_error (flexeval.schema.config_schema.Config attribute) raise_on_metric_error (flexeval.schema.config_schema.Config attribute) random_seed_conversation_sampling (flexeval.schema.config_schema.Config attribute) relative_object_position (flexeval.schema.eval_schema.DependsOnItem attribute) rubric (flexeval.schema.eval_schema.Metrics attribute) rubric_paths (flexeval.schema.evalrun_schema.EvalRun attribute) rubrics (flexeval.schema.rubric_schema.RubricsCollection attribute) run() (in module flexeval) (in module flexeval.cli) (in module flexeval.runner) run_eval_by_name() (in module flexeval.cli) run_from_name_args() (in module flexeval.runner) S save_completion() (in module flexeval.completions) save_metrics() (in module flexeval.metrics.save) set_up_logging() (in module flexeval.log_utils) string_length() (in module flexeval.configuration.function_metrics) summarize_metrics() (in module flexeval.cli) T Thread (class in flexeval.classes.thread) tool_was_called() (in module flexeval.configuration.function_metrics) ToolCall (class in flexeval.classes.tool_call) Turn (class in flexeval.classes.turn) type (flexeval.schema.eval_schema.DependsOnItem attribute) V validate_assignment (flexeval.schema.config_schema.Config.Config attribute) validate_python_module() (in module flexeval.schema.schema_utils) value_counts_by_tool_name() (in module flexeval.configuration.function_metrics) verify_checkpoints_table_exists() (in module flexeval.data_loader) visualize_graph() (in module flexeval.helpers)
