Skip to main content

Function: createCustomLlmMetric()

function createCustomLlmMetric(params: CreateCustomLlmMetricParams): Promise<{
  cot_enabled?: null | boolean;
  finetuned_scorer?: null | {
    executor?:
      | null
      | "agentic_session_success"
      | "agentic_workflow_success"
      | "action_completion_luna"
      | "action_advancement_luna"
      | "bleu"
      | "chunk_attribution_utilization_luna"
      | "chunk_attribution_utilization"
      | "completeness_luna"
      | "completeness"
      | "context_adherence"
      | "context_adherence_luna"
      | "context_relevance"
      | "correctness"
      | "ground_truth_adherence"
      | "input_pii"
      | "input_sexist"
      | "input_sexist_luna"
      | "input_tone"
      | "input_toxicity"
      | "input_toxicity_luna"
      | "instruction_adherence"
      | "output_pii"
      | "output_sexist"
      | "output_sexist_luna"
      | "output_tone"
      | "output_toxicity"
      | "output_toxicity_luna"
      | "prompt_injection"
      | "prompt_injection_luna"
      | "prompt_perplexity"
      | "rouge"
      | "tool_error_rate"
      | "tool_error_rate_luna"
      | "tool_selection_quality"
      | "tool_selection_quality_luna"
      | "uncertainty";
    id: string;
    lora_task_id: number;
    name: string;
    prompt: string;
  };
  generated_scorer?: null | {
    chain_poll_template: {
      explanation_field_name?: string;
      metric_description?: null | string;
      metric_few_shot_examples?: object[];
      metric_system_prompt?: null | string;
      template: string;
      value_field_name?: string;
    };
    id: string;
    instructions?: null | string;
    name: string;
    user_prompt?: null | string;
  };
  id: string;
  input_type?:
    | null
    | "basic"
    | "llm_spans"
    | "retriever_spans"
    | "sessions_normalized"
    | "sessions_trace_io_only"
    | "tool_spans"
    | "trace_input_only"
    | "trace_io_only"
    | "trace_normalized"
    | "trace_output_only";
  model_name?: null | string;
  num_judges?: null | number;
  output_type?:
    | null
    | "boolean"
    | "categorical"
    | "count"
    | "discrete"
    | "freeform"
    | "percentage"
    | "multilabel";
  registered_scorer?: null | {
    id: string;
    name: string;
  };
  scoreable_node_types?: null | string[];
  version: number;
}>;
Defined in: src/utils/metrics.ts:30 Creates a custom LLM metric.

Parameters

params

CreateCustomLlmMetricParams The parameters for creating the custom LLM metric.

Returns

Promise<{ cot_enabled?: null | boolean; finetuned_scorer?: | null | { executor?: | null | "agentic_session_success" | "agentic_workflow_success" | "action_completion_luna" | "action_advancement_luna" | "bleu" | "chunk_attribution_utilization_luna" | "chunk_attribution_utilization" | "completeness_luna" | "completeness" | "context_adherence" | "context_adherence_luna" | "context_relevance" | "correctness" | "ground_truth_adherence" | "input_pii" | "input_sexist" | "input_sexist_luna" | "input_tone" | "input_toxicity" | "input_toxicity_luna" | "instruction_adherence" | "output_pii" | "output_sexist" | "output_sexist_luna" | "output_tone" | "output_toxicity" | "output_toxicity_luna" | "prompt_injection" | "prompt_injection_luna" | "prompt_perplexity" | "rouge" | "tool_error_rate" | "tool_error_rate_luna" | "tool_selection_quality" | "tool_selection_quality_luna" | "uncertainty"; id: string; lora_task_id: number; name: string; prompt: string; }; generated_scorer?: | null | { chain_poll_template: { explanation_field_name?: string; metric_description?: null | string; metric_few_shot_examples?: object[]; metric_system_prompt?: null | string; template: string; value_field_name?: string; }; id: string; instructions?: null | string; name: string; user_prompt?: null | string; }; id: string; input_type?: | null | "basic" | "llm_spans" | "retriever_spans" | "sessions_normalized" | "sessions_trace_io_only" | "tool_spans" | "trace_input_only" | "trace_io_only" | "trace_normalized" | "trace_output_only"; model_name?: null | string; num_judges?: null | number; output_type?: | null | "boolean" | "categorical" | "count" | "discrete" | "freeform" | "percentage" | "multilabel"; registered_scorer?: | null | { id: string; name: string; }; scoreable_node_types?: null | string[]; version: number; }> A promise that resolves when the metric is created.