API ReferenceClass: GalileoApiClient
Defined in: src/api-client/galileo-client.ts:52Extends
BaseClient
Constructors
Constructor
Returns
GalileoApiClient
Inherited from
Properties
apiUrl
Inherited from
client
Inherited from
datasetId
experimentId
logStreamId
projectId
projectScoped
projectType
runId
sessionId?
token
Inherited from
Methods
appendRowsToDatasetContent()
Parameters
datasetId
string
etag
string
rows
object[]
Returns
Promise<void>
createDataset()
Parameters
name
string
filePath
string
format
any
Returns
Promise<{
column_names: null | string[];
created_at: string;
created_by_user: | null
| {
email: string;
first_name?: null | string;
id: string;
last_name?: null | string;
};
current_version_index: number;
draft: boolean;
id: string;
name: string;
num_rows: null | number;
permissions?: object[];
project_count: number;
updated_at: string;
}>
createExperiment()
Parameters
name
string
Returns
Promise<Experiment>
createGlobalPromptTemplate()
Parameters
template
Message[]
name
string
Returns
Promise<{
all_available_versions: number[];
all_versions?: object[];
created_at: string;
created_by_user: | null
| {
email: string;
first_name?: null | string;
id: string;
last_name?: null | string;
};
id: string;
max_version: number;
name: string;
permissions?: object[];
selected_version: {
content_changed: boolean;
created_at: string;
created_by_user: | null
| {
email: string;
first_name?: null | string;
id: string;
last_name?: null | string;
};
id: string;
lines_added?: number;
lines_edited?: number;
lines_removed?: number;
model_changed: boolean;
output_type?: null | string;
raw?: boolean;
settings: {
deployment_name?: null | string;
echo?: boolean;
frequency_penalty?: number;
known_models?: object[];
logprobs?: boolean;
max_tokens?: number;
model_alias?: string;
n?: number;
presence_penalty?: number;
response_format?: | null
| {
[key: string]: string;
};
stop_sequences?: null | string[];
temperature?: number;
tool_choice?: | null
| string
| {
function: {
name: string;
};
type?: string;
};
tools?: null | object[];
top_k?: number;
top_logprobs?: number;
top_p?: number;
};
settings_changed: boolean;
template: string | object & object[];
updated_at: string;
version: number;
};
selected_version_id: string;
template: string;
total_versions: number;
updated_at: string;
}>
createLlmScorerVersion()
Parameters
scorerId
string
instructions?
string
chainPollTemplate?
explanation_field_name?
string
Explanation Field Name
Description
Field name to look for in the chainpoll response, for the explanation.
Default
metric_description?
null | string
Metric Description
Description
Description of what the metric should do.
metric_few_shot_examples?
object[]
Metric Few Shot Examples
Description
Few-shot examples for the metric.
metric_system_prompt?
null | string
Metric System Prompt
Description
System prompt for the metric.
template
string
Template
Description
Chainpoll prompt template.
value_field_name?
string
Value Field Name
Description
Field name to look for in the chainpoll response, for the rating.
Default
userPrompt?
string
scoreableNodeTypes?
StepType[]
cotEnabled?
boolean
modelName?
string
numJudges?
number
outputType?
OutputType
Returns
Promise<{
cot_enabled?: null | boolean;
generated_scorer?: | null
| {
chain_poll_template: {
explanation_field_name?: string;
metric_description?: null | string;
metric_few_shot_examples?: object[];
metric_system_prompt?: null | string;
template: string;
value_field_name?: string;
};
id: string;
instructions?: null | string;
name: string;
user_prompt?: null | string;
};
id: string;
input_type?: | null
| "basic"
| "llm_spans"
| "retriever_spans"
| "sessions_normalized"
| "sessions_trace_io_only"
| "tool_spans"
| "trace_input_only"
| "trace_io_only"
| "trace_normalized"
| "trace_output_only";
model_name?: null | string;
num_judges?: null | number;
output_type?: | null
| "boolean"
| "categorical"
| "count"
| "discrete"
| "freeform"
| "percentage";
registered_scorer?: | null
| {
id: string;
name: string;
};
scoreable_node_types?: null | string[];
version: number;
}>
createLogStream()
Parameters
name
string
Returns
Promise<LogStream>
createProject()
Parameters
name
string
Returns
Promise<Project>
createPromptRunJob()
Parameters
experimentId
string
projectId
string
promptTemplateVersionId
string
datasetId
string
scorers?
object[]
promptSettings?
PromptRunSettings
Returns
Promise<{
dataset_id?: null | string;
dataset_version_index?: null | number;
epoch?: number;
feature_names?: null | string[];
job_id?: null | string;
job_name?: string;
labels?: string[] | string[][];
link: string;
message: string;
metric_critique_configuration?: | null
| {
critique_ids: string[];
metric_name: string;
project_type: "prompt_evaluation" | "llm_monitor" | "gen_ai";
recompute_settings?: | null
| {
mode: "runs";
run_ids: string[];
}
| {
mode: "project";
}
| {
filters: unknown[];
mode: "observe_filters";
}
| {
filters: unknown[];
mode: "log_stream_filters";
run_id: string;
};
scorer_id?: null | string;
};
migration_name?: null | string;
monitor_batch_id?: null | string;
ner_labels?: null | string[];
non_inference_logged?: boolean;
process_existing_inference_runs?: boolean;
project_id: string;
prompt_customized_scorers_configuration?: | null
| (
| {
aggregate_keys?: string[];
aggregates?: | null
| {
[key: string]: unknown;
};
chainpoll_template?: {
explanation_field_name?: string;
metric_description?: string;
metric_few_shot_examples?: object[];
metric_system_prompt?: string;
template?: string;
value_field_name?: string;
};
cot_enabled?: null | boolean;
description?: null | string;
extra?: | null
| {
[key: string]: unknown;
};
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
generated_scorer_id?: null | string;
indices?: null | number[];
input_type?: | null
| "basic"
| "llm_spans"
| "retriever_spans"
| "sessions_normalized"
| "sessions_trace_io_only"
| "tool_spans"
| "trace_input_only"
| "trace_io_only"
| "trace_normalized"
| "trace_output_only";
lora_task_id?: null | number;
metric_name?: null | string;
model_alias?: string;
name?: "agentic_session_success";
num_judges?: number;
output_type?: | null
| "boolean"
| "categorical"
| "count"
| "discrete"
| "freeform"
| "percentage";
prompt?: null | string;
regex_field?: string;
registered_scorer_id?: null | string;
scoreable_node_types?: | null
| (
| "agent"
| "llm"
| "retriever"
| "tool"
| "workflow"
| "trace"
| "session"
| "chain"
| "chat")[];
scorer_name: "_customized_agentic_session_success";
scores?: null | unknown[];
sub_scorers?: (
| "_completeness_gpt"
| "_context_adherence_luna"
| "_context_relevance"
| "_chunk_attribution_utilization_gpt"
| "_factuality"
| "_groundedness"
| "_latency"
| "_prompt_perplexity"
| "_protect_status"
| "_pii"
| "_input_pii"
| "_sexist"
| "_input_sexist"
| "_sexist_gpt"
| "_input_sexist_gpt"
| "_tone"
| "_input_tone"
| "_toxicity"
| "_toxicity_gpt"
| "_input_toxicity"
| "_input_toxicity_gpt"
| "_user_registered"
| "_user_submitted"
| "_user_generated"
| "_user_finetuned"
| "_uncertainty"
| "_bleu"
| "_cost"
| "_rouge"
| "_prompt_injection_gpt"
| "_prompt_injection"
| "_rag_nli"
| "_adherence_nli"
| "_completeness_nli"
| "_chunk_attribution_utilization_nli"
| "_instruction_adherence"
| "_ground_truth_adherence"
| "_tool_selection_quality"
| "_tool_selection_quality_luna"
| "_tool_error_rate"
| "_tool_error_rate_luna"
| "_action_completion_luna"
| "_agentic_session_success"
| "_action_advancement_luna"
| "_agentic_workflow_success"
| "_generic_wizard"
| "_customized_completeness_gpt"
| "_customized_factuality"
| "_customized_groundedness"
| "_customized_chunk_attribution_utilization_gpt"
| "_customized_instruction_adherence"
| "_customized_ground_truth_adherence"
| "_customized_prompt_injection_gpt"
| "_customized_tool_selection_quality"
| "_customized_tool_error_rate"
| "_customized_agentic_session_success"
| "_customized_agentic_workflow_success"
| "_customized_sexist_gpt"
| "_customized_input_sexist_gpt"
| "_customized_toxicity_gpt"
| "_customized_input_toxicity_gpt")[];
}
| {
aggregate_keys?: string[];
aggregates?: | null
| {
[key: string]: unknown;
};
chainpoll_template?: {
explanation_field_name?: string;
metric_description?: string;
metric_few_shot_examples?: object[];
metric_system_prompt?: string;
template?: string;
value_field_name?: string;
};
cot_enabled?: null | boolean;
description?: null | string;
extra?: | null
| {
[key: string]: unknown;
};
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
generated_scorer_id?: null | string;
indices?: null | number[];
input_type?: | null
| "basic"
| "llm_spans"
| "retriever_spans"
| "sessions_normalized"
| "sessions_trace_io_only"
| "tool_spans"
| "trace_input_only"
| "trace_io_only"
| "trace_normalized"
| "trace_output_only";
lora_task_id?: null | number;
metric_name?: null | string;
model_alias?: string;
name?: "agentic_workflow_success";
num_judges?: number;
output_type?: | null
| "boolean"
| "categorical"
| "count"
| "discrete"
| "freeform"
| "percentage";
prompt?: null | string;
regex_field?: string;
registered_scorer_id?: null | string;
scoreable_node_types?: | null
| (
| "agent"
| "llm"
| "retriever"
| "tool"
| "workflow"
| "trace"
| "session"
| "chain"
| "chat")[];
scorer_name: "_customized_agentic_workflow_success";
scores?: null | unknown[];
sub_scorers?: (
| "_completeness_gpt"
| "_context_adherence_luna"
| "_context_relevance"
| "_chunk_attribution_utilization_gpt"
| "_factuality"
| "_groundedness"
| "_latency"
| "_prompt_perplexity"
| "_protect_status"
| "_pii"
| "_input_pii"
| "_sexist"
| "_input_sexist"
| "_sexist_gpt"
| "_input_sexist_gpt"
| "_tone"
| "_input_tone"
| "_toxicity"
| "_toxicity_gpt"
| "_input_toxicity"
| "_input_toxicity_gpt"
| "_user_registered"
| "_user_submitted"
| "_user_generated"
| "_user_finetuned"
| "_uncertainty"
| "_bleu"
| "_cost"
| "_rouge"
| "_prompt_injection_gpt"
| "_prompt_injection"
| "_rag_nli"
| "_adherence_nli"
| "_completeness_nli"
| "_chunk_attribution_utilization_nli"
| "_instruction_adherence"
| "_ground_truth_adherence"
| "_tool_selection_quality"
| "_tool_selection_quality_luna"
| "_tool_error_rate"
| "_tool_error_rate_luna"
| "_action_completion_luna"
| "_agentic_session_success"
| "_action_advancement_luna"
| "_agentic_workflow_success"
| "_generic_wizard"
| "_customized_completeness_gpt"
| "_customized_factuality"
| "_customized_groundedness"
| "_customized_chunk_attribution_utilization_gpt"
| "_customized_instruction_adherence"
| "_customized_ground_truth_adherence"
| "_customized_prompt_injection_gpt"
| "_customized_tool_selection_quality"
| "_customized_tool_error_rate"
| "_customized_agentic_session_success"
| "_customized_agentic_workflow_success"
| "_customized_sexist_gpt"
| "_customized_input_sexist_gpt"
| "_customized_toxicity_gpt"
| "_customized_input_toxicity_gpt")[];
}
| {
aggregate_keys?: string[];
aggregates?: | null
| {
[key: string]: unknown;
};
chainpoll_template?: {
explanation_field_name?: string;
metric_description?: null | string;
metric_few_shot_examples?: object[];
metric_system_prompt?: null | string;
template?: string;
value_field_name?: string;
};
cot_enabled?: null | boolean;
description?: null | string;
extra?: | null
| {
[key: string]: unknown;
};
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
generated_scorer_id?: null | string;
indices?: null | number[];
input_type?: | null
| "basic"
| "llm_spans"
| "retriever_spans"
| "sessions_normalized"
| "sessions_trace_io_only"
| "tool_spans"
| "trace_input_only"
| "trace_io_only"
| "trace_normalized"
| "trace_output_only";
lora_task_id?: null | number;
metric_name?: null | string;
model_alias?: string;
name?: "chunk_attribution_utilization";
num_judges?: number;
output_type?: | null
| "boolean"
| "categorical"
| "count"
| "discrete"
| "freeform"
| "percentage";
prompt?: null | string;
regex_field?: string;
registered_scorer_id?: null | string;
scoreable_node_types?: | null
| (
| "agent"
| "llm"
| "retriever"
| "tool"
| "workflow"
| "trace"
| "session"
| "chain"
| "chat")[];
scorer_name: "_customized_chunk_attribution_utilization_gpt";
scores?: null | unknown[];
sub_scorers?: (
| "_completeness_gpt"
| "_context_adherence_luna"
| "_context_relevance"
| "_chunk_attribution_utilization_gpt"
| "_factuality"
| "_groundedness"
| "_latency"
| "_prompt_perplexity"
| "_protect_status"
| "_pii"
| "_input_pii"
| "_sexist"
| "_input_sexist"
| "_sexist_gpt"
| "_input_sexist_gpt"
| "_tone"
| "_input_tone"
| "_toxicity"
| "_toxicity_gpt"
| "_input_toxicity"
| "_input_toxicity_gpt"
| "_user_registered"
| "_user_submitted"
| "_user_generated"
| "_user_finetuned"
| "_uncertainty"
| "_bleu"
| "_cost"
| "_rouge"
| "_prompt_injection_gpt"
| "_prompt_injection"
| "_rag_nli"
| "_adherence_nli"
| "_completeness_nli"
| "_chunk_attribution_utilization_nli"
| "_instruction_adherence"
| "_ground_truth_adherence"
| "_tool_selection_quality"
| "_tool_selection_quality_luna"
| "_tool_error_rate"
| "_tool_error_rate_luna"
| "_action_completion_luna"
| "_agentic_session_success"
| "_action_advancement_luna"
| "_agentic_workflow_success"
| "_generic_wizard"
| "_customized_completeness_gpt"
| "_customized_factuality"
| "_customized_groundedness"
| "_customized_chunk_attribution_utilization_gpt"
| "_customized_instruction_adherence"
| "_customized_ground_truth_adherence"
| "_customized_prompt_injection_gpt"
| "_customized_tool_selection_quality"
| "_customized_tool_error_rate"
| "_customized_agentic_session_success"
| "_customized_agentic_workflow_success"
| "_customized_sexist_gpt"
| "_customized_input_sexist_gpt"
| "_customized_toxicity_gpt"
| "_customized_input_toxicity_gpt")[];
}
| {
aggregate_keys?: string[];
aggregates?: | null
| {
[key: string]: unknown;
};
chainpoll_template?: {
explanation_field_name?: string;
metric_description?: null | string;
metric_few_shot_examples?: object[];
metric_system_prompt?: null | string;
template?: string;
value_field_name?: string;
};
cot_enabled?: null | boolean;
description?: null | string;
extra?: | null
| {
[key: string]: unknown;
};
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
generated_scorer_id?: null | string;
indices?: null | number[];
input_type?: | null
| "basic"
| "llm_spans"
| "retriever_spans"
| "sessions_normalized"
| "sessions_trace_io_only"
| "tool_spans"
| "trace_input_only"
| "trace_io_only"
| "trace_normalized"
| "trace_output_only";
lora_task_id?: null | number;
metric_name?: null | string;
model_alias?: string;
name?: "completeness";
num_judges?: number;
output_type?: | null
| "boolean"
| "categorical"
| "count"
| "discrete"
| "freeform"
| "percentage";
prompt?: null | string;
regex_field?: string;
registered_scorer_id?: null | string;
scoreable_node_types?: | null
| (
| "agent"
| "llm"
| "retriever"
| "tool"
| "workflow"
| "trace"
| "session"
| "chain"
| "chat")[];
scorer_name: "_customized_completeness_gpt";
scores?: null | unknown[];
sub_scorers?: (
| "_completeness_gpt"
| "_context_adherence_luna"
| "_context_relevance"
| "_chunk_attribution_utilization_gpt"
| "_factuality"
| "_groundedness"
| "_latency"
| "_prompt_perplexity"
| "_protect_status"
| "_pii"
| "_input_pii"
| "_sexist"
| "_input_sexist"
| "_sexist_gpt"
| "_input_sexist_gpt"
| "_tone"
| "_input_tone"
| "_toxicity"
| "_toxicity_gpt"
| "_input_toxicity"
| "_input_toxicity_gpt"
| "_user_registered"
| "_user_submitted"
| "_user_generated"
| "_user_finetuned"
| "_uncertainty"
| "_bleu"
| "_cost"
| "_rouge"
| "_prompt_injection_gpt"
| "_prompt_injection"
| "_rag_nli"
| "_adherence_nli"
| "_completeness_nli"
| "_chunk_attribution_utilization_nli"
| "_instruction_adherence"
| "_ground_truth_adherence"
| "_tool_selection_quality"
| "_tool_selection_quality_luna"
| "_tool_error_rate"
| "_tool_error_rate_luna"
| "_action_completion_luna"
| "_agentic_session_success"
| "_action_advancement_luna"
| "_agentic_workflow_success"
| "_generic_wizard"
| "_customized_completeness_gpt"
| "_customized_factuality"
| "_customized_groundedness"
| "_customized_chunk_attribution_utilization_gpt"
| "_customized_instruction_adherence"
| "_customized_ground_truth_adherence"
| "_customized_prompt_injection_gpt"
| "_customized_tool_selection_quality"
| "_customized_tool_error_rate"
| "_customized_agentic_session_success"
| "_customized_agentic_workflow_success"
| "_customized_sexist_gpt"
| "_customized_input_sexist_gpt"
| "_customized_toxicity_gpt"
| "_customized_input_toxicity_gpt")[];
}
| {
aggregate_keys?: string[];
aggregates?: | null
| {
[key: string]: unknown;
};
chainpoll_template?: {
explanation_field_name?: string;
metric_description?: null | string;
metric_few_shot_examples?: object[];
metric_system_prompt?: string;
template?: string;
value_field_name?: string;
};
cot_enabled?: null | boolean;
description?: null | string;
extra?: | null
| {
[key: string]: unknown;
};
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
function_explanation_param_name?: string;
generated_scorer_id?: null | string;
indices?: null | number[];
input_type?: | null
| "basic"
| "llm_spans"
| "retriever_spans"
| "sessions_normalized"
| "sessions_trace_io_only"
| "tool_spans"
| "trace_input_only"
| "trace_io_only"
| "trace_normalized"
| "trace_output_only";
lora_task_id?: null | number;
metric_name?: null | string;
model_alias?: string;
name?: "correctness";
num_judges?: number;
output_type?: | null
| "boolean"
| "categorical"
| "count"
| "discrete"
| "freeform"
| "percentage";
prompt?: null | string;
regex_field?: string;
registered_scorer_id?: null | string;
scoreable_node_types?: | null
| (
| "agent"
| "llm"
| "retriever"
| "tool"
| "workflow"
| "trace"
| "session"
| "chain"
| "chat")[];
scorer_name: "_customized_factuality";
scores?: null | unknown[];
sub_scorers?: (
| "_completeness_gpt"
| "_context_adherence_luna"
| "_context_relevance"
| "_chunk_attribution_utilization_gpt"
| "_factuality"
| "_groundedness"
| "_latency"
| "_prompt_perplexity"
| "_protect_status"
| "_pii"
| "_input_pii"
| "_sexist"
| "_input_sexist"
| "_sexist_gpt"
| "_input_sexist_gpt"
| "_tone"
| "_input_tone"
| "_toxicity"
| "_toxicity_gpt"
| "_input_toxicity"
| "_input_toxicity_gpt"
| "_user_registered"
| "_user_submitted"
| "_user_generated"
| "_user_finetuned"
| "_uncertainty"
| "_bleu"
| "_cost"
| "_rouge"
| "_prompt_injection_gpt"
| "_prompt_injection"
| "_rag_nli"
| "_adherence_nli"
| "_completeness_nli"
| "_chunk_attribution_utilization_nli"
| "_instruction_adherence"
| "_ground_truth_adherence"
| "_tool_selection_quality"
| "_tool_selection_quality_luna"
| "_tool_error_rate"
| "_tool_error_rate_luna"
| "_action_completion_luna"
| "_agentic_session_success"
| "_action_advancement_luna"
| "_agentic_workflow_success"
| "_generic_wizard"
| "_customized_completeness_gpt"
| "_customized_factuality"
| "_customized_groundedness"
| "_customized_chunk_attribution_utilization_gpt"
| "_customized_instruction_adherence"
| "_customized_ground_truth_adherence"
| "_customized_prompt_injection_gpt"
| "_customized_tool_selection_quality"
| "_customized_tool_error_rate"
| "_customized_agentic_session_success"
| "_customized_agentic_workflow_success"
| "_customized_sexist_gpt"
| "_customized_input_sexist_gpt"
| "_customized_toxicity_gpt"
| "_customized_input_toxicity_gpt")[];
}
| {
aggregate_keys?: string[];
aggregates?: | null
| {
[key: string]: unknown;
};
chainpoll_template?: {
explanation_field_name?: string;
metric_description?: string;
metric_few_shot_examples?: object[];
metric_system_prompt?: string;
template?: string;
value_field_name?: string;
};
cot_enabled?: null | boolean;
description?: null | string;
extra?: | null
| {
[key: string]: unknown;
};
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
generated_scorer_id?: null | string;
indices?: null | number[];
input_type?: | null
| "basic"
| "llm_spans"
| "retriever_spans"
| "sessions_normalized"
| "sessions_trace_io_only"
| "tool_spans"
| "trace_input_only"
| "trace_io_only"
| "trace_normalized"
| "trace_output_only";
lora_task_id?: null | number;
metric_name?: null | string;
model_alias?: string;
name?: "context_adherence";
num_judges?: number;
output_type?: | null
| "boolean"
| "categorical"
| "count"
| "discrete"
| "freeform"
| "percentage";
prompt?: null | string;
regex_field?: string;
registered_scorer_id?: null | string;
scoreable_node_types?: | null
| (
| "agent"
| "llm"
| "retriever"
| "tool"
| "workflow"
| "trace"
| "session"
| "chain"
| "chat")[];
scorer_name: "_customized_groundedness";
scores?: null | unknown[];
sub_scorers?: (
| "_completeness_gpt"
| "_context_adherence_luna"
| "_context_relevance"
| "_chunk_attribution_utilization_gpt"
| "_factuality"
| "_groundedness"
| "_latency"
| "_prompt_perplexity"
| "_protect_status"
| "_pii"
| "_input_pii"
| "_sexist"
| "_input_sexist"
| "_sexist_gpt"
| "_input_sexist_gpt"
| "_tone"
| "_input_tone"
| "_toxicity"
| "_toxicity_gpt"
| "_input_toxicity"
| "_input_toxicity_gpt"
| "_user_registered"
| "_user_submitted"
| "_user_generated"
| "_user_finetuned"
| "_uncertainty"
| "_bleu"
| "_cost"
| "_rouge"
| "_prompt_injection_gpt"
| "_prompt_injection"
| "_rag_nli"
| "_adherence_nli"
| "_completeness_nli"
| "_chunk_attribution_utilization_nli"
| "_instruction_adherence"
| "_ground_truth_adherence"
| "_tool_selection_quality"
| "_tool_selection_quality_luna"
| "_tool_error_rate"
| "_tool_error_rate_luna"
| "_action_completion_luna"
| "_agentic_session_success"
| "_action_advancement_luna"
| "_agentic_workflow_success"
| "_generic_wizard"
| "_customized_completeness_gpt"
| "_customized_factuality"
| "_customized_groundedness"
| "_customized_chunk_attribution_utilization_gpt"
| "_customized_instruction_adherence"
| "_customized_ground_truth_adherence"
| "_customized_prompt_injection_gpt"
| "_customized_tool_selection_quality"
| "_customized_tool_error_rate"
| "_customized_agentic_session_success"
| "_customized_agentic_workflow_success"
| "_customized_sexist_gpt"
| "_customized_input_sexist_gpt"
| "_customized_toxicity_gpt"
| "_customized_input_toxicity_gpt")[];
}
| {
aggregate_keys?: string[];
aggregates?: | null
| {
[key: string]: unknown;
};
chainpoll_template?: {
explanation_field_name?: string;
metric_description?: string;
metric_few_shot_examples?: object[];
metric_system_prompt?: string;
template?: string;
value_field_name?: string;
};
cot_enabled?: null | boolean;
description?: null | string;
extra?: | null
| {
[key: string]: unknown;
};
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
function_explanation_param_name?: string;
generated_scorer_id?: null | string;
indices?: null | number[];
input_type?: | null
| "basic"
| "llm_spans"
| "retriever_spans"
| "sessions_normalized"
| "sessions_trace_io_only"
| "tool_spans"
| "trace_input_only"
| "trace_io_only"
| "trace_normalized"
| "trace_output_only";
lora_task_id?: null | number;
metric_name?: null | string;
model_alias?: string;
name?: "instruction_adherence";
num_judges?: number;
output_type?: | null
| "boolean"
| "categorical"
| "count"
| "discrete"
| "freeform"
| "percentage";
prompt?: null | string;
regex_field?: string;
registered_scorer_id?: null | string;
scoreable_node_types?: | null
| (
| "agent"
| "llm"
| "retriever"
| "tool"
| "workflow"
| "trace"
| "session"
| "chain"
| "chat")[];
scorer_name: "_customized_instruction_adherence";
scores?: null | unknown[];
sub_scorers?: (
| "_completeness_gpt"
| "_context_adherence_luna"
| "_context_relevance"
| "_chunk_attribution_utilization_gpt"
| "_factuality"
| "_groundedness"
| "_latency"
| "_prompt_perplexity"
| "_protect_status"
| "_pii"
| "_input_pii"
| "_sexist"
| "_input_sexist"
| "_sexist_gpt"
| "_input_sexist_gpt"
| "_tone"
| "_input_tone"
| "_toxicity"
| "_toxicity_gpt"
| "_input_toxicity"
| "_input_toxicity_gpt"
| "_user_registered"
| "_user_submitted"
| "_user_generated"
| "_user_finetuned"
| "_uncertainty"
| "_bleu"
| "_cost"
| "_rouge"
| "_prompt_injection_gpt"
| "_prompt_injection"
| "_rag_nli"
| "_adherence_nli"
| "_completeness_nli"
| "_chunk_attribution_utilization_nli"
| "_instruction_adherence"
| "_ground_truth_adherence"
| "_tool_selection_quality"
| "_tool_selection_quality_luna"
| "_tool_error_rate"
| "_tool_error_rate_luna"
| "_action_completion_luna"
| "_agentic_session_success"
| "_action_advancement_luna"
| "_agentic_workflow_success"
| "_generic_wizard"
| "_customized_completeness_gpt"
| "_customized_factuality"
| "_customized_groundedness"
| "_customized_chunk_attribution_utilization_gpt"
| "_customized_instruction_adherence"
| "_customized_ground_truth_adherence"
| "_customized_prompt_injection_gpt"
| "_customized_tool_selection_quality"
| "_customized_tool_error_rate"
| "_customized_agentic_session_success"
| "_customized_agentic_workflow_success"
| "_customized_sexist_gpt"
| "_customized_input_sexist_gpt"
| "_customized_toxicity_gpt"
| "_customized_input_toxicity_gpt")[];
}
| {
aggregate_keys?: string[];
aggregates?: | null
| {
[key: string]: unknown;
};
chainpoll_template?: {
explanation_field_name?: string;
metric_description?: string;
metric_few_shot_examples?: object[];
metric_system_prompt?: string;
template?: string;
value_field_name?: string;
};
cot_enabled?: null | boolean;
description?: null | string;
extra?: | null
| {
[key: string]: unknown;
};
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
generated_scorer_id?: null | string;
indices?: null | number[];
input_type?: | null
| "basic"
| "llm_spans"
| "retriever_spans"
| "sessions_normalized"
| "sessions_trace_io_only"
| "tool_spans"
| "trace_input_only"
| "trace_io_only"
| "trace_normalized"
| "trace_output_only";
lora_task_id?: null | number;
metric_name?: null | string;
model_alias?: string;
name?: "ground_truth_adherence";
num_judges?: number;
output_type?: | null
| "boolean"
| "categorical"
| "count"
| "discrete"
| "freeform"
| "percentage";
prompt?: null | string;
regex_field?: string;
registered_scorer_id?: null | string;
scoreable_node_types?: | null
| (
| "agent"
| "llm"
| "retriever"
| "tool"
| "workflow"
| "trace"
| "session"
| "chain"
| "chat")[];
scorer_name: "_customized_ground_truth_adherence";
scores?: null | unknown[];
sub_scorers?: (
| "_completeness_gpt"
| "_context_adherence_luna"
| "_context_relevance"
| "_chunk_attribution_utilization_gpt"
| "_factuality"
| "_groundedness"
| "_latency"
| "_prompt_perplexity"
| "_protect_status"
| "_pii"
| "_input_pii"
| "_sexist"
| "_input_sexist"
| "_sexist_gpt"
| "_input_sexist_gpt"
| "_tone"
| "_input_tone"
| "_toxicity"
| "_toxicity_gpt"
| "_input_toxicity"
| "_input_toxicity_gpt"
| "_user_registered"
| "_user_submitted"
| "_user_generated"
| "_user_finetuned"
| "_uncertainty"
| "_bleu"
| "_cost"
| "_rouge"
| "_prompt_injection_gpt"
| "_prompt_injection"
| "_rag_nli"
| "_adherence_nli"
| "_completeness_nli"
| "_chunk_attribution_utilization_nli"
| "_instruction_adherence"
| "_ground_truth_adherence"
| "_tool_selection_quality"
| "_tool_selection_quality_luna"
| "_tool_error_rate"
| "_tool_error_rate_luna"
| "_action_completion_luna"
| "_agentic_session_success"
| "_action_advancement_luna"
| "_agentic_workflow_success"
| "_generic_wizard"
| "_customized_completeness_gpt"
| "_customized_factuality"
| "_customized_groundedness"
| "_customized_chunk_attribution_utilization_gpt"
| "_customized_instruction_adherence"
| "_customized_ground_truth_adherence"
| "_customized_prompt_injection_gpt"
| "_customized_tool_selection_quality"
| "_customized_tool_error_rate"
| "_customized_agentic_session_success"
| "_customized_agentic_workflow_success"
| "_customized_sexist_gpt"
| "_customized_input_sexist_gpt"
| "_customized_toxicity_gpt"
| "_customized_input_toxicity_gpt")[];
}
| {
aggregate_keys?: string[];
aggregates?: | null
| {
[key: string]: unknown;
};
chainpoll_template?: {
explanation_field_name?: string;
metric_description?: string;
metric_few_shot_examples?: object[];
metric_system_prompt?: string;
template?: string;
value_field_name?: string;
};
cot_enabled?: null | boolean;
description?: null | string;
extra?: | null
| {
[key: string]: unknown;
};
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
generated_scorer_id?: null | string;
indices?: null | number[];
input_type?: | null
| "basic"
| "llm_spans"
| "retriever_spans"
| "sessions_normalized"
| "sessions_trace_io_only"
| "tool_spans"
| "trace_input_only"
| "trace_io_only"
| "trace_normalized"
| "trace_output_only";
lora_task_id?: null | number;
metric_name?: null | string;
model_alias?: string;
name?: "prompt_injection";
num_judges?: number;
output_type?: | null
| "boolean"
| "categorical"
| "count"
| "discrete"
| "freeform"
| "percentage";
prompt?: null | string;
regex_field?: string;
registered_scorer_id?: null | string;
scoreable_node_types?: | null
| (
| "agent"
| "llm"
| "retriever"
| "tool"
| "workflow"
| "trace"
| "session"
| "chain"
| "chat")[];
scorer_name: "_customized_prompt_injection_gpt";
scores?: null | unknown[];
sub_scorers?: (
| "_completeness_gpt"
| "_context_adherence_luna"
| "_context_relevance"
| "_chunk_attribution_utilization_gpt"
| "_factuality"
| "_groundedness"
| "_latency"
| "_prompt_perplexity"
| "_protect_status"
| "_pii"
| "_input_pii"
| "_sexist"
| "_input_sexist"
| "_sexist_gpt"
| "_input_sexist_gpt"
| "_tone"
| "_input_tone"
| "_toxicity"
| "_toxicity_gpt"
| "_input_toxicity"
| "_input_toxicity_gpt"
| "_user_registered"
| "_user_submitted"
| "_user_generated"
| "_user_finetuned"
| "_uncertainty"
| "_bleu"
| "_cost"
| "_rouge"
| "_prompt_injection_gpt"
| "_prompt_injection"
| "_rag_nli"
| "_adherence_nli"
| "_completeness_nli"
| "_chunk_attribution_utilization_nli"
| "_instruction_adherence"
| "_ground_truth_adherence"
| "_tool_selection_quality"
| "_tool_selection_quality_luna"
| "_tool_error_rate"
| "_tool_error_rate_luna"
| "_action_completion_luna"
| "_agentic_session_success"
| "_action_advancement_luna"
| "_agentic_workflow_success"
| "_generic_wizard"
| "_customized_completeness_gpt"
| "_customized_factuality"
| "_customized_groundedness"
| "_customized_chunk_attribution_utilization_gpt"
| "_customized_instruction_adherence"
| "_customized_ground_truth_adherence"
| "_customized_prompt_injection_gpt"
| "_customized_tool_selection_quality"
| "_customized_tool_error_rate"
| "_customized_agentic_session_success"
| "_customized_agentic_workflow_success"
| "_customized_sexist_gpt"
| "_customized_input_sexist_gpt"
| "_customized_toxicity_gpt"
| "_customized_input_toxicity_gpt")[];
}
| {
aggregate_keys?: string[];
aggregates?: | null
| {
[key: string]: unknown;
};
chainpoll_template?: {
explanation_field_name?: string;
metric_description?: string;
metric_few_shot_examples?: object[];
metric_system_prompt?: string;
template?: string;
value_field_name?: string;
};
cot_enabled?: null | boolean;
description?: null | string;
extra?: | null
| {
[key: string]: unknown;
};
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
generated_scorer_id?: null | string;
indices?: null | number[];
input_type?: | null
| "basic"
| "llm_spans"
| "retriever_spans"
| "sessions_normalized"
| "sessions_trace_io_only"
| "tool_spans"
| "trace_input_only"
| "trace_io_only"
| "trace_normalized"
| "trace_output_only";
lora_task_id?: null | number;
metric_name?: null | string;
model_alias?: string;
name?: "output_sexist";
num_judges?: number;
output_type?: | null
| "boolean"
| "categorical"
| "count"
| "discrete"
| "freeform"
| "percentage";
prompt?: null | string;
regex_field?: string;
registered_scorer_id?: null | string;
scoreable_node_types?: | null
| (
| "agent"
| "llm"
| "retriever"
| "tool"
| "workflow"
| "trace"
| "session"
| "chain"
| "chat")[];
scorer_name: "_customized_sexist_gpt";
scores?: null | unknown[];
sub_scorers?: (
| "_completeness_gpt"
| "_context_adherence_luna"
| "_context_relevance"
| "_chunk_attribution_utilization_gpt"
| "_factuality"
| "_groundedness"
| "_latency"
| "_prompt_perplexity"
| "_protect_status"
| "_pii"
| "_input_pii"
| "_sexist"
| "_input_sexist"
| "_sexist_gpt"
| "_input_sexist_gpt"
| "_tone"
| "_input_tone"
| "_toxicity"
| "_toxicity_gpt"
| "_input_toxicity"
| "_input_toxicity_gpt"
| "_user_registered"
| "_user_submitted"
| "_user_generated"
| "_user_finetuned"
| "_uncertainty"
| "_bleu"
| "_cost"
| "_rouge"
| "_prompt_injection_gpt"
| "_prompt_injection"
| "_rag_nli"
| "_adherence_nli"
| "_completeness_nli"
| "_chunk_attribution_utilization_nli"
| "_instruction_adherence"
| "_ground_truth_adherence"
| "_tool_selection_quality"
| "_tool_selection_quality_luna"
| "_tool_error_rate"
| "_tool_error_rate_luna"
| "_action_completion_luna"
| "_agentic_session_success"
| "_action_advancement_luna"
| "_agentic_workflow_success"
| "_generic_wizard"
| "_customized_completeness_gpt"
| "_customized_factuality"
| "_customized_groundedness"
| "_customized_chunk_attribution_utilization_gpt"
| "_customized_instruction_adherence"
| "_customized_ground_truth_adherence"
| "_customized_prompt_injection_gpt"
| "_customized_tool_selection_quality"
| "_customized_tool_error_rate"
| "_customized_agentic_session_success"
| "_customized_agentic_workflow_success"
| "_customized_sexist_gpt"
| "_customized_input_sexist_gpt"
| "_customized_toxicity_gpt"
| "_customized_input_toxicity_gpt")[];
}
| {
aggregate_keys?: string[];
aggregates?: | null
| {
[key: string]: unknown;
};
chainpoll_template?: {
explanation_field_name?: string;
metric_description?: string;
metric_few_shot_examples?: object[];
metric_system_prompt?: string;
template?: string;
value_field_name?: string;
};
cot_enabled?: null | boolean;
description?: null | string;
extra?: | null
| {
[key: string]: unknown;
};
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
generated_scorer_id?: null | string;
indices?: null | number[];
input_type?: | null
| "basic"
| "llm_spans"
| "retriever_spans"
| "sessions_normalized"
| "sessions_trace_io_only"
| "tool_spans"
| "trace_input_only"
| "trace_io_only"
| "trace_normalized"
| "trace_output_only";
lora_task_id?: null | number;
metric_name?: null | string;
model_alias?: string;
name?: "input_sexist";
num_judges?: number;
output_type?: | null
| "boolean"
| "categorical"
| "count"
| "discrete"
| "freeform"
| "percentage";
prompt?: null | string;
regex_field?: string;
registered_scorer_id?: null | string;
scoreable_node_types?: | null
| (
| "agent"
| "llm"
| "retriever"
| "tool"
| "workflow"
| "trace"
| "session"
| "chain"
| "chat")[];
scorer_name: "_customized_input_sexist_gpt";
scores?: null | unknown[];
sub_scorers?: (
| "_completeness_gpt"
| "_context_adherence_luna"
| "_context_relevance"
| "_chunk_attribution_utilization_gpt"
| "_factuality"
| "_groundedness"
| "_latency"
| "_prompt_perplexity"
| "_protect_status"
| "_pii"
| "_input_pii"
| "_sexist"
| "_input_sexist"
| "_sexist_gpt"
| "_input_sexist_gpt"
| "_tone"
| "_input_tone"
| "_toxicity"
| "_toxicity_gpt"
| "_input_toxicity"
| "_input_toxicity_gpt"
| "_user_registered"
| "_user_submitted"
| "_user_generated"
| "_user_finetuned"
| "_uncertainty"
| "_bleu"
| "_cost"
| "_rouge"
| "_prompt_injection_gpt"
| "_prompt_injection"
| "_rag_nli"
| "_adherence_nli"
| "_completeness_nli"
| "_chunk_attribution_utilization_nli"
| "_instruction_adherence"
| "_ground_truth_adherence"
| "_tool_selection_quality"
| "_tool_selection_quality_luna"
| "_tool_error_rate"
| "_tool_error_rate_luna"
| "_action_completion_luna"
| "_agentic_session_success"
| "_action_advancement_luna"
| "_agentic_workflow_success"
| "_generic_wizard"
| "_customized_completeness_gpt"
| "_customized_factuality"
| "_customized_groundedness"
| "_customized_chunk_attribution_utilization_gpt"
| "_customized_instruction_adherence"
| "_customized_ground_truth_adherence"
| "_customized_prompt_injection_gpt"
| "_customized_tool_selection_quality"
| "_customized_tool_error_rate"
| "_customized_agentic_session_success"
| "_customized_agentic_workflow_success"
| "_customized_sexist_gpt"
| "_customized_input_sexist_gpt"
| "_customized_toxicity_gpt"
| "_customized_input_toxicity_gpt")[];
}
| {
aggregate_keys?: string[];
aggregates?: | null
| {
[key: string]: unknown;
};
chainpoll_template?: {
explanation_field_name?: string;
metric_description?: string;
metric_few_shot_examples?: object[];
metric_system_prompt?: string;
template?: string;
value_field_name?: string;
};
cot_enabled?: null | boolean;
description?: null | string;
extra?: | null
| {
[key: string]: unknown;
};
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
generated_scorer_id?: null | string;
indices?: null | number[];
input_type?: | null
| "basic"
| "llm_spans"
| "retriever_spans"
| "sessions_normalized"
| "sessions_trace_io_only"
| "tool_spans"
| "trace_input_only"
| "trace_io_only"
| "trace_normalized"
| "trace_output_only";
lora_task_id?: null | number;
metric_name?: null | string;
model_alias?: string;
name?: "tool_selection_quality";
num_judges?: number;
output_type?: | null
| "boolean"
| "categorical"
| "count"
| "discrete"
| "freeform"
| "percentage";
prompt?: null | string;
regex_field?: string;
registered_scorer_id?: null | string;
scoreable_node_types?: | null
| (
| "agent"
| "llm"
| "retriever"
| "tool"
| "workflow"
| "trace"
| "session"
| "chain"
| "chat")[];
scorer_name: "_customized_tool_selection_quality";
scores?: null | unknown[];
sub_scorers?: (
| "_completeness_gpt"
| "_context_adherence_luna"
| "_context_relevance"
| "_chunk_attribution_utilization_gpt"
| "_factuality"
| "_groundedness"
| "_latency"
| "_prompt_perplexity"
| "_protect_status"
| "_pii"
| "_input_pii"
| "_sexist"
| "_input_sexist"
| "_sexist_gpt"
| "_input_sexist_gpt"
| "_tone"
| "_input_tone"
| "_toxicity"
| "_toxicity_gpt"
| "_input_toxicity"
| "_input_toxicity_gpt"
| "_user_registered"
| "_user_submitted"
| "_user_generated"
| "_user_finetuned"
| "_uncertainty"
| "_bleu"
| "_cost"
| "_rouge"
| "_prompt_injection_gpt"
| "_prompt_injection"
| "_rag_nli"
| "_adherence_nli"
| "_completeness_nli"
| "_chunk_attribution_utilization_nli"
| "_instruction_adherence"
| "_ground_truth_adherence"
| "_tool_selection_quality"
| "_tool_selection_quality_luna"
| "_tool_error_rate"
| "_tool_error_rate_luna"
| "_action_completion_luna"
| "_agentic_session_success"
| "_action_advancement_luna"
| "_agentic_workflow_success"
| "_generic_wizard"
| "_customized_completeness_gpt"
| "_customized_factuality"
| "_customized_groundedness"
| "_customized_chunk_attribution_utilization_gpt"
| "_customized_instruction_adherence"
| "_customized_ground_truth_adherence"
| "_customized_prompt_injection_gpt"
| "_customized_tool_selection_quality"
| "_customized_tool_error_rate"
| "_customized_agentic_session_success"
| "_customized_agentic_workflow_success"
| "_customized_sexist_gpt"
| "_customized_input_sexist_gpt"
| "_customized_toxicity_gpt"
| "_customized_input_toxicity_gpt")[];
}
| {
aggregate_keys?: string[];
aggregates?: | null
| {
[key: string]: unknown;
};
chainpoll_template?: {
explanation_field_name?: string;
metric_description?: string;
metric_few_shot_examples?: object[];
metric_system_prompt?: string;
template?: string;
value_field_name?: string;
};
cot_enabled?: null | boolean;
description?: null | string;
extra?: | null
| {
[key: string]: unknown;
};
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
generated_scorer_id?: null | string;
indices?: null | number[];
input_type?: | null
| "basic"
| "llm_spans"
| "retriever_spans"
| "sessions_normalized"
| "sessions_trace_io_only"
| "tool_spans"
| "trace_input_only"
| "trace_io_only"
| "trace_normalized"
| "trace_output_only";
lora_task_id?: null | number;
metric_name?: null | string;
model_alias?: string;
name?: "tool_error_rate";
num_judges?: number;
output_type?: | null
| "boolean"
| "categorical"
| "count"
| "discrete"
| "freeform"
| "percentage";
prompt?: null | string;
regex_field?: string;
registered_scorer_id?: null | string;
scoreable_node_types?: | null
| (
| "agent"
| "llm"
| "retriever"
| "tool"
| "workflow"
| "trace"
| "session"
| "chain"
| "chat")[];
scorer_name: "_customized_tool_error_rate";
scores?: null | unknown[];
sub_scorers?: (
| "_completeness_gpt"
| "_context_adherence_luna"
| "_context_relevance"
| "_chunk_attribution_utilization_gpt"
| "_factuality"
| "_groundedness"
| "_latency"
| "_prompt_perplexity"
| "_protect_status"
| "_pii"
| "_input_pii"
| "_sexist"
| "_input_sexist"
| "_sexist_gpt"
| "_input_sexist_gpt"
| "_tone"
| "_input_tone"
| "_toxicity"
| "_toxicity_gpt"
| "_input_toxicity"
| "_input_toxicity_gpt"
| "_user_registered"
| "_user_submitted"
| "_user_generated"
| "_user_finetuned"
| "_uncertainty"
| "_bleu"
| "_cost"
| "_rouge"
| "_prompt_injection_gpt"
| "_prompt_injection"
| "_rag_nli"
| "_adherence_nli"
| "_completeness_nli"
| "_chunk_attribution_utilization_nli"
| "_instruction_adherence"
| "_ground_truth_adherence"
| "_tool_selection_quality"
| "_tool_selection_quality_luna"
| "_tool_error_rate"
| "_tool_error_rate_luna"
| "_action_completion_luna"
| "_agentic_session_success"
| "_action_advancement_luna"
| "_agentic_workflow_success"
| "_generic_wizard"
| "_customized_completeness_gpt"
| "_customized_factuality"
| "_customized_groundedness"
| "_customized_chunk_attribution_utilization_gpt"
| "_customized_instruction_adherence"
| "_customized_ground_truth_adherence"
| "_customized_prompt_injection_gpt"
| "_customized_tool_selection_quality"
| "_customized_tool_error_rate"
| "_customized_agentic_session_success"
| "_customized_agentic_workflow_success"
| "_customized_sexist_gpt"
| "_customized_input_sexist_gpt"
| "_customized_toxicity_gpt"
| "_customized_input_toxicity_gpt")[];
}
| {
aggregate_keys?: string[];
aggregates?: | null
| {
[key: string]: unknown;
};
chainpoll_template?: {
explanation_field_name?: string;
metric_description?: string;
metric_few_shot_examples?: object[];
metric_system_prompt?: string;
template?: string;
value_field_name?: string;
};
cot_enabled?: null | boolean;
description?: null | string;
extra?: | null
| {
[key: string]: unknown;
};
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
generated_scorer_id?: null | string;
indices?: null | number[];
input_type?: | null
| "basic"
| "llm_spans"
| "retriever_spans"
| "sessions_normalized"
| "sessions_trace_io_only"
| "tool_spans"
| "trace_input_only"
| "trace_io_only"
| "trace_normalized"
| "trace_output_only";
lora_task_id?: null | number;
metric_name?: null | string;
model_alias?: string;
name?: "output_toxicity";
num_judges?: number;
output_type?: | null
| "boolean"
| "categorical"
| "count"
| "discrete"
| "freeform"
| "percentage";
prompt?: null | string;
regex_field?: string;
registered_scorer_id?: null | string;
scoreable_node_types?: | null
| (
| "agent"
| "llm"
| "retriever"
| "tool"
| "workflow"
| "trace"
| "session"
| "chain"
| "chat")[];
scorer_name: "_customized_toxicity_gpt";
scores?: null | unknown[];
sub_scorers?: (
| "_completeness_gpt"
| "_context_adherence_luna"
| "_context_relevance"
| "_chunk_attribution_utilization_gpt"
| "_factuality"
| "_groundedness"
| "_latency"
| "_prompt_perplexity"
| "_protect_status"
| "_pii"
| "_input_pii"
| "_sexist"
| "_input_sexist"
| "_sexist_gpt"
| "_input_sexist_gpt"
| "_tone"
| "_input_tone"
| "_toxicity"
| "_toxicity_gpt"
| "_input_toxicity"
| "_input_toxicity_gpt"
| "_user_registered"
| "_user_submitted"
| "_user_generated"
| "_user_finetuned"
| "_uncertainty"
| "_bleu"
| "_cost"
| "_rouge"
| "_prompt_injection_gpt"
| "_prompt_injection"
| "_rag_nli"
| "_adherence_nli"
| "_completeness_nli"
| "_chunk_attribution_utilization_nli"
| "_instruction_adherence"
| "_ground_truth_adherence"
| "_tool_selection_quality"
| "_tool_selection_quality_luna"
| "_tool_error_rate"
| "_tool_error_rate_luna"
| "_action_completion_luna"
| "_agentic_session_success"
| "_action_advancement_luna"
| "_agentic_workflow_success"
| "_generic_wizard"
| "_customized_completeness_gpt"
| "_customized_factuality"
| "_customized_groundedness"
| "_customized_chunk_attribution_utilization_gpt"
| "_customized_instruction_adherence"
| "_customized_ground_truth_adherence"
| "_customized_prompt_injection_gpt"
| "_customized_tool_selection_quality"
| "_customized_tool_error_rate"
| "_customized_agentic_session_success"
| "_customized_agentic_workflow_success"
| "_customized_sexist_gpt"
| "_customized_input_sexist_gpt"
| "_customized_toxicity_gpt"
| "_customized_input_toxicity_gpt")[];
}
| {
aggregate_keys?: string[];
aggregates?: | null
| {
[key: string]: unknown;
};
chainpoll_template?: {
explanation_field_name?: string;
metric_description?: string;
metric_few_shot_examples?: object[];
metric_system_prompt?: string;
template?: string;
value_field_name?: string;
};
cot_enabled?: null | boolean;
description?: null | string;
extra?: | null
| {
[key: string]: unknown;
};
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
generated_scorer_id?: null | string;
indices?: null | number[];
input_type?: | null
| "basic"
| "llm_spans"
| "retriever_spans"
| "sessions_normalized"
| "sessions_trace_io_only"
| "tool_spans"
| "trace_input_only"
| "trace_io_only"
| "trace_normalized"
| "trace_output_only";
lora_task_id?: null | number;
metric_name?: null | string;
model_alias?: string;
name?: "input_toxicity";
num_judges?: number;
output_type?: | null
| "boolean"
| "categorical"
| "count"
| "discrete"
| "freeform"
| "percentage";
prompt?: null | string;
regex_field?: string;
registered_scorer_id?: null | string;
scoreable_node_types?: | null
| (
| "agent"
| "llm"
| "retriever"
| "tool"
| "workflow"
| "trace"
| "session"
| "chain"
| "chat")[];
scorer_name: "_customized_input_toxicity_gpt";
scores?: null | unknown[];
sub_scorers?: (
| "_completeness_gpt"
| "_context_adherence_luna"
| "_context_relevance"
| "_chunk_attribution_utilization_gpt"
| "_factuality"
| "_groundedness"
| "_latency"
| "_prompt_perplexity"
| "_protect_status"
| "_pii"
| "_input_pii"
| "_sexist"
| "_input_sexist"
| "_sexist_gpt"
| "_input_sexist_gpt"
| "_tone"
| "_input_tone"
| "_toxicity"
| "_toxicity_gpt"
| "_input_toxicity"
| "_input_toxicity_gpt"
| "_user_registered"
| "_user_submitted"
| "_user_generated"
| "_user_finetuned"
| "_uncertainty"
| "_bleu"
| "_cost"
| "_rouge"
| "_prompt_injection_gpt"
| "_prompt_injection"
| "_rag_nli"
| "_adherence_nli"
| "_completeness_nli"
| "_chunk_attribution_utilization_nli"
| "_instruction_adherence"
| "_ground_truth_adherence"
| "_tool_selection_quality"
| "_tool_selection_quality_luna"
| "_tool_error_rate"
| "_tool_error_rate_luna"
| "_action_completion_luna"
| "_agentic_session_success"
| "_action_advancement_luna"
| "_agentic_workflow_success"
| "_generic_wizard"
| "_customized_completeness_gpt"
| "_customized_factuality"
| "_customized_groundedness"
| "_customized_chunk_attribution_utilization_gpt"
| "_customized_instruction_adherence"
| "_customized_ground_truth_adherence"
| "_customized_prompt_injection_gpt"
| "_customized_tool_selection_quality"
| "_customized_tool_error_rate"
| "_customized_agentic_session_success"
| "_customized_agentic_workflow_success"
| "_customized_sexist_gpt"
| "_customized_input_sexist_gpt"
| "_customized_toxicity_gpt"
| "_customized_input_toxicity_gpt")[];
})[];
prompt_dataset_id?: null | string;
prompt_finetuned_scorers_configuration?: null | object[];
prompt_generated_scorers_configuration?: null | string[];
prompt_optimization_configuration?: | null
| {
evaluation_criteria: string;
evaluation_model_alias: string;
generation_model_alias: string;
includes_target: boolean;
integration_name?: | "custom"
| "anthropic"
| "aws_bedrock"
| "aws_sagemaker"
| "azure"
| "databricks"
| "mistral"
| "nvidia"
| "openai"
| "vegas_gateway"
| "vertex_ai"
| "writer";
iterations: number;
max_tokens: number;
num_rows: number;
prompt: string;
task_description: string;
temperature: number;
};
prompt_registered_scorers_configuration?: null | object[];
prompt_scorer_settings?: | null
| {
aggregate_keys?: null | string[];
aggregates?: | null
| {
[key: string]: unknown;
};
chainpoll_template?: | null
| {
explanation_field_name?: string;
metric_description?: null | string;
metric_few_shot_examples?: object[];
metric_system_prompt?: null | string;
template: string;
value_field_name?: string;
};
cot_enabled?: null | boolean;
description?: null | string;
extra?: | null
| {
[key: string]: unknown;
};
filters?: | null
| (
| {
case_sensitive?: boolean;
filter_type?: "string";
name: "node_name";
operator: "eq" | "ne" | "contains";
value: string;
}
| {
filter_type?: "map";
key: string;
name: "metadata";
operator: "eq" | "ne" | "one_of" | "not_in";
value: string | string[];
})[];
generated_scorer_id?: null | string;
indices?: null | number[];
input_type?: | null
| "basic"
| "llm_spans"
| "retriever_spans"
| "sessions_normalized"
| "sessions_trace_io_only"
| "tool_spans"
| "trace_input_only"
| "trace_io_only"
| "trace_normalized"
| "trace_output_only";
lora_task_id?: null | number;
metric_name?: null | string;
model_alias?: null | string;
name?: string;
num_judges?: null | number;
output_type?: | null
| "boolean"
| "categorical"
| "count"
| "discrete"
| "freeform"
| "percentage";
prompt?: null | string;
regex_field?: string;
registered_scorer_id?: null | string;
scoreable_node_types?: | null
| (
| "agent"
| "llm"
| "retriever"
| "tool"
| "workflow"
| "trace"
| "session"
| "chain"
| "chat")[];
scorer_name?: string;
scores?: null | unknown[];
sub_scorers?: (
| "_completeness_gpt"
| "_context_adherence_luna"
| "_context_relevance"
| "_chunk_attribution_utilization_gpt"
| "_factuality"
| "_groundedness"
| "_latency"
| "_prompt_perplexity"
| "_protect_status"
| "_pii"
| "_input_pii"
| "_sexist"
| "_input_sexist"
| "_sexist_gpt"
| "_input_sexist_gpt"
| "_tone"
| "_input_tone"
| "_toxicity"
| "_toxicity_gpt"
| "_input_toxicity"
| "_input_toxicity_gpt"
| "_user_registered"
| "_user_submitted"
| "_user_generated"
| "_user_finetuned"
| "_uncertainty"
| "_bleu"
| "_cost"
| "_rouge"
| "_prompt_injection_gpt"
| "_prompt_injection"
| "_rag_nli"
| "_adherence_nli"
| "_completeness_nli"
| "_chunk_attribution_utilization_nli"
| "_instruction_adherence"
| "_ground_truth_adherence"
| "_tool_selection_quality"
| "_tool_selection_quality_luna"
| "_tool_error_rate"
| "_tool_error_rate_luna"
| "_action_completion_luna"
| "_agentic_session_success"
| "_action_advancement_luna"
| "_agentic_workflow_success"
| "_generic_wizard"
| "_customized_completeness_gpt"
| "_customized_factuality"
| "_customized_groundedness"
| "_customized_chunk_attribution_utilization_gpt"
| "_customized_instruction_adherence"
| "_customized_ground_truth_adherence"
| "_customized_prompt_injection_gpt"
| "_customized_tool_selection_quality"
| "_customized_tool_error_rate"
| "_customized_agentic_session_success"
| "_customized_agentic_workflow_success"
| "_customized_sexist_gpt"
| "_customized_input_sexist_gpt"
| "_customized_toxicity_gpt"
| "_customized_input_toxicity_gpt")[];
};
prompt_scorers_configuration?: | null
| {
action_advancement_luna?: boolean;
action_completion_luna?: boolean;
adherence_nli?: boolean;
agentic_session_success?: boolean;
agentic_workflow_success?: boolean;
bleu?: boolean;
chunk_attribution_utilization_gpt?: boolean;
chunk_attribution_utilization_nli?: boolean;
completeness_gpt?: boolean;
completeness_nli?: boolean;
context_adherence_luna?: boolean;
context_relevance?: boolean;
cost?: boolean;
factuality?: boolean;
ground_truth_adherence?: boolean;
groundedness?: boolean;
input_pii?: boolean;
input_sexist?: boolean;
input_sexist_gpt?: boolean;
input_tone?: boolean;
input_toxicity?: boolean;
input_toxicity_gpt?: boolean;
instruction_adherence?: boolean;
latency?: boolean;
pii?: boolean;
prompt_injection?: boolean;
prompt_injection_gpt?: boolean;
prompt_perplexity?: boolean;
protect_status?: boolean;
rouge?: boolean;
sexist?: boolean;
sexist_gpt?: boolean;
tone?: boolean;
tool_error_rate?: boolean;
tool_error_rate_luna?: boolean;
tool_selection_quality?: boolean;
tool_selection_quality_luna?: boolean;
toxicity?: boolean;
toxicity_gpt?: boolean;
uncertainty?: boolean;
};
prompt_settings?: | null
| {
deployment_name?: null | string;
echo?: boolean;
frequency_penalty?: number;
known_models?: object[];
logprobs?: boolean;
max_tokens?: number;
model_alias?: string;
n?: number;
presence_penalty?: number;
response_format?: | null
| {
[key: string]: string;
};
stop_sequences?: null | string[];
temperature?: number;
tool_choice?: | null
| string
| {
function: {
name: string;
};
type?: string;
};
tools?: null | object[];
top_k?: number;
top_logprobs?: number;
top_p?: number;
};
prompt_template_version_id?: null | string;
protect_scorer_payload?: null | string;
protect_trace_id?: null | string;
resource_limits?: | null
| {
cpu_time?: number;
memory_mb?: number;
};
run_id: string;
scorer_config?: | null
| {
cot_enabled?: null | boolean;
filters?: | null
| (
| {
case_sensitive?: boolean;
filter_type?: "string";
name: "node_name";
operator: "eq" | "ne" | "contains";
value: string;
}
| {
filter_type?: "map";
key: string;
name: "metadata";
operator: "eq" | "ne" | "one_of" | "not_in";
value: string | string[];
})[];
id: string;
input_type?: | null
| "basic"
| "llm_spans"
| "retriever_spans"
| "sessions_normalized"
| "sessions_trace_io_only"
| "tool_spans"
| "trace_input_only"
| "trace_io_only"
| "trace_normalized"
| "trace_output_only";
model_name?: null | string;
model_type?: null | "llm" | "code" | "slm";
name?: null | string;
num_judges?: null | number;
output_type?: | null
| "boolean"
| "categorical"
| "count"
| "discrete"
| "freeform"
| "percentage";
scoreable_node_types?: null | string[];
scorer_type: "llm" | "code" | "preset";
scorer_version?: | null
| {
cot_enabled?: null | boolean;
generated_scorer?: | null
| {
chain_poll_template: {
explanation_field_name?: string;
metric_description?: null | string;
metric_few_shot_examples?: …[];
metric_system_prompt?: null | string;
template: string;
value_field_name?: string;
};
id: string;
instructions?: null | string;
name: string;
user_prompt?: null | string;
};
id: string;
input_type?: | null
| "basic"
| "llm_spans"
| "retriever_spans"
| "sessions_normalized"
| "sessions_trace_io_only"
| "tool_spans"
| "trace_input_only"
| "trace_io_only"
| "trace_normalized"
| "trace_output_only";
model_name?: null | string;
num_judges?: null | number;
output_type?: | null
| "boolean"
| "categorical"
| "count"
| "discrete"
| "freeform"
| "percentage";
registered_scorer?: | null
| {
id: string;
name: string;
};
scoreable_node_types?: null | string[];
version: number;
};
};
scorers?: | null
| object[]
| (
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
model_name?: null | string;
name: "agentic_session_success";
num_judges?: null | number;
type?: "luna" | "plus";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
model_name?: null | string;
name: "agentic_workflow_success";
num_judges?: null | number;
type?: "luna" | "plus";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
name: "bleu";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
model_name?: null | string;
name: "chunk_attribution_utilization";
type?: "luna" | "plus";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
model_name?: null | string;
name: "completeness";
num_judges?: null | number;
type?: "luna" | "plus";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
model_name?: null | string;
name: "context_adherence";
num_judges?: null | number;
type?: "luna" | "plus";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
name: "context_relevance";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
model_name?: null | string;
name: "correctness";
num_judges?: null | number;
type?: "plus";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
model_name?: null | string;
name: "ground_truth_adherence";
num_judges?: null | number;
type?: "plus";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
name: "input_pii";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
model_name?: null | string;
name: "input_sexist";
num_judges?: null | number;
type?: "luna" | "plus";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
name: "input_tone";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
model_name?: null | string;
name: "input_toxicity";
num_judges?: null | number;
type?: "luna" | "plus";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
model_name?: null | string;
name: "instruction_adherence";
num_judges?: null | number;
type?: "plus";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
name: "output_pii";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
model_name?: null | string;
name: "output_sexist";
num_judges?: null | number;
type?: "luna" | "plus";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
name: "output_tone";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
model_name?: null | string;
name: "output_toxicity";
num_judges?: null | number;
type?: "luna" | "plus";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
model_name?: null | string;
name: "prompt_injection";
num_judges?: null | number;
type?: "luna" | "plus";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
name: "prompt_perplexity";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
name: "rouge";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
model_name?: null | string;
name: "tool_error_rate";
type?: "luna" | "plus";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
model_name?: null | string;
name: "tool_selection_quality";
num_judges?: null | number;
type?: "luna" | "plus";
}
| {
filters?: | null
| (
| {
case_sensitive?: … | … | …;
filter_type?: … | …;
name: "node_name";
operator: … | … | …;
value: string;
}
| {
filter_type?: … | …;
key: string;
name: "metadata";
operator: … | … | … | …;
value: … | …;
})[];
name: "uncertainty";
})[];
segment_filters?: null | object[];
should_retry?: boolean;
sub_scorers?: (
| "_completeness_gpt"
| "_context_adherence_luna"
| "_context_relevance"
| "_chunk_attribution_utilization_gpt"
| "_factuality"
| "_groundedness"
| "_latency"
| "_prompt_perplexity"
| "_protect_status"
| "_pii"
| "_input_pii"
| "_sexist"
| "_input_sexist"
| "_sexist_gpt"
| "_input_sexist_gpt"
| "_tone"
| "_input_tone"
| "_toxicity"
| "_toxicity_gpt"
| "_input_toxicity"
| "_input_toxicity_gpt"
| "_user_registered"
| "_user_submitted"
| "_user_generated"
| "_user_finetuned"
| "_uncertainty"
| "_bleu"
| "_cost"
| "_rouge"
| "_prompt_injection_gpt"
| "_prompt_injection"
| "_rag_nli"
| "_adherence_nli"
| "_completeness_nli"
| "_chunk_attribution_utilization_nli"
| "_instruction_adherence"
| "_ground_truth_adherence"
| "_tool_selection_quality"
| "_tool_selection_quality_luna"
| "_tool_error_rate"
| "_tool_error_rate_luna"
| "_action_completion_luna"
| "_agentic_session_success"
| "_action_advancement_luna"
| "_agentic_workflow_success"
| "_generic_wizard"
| "_customized_completeness_gpt"
| "_customized_factuality"
| "_customized_groundedness"
| "_customized_chunk_attribution_utilization_gpt"
| "_customized_instruction_adherence"
| "_customized_ground_truth_adherence"
| "_customized_prompt_injection_gpt"
| "_customized_tool_selection_quality"
| "_customized_tool_error_rate"
| "_customized_agentic_session_success"
| "_customized_agentic_workflow_success"
| "_customized_sexist_gpt"
| "_customized_input_sexist_gpt"
| "_customized_toxicity_gpt"
| "_customized_input_toxicity_gpt")[];
task_type?: null | 0 | 1 | 2 | 4 | 6 | 3 | 5 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17;
tasks?: null | string[];
upload_data_in_separate_task?: boolean;
user_id?: null | string;
xray?: boolean;
}>
createPromptTemplate()
Parameters
template
Message[]
name
string
Returns
Promise<{
all_available_versions: number[];
all_versions?: object[];
created_at: string;
created_by_user: | null
| {
email: string;
first_name?: null | string;
id: string;
last_name?: null | string;
};
id: string;
max_version: number;
name: string;
permissions?: object[];
selected_version: {
content_changed: boolean;
created_at: string;
created_by_user: | null
| {
email: string;
first_name?: null | string;
id: string;
last_name?: null | string;
};
id: string;
lines_added?: number;
lines_edited?: number;
lines_removed?: number;
model_changed: boolean;
output_type?: null | string;
raw?: boolean;
settings: {
deployment_name?: null | string;
echo?: boolean;
frequency_penalty?: number;
known_models?: object[];
logprobs?: boolean;
max_tokens?: number;
model_alias?: string;
n?: number;
presence_penalty?: number;
response_format?: | null
| {
[key: string]: string;
};
stop_sequences?: null | string[];
temperature?: number;
tool_choice?: | null
| string
| {
function: {
name: string;
};
type?: string;
};
tools?: null | object[];
top_k?: number;
top_logprobs?: number;
top_p?: number;
};
settings_changed: boolean;
template: string | object & object[];
updated_at: string;
version: number;
};
selected_version_id: string;
template: string;
total_versions: number;
updated_at: string;
}>
createRunScorerSettings()
Parameters
experimentId
string
projectId
string
scorers
object[]
Returns
Promise<void>
createScorer()
Parameters
name
string
scorerType
ScorerTypes
description?
string
tags?
string[]
defaults?
cot_enabled?
null | boolean
Cot Enabled
Description
Whether to enable chain of thought for this scorer. Defaults to False for llm scorers.
filters?
|null
| (
| {
case_sensitive?: boolean;
filter_type?: "string";
name: "node_name";
operator: "eq" | "ne" | "contains";
value: string;
}
| {
filter_type?: "map";
key: string;
name: "metadata";
operator: "eq" | "ne" | "one_of" | "not_in";
value: string | string[];
})[]
Filters
Description
List of filters to apply to the scorer.
input_type?
|null
| "basic"
| "llm_spans"
| "retriever_spans"
| "sessions_normalized"
| "sessions_trace_io_only"
| "tool_spans"
| "trace_input_only"
| "trace_io_only"
| "trace_normalized"
| "trace_output_only"
Description
What type of input to use for model-based scorers (sessions_normalized, trace_io_only, etc..).
model_name?
null | string
Model Name
num_judges?
null | number
Num Judges
output_type?
|null
| "boolean"
| "categorical"
| "count"
| "discrete"
| "freeform"
| "percentage"
Description
What type of output to use for model-based scorers (boolean, categorical, etc.).
scoreable_node_types?
null | string[]
Scoreable Node Types
Description
List of node types that can be scored by this scorer. Defaults to llm/chat.
modelType?
"llm" | "code" | "slm"
defaultVersionId?
string
Returns
Promise<Scorer>
createSession()
Parameters
__namedParameters
externalId?
string
name?
string
previousSessionId?
string
Returns
Promise<{
external_id?: null | string;
id: string;
name: null | string;
previous_session_id?: null | string;
project_id: string;
project_name: string;
}>
deleteDataset()
Parameters
id
string
Returns
Promise<void>
deleteGlobalPromptTemplate()
Parameters
id
string
Returns
Promise<void>
deleteScorer()
Parameters
scorerId
string
Returns
Promise<void>
extendDataset()
Parameters
params
count?
number
Count
Default
data_types?
|null
| (
| "General Query"
| "Prompt Injection"
| "Off-Topic Query"
| "Toxic Content in Query"
| "Multiple Questions in Query"
| "Sexist Content in Query")[]
Data Types
examples?
string[]
Examples
instructions?
null | string
Instructions
prompt?
null | string
Prompt
prompt_settings?
{deployment_name?: null | string;
echo?: boolean;
frequency_penalty?: number;
known_models?: object[];
logprobs?: boolean;
max_tokens?: number;
model_alias?: string;
n?: number;
presence_penalty?: number;
response_format?: | null
| {
[key: string]: string;
};
stop_sequences?: null | string[];
temperature?: number;
tool_choice?: | null
| string
| {
function: {
name: string;
};
type?: string;
};
tools?: null | object[];
top_k?: number;
top_logprobs?: number;
top_p?: number;
}
Description
Only the model is used.
prompt_settings.deployment_name?
null | string
Deployment Name
prompt_settings.echo?
boolean
Echo
Default
prompt_settings.frequency_penalty?
number
Frequency Penalty
Default
prompt_settings.known_models?
object[]
Known Models
prompt_settings.logprobs?
boolean
Logprobs
Default
prompt_settings.max_tokens?
number
Max Tokens
Default
prompt_settings.model_alias?
string
Model Alias
Default
prompt_settings.n?
number
N
Default
prompt_settings.presence_penalty?
number
Presence Penalty
Default
prompt_settings.response_format?
|null
| {
[key: string]: string;
}
Response Format
prompt_settings.stop_sequences?
null | string[]
Stop Sequences
prompt_settings.temperature?
number
Temperature
Default
prompt_settings.tool_choice?
|null
| string
| {
function: {
name: string;
};
type?: string;
}
Tool Choice
prompt_settings.tools?
null | object[]
Tools
prompt_settings.top_k?
number
Top K
Default
prompt_settings.top_logprobs?
number
Top Logprobs
Default
prompt_settings.top_p?
number
Top P
Default
source_dataset?
|null
| {
dataset_id: string;
dataset_version_index?: null | number;
row_ids?: null | string[];
}
Returns
Promise<{
dataset_id: string;
}>
getApiUrl()
Parameters
projectType
string
Returns
string
Inherited from
getAuthHeader()
Parameters
token
string
Returns
object
Authorization
Inherited from
getDataset()
Parameters
id
string
Returns
Promise<{
column_names: null | string[];
created_at: string;
created_by_user: | null
| {
email: string;
first_name?: null | string;
id: string;
last_name?: null | string;
};
current_version_index: number;
draft: boolean;
id: string;
name: string;
num_rows: null | number;
permissions?: object[];
project_count: number;
updated_at: string;
}>
getDatasetByName()
Parameters
name
string
Returns
Promise<{
column_names: null | string[];
created_at: string;
created_by_user: | null
| {
email: string;
first_name?: null | string;
id: string;
last_name?: null | string;
};
current_version_index: number;
draft: boolean;
id: string;
name: string;
num_rows: null | number;
permissions?: object[];
project_count: number;
updated_at: string;
}>
getDatasetContent()
Parameters
datasetId
string
Returns
Promise<object[]>
getDatasetEtag()
Parameters
id
string
Returns
Promise<string>
getDatasets()
Returns
Promise<object[]>
getExperiment()
Parameters
id
string
Returns
Promise<Experiment>
getExperiments()
Returns
Promise<Experiment[]>
getExtendDatasetStatus()
Parameters
datasetId
string
Returns
Promise<{
progress_message?: null | string;
steps_completed?: null | number;
steps_total?: null | number;
}>
getGlobalPromptTemplate()
Parameters
id
string
Returns
Promise<{
all_available_versions: number[];
all_versions?: object[];
created_at: string;
created_by_user: | null
| {
email: string;
first_name?: null | string;
id: string;
last_name?: null | string;
};
id: string;
max_version: number;
name: string;
permissions?: object[];
selected_version: {
content_changed: boolean;
created_at: string;
created_by_user: | null
| {
email: string;
first_name?: null | string;
id: string;
last_name?: null | string;
};
id: string;
lines_added?: number;
lines_edited?: number;
lines_removed?: number;
model_changed: boolean;
output_type?: null | string;
raw?: boolean;
settings: {
deployment_name?: null | string;
echo?: boolean;
frequency_penalty?: number;
known_models?: object[];
logprobs?: boolean;
max_tokens?: number;
model_alias?: string;
n?: number;
presence_penalty?: number;
response_format?: | null
| {
[key: string]: string;
};
stop_sequences?: null | string[];
temperature?: number;
tool_choice?: | null
| string
| {
function: {
name: string;
};
type?: string;
};
tools?: null | object[];
top_k?: number;
top_logprobs?: number;
top_p?: number;
};
settings_changed: boolean;
template: string | object & object[];
updated_at: string;
version: number;
};
selected_version_id: string;
template: string;
total_versions: number;
updated_at: string;
}>
getGlobalPromptTemplateVersion()
Parameters
id
string
version
number
Returns
Promise<{
content_changed: boolean;
created_at: string;
created_by_user: | null
| {
email: string;
first_name?: null | string;
id: string;
last_name?: null | string;
};
id: string;
lines_added?: number;
lines_edited?: number;
lines_removed?: number;
model_changed: boolean;
output_type?: null | string;
raw?: boolean;
settings: {
deployment_name?: null | string;
echo?: boolean;
frequency_penalty?: number;
known_models?: object[];
logprobs?: boolean;
max_tokens?: number;
model_alias?: string;
n?: number;
presence_penalty?: number;
response_format?: | null
| {
[key: string]: string;
};
stop_sequences?: null | string[];
temperature?: number;
tool_choice?: | null
| string
| {
function: {
name: string;
};
type?: string;
};
tools?: null | object[];
top_k?: number;
top_logprobs?: number;
top_p?: number;
};
settings_changed: boolean;
template: string | object & object[];
updated_at: string;
version: number;
}>
getLogStream()
Parameters
id
string
Returns
Promise<LogStream>
getLogStreamByName()
Parameters
name
string
Returns
Promise<LogStream>
getLogStreams()
Returns
Promise<LogStream[]>
getProject()
Parameters
id
string
Returns
Promise<Project>
getProjectByName()
Parameters
name
string
Returns
Promise<Project>
getProjectIdByName()
Parameters
name
string
Returns
Promise<string>
getProjects()
Returns
Promise<Project[]>
getPromptTemplate()
Parameters
id
string
Returns
Promise<{
all_available_versions: number[];
all_versions?: object[];
created_at: string;
created_by_user: | null
| {
email: string;
first_name?: null | string;
id: string;
last_name?: null | string;
};
id: string;
max_version: number;
name: string;
permissions?: object[];
selected_version: {
content_changed: boolean;
created_at: string;
created_by_user: | null
| {
email: string;
first_name?: null | string;
id: string;
last_name?: null | string;
};
id: string;
lines_added?: number;
lines_edited?: number;
lines_removed?: number;
model_changed: boolean;
output_type?: null | string;
raw?: boolean;
settings: {
deployment_name?: null | string;
echo?: boolean;
frequency_penalty?: number;
known_models?: object[];
logprobs?: boolean;
max_tokens?: number;
model_alias?: string;
n?: number;
presence_penalty?: number;
response_format?: | null
| {
[key: string]: string;
};
stop_sequences?: null | string[];
temperature?: number;
tool_choice?: | null
| string
| {
function: {
name: string;
};
type?: string;
};
tools?: null | object[];
top_k?: number;
top_logprobs?: number;
top_p?: number;
};
settings_changed: boolean;
template: string | object & object[];
updated_at: string;
version: number;
};
selected_version_id: string;
template: string;
total_versions: number;
updated_at: string;
}>
getPromptTemplates()
Returns
Promise<object[]>
getPromptTemplateVersion()
Parameters
id
string
version
number
Returns
Promise<{
content_changed: boolean;
created_at: string;
created_by_user: | null
| {
email: string;
first_name?: null | string;
id: string;
last_name?: null | string;
};
id: string;
lines_added?: number;
lines_edited?: number;
lines_removed?: number;
model_changed: boolean;
output_type?: null | string;
raw?: boolean;
settings: {
deployment_name?: null | string;
echo?: boolean;
frequency_penalty?: number;
known_models?: object[];
logprobs?: boolean;
max_tokens?: number;
model_alias?: string;
n?: number;
presence_penalty?: number;
response_format?: | null
| {
[key: string]: string;
};
stop_sequences?: null | string[];
temperature?: number;
tool_choice?: | null
| string
| {
function: {
name: string;
};
type?: string;
};
tools?: null | object[];
top_k?: number;
top_logprobs?: number;
top_p?: number;
};
settings_changed: boolean;
template: string | object & object[];
updated_at: string;
version: number;
}>
getPromptTemplateVersionByName()
Parameters
name
string
version?
number
Returns
Promise<{
content_changed: boolean;
created_at: string;
created_by_user: | null
| {
email: string;
first_name?: null | string;
id: string;
last_name?: null | string;
};
id: string;
lines_added?: number;
lines_edited?: number;
lines_removed?: number;
model_changed: boolean;
output_type?: null | string;
raw?: boolean;
settings: {
deployment_name?: null | string;
echo?: boolean;
frequency_penalty?: number;
known_models?: object[];
logprobs?: boolean;
max_tokens?: number;
model_alias?: string;
n?: number;
presence_penalty?: number;
response_format?: | null
| {
[key: string]: string;
};
stop_sequences?: null | string[];
temperature?: number;
tool_choice?: | null
| string
| {
function: {
name: string;
};
type?: string;
};
tools?: null | object[];
top_k?: number;
top_logprobs?: number;
top_p?: number;
};
settings_changed: boolean;
template: string | object & object[];
updated_at: string;
version: number;
}>
getScorers()
Parameters
options?
names?
string[]
type?
ScorerTypes
Returns
Promise<Scorer[]>
getScorerVersion()
Parameters
scorer_id
string
version
number
Returns
Promise<{
cot_enabled?: null | boolean;
generated_scorer?: | null
| {
chain_poll_template: {
explanation_field_name?: string;
metric_description?: null | string;
metric_few_shot_examples?: object[];
metric_system_prompt?: null | string;
template: string;
value_field_name?: string;
};
id: string;
instructions?: null | string;
name: string;
user_prompt?: null | string;
};
id: string;
input_type?: | null
| "basic"
| "llm_spans"
| "retriever_spans"
| "sessions_normalized"
| "sessions_trace_io_only"
| "tool_spans"
| "trace_input_only"
| "trace_io_only"
| "trace_normalized"
| "trace_output_only";
model_name?: null | string;
num_judges?: null | number;
output_type?: | null
| "boolean"
| "categorical"
| "count"
| "discrete"
| "freeform"
| "percentage";
registered_scorer?: | null
| {
id: string;
name: string;
};
scoreable_node_types?: null | string[];
version: number;
}>
healthCheck()
Returns
Promise<boolean>
Inherited from
ingestTraces()
Parameters
traces
any[]
Returns
Promise<void>
init()
Parameters
params
Partial<GalileoApiClientParams> = {}
Returns
Promise<void>
initializeClient()
Returns
void
Inherited from
listGlobalPromptTemplates()
Parameters
name_filter
string
limit
number
starting_token
number
Returns
Promise<object[]>
makeRequest()
Type Parameters
T
T
Parameters
request_method
Method
endpoint
Routes
data?
null | string | Record<string, any>
params?
Record<string, unknown>
extraHeaders?
Record<string, string>
Returns
Promise<T>
Inherited from
makeRequestRaw()
Type Parameters
T
T
Parameters
request_method
Method
endpoint
Routes
data?
null | string | Record<string, any>
params?
Record<string, unknown>
extraHeaders?
Record<string, string>
Returns
Promise<AxiosResponse<T, any>>
Inherited from
processResponse()
Type Parameters
T
T
Parameters
data
undefined | T
error
unknown
Returns
T
Inherited from
refreshTokenIfNeeded()
Parameters
endpoint
Routes
Returns
Promise<void>
Inherited from
validateResponse()
Parameters
response
AxiosResponse
Returns
void