Run Experiments in Code

from galileo import Message, MessageRole
from galileo.prompts import create_prompt
from galileo.experiments import run_experiment
from galileo.datasets import get_dataset
from galileo import GalileoMetrics

from dotenv import load_dotenv
load_dotenv()

project = "my-project"

# 1a. If the prompt  does not exist, create it:
prompt = create_prompt(
    name="geography-prompt",
    template=[
        Message(role=MessageRole.system,
                content="""
                You are a geography expert.
                Respond with only the continent name.
                """),
        Message(role=MessageRole.user, content="{{input}}")
    ]
)

# 1b. (OPTIONAL) If the prompt  already exists, fetch it:
# prompt = get_prompt(name="geography-prompt")

# 2. Run the experiment and get results
results = run_experiment(
    "geography-experiment",
    # Name of a dataset you created
    dataset=get_dataset(name="countries"),
    prompt_template=prompt,
    # Optional
    prompt_settings={
        "max_tokens": 256,
        # Make sure you have an integration set up
        # for the model alias you're using
        "model_alias": "GPT-4o",
        "temperature": 0.8
    },
    metrics=[GalileoMetrics.correctness],
    project=project
)

from galileo import log
from galileo.experiments import run_experiment
from galileo.datasets import get_dataset
from galileo import GalileoMetrics

dataset = get_dataset(name="countries")

@log(span_type="llm", name= "My Span")
def llm_call(input):
  # Custom function code
    return result

results = run_experiment(
    "geography-experiment",
    dataset=dataset,
    function=llm_call,
    metrics=[GalileoMetrics.correctness],
    project="my-project",
)

import os
from galileo.experiments import run_experiment
from galileo.datasets import get_dataset
from galileo.openai import openai
from galileo import GalileoMetrics

client = openai.OpenAI(api_key=os.environ["OPENAI_API_KEY"])
dataset = get_dataset(name="countries")

def llm_call(input):
    return client.chat.completions.create(
        model="gpt-4o",
        messages=[
          {
            "role": "system",
            "content": "You are a geography expert."
          },
          {
            "role": "user",
            "content": f"""
            Which continent does the following country belong to: {input}
            """
          }
        ],
    ).choices[0].message.content

results = run_experiment(
    "geography-experiment",
    dataset=dataset,
    function=llm_call,
    metrics=[GalileoMetrics.correctness],
    project="my-project",
)

from galileo import galileo_context

# Get the current logger
galileo_logger=galileo_context.get_logger_instance()

has_existing_trace = galileo_logger.current_parent() is not None

def process_message(input):
    # Get the Galileo logger instance
    galileo_logger = galileo_context.get_logger_instance()

    # If there is a current parent trace, we are in an experiment
    # Otherwise, we start a new trace for the chat workflow
    is_in_experiment = False
    if not galileo_logger.current_parent():
        galileo_logger.start_trace(
            input=input,
            name="Chat Workflow"
        )
    else:
        is_in_experiment = True

    # Your code goes here to process the input and create log spans as needed
    # You can also pass this log to other functions, or access it in those using
    # galileo_context.get_logger_instance()

    # If we are not in an experiment, we conclude and flush the trace
    if not is_in_experiment:
        galileo_logger.conclude("Some output")
        galileo_logger.flush()

import os
from galileo.experiments import run_experiment
from galileo.openai import openai
from galileo import GalileoMetrics

dataset = [
  {
    "input": "Spain"
  }
]

def llm_call(input):
  client = openai.OpenAI(api_key=os.environ["OPENAI_API_KEY"])
  return client.chat.completions.create(
        model="gpt-4",
        messages=[
          {
            "role": "system",
            "content": "You are a geography expert"
          },
          {
            "role": "user",
            "content": f"""
            Which continent does the following country belong to: {input}
            """
          }
        ],
    ).choices[0].message.content

results = run_experiment(
    "geography-experiment",
    dataset=dataset,
    function=llm_call,
    metrics=[GalileoMetrics.correctness],
    project="my-project"
)

import os
from galileo import Trace, Span
from galileo.experiments import run_experiment
from galileo.openai import openai
from galileo.schema.metrics import LocalMetricConfig

# 1. Scorer Function
def brevity_rank(step: Span | Trace) -> str:
    """Rank response brevity based on word count."""
    word_count = len(step.output.content.split(" "))
    if word_count <= 3:
        return "Terse"
    if word_count <= 5:
        return "Temperate"
    return "Talkative"

# 2. Configure the Local Metric
terseness = LocalMetricConfig[str](
    name="Terseness",
    scorer_fn=brevity_rank
)

# 3. Dataset
countries_dataset = [
    {"input": "Indonesia"},
    {"input": "New Zealand"},
    {"input": "Greenland"},
    {"input": "China"},
]

# 4. LLM-Call Function
def llm_call(input):
    client = openai.OpenAI(api_key=os.environ["OPENAI_API_KEY"])
    return (
        client.chat.completions.create(
            model="gpt-4o",
            messages=[
                {
                    "role": "system",
                    "content": """
                    You are a geography expert. Always answer as succinctly as possible.
                    """
                },
                {
                    "role": "user",
                    "content": f"""
                    Which continent does the following country belong to: {input}
                    """
                },
            ],
        )
        .choices[0]
        .message.content
    )

# 5. Run the Experiment!
results = run_experiment(
    "terseness-custom-metric",
    dataset=countries_dataset,
    function=llm_call,
    metrics=[terseness],  # You can add multiple custom metrics here
    project="My first project",
)