import ddtrace.llmobs.experimentation as dne

dne.init(project_name="example")

dataset = dne.Dataset(
    name="capitals-of-the-world",
    data=[
        {"input": "What is the capital of China?", "expected_output": "Beijing"},
        {
            "input": "Which city serves as the capital of South Africa?",
            "expected_output": "Pretoria",
        },
        {
            "input": "What is the capital of Switzerland?",
            "expected_output": "Bern",
        },
        {
            "input": "Name the capital city of a country that starts with 'Z'."  # Open-ended question
        }
    ],
)

Dataset.pull(name: str) -> Dataset

import ddtrace.llmobs.experimentation as dne

dne.init(project_name="example")

dataset = dne.Dataset.pull("capitals-of-the-world")

Dataset.from_csv(
        cls,
        filepath: str,
        name: str,
        description: str = "",
        delimiter: str = ",",
        input_columns: List[str] = None,
        expected_output_columns: List[str] = None,
    ) -> Dataset:

import ddtrace.llmobs.experimentation as dne

dne.init(project_name="example")


dataset = dne.Dataset.from_csv(
path="data.csv", 
name="my_dataset", 
input_columns=["question", "category", "difficulty"], 
expected_output_columns=["answer"]
)

Dataset.push(overwrite: boolean = None, new_version: boolean = None)

import ddtrace.llmobs.experimentation as dne

dne.init(project_name="example")

dataset = dne.Dataset(...)

dataset.push()

Dataset.as_dataframe(multiindex: bool = True) -> pd.DataFrame

Experiment(
    name: str,
    task: Callable,
    dataset: Dataset,
    evaluators: List[Callable],
    tags: List[str] = [],
    description: str = "",
    metadata: Dict[str, Any] = {},
    config: Optional[Dict[str, Any]] = None
)

Experiment.run(jobs: int = 10, raise_errors: bool = False, sample_size: int = None) -> ExperimentResults

# To test, run top 10 rows and see if it throws errors
results = experiment.run(raise_errors=True, sample_size=10)

# If it's acceptable after that, run the whole thing
results = experiment.run()

Experiment.run_evaluations(evaluators: Optional[List[Callable]] = None, raise_errors: bool = False) -> ExperimentResults

ExperimentResults.as_dataframe(multiindex: bool = True) -> pd.DataFrame

import ddtrace.llmobs.experimentation as dne

dne.init(project_name="example")

@dne.task
def process(input: Dict[str, Any], config: Optional[Dict[str, Any]] = None) -> Any:
    # Your business logic

import ddtrace.llmobs.experimentation as dne

dne.init(project_name="example")

@dne.evaluator
def evaluate(input: Any, output: Any, expected_output: Any) -> Any:
    # Your evaluation logic