Serialization

Overview

Strands Evals provides JSON serialization for experiments and reports, enabling you to save, load, version, and share evaluation work.

Saving Experiments

from strands_evals import Experiment

# Save to file
experiment.to_file("my_experiment.json")
experiment.to_file("my_experiment")  # .json added automatically

# Relative path
experiment.to_file("experiments/baseline.json")

# Absolute path
experiment.to_file("/path/to/experiments/baseline.json")

Loading Experiments

# Load from file
experiment = Experiment.from_file("my_experiment.json")

print(f"Loaded {len(experiment.cases)} cases")
print(f"Evaluators: {[e.get_type_name() for e in experiment.evaluators]}")

Custom Evaluators

Pass custom evaluator classes when loading:

from strands_evals.evaluators import Evaluator

class CustomEvaluator(Evaluator):
    def evaluate(self, evaluation_case):
        # Custom logic — must return list[EvaluationOutput]
        return [EvaluationOutput(score=1.0, test_pass=True, reason="...")]

# Save with custom evaluator
experiment = Experiment(
    cases=cases,
    evaluators=[CustomEvaluator()]
)
experiment.to_file("custom.json")

# Load with custom evaluator class
loaded = Experiment.from_file(
    "custom.json",
    custom_evaluators=[CustomEvaluator]
)

Dictionary Conversion

# To dictionary
experiment_dict = experiment.to_dict()

# From dictionary
experiment = Experiment.from_dict(experiment_dict)

# With custom evaluators
experiment = Experiment.from_dict(
    experiment_dict,
    custom_evaluators=[CustomEvaluator]
)

Saving Reports

run_evaluations returns a single EvaluationReport. When the experiment runs multiple evaluators, each row in report.cases carries an evaluator key naming the evaluator that produced it — use that to filter by evaluator if needed.

# Run evaluation
report = experiment.run_evaluations(task_function)

# Save the full report directly (writes JSON; .json is added if missing)
report.to_file("report")

# Or build a custom JSON payload
import json

report_data = {
    "overall_score": report.overall_score,
    "scores": report.scores,
    "test_passes": report.test_passes,
    "reasons": report.reasons,
    "cases": report.cases,
}

with open("report_summary.json", "w") as f:
    json.dump(report_data, f, indent=2)

Versioning Strategies

Timestamp Versioning

from datetime import datetime

timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
experiment.to_file(f"experiment_{timestamp}.json")

Semantic Versioning

experiment.to_file("experiment_v1.json")
experiment.to_file("experiment_v2.json")

Organizing Files

Directory Structure

experiments/
├── baseline/
│   ├── experiment.json
│   └── reports/
├── iteration_1/
│   ├── experiment.json
│   └── reports/
└── final/
    ├── experiment.json
    └── reports/

Organized Saving

from pathlib import Path

base_dir = Path("experiments/iteration_1")
base_dir.mkdir(parents=True, exist_ok=True)

# Save experiment
experiment.to_file(base_dir / "experiment.json")

# Save reports
reports_dir = base_dir / "reports"
reports_dir.mkdir(exist_ok=True)

Saving Experiments with Reports

from pathlib import Path

def save_with_report(experiment, report, base_name):
    base_path = Path(f"evaluations/{base_name}")
    base_path.mkdir(parents=True, exist_ok=True)

    # Save experiment definition and the report side by side
    experiment.to_file(str(base_path / "experiment.json"))
    report.to_file(str(base_path / "report.json"))

# Usage
report = experiment.run_evaluations(task_function)
save_with_report(experiment, report, "baseline_20250115")

Error Handling

from pathlib import Path

def safe_load(path, custom_evaluators=None):
    try:
        file_path = Path(path)

        if not file_path.exists():
            raise FileNotFoundError(f"File not found: {path}")

        if file_path.suffix != ".json":
            raise ValueError(f"Expected .json file, got: {file_path.suffix}")

        experiment = Experiment.from_file(path, custom_evaluators=custom_evaluators)
        print(f"✓ Loaded {len(experiment.cases)} cases")
        return experiment

    except Exception as e:
        print(f"✗ Failed to load: {e}")
        return None

Best Practices

1. Use Consistent Naming

# Good
experiment.to_file("customer_service_baseline_v1.json")

# Less helpful
experiment.to_file("test.json")

2. Validate After Loading

experiment = Experiment.from_file("experiment.json")

assert len(experiment.cases) > 0, "No cases loaded"
assert len(experiment.evaluators) > 0, "No evaluators loaded"

3. Include Metadata

experiment_data = experiment.to_dict()
experiment_data["metadata"] = {
    "created_date": datetime.now().isoformat(),
    "description": "Baseline evaluation",
    "version": "1.0"
}

with open("experiment.json", "w") as f:
    json.dump(experiment_data, f, indent=2)

Experiment Management: Organize experiments
Experiment Generator: Generate experiments
Quickstart Guide: Get started with Strands Evals