Test Runs - Freeplay Introduction

Test Runs in Freeplay provide a structured way for you to run batch tests of your LLM prompts and chains. All methods associated with the Test Runs concept in Freeplay are accessible via the client.test_runs namespace. Test runs can be completed using Completion or Trace datasets. We will focus on code in this section, but for more detail on the Test Runs concept see Test Runs.

Methods Overview

Method Name	Parameters	Description
`create`	`project_id:` string `testlist:` string (dataset name) `include_outputs:` bool (optional, defaults to False) `name:` string (optional) `description:` string (optional)	Instantiate a Test Run object server side and get an Id to reference your Test Run instance. To get expected outputs with your test cases, set `include_outputs=True`.

The testlist parameter accepts the name of a dataset stored in Freeplay. This parameter name is preserved for backwards compatibility. In the UI and documentation, we use “dataset” to refer to this concept.

Step by Step Usage

Create a new Test Run

from freeplay import Freeplay, RecordPayload, TestRunInfo
from openai import OpenAI

# create a new test run

test_run = fpClient.test_runs.create(
project_id=project_id,
testlist=<dataset name> # TODO fill in with the name of the dataset stored in Freeplay
name="mytestrun", # Name of the test run in Freeplay
description="this is a test test!"
)

import Freeplay, { getSessionInfo, getCallInfo, getTestRunInfo} from "freeplay";

// create a test run
const testRun = await fpClient.testRuns.create({
    projectId: fpProjectId,
    testList: 'test-list-name',
  	name: 'my test run',
  	description: 'this is a test test'
});

// see full implmentation in Iterate over each Test Case

val testRun = fpClient.testRuns().create(
  projectId,
  "test-list",
  "my test run", "this is a test test!"
 ).await()

Retrieve your Prompts

Retrieve the prompts needed for your Test Run

# get the prompt associated with the test run
template_prompt = fpClient.prompts.get(
  project_id=project_id,
  template_name="template-name",
  environment="latest"
)

// fetch the prompt template for the test run
let templatePrompt = await fpClient.prompts.get({
  projectId: fpProjectId,
  templateName: "template-name",
  environment: "latest",
});

// see full implmentation in Iterate over each test Case

val templatePrompt = fpClient.prompts().get(
  projectId,
  "template-name",
  "prod").await()

Iterate over each Test Case

For the code you want to test: loop over each test case from the dataset, make an LLM call, and record the results with a link to your test run.

# iterate over each test case
for test_case in test_run.test_cases:
    # format the prompt with the test case variables
    formatted_prompt = template_prompt.bind(test_case.variables).format()

    # make your llm call
    s = time.time()
    openai_client = OpenAI(api_key=openai_key)
    chat_response = openai_client.chat.completions.create(
        model=formatted_prompt.prompt_info.model,
        messages=formatted_prompt.llm_prompt,
        **formatted_prompt.prompt_info.model_parameters
    )
    e = time.time()

    # append the results to the messages
    all_messages = formatted_prompt.all_messages({
        'role': chat_response.choices[0].message.role,
        'content': chat_response.choices[0].message.content
    })
    call_info = CallInfo.from_prompt_info(
        formatted_prompt.prompt_info,
        start_time=s,
        end_time=e,
        usage=UsageTokens(
            chat_response.usage.prompt_tokens,
            chat_response.usage.completion_tokens
        )
    )

    # create a session which will create a UID
    session = fp_client.sessions.create()
    # build the record payload
    payload = RecordPayload(
        project_id=project_id,
        all_messages=all_messages,
        inputs=test_case.variables, # Variables from the test case are the inputs
        session_info=session.session_info,
        # IMPORTANT: link the record call to the test run and test case
        test_run_info=test_run.get_test_run_info(test_case.id),
        prompt_version_info=formatted_prompt.prompt_info, # log the prompt information
        call_info=call_info
    )
    # record the results to freeplay
    fpClient.recordings.create(payload)

for (const testCase of testRun.testCases) {
    // create a formatted prompt from the test case
    const formattedPrompt = templatePrompt.bind(testCase.variables).format();

    // make the llm call
    let start = new Date();
    const chatCompletion = await openai.chat.completions.create({
        messages: formattedPrompt.llmPrompt,
        model: formattedPrompt.promptInfo.model,
        ...formattedPrompt.promptInfo.modelParameters
    });
    let end = new Date();
    console.log(chatCompletion.choices[0].message);

    // update the messages
    let messages = formattedPrompt.allMessages({
        role: chatCompletion.choices[0].message.role,
        content: chatCompletion.choices[0].message.content,
    });

    // create a session
    let session = fpClient.sessions.create({});

    // record the test case interaction with Freeplay
  await fpClient.recordings.create({
    projectId,
    allMessages: messages,
    inputs: testCase.variables,
    sessionInfo: getSessionInfo(session),
    promptVersionInfo: formattedPrompt.promptInfo,
    callInfo: getCallInfo(formattedPrompt.promptInfo, start, end),
    testRunInfo: getTestRunInfo(testRun, testCase.id)
  });
}

import ai.freeplay.client.thin.Freeplay;
import ai.freeplay.client.thin.resources.prompts.ChatMessage;
import ai.freeplay.client.thin.resources.prompts.FormattedPrompt;
import ai.freeplay.client.thin.resources.prompts.TemplatePrompt;
import ai.freeplay.client.thin.resources.recordings.CallInfo;
import ai.freeplay.client.thin.resources.recordings.RecordInfo;
import ai.freeplay.client.thin.resources.recordings.RecordResponse;
import ai.freeplay.client.thin.resources.sessions.SessionInfo;
import ai.freeplay.client.thin.resources.testruns.TestCase;
import ai.freeplay.client.thin.resources.testruns.TestRun;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;

import java.net.http.HttpResponse;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;

import static ai.freeplay.example.java.ThinExampleUtils.callOpenAI;
import static java.lang.String.format;
import static ai.freeplay.client.thin.Freeplay.Config;
import static java.util.stream.Collectors.toList;

public class TestClass {
    private static final ObjectMapper objectMapper = new ObjectMapper();

    public static void main(String[] args) throws ExecutionException, InterruptedException {
        // set your environment variables
        String openaiApiKey = System.getenv("OPENAI_API_KEY");
        String freeplayApiKey = System.getenv("FREEPLAY_API_KEY");
        String projectId = System.getenv("FREEPLAY_PROJECT_ID");
        String customerDomain = System.getenv("FREEPLAY_CUSTOMER_NAME");
        // create your url from your customer domain
        String baseUrl = format("https://%s.freeplay.ai/api", customerDomain);

        Freeplay fpClient = new Freeplay(Config()
                .freeplayAPIKey(freeplayApiKey)
                .customerDomain(customerDomain)
        );
        // create the test run
        List<RecordResponse> recordResponses = fpClient.testRuns().create(projectId, "test-list-name")
                .thenCompose(testRun ->
                        fpClient.prompts().get(projectId, "template-name", "latest")
                                .thenCompose(templatePrompt -> {

                                    var futures =
                                            testRun.getTestCases().stream()
                                                    .map(testCase ->
                                                            handleTestCase(
                                                                    fpClient,
                                                                    openaiApiKey,
                                                                    templatePrompt,
                                                                    testRun,
                                                                    testCase))
                                                    .collect(toList());

                                    return CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new))
                                            .thenApply((Void v) -> futures.stream().map(CompletableFuture::join).collect(toList()));
                                })
                ).join();

        System.out.println("Record responses: " + recordResponses);
    }

    private static CompletableFuture<RecordResponse> handleTestCase(
            Freeplay fpClient,
            String openaiApiKey,
            TemplatePrompt templatePrompt,
            TestRun testRun,
            TestCase testCase
    ) {
        FormattedPrompt<String> formattedPrompt =
                templatePrompt.bind(testCase.getVariables()).format();

        long startTime = System.currentTimeMillis();
        return callOpenAI(
                objectMapper,
                openaiApiKey,
                formattedPrompt.getPromptInfo().getModel(),
                formattedPrompt.getPromptInfo().getModelParameters(),
                formattedPrompt.getBoundMessages()
        ).thenCompose((HttpResponse<String> response) ->
                recordOpenAI(fpClient, testRun, testCase, formattedPrompt, startTime, response)
        );
    }

    private static CompletableFuture<RecordResponse> recordOpenAI(
            Freeplay fpClient,
            TestRun testRun,
            TestCase testCase,
            FormattedPrompt<String> formattedPrompt,
            long startTime,
            HttpResponse<String> response
    ) {
        JsonNode bodyNode;
        try {
            bodyNode = objectMapper.readTree(response.body());
        } catch (JsonProcessingException e) {
            throw new RuntimeException("Unable to parse response body.", e);
        }

        // add the returned message to the list of messages
        String role = bodyNode.path("choices").path(0).path("message").path("role").asText();
        String content = bodyNode.path("choices").path(0).path("message").path("content").asText();
        List<ChatMessage> allMessages = formattedPrompt.allMessages(
                new ChatMessage(role, content)
        );

        CallInfo callInfo = CallInfo.from(
                formattedPrompt.getPromptInfo(),
                startTime,
                System.currentTimeMillis()
        );
        SessionInfo sessionInfo = fpClient.sessions().create().getSessionInfo();

        System.out.println("Completion: " + content);

        return fpClient.recordings().create(
          new RecordInfo(projectId, allMessages)
            .inputs(testCase.getVariables())
            .promptVersionInfo(formattedPrompt.getPromptInfo())
            .callInfo(callInfo)
            .toolSchema(formattedPrompt.getToolSchema())
            .testRunInfo(testRun.getTestRunInfo(testCase.getTestCaseId())));
    }
}

for (testCase in testRun.testCases) {
  // format the prompt with test case variables
  val formattedPrompt = templatePrompt.bind(testCase.variables).format<String>()

  // make your llm call
  val startTime = System.currentTimeMillis()
  val llmResponse = callOpenAI(
    objectMapper,
    anthropicApiKey,
    formattedPrompt.promptInfo.model,
    formattedPrompt.promptInfo.modelParameters,
    formattedPrompt.formattedPrompt
  ).await()

  val bodyNode = objectMapper.readTree(llmResponse.body())

  println("Recording the result")
 	// append the results to your message set
  val allMessages = formattedPrompt.allMessages(
    ChatMessage("Assistant", bodyNode.path("completion").asText())
  )
  val callInfo = CallInfo.from(
    formattedPrompt.getPromptInfo(),
    startTime,
    System.currentTimeMillis()
  )
  // create a session
  val sessionInfo = fpClient.sessions().create().sessionInfo

  // record the test case results
  val recordResponse = fpClient.recordings().create(
 		RecordInfo(
      projectId,
      allMessages
    ).inputs(testCase.variables)
      .sessionInfo(sessionInfo)
      .promptVersionInfo(formattedPrompt.getPromptInfo())
      .callInfo(callInfo)
      .testRunInfo(testRun.getTestRunInfo(testCase.testCaseId))
  ).await()
  println("Recorded with completionId ${recordResponse.completionId}")
}

Agent Test Runs

To execute tests in code, iterate through each test case in your dataset and run it through your full agent workflow. This end-to-end testing approach is particularly valuable for agentic systems, where the goal is to observe how changes—whether to prompts, tools, or orchestration logic—affect the final output. Trace-level tests allow you to simulate production-like behavior and evaluate the agent holistically. As each test case runs, its input is passed into your system, and the resulting trace is logged for evaluation and analysis. See the full example here. Below is an psuedo code example to show the general logic for test-runs:

##################################
# Preare the test
##################################
trace_dataset = "Your dataset name that targets an agent"
test_name = "Name of the test you are running"

# Create the test
test_run = fp_client.test_runs.create(
  project_id=project_id,
  testlist=trace_dataset,
  name=test_name
)

##################################
# Initialize your code or agents
##################################
"""
NOTE: It is important that all agents get passed the test_run_info and it is used when RecordPayload is called. This is how Freeplay tracks the test run
"""

my_agent = MyAgent(<initialize your system code>)

################################################
# Loop over all test cases and execute system code
################################################

for test_case in test_run.trace_test_cases:
  
  # Initialize a Freeplay Session
  session = fp_client.sessions.create()

  # NOTE: Get the test case information; This must be passed with any record calls
  test_case_info = test_run.get_test_run_info(test_case.id)
  
  # Create your Agent
  question = test_case.input
  trace_info = session.create_trace(
      input=question,
      agent_name="MyAgent", # Agent's name
      custom_metadata={
          "version": "1.0.0" # custom dict[str, str]
      }
  )

  ################################################
  # Execute Agent
  ################################################

  # Run your system code using the agents inputs/outputs
  # This is where all the sub-agent processes and recordings happen
  MyAgent.run(
    input=question,
    test_run_info=test_run_info # NOTE: This must be passed to record calls
  )

  # Record the agent output, pass the test run info here as well
  trace_info.record_output(
    project_id,
    completion.choices[0].message.content,
    eval_results={
        'evaluation_score': 0.48,
        'is_high_quality': True
    },
    test_run_info=test_run_info 
  )

​Methods Overview

​Step by Step Usage

​Create a new Test Run

​Retrieve your Prompts

​Iterate over each Test Case

​Agent Test Runs

Methods Overview

Step by Step Usage

Create a new Test Run

Retrieve your Prompts

Iterate over each Test Case

Agent Test Runs