Run a Test with Tools Programmatically

1. Setup Freeplay and LLM SDK

Initialize Freeplay and your AI Provider’s SDK (OpenAI in this example).

2. Fetch raw prompt

Use Freeplay SDK to pull in your raw prompt. This prompt template contains the tool schema you saved in Freeplay web app. We are using the raw prompt to bind with test cases specific variable and history.

3. Create a test run

With Freeplay SDK, create a test run. We’ll use this down below to associate test related data to this run.

4. Format prompt with test case variables

For each test case in a test run, we’ll bind its variable and history with the prompt we fetched

5. Call LLM with the tools

Create a new completion and pass in the tool schema from formatted prompt when creating a new completion.

6. Capture test run details with Freeplay

Record the eval result with its associated messages with tool calls and schema.

Examples

import os
import time
from openai import OpenAI
from freeplay import Freeplay, RecordPayload, CallInfo

fp_client = Freeplay(freeplay_api_key=os.environ['FREEPLAY_API_KEY'])
openai_client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))

project_id = os.environ['FREEPLAY_PROJECT_ID']
template_prompt = fp_client.prompts.get(
    project_id=project_id,
    template_name='your-prompt',
    environment='latest'
)

test_run = fp_client.test_runs.create(
    project_id,
    "Name of your dataset",
    include_outputs=True,
    name=f'My Example Test Run',
    description='Run from examples',
    flavor_name=template_prompt.prompt_info.flavor_name
)

for test_case in test_run.test_cases:
    formatted_prompt = template_prompt.bind(test_case.variables, history=test_case.history).format()

    start = time.time()
    completion = openai_client.chat.completions.create(
        messages=formatted_prompt.llm_prompt,
        model=formatted_prompt.prompt_info.model,
        tools=formatted_prompt.tool_schema,
        **formatted_prompt.prompt_info.model_parameters
    )
    end = time.time()

    session = fp_client.sessions.create()
    all_messages = formatted_prompt.all_messages(completion.choices[0].message)
    # Handle tool call and append its result to all_messages.
    # Look at OpenAI Recipe: https://docs.freeplay.ai/developer-resources/recipes/using-tools-with-openai
    # Anthropic: https://docs.freeplay.ai/developer-resources/recipes/using-tools-with-anthropic

    fp_client.recordings.create(
        RecordPayload(
            project_id=project_id,
            all_messages=all_messages,
			tool_schema=formatted_prompt.tool_schema,
            session_info=session.session_info,
            inputs=test_case.variables,
            prompt_version_info=formatted_prompt.prompt_info,
            call_info=CallInfo.from_prompt_info(formatted_prompt.prompt_info, start, end),
            test_run_info=test_run.get_test_run_info(test_case.id),
            eval_results={
                'f1-score': 0.48,
                'is_non_empty': True
            }
        )
    )

import Freeplay, { getSessionInfo, getTestRunInfo } from "freeplay";
import OpenAI from "openai";

const fpClient = new Freeplay({
  freeplayApiKey: process.env["FREEPLAY_API_KEY"],
  baseUrl: `${process.env["FREEPLAY_API_URL"]}/api`,
});

const openaiClient = new OpenAI({
  apiKey: process.env["OPENAI_API_KEY"],
});

const projectId = process.env["FREEPLAY_PROJECT_ID"];
const templatePrompt = await fpClient.prompts.get({
  projectId,
  templateName: "your-prompt",
  environment: "latest",
});

const testRun = await fpClient.testRuns.create({
  projectId,
  testList: "Name of your dataset",
  includeOutputs: true,
  name: "My Example Test Run",
  description: "Run from examples",
  flavorName: templatePrompt.promptInfo.flavorName,
});

for await (const testCase of testRun.testCases) {
  const formattedPrompt = templatePrompt
    .bind(testCase.variables, testCase.history)
    .format();

  const start = new Date();
  const completion = await openaiClient.chat.completions.create({
    messages: formattedPrompt.llmPrompt,
    model: formattedPrompt.promptInfo.model,
    tools: formattedPrompt.toolSchema,
    ...formattedPrompt.promptInfo.modelParameters,
  });
  const end = new Date();

  const session = fpClient.sessions.create();
  const messages = formattedPrompt.allMessages(completion.choices[0].message);

  // Handle tool call and append its result to all_messages.
  // Look at OpenAI Recipe: https://docs.freeplay.ai/developer-resources/recipes/using-tools-with-openai
  // Anthropic: https://docs.freeplay.ai/developer-resources/recipes/using-tools-with-anthropic

  await fpClient.recordings.create({
    projectId,
    allMessages: messages,
    toolSchema: formattedPrompt.toolSchema,
    sessionInfo: getSessionInfo(session),
    inputs: testCase.variables,
    promptVersionInfo: formattedPrompt.promptInfo,
    callInfo: {
      provider: formattedPrompt.promptInfo.provider,
      model: formattedPrompt.promptInfo.model,
      startTime: start,
      endTime: end,
      modelParameters: formattedPrompt.promptInfo.modelParameters,
    },
    testRunInfo: getTestRunInfo(testRun, testCase.id),
    evalResults: {
      "f1-score": 0.48,
      is_non_empty: true,
    },
  });
}

package ai.freeplay.example.kotlin

import ai.freeplay.client.thin.Freeplay
import ai.freeplay.client.thin.resources.prompts.*
import ai.freeplay.client.thin.resources.recordings.*
import ai.freeplay.example.java.ThinExampleUtils.callOpenAIWithTools
import com.fasterxml.jackson.databind.ObjectMapper
import java.time.Instant

class TestRunToolsExample {
    companion object {
        private val objectMapper = ObjectMapper()

        @JvmStatic
        fun main(args: Array<String>) {
            // Initialize clients
            val fpClient = Freeplay(
                Freeplay.Config()
                    .freeplayAPIKey(System.getenv("FREEPLAY_API_KEY"))
                    .baseUrl("${System.getenv("FREEPLAY_API_URL")}/api")
            )

            val projectId = System.getenv("FREEPLAY_PROJECT_ID")

            // Get template prompt
            val templatePrompt = fpClient.prompts()
                .get(projectId, "your-prompt", "latest")
                .join()

            // Create test run
            val testRun = fpClient.testRuns().createRequest(projectId, "Name of your dataset")
                .name("My Example Test Run")
                .description("Run from examples")
                .includeOutputs(true)
                .flavorName(templatePrompt.promptInfo.flavorName)
                .build()
                .let { fpClient.testRuns().create(it).join() }

            // Process each test case
            testRun.testCases.forEach { testCase ->
                val formattedPrompt = templatePrompt
                    .bind(testCase.variables, testCase.history)
                    .format<List<ChatMessage>>()

                val startTime = Instant.now().toEpochMilli()
                val response = callOpenAIWithTools(
                    objectMapper,
                    System.getenv("OPENAI_API_KEY"),
                    formattedPrompt.promptInfo.model,
                    formattedPrompt.promptInfo.modelParameters,
                    formattedPrompt.formattedPrompt,
                    formattedPrompt.toolSchema
                ).join()

                val session = fpClient.sessions().create()

                val bodyNode = objectMapper.readTree(response.body())
                val message = objectMapper.convertValue(bodyNode["choices"][0]["message"], Object::class.java)
                val allMessages = formattedPrompt.allMessages(message)

                // Handle tool call and append its result to all_messages.
                // Look at OpenAI Recipe: https://docs.freeplay.ai/developer-resources/recipes/using-tools-with-openai
                // Anthropic: https://docs.freeplay.ai/developer-resources/recipes/using-tools-with-anthropic

                fpClient.recordings().create(
  RecordInfo(
              projectId,
              allMessages
          ).inputs(testCase.variables)
              .sessionInfo(session.sessionInfo)
              .promptVersionInfo(formattedPrompt.promptInfo)
              .callInfo( CallInfo.from(
                              formattedPrompt.promptInfo,
                              startTime,
                              System.currentTimeMillis()
                          ),)
              .toolSchema(formattedPrompt.toolSchema)
              .testRunInfo(testRun.getTestRunInfo(testCase.testCaseId))
              .evalResults(mapOf(
                "f1-score" to 0.48,
                 "is_non_empty" to true
               ))
                ).join()
            }
        }
    }
}

​1. Setup Freeplay and LLM SDK

​2. Fetch raw prompt

​3. Create a test run

​4. Format prompt with test case variables

​5. Call LLM with the tools

​6. Capture test run details with Freeplay

​Examples