> ## Documentation Index
> Fetch the complete documentation index at: https://docs.freeplay.ai/llms.txt
> Use this file to discover all available pages before exploring further.

# Run a Test with Tools Programmatically

> Run programmatic test runs with tool calling functionality using the Freeplay SDK.

### 1. Setup Freeplay and LLM SDK

Initialize Freeplay and your AI Provider's SDK (OpenAI in this example).

### 2. Fetch raw prompt

Use Freeplay SDK to pull in your raw prompt. This prompt template contains the tool schema you saved in Freeplay web app.

We are using the raw prompt to bind with test cases specific variable and history.

### 3. Create a test run

With Freeplay SDK, create a test run. We'll use this down below to associate test related data to this run.

### 4. Format prompt with test case variables

For each test case in a test run, we'll bind its variable and history with the prompt we fetched

### 5. Call LLM with the tools

Create a new completion and pass in the tool schema from formatted prompt when creating a new completion.

### 6. Capture test run details with Freeplay

Record the eval result with its associated messages with tool calls and schema.

## Examples

<CodeGroup>
  ```python Python theme={null}
  import os
  import time
  from openai import OpenAI
  from freeplay import Freeplay, RecordPayload, CallInfo

  fp_client = Freeplay(freeplay_api_key=os.environ['FREEPLAY_API_KEY'])
  openai_client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))

  project_id = os.environ['FREEPLAY_PROJECT_ID']
  template_prompt = fp_client.prompts.get(
      project_id=project_id,
      template_name='your-prompt',
      environment='latest'
  )

  test_run = fp_client.test_runs.create(
      project_id,
      "Name of your dataset",
      include_outputs=True,
      name=f'My Example Test Run',
      description='Run from examples',
      flavor_name=template_prompt.prompt_info.flavor_name
  )

  for test_case in test_run.test_cases:
      formatted_prompt = template_prompt.bind(test_case.variables, history=test_case.history).format()

      start = time.time()
      completion = openai_client.chat.completions.create(
          messages=formatted_prompt.llm_prompt,
          model=formatted_prompt.prompt_info.model,
          tools=formatted_prompt.tool_schema,
          **formatted_prompt.prompt_info.model_parameters
      )
      end = time.time()

      session = fp_client.sessions.create()
      all_messages = formatted_prompt.all_messages(completion.choices[0].message)
      # Handle tool call and append its result to all_messages.
      # Look at OpenAI Recipe: https://docs.freeplay.ai/developer-resources/recipes/using-tools-with-openai
      # Anthropic: https://docs.freeplay.ai/developer-resources/recipes/using-tools-with-anthropic

      fp_client.recordings.create(
          RecordPayload(
              project_id=project_id,
              all_messages=all_messages,
  			tool_schema=formatted_prompt.tool_schema,
              session_info=session.session_info,
              inputs=test_case.variables,
              prompt_version_info=formatted_prompt.prompt_info,
              call_info=CallInfo.from_prompt_info(formatted_prompt.prompt_info, start, end),
              test_run_info=test_run.get_test_run_info(test_case.id),
              eval_results={
                  'f1-score': 0.48,
                  'is_non_empty': True
              }
          )
      )
  ```

  ```javascript Node theme={null}
  import Freeplay, { getSessionInfo, getTestRunInfo } from "freeplay";
  import OpenAI from "openai";

  const fpClient = new Freeplay({
    freeplayApiKey: process.env["FREEPLAY_API_KEY"],
    baseUrl: `${process.env["FREEPLAY_API_URL"]}/api`,
  });

  const openaiClient = new OpenAI({
    apiKey: process.env["OPENAI_API_KEY"],
  });

  const projectId = process.env["FREEPLAY_PROJECT_ID"];
  const templatePrompt = await fpClient.prompts.get({
    projectId,
    templateName: "your-prompt",
    environment: "latest",
  });

  const testRun = await fpClient.testRuns.create({
    projectId,
    testList: "Name of your dataset",
    includeOutputs: true,
    name: "My Example Test Run",
    description: "Run from examples",
    flavorName: templatePrompt.promptInfo.flavorName,
  });

  for await (const testCase of testRun.testCases) {
    const formattedPrompt = templatePrompt
      .bind(testCase.variables, testCase.history)
      .format();

    const start = new Date();
    const completion = await openaiClient.chat.completions.create({
      messages: formattedPrompt.llmPrompt,
      model: formattedPrompt.promptInfo.model,
      tools: formattedPrompt.toolSchema,
      ...formattedPrompt.promptInfo.modelParameters,
    });
    const end = new Date();

    const session = fpClient.sessions.create();
    const messages = formattedPrompt.allMessages(completion.choices[0].message);

    // Handle tool call and append its result to all_messages.
    // Look at OpenAI Recipe: https://docs.freeplay.ai/developer-resources/recipes/using-tools-with-openai
    // Anthropic: https://docs.freeplay.ai/developer-resources/recipes/using-tools-with-anthropic

    await fpClient.recordings.create({
      projectId,
      allMessages: messages,
      toolSchema: formattedPrompt.toolSchema,
      sessionInfo: getSessionInfo(session),
      inputs: testCase.variables,
      promptVersionInfo: formattedPrompt.promptInfo,
      callInfo: {
        provider: formattedPrompt.promptInfo.provider,
        model: formattedPrompt.promptInfo.model,
        startTime: start,
        endTime: end,
        modelParameters: formattedPrompt.promptInfo.modelParameters,
      },
      testRunInfo: getTestRunInfo(testRun, testCase.id),
      evalResults: {
        "f1-score": 0.48,
        is_non_empty: true,
      },
    });
  }
  ```

  ```java Kotlin theme={null}
  package ai.freeplay.example.kotlin

  import ai.freeplay.client.thin.Freeplay
  import ai.freeplay.client.thin.resources.prompts.*
  import ai.freeplay.client.thin.resources.recordings.*
  import ai.freeplay.example.java.ThinExampleUtils.callOpenAIWithTools
  import com.fasterxml.jackson.databind.ObjectMapper
  import java.time.Instant

  class TestRunToolsExample {
      companion object {
          private val objectMapper = ObjectMapper()

          @JvmStatic
          fun main(args: Array<String>) {
              // Initialize clients
              val fpClient = Freeplay(
                  Freeplay.Config()
                      .freeplayAPIKey(System.getenv("FREEPLAY_API_KEY"))
                      .baseUrl("${System.getenv("FREEPLAY_API_URL")}/api")
              )

              val projectId = System.getenv("FREEPLAY_PROJECT_ID")

              // Get template prompt
              val templatePrompt = fpClient.prompts()
                  .get(projectId, "your-prompt", "latest")
                  .join()

              // Create test run
              val testRun = fpClient.testRuns().createRequest(projectId, "Name of your dataset")
                  .name("My Example Test Run")
                  .description("Run from examples")
                  .includeOutputs(true)
                  .flavorName(templatePrompt.promptInfo.flavorName)
                  .build()
                  .let { fpClient.testRuns().create(it).join() }

              // Process each test case
              testRun.testCases.forEach { testCase ->
                  val formattedPrompt = templatePrompt
                      .bind(testCase.variables, testCase.history)
                      .format<List<ChatMessage>>()

                  val startTime = Instant.now().toEpochMilli()
                  val response = callOpenAIWithTools(
                      objectMapper,
                      System.getenv("OPENAI_API_KEY"),
                      formattedPrompt.promptInfo.model,
                      formattedPrompt.promptInfo.modelParameters,
                      formattedPrompt.formattedPrompt,
                      formattedPrompt.toolSchema
                  ).join()

                  val session = fpClient.sessions().create()

                  val bodyNode = objectMapper.readTree(response.body())
                  val message = objectMapper.convertValue(bodyNode["choices"][0]["message"], Object::class.java)
                  val allMessages = formattedPrompt.allMessages(message)

                  // Handle tool call and append its result to all_messages.
                  // Look at OpenAI Recipe: https://docs.freeplay.ai/developer-resources/recipes/using-tools-with-openai
                  // Anthropic: https://docs.freeplay.ai/developer-resources/recipes/using-tools-with-anthropic

                  fpClient.recordings().create(
    RecordInfo(
                projectId,
                allMessages
            ).inputs(testCase.variables)
                .sessionInfo(session.sessionInfo)
                .promptVersionInfo(formattedPrompt.promptInfo)
                .callInfo( CallInfo.from(
                                formattedPrompt.promptInfo,
                                startTime,
                                System.currentTimeMillis()
                            ),)
                .toolSchema(formattedPrompt.toolSchema)
                .testRunInfo(testRun.getTestRunInfo(testCase.testCaseId))
                .evalResults(mapOf(
                  "f1-score" to 0.48,
                   "is_non_empty" to true
                 ))
                  ).join()
              }
          }
      }
  }
  ```
</CodeGroup>