Test Run

from freeplay import Freeplay, RecordPayload, TestRunInfo
from openai import OpenAI

# create your a freeplay client object
fpClient = Freeplay(
    freeplay_api_key=os.getenv("FREEPLAY_API_KEY"),
    api_base="https://acme.freeplay.ai/api"
)

# create a new test run
test_run = fpClient.test_runs.create(project_id=project_id, testlist="test-list-name")

# get the prompt associated with the test run
template_prompt = fpClient.prompts.get(project_id=project_id,
                                       template_name="template-name",
                                       environment="latest"
                                       )

# iterate over each test case
for test_case in test_run.test_cases:
    # format the prompt with the test case variables
    formatted_prompt = template_prompt.bind(test_case.variables).format()

    # make your llm call
    s = time.time()
    openai_client = OpenAI(api_key=openai_key)
    chat_response = openai_client.chat.completions.create(
        model=formatted_prompt.prompt_info.model,
        messages=formatted_prompt.llm_prompt,
        **formatted_prompt.prompt_info.model_parameters
    )
    e = time.time()

    # append the results to the messages
    all_messages = formatted_prompt.all_messages({
        'role': chat_response.choices[0].message.role,
        'content': chat_response.choices[0].message.content
    })

    # create a session which will create a UID
    session = fp_client.sessions.create()
    # build the record payload
    payload = RecordPayload(
				project_id=project_id,
        all_messages=all_messages,
        inputs=test_case.variables, # the variables from the test case are the inputs
        session_info=session, # use the session object created above
        test_run_info=test_run.get_test_run_info(test_case.id), # link the record call to the test run and test case
        prompt_version_info=formatted_prompt.prompt_info, # log the prompt information 
        call_info=CallInfo.from_prompt_info(formatted_prompt.prompt_info, start_time=s, end_time=e) # log call information
    )
    # record the results to freeplay
    fpClient.recordings.create(payload)

import Freeplay, { getSessionInfo, getCallInfo, getTestRunInfo} from "freeplay";

// create your freeplay client
const fpClient = new Freeplay({
    freeplayApiKey: process.env["FREEPLAY_API_KEY"],
    baseUrl: "https://acme.freeplay.ai/api",
});

// create a test run
const testRun = await fpClient.testRuns.create({
    projectId: fpProjectId,
    testList: 'test-list-name'
});

// fetch the prompt template for the test run
let templatePrompt = await fpClient.prompts.get({
    projectId: fpProjectId,
    templateName: "template-name",
    environment: "latest",
});

for (const testCase of testRun.testCases) {
    // create a formatted prompt from the test case
    const formattedPrompt = templatePrompt.bind(testCase.variables).format();

    // make the llm call
    let start = new Date();
    const chatCompletion = await openai.chat.completions.create({
        messages: formattedPrompt.llmPrompt,
        model: formattedPrompt.promptInfo.model,
        ...formattedPrompt.promptInfo.modelParameters
    });
    let end = new Date();
    console.log(chatCompletion.choices[0].message);

    // update the messages
    let messages = formattedPrompt.allMessages({
        role: chatCompletion.choices[0].message.role,
        content: chatCompletion.choices[0].message.content,
    });

    // create a session
    let session = fpClient.sessions.create({});

    // record the test case interaction with Freeplay
  await fpClient.recordings.create({
				projectId: fpProjectId,
        allMessages: messages,
        inputs: testCase.variables,
        sessionInfo: getSessionInfo(session),
        promptVersionInfo: formattedPrompt.promptInfo,
        callInfo: getCallInfo(formattedPrompt.promptInfo, start, end),
        testRunInfo: getTestRunInfo(testRun, testCase.id)
    });
}

import ai.freeplay.client.thin.Freeplay;
import ai.freeplay.client.thin.resources.prompts.ChatMessage;
import ai.freeplay.client.thin.resources.prompts.FormattedPrompt;
import ai.freeplay.client.thin.resources.prompts.TemplatePrompt;
import ai.freeplay.client.thin.resources.recordings.CallInfo;
import ai.freeplay.client.thin.resources.recordings.RecordInfo;
import ai.freeplay.client.thin.resources.recordings.RecordResponse;
import ai.freeplay.client.thin.resources.sessions.SessionInfo;
import ai.freeplay.client.thin.resources.testruns.TestCase;
import ai.freeplay.client.thin.resources.testruns.TestRun;

// create the freeplay client
val fpClient = Freeplay(
  Freeplay.Config()
    .freeplayAPIKey(freeplayApiKey)
    .customerDomain(customerDomain)
)
  
val testRun = fpClient.testRuns().create(projectId, "test-list").await()

val templatePrompt = fpClient.prompts().get(projectId, "template-name", "prod").await()
  
for (testCase in testRun.testCases) {
  // format the prompt with test case variables
  val formattedPrompt = templatePrompt.bind(testCase.variables).format<String>()
	
  // make your llm call
  val startTime = System.currentTimeMillis()
  val llmResponse = callOpenAI(
    objectMapper,
    anthropicApiKey,
    formattedPrompt.promptInfo.model,
    formattedPrompt.promptInfo.modelParameters,
    formattedPrompt.formattedPrompt
  ).await()

  val bodyNode = objectMapper.readTree(llmResponse.body())

  println("Recording the result")
 	// append the results to your message set
  val allMessages = formattedPrompt.allMessages(
    ChatMessage("Assistant", bodyNode.path("completion").asText())
  )
  val callInfo = CallInfo.from(
    formattedPrompt.getPromptInfo(),
    startTime,
    System.currentTimeMillis()
  )
  // create a session
  val sessionInfo = fpClient.sessions().create().sessionInfo

  // record the test case results
  val recordResponse = fpClient.recordings().create(
        RecordInfo(
            projectId,
            allMessages
        ).inputs(variables)
            .sessionInfo(session.sessionInfo)
            .promptVersionInfo(prompt.promptInfo)
            .callInfo(callInfo)
            .traceInfo(trace)
            .testRunInfo(testRun.getTestRunInfo(testCase.testCaseId))
    ).await()
  println("Recorded with completionId ${recordResponse.completionId}")
}