Test Run

from freeplay import Freeplay, RecordPayload, TestRunInfo
from openai import OpenAI

# create your a freeplay client object
fpClient = Freeplay(
    freeplay_api_key=os.getenv("FREEPLAY_API_KEY"),
    api_base="https://acme.freeplay.ai/api"
)

# create a new test run
test_run = fpClient.test_runs.create(project_id=project_id, testlist="test-list-name")

# get the prompt associated with the test run
template_prompt = fpClient.prompts.get(project_id=project_id,
                                       template_name="template-name",
                                       environment="latest"
                                       )

# iterate over each test case
for test_case in test_run.test_cases:
    # format the prompt with the test case variables
    formatted_prompt = template_prompt.bind(test_case.variables).format()

    # make your llm call
    s = time.time()
    openai_client = OpenAI(api_key=openai_key)
    chat_response = openai_client.chat.completions.create(
        model=formatted_prompt.prompt_info.model,
        messages=formatted_prompt.llm_prompt,
        **formatted_prompt.prompt_info.model_parameters
    )
    e = time.time()

    # append the results to the messages
    all_messages = formatted_prompt.all_messages({
        'role': chat_response.choices[0].message.role,
        'content': chat_response.choices[0].message.content
    })

    # create a session which will create a UID
    session = fp_client.sessions.create()
    # build the record payload
    payload = RecordPayload(
				project_id=project_id,
        all_messages=all_messages,
        inputs=test_case.variables, # the variables from the test case are the inputs
        session_info=session, # use the session object created above
        test_run_info=test_run.get_test_run_info(test_case.id), # link the record call to the test run and test case
        prompt_version_info=formatted_prompt.prompt_info, # log the prompt information 
        call_info=CallInfo.from_prompt_info(formatted_prompt.prompt_info, start_time=s, end_time=e) # log call information
    )
    # record the results to freeplay
    fpClient.recordings.create(payload)