Skip to main content

​
1. Configure Freeplay Client

​
2. Create a Test Run

Instantiate your Test Run which will fetch the Test Cases and create a new Test Run ID

​
3. Fetch Prompt

Fetch your prompt template. Don’t format it yet, we will bind and format for each Test Case

​
4. Loop over Test Cases

Loop over each Test Case, make a completion and record to Freeplay

​
Examples

from freeplay import Freeplay, RecordPayload, ResponseInfo, TestRunInfo
from openai import OpenAI

# create your a freeplay client object
fpClient = Freeplay(
    freeplay_api_key=os.getenv("FREEPLAY_API_KEY"),
    api_base="https://acme.freeplay.ai/api"
)

# create a new test run
test_run = fpClient.test_runs.create(project_id=project_id, testlist="test-list-name")

# get the prompt associated with the test run
template_prompt = fpClient.prompts.get(project_id=project_id,
                                       template_name="template-name",
                                       environment="latest"
                                       )

# iterate over each test case
for test_case in test_run.test_cases:
    # format the prompt with the test case variables
    formatted_prompt = template_prompt.bind(test_case.variables).format()

    # make your llm call
    s = time.time()
    openaiClient = OpenAI(api_key=openai_key)
    chat_response = openaiClient.chat.completions.create(
        model=formatted_prompt.prompt_info.model,
        messages=formatted_prompt.llm_prompt,
        **formatted_prompt.prompt_info.model_parameters
    )
    e = time.time()

    # append the results to the messages
    all_messages = formatted_prompt.all_messages(
        {'role': chat_response.choices[0].message.role,
         'content': chat_response.choices[0].message.content}
    )

    # create a session which will create a UID
    session = fpClient.sessions.create()
    # build the record payload
    payload = RecordPayload(
				project_id=project_id,
        all_messages=all_messages,
        inputs=test_case.variables, # the variables from the test case are the inputs
        session_info=session, # use the session object created above
        test_run_info=test_run.get_test_run_info(test_case.id), # link the record call to the test run and test case
        prompt_version_info=formatted_prompt.prompt_info, # log the prompt information 
        call_info=CallInfo.from_prompt_info(formatted_prompt.prompt_info, start_time=s, end_time=e), # log call information
        response_info=ResponseInfo(
            is_complete=chat_response.choices[0].finish_reason == 'stop'
        )
    )
    # record the results to freeplay
    fpClient.recordings.create(payload)