Skip to main content
Freeplay allows you to record LLM interactions from any model or provider, including hosts or formats Freeplay doesn’t natively support in our application (see that list here). When calling other models, you’ll retrieve a Freeplay prompt template in our common format and reformat it as needed for the LLM you want to use. Here is an example of calling Mistral 7B hosted on BaseTen 
from freeplay import Freeplay, RecordPayload, CallInfo
from freeplay.llm_parameters import LLMParameters
import os
import requests
import time

# retrieve your prompt template from freeplay
prompt_vars = {"keyA": "valueA"}
prompt = fpClient.prompts.get(project_id=project_id,
                                        template_name="template_name",
                                        environment="latest")
# bind your variables to the prompt
formatted_prompt = prompt.bind(prompt_vars).format()
# customize the messages for your provider API
# In this case, mistral does not support system messages
# we need to merge the system message into the initial user message
messages = [{'role': 'user',
             'content': formatted_prompt.messages[0]['content'] + '' + formatted_prompt.messages[1]['content']}]

# make your LLM call to your custom provider
# call mistral 7b hosted with baseten
s = time.time()
baseten_url = "https://model-xyz.api.baseten.co/production/predict"
headers = {
    "Authorization": "Api-Key " + baseten_key,
}
data = {'messages': messages}
req = requests.post(
    url=baseten_url,
    headers=headers,
    json=data
)
e = time.time()

# add the response to ongoing list of messages
resText = req.json()
messages.append({'role': 'assistant', 'content': resText})

# create a freeplay session
session = fpClient.sessions.create()

# Construct the CallInfo from scratch
call_info=CallInfo(
        provider="mistral",
        model="mistral-7b",
        start_time=s,
        end_time=e,
        model_parameters=LLMParameters(
            {"paramA": "valueA", "paramB": "valueB"}
        ),
        usage=UsageTokens(prompt_tokens=123, completion_tokens=456)
  )

# record the LLM interaction with Freeplay
payload = RecordPayload(
    project_id=project_id,
    all_messages=messages,
    inputs=prompt_vars,
    session_info=session.session_info,
    prompt_version_info=prompt.prompt_info,
    call_info=call_info
)

fpClient.recordings.create(payload)