Calling Any Model - Freeplay Introduction

Freeplay allows you to record LLM interactions from any model or provider, including hosts or formats Freeplay doesn’t natively support in our application (see that list here). When calling other models, you’ll retrieve a Freeplay prompt template in our common format and reformat it as needed for the LLM you want to use. Here is an example of calling Mistral 7B hosted on BaseTen

from freeplay import Freeplay, RecordPayload, CallInfo
from freeplay.llm_parameters import LLMParameters
import os
import requests
import time

# retrieve your prompt template from freeplay
prompt_vars = {"keyA": "valueA"}
prompt = fpClient.prompts.get(project_id=project_id,
                                        template_name="template_name",
                                        environment="latest")
# bind your variables to the prompt
formatted_prompt = prompt.bind(prompt_vars).format()
# customize the messages for your provider API
# In this case, mistral does not support system messages
# we need to merge the system message into the initial user message
messages = [{'role': 'user',
             'content': formatted_prompt.messages[0]['content'] + '' + formatted_prompt.messages[1]['content']}]

# make your LLM call to your custom provider
# call mistral 7b hosted with baseten
s = time.time()
baseten_url = "https://model-xyz.api.baseten.co/production/predict"
headers = {
    "Authorization": "Api-Key " + baseten_key,
}
data = {'messages': messages}
req = requests.post(
    url=baseten_url,
    headers=headers,
    json=data
)
e = time.time()

# add the response to ongoing list of messages
resText = req.json()
messages.append({'role': 'assistant', 'content': resText})

# create a freeplay session
session = fpClient.sessions.create()

# Construct the CallInfo from scratch
call_info=CallInfo(
        provider="mistral",
        model="mistral-7b",
        start_time=s,
        end_time=e,
        model_parameters=LLMParameters(
            {"paramA": "valueA", "paramB": "valueB"}
        ),
        usage=UsageTokens(prompt_tokens=123, completion_tokens=456)
  )

# record the LLM interaction with Freeplay
payload = RecordPayload(
    project_id=project_id,
    all_messages=messages,
    inputs=prompt_vars,
    session_info=session.session_info,
    prompt_version_info=prompt.prompt_info,
    call_info=call_info
)

fpClient.recordings.create(payload)

import Freeplay from "freeplay";
import axios from "axios";

// configure freeplay client
const fpClient = Freeplay({..});

/* PROMPT FETCH */
// set the prompt variables
let promptVars = {"keyA": "valueA"};
// fetch your prompt from freeplay
let promptTemplate = await fpClient.prompts.get({
    projectId,
    templateName: "template_name",
    environment: "latest"
});
// format the prompt
let formattedPrompt = promptTemplate.bind(promptVars).format();

/* LLM CALL */
// build the messages for mistral
// mistral does not support system messages
// we need to merge the system message into the initial user message
let messages = [{'role': 'user', 'content': formattedPrompt.messages[0]['content'] + ' ' + formattedPrompt.messages[1]['content']}];

// configure baseten requests
const basetenUrl = "https://model-xyz.api.baseten.co/production/predict";
const headers = {
    "Authorization": "Api-Key " + basetenKey
};
let data = {'messages': messages};

async function pingBaseTen(data) {
    // send the requests
    let start = new Date();
    let responseData;
    const response = await axios.post(basetenUrl, data, {headers: headers})
    responseData = response.data;
    console.log(responseData);
    let end = new Date();
    return [responseData, start, end];
}

let [responseData, start, end] = await pingBaseTen(data);

messages.push({'role': 'assistant', 'content': responseData});
console.log(messages);

/* RECORD */
// create a session
let session = fpClient.sessions.create({});

// log the interaction with freeplay
fpClient.recordings.create({
    projectId,
    allMessages: messages,
    inputs: promptVars,
    sessionInfo: session,
    promptVersionInfo: formattedPrompt.promptInfo,
    callInfo: {provider: "mistral", model: "mistral-7B", startTime: start, endTime: end, modelParams: {'paramA': 'valueA', 'paramB': 'valueB'}}
});

import ai.freeplay.client.thin.Freeplay;
import ai.freeplay.client.thin.resources.prompts.FormattedPrompt;
import ai.freeplay.client.thin.resources.recordings.*;
import ai.freeplay.client.thin.resources.recordings.CallInfo;
import ai.freeplay.client.thin.resources.sessions.SessionInfo;
import ai.freeplay.client.thin.resources.prompts.ChatMessage;
import ai.freeplay.example.java.ThinExampleUtils.Tuple2;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;

import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.List;
import java.net.http.HttpResponse;
import java.util.concurrent.CompletableFuture;

import static java.lang.String.format;
import static ai.freeplay.client.thin.Freeplay.Config;
import static java.net.http.HttpRequest.BodyPublishers.ofString;

public class CustomModel {
    private static final ObjectMapper objectMapper = new ObjectMapper();

    public static void main(String[] args) {

        // set your environment variables
        String basetenApiKey = System.getenv("BASETEN_KEY");
        String freeplayApiKey = System.getenv("FREEPLAY_API_KEY");
        String projectId = System.getenv("FREEPLAY_PROJECT_ID");
        String customerDomain = System.getenv("FREEPLAY_CUSTOMER_NAME");
        // create your url from your customer domain
        String baseUrl = format("https://%s.freeplay.ai/api", customerDomain);

        // initialize the freeplay client
        Freeplay fpClient = new Freeplay(Config()
                .freeplayAPIKey(freeplayApiKey)
                .customerDomain(customerDomain)
        );

        // create the prompt variables
        Map<String, Object> promptVars = Map.of("keyA", "varA");
        /* FETCH PROMPT */
        var promptFuture = fpClient.prompts().<String>getFormatted(
                projectId,
                "template-name",
                "latest",
                promptVars
        );

        /* LLM CALL */
        // set timer to be used to measure latency
        long startTime = System.currentTimeMillis();
        var llmFuture = promptFuture.thenCompose((FormattedPrompt<String> formattedPrompt) ->

                callMistral(
                        objectMapper,
                        basetenApiKey,
                        formattedPrompt.getPromptInfo().getModel(), // get the model name
                        formattedPrompt.getPromptInfo().getModelParameters(), // get the model parameters
                        formattedPrompt.getBoundMessages() // get the messages, formatted for your specified provider
                ).thenApply((HttpResponse<String> response) ->
                        new Tuple2<>(formattedPrompt, response)
                        )
                );

        /* RECORD TO FREEPLAY */
        var recordFuture = llmFuture.thenCompose((Tuple2<FormattedPrompt<String>, HttpResponse<String>> promptAndResponse) ->
                recordResult(
                        fpClient,
                        promptAndResponse.first, promptVars, startTime, promptAndResponse.second
                )
        );

        recordFuture.thenApply(recordResponse -> {
            System.out.println("Recorded call succeeded with completionId: " + recordResponse.getCompletionId());
            return null;
        }).exceptionally(exception -> {
            System.out.println("Got exception: " + exception.getMessage());
            return new RecordResponse(null);
        }).join();
    }

    public static CompletableFuture<RecordResponse> recordResult(
            Freeplay fpClient,
            FormattedPrompt<String> formattedPrompt,
            Map<String, Object> promptVars,
            long startTime,
            HttpResponse<String> response
    ) {
        JsonNode bodyNode;
        try{
            bodyNode = objectMapper.readTree(response.body());
            System.out.println(bodyNode);
        } catch (JsonProcessingException e){
            throw new RuntimeException("Unable to parse response body", e);
        }

        // add the returned message to the list of messages
        String role = "assistant";
        String content = bodyNode.asText();
        List<ChatMessage> allMessages = formattedPrompt.allMessages(
                new ChatMessage(role, content)
        );

        // construct the call info from scratch
        CallInfo callInfo = new CallInfo(
                "mistral",
                "mistral-7b",
                startTime,
                System.currentTimeMillis(),
                formattedPrompt.getPromptInfo().getModelParameters()
        );
        // construct the session info from the session object
        SessionInfo sessionInfo = fpClient.sessions().create().getSessionInfo();

        System.out.println("Completion: " + content);

        // make the record call to freeplay
        return fpClient.recordings().create(
            new RecordInfo(projectId, allMessages)
                .inputs(promptVars)
                .sessionInfo(sessionInfo)
                .promptVersionInfo(formattedPrompt.getPromptInfo())
                .callInfo(callInfo)
        );
    }

    public static CompletableFuture<HttpResponse<String>> callMistral(
            ObjectMapper objectMapper,
            String basetenApiKey,
            String model,
            Map<String, Object> llmParameters,
            List<ChatMessage> messages
    ) {
        try {
            String basetenChatURL = "https://model-xyz.api.baseten.co/production/predict";

            Map<String, Object> bodyMap = new LinkedHashMap<>();
            bodyMap.put("messages", messages);

            bodyMap.putAll(llmParameters);
            String body = objectMapper.writeValueAsString(bodyMap);

            HttpRequest.Builder requestBuilder = HttpRequest
                    .newBuilder(new URI(basetenChatURL))
                    .header("Content-Type", "application/json")
                    .header("Authorization", format("Api-Key %s", basetenApiKey))
                    .POST(ofString(body));

            return HttpClient.newBuilder()
                    .build()
                    .sendAsync(requestBuilder.build(), HttpResponse.BodyHandlers.ofString());
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
}

package ai.freeplay.example.kotlin

import ai.freeplay.client.thin.Freeplay
import ai.freeplay.client.thin.resources.prompts.ChatMessage
import ai.freeplay.client.thin.resources.recordings.CallInfo
import ai.freeplay.client.thin.resources.recordings.RecordInfo
import ai.freeplay.example.java.ThinExampleUtils.callOpenAI
import com.fasterxml.jackson.databind.ObjectMapper
import kotlinx.coroutines.future.await
import kotlinx.coroutines.runBlocking

// for baseten mistral call
import java.net.URI
import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
import java.net.http.HttpRequest.BodyPublishers
import java.util.concurrent.CompletableFuture

private val objectMapper = ObjectMapper()

fun callMistral(
        objectMapper: ObjectMapper,
        basetenApiKey: String,
        model: String,
        llmParameters: Map<String, Any>,
        messages: List<ChatMessage>
): CompletableFuture<HttpResponse<String>> {
    return try {
        val basetenChatURL = "https://model-xyz.api.baseten.co/production/predict"

        val bodyMap = LinkedHashMap<String, Any>()
        bodyMap["messages"] = messages
        bodyMap.putAll(llmParameters)
        val body = objectMapper.writeValueAsString(bodyMap)

        val requestBuilder = HttpRequest.newBuilder(URI(basetenChatURL))
                .header("Content-Type", "application/json")
                .header("Authorization", "Api-Key $basetenApiKey")
                .POST(BodyPublishers.ofString(body))

        HttpClient.newBuilder()
                .build()
                .sendAsync(requestBuilder.build(), HttpResponse.BodyHandlers.ofString())
    } catch (e: Exception) {
        throw RuntimeException(e)
    }
}

fun main(): Unit = runBlocking {
    // set your environment variables
    val openaiApiKey = System.getenv("OPENAI_API_KEY")
    val freeplayApiKey = System.getenv("FREEPLAY_API_KEY")
    val projectId = System.getenv("FREEPLAY_PROJECT_ID")
    val customerDomain = System.getenv("FREEPLAY_CUSTOMER_NAME")
    val basetenApiKey = System.getenv("BASETEN_KEY")

    // create your url from your customer domain
    val baseUrl = String.format("https://%s.freeplay.ai/api", customerDomain)

    // create the freeplay client
    val fpClient = Freeplay(
            Freeplay.Config()
                    .freeplayAPIKey(freeplayApiKey)
                    .customerDomain(customerDomain)
    )

    // set the prompt variables
    val promptVars = mapOf("keyA" to "varA")

    /* PROMPT FETCH */
    val formattedPrompt = fpClient.prompts().getFormatted<String>(
            projectId,
            "template-name",
            "latest",
            promptVars
    ).await()

    /* LLM CALL */
    // set timer to measure latency
    val startTime = System.currentTimeMillis()
    val llmResponse = callMistral(
            objectMapper,
            basetenApiKey,
            formattedPrompt.promptInfo.model, // get the model name
            formattedPrompt.promptInfo.modelParameters, // get the model params
            formattedPrompt.getBoundMessages()
    ).await()
    val endTime = System.currentTimeMillis()

    // add the LLM response to the message set
    val bodyNode = objectMapper.readTree(llmResponse.body())
    println(bodyNode)
    val role = "assistant"
    val content = bodyNode.asText()
    println("Completion: " + content)

    val allMessage: List<ChatMessage> = formattedPrompt.allMessages(
            ChatMessage(role, content)
    )

    /* RECORD */
    // construct the call info from the prompt object
    val callInfo = CallInfo(
            "mistral", // hard code the provider
            "mistral-7b", // hard code the model
            startTime,
            System.currentTimeMillis(),
            formattedPrompt.promptInfo.modelParameters
    )

    // create the session and get the session info
    val sessionInfo = fpClient.sessions().create().sessionInfo

    // build the final record payload
    val recordResponse = fpClient.recordings().create(
      RecordInfo(
        projectId,
        allMessage,
        promptVars,
        sessionInfo,
        formattedPrompt.getPromptInfo(),
        callInfo
      )
    ).await()
    println("Completion Record Succeeded with ${recordResponse.completionId}")

}