Using Structured Outputs (JSON mode) with Text Gen endpoints

1
from openai import OpenAI
2
import os
3
4
client = OpenAI(
5
    base_url="https://text.octoai.run/v1",
6
    api_key=os.environ["OCTOAI_API_KEY"],
7
)
8
9
model = "meta-llama-3.1-8b-instruct"

1
import json
2
3
def generate_json_object():
4
    response = client.chat.completions.create(
5
        model=model,
6
        messages=[
7
            {
8
                "role": "system",
9
                "content": "Generate a JSON object, without any additional text or comments.",
10
            },
11
            {"role": "user", "content": "who won the world cup in 2022? answer in JSON"},
12
        ],
13
        max_tokens=max_tokens,
14
        response_format={
15
            "type": "json_object",
16
        },
17
        temperature=0,
18
    )
19
20
    content = response.choices[0].message.content
21
    data = json.loads(content)
22
    return data

1
from pydantic import BaseModel
2
from jsonschema import validate
3
4
class Output(BaseModel):
5
    answer: str
6
7
def generate_json_schema_strict_false():
8
    schema = Output.model_json_schema()
9
    response = client.chat.completions.create(
10
        model=model,
11
        messages=[
12
            {
13
                "role": "system",
14
                "content": "Generate a JSON object, without any additional text or comments.",
15
            },
16
            {"role": "user", "content": "who won the world cup in 2022?"},
17
        ],
18
        response_format={
19
            "type": "json_schema",
20
            "json_schema": {"name": "output", "schema": schema, "strict": False},
21
        },
22
        temperature=0,
23
    )
24
    content = response.choices[0].message.content
25
    data = json.loads(content)
26
    validate(instance=data, schema=schema)
27
    return data

1
from textwrap import dedent
2
3
math_tutor_prompt = """
4
    You are a helpful math tutor. You will be provided with a math problem,
5
    and your goal will be to output a step by step solution, along with a final answer.
6
    For each step, just provide the output as an equation use the explanation field to detail the reasoning.
7
"""
8
9
question = "how can I solve 8x + 7 = -23"
10
11
schema = {
12
    "type": "object",
13
    "properties": {
14
        "steps": {
15
            "type": "array",
16
            "items": {
17
                "type": "object",
18
                "properties": {
19
                    "explanation": {"type": "string"},
20
                    "output": {"type": "string"},
21
                },
22
                "required": ["explanation", "output"],
23
                "additionalProperties": False,
24
            },
25
        },
26
        "final_answer": {"type": "string"},
27
    },
28
    "required": ["steps", "final_answer"],
29
    "additionalProperties": False,
30
}
31
32
def generate_json_schema_strict_true():
33
    response = client.chat.completions.create(
34
        model=model,
35
        messages=[
36
            {"role": "system", "content": dedent(math_tutor_prompt)},
37
            {"role": "user", "content": question},
38
        ],
39
        response_format={
40
            "type": "json_schema",
41
            "json_schema": {"name": "math_reasoning", "schema": schema, "strict": True},
42
        },
43
        temperature=0,
44
    )
45
    content = response.choices[0].message.content
46
    data = json.loads(content)
47
    validate(instance=data, schema=schema)
48
    return data