Implement batched server parity and observability
This commit is contained in:
Georg Bauer
154
scripts/endpoint_continuation.py
Normal file
154
scripts/endpoint_continuation.py
Normal file
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
|
||||
base_url = sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:9000"
|
||||
prompt = "Call echo with text hi. After its result, reply with only DONE."
|
||||
parameters = {
|
||||
"type": "object",
|
||||
"properties": {"text": {"type": "string"}},
|
||||
"required": ["text"],
|
||||
}
|
||||
chat_tool = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "echo",
|
||||
"description": "Echo text",
|
||||
"parameters": parameters,
|
||||
},
|
||||
}
|
||||
anthropic_tool = {
|
||||
"name": "echo",
|
||||
"description": "Echo text",
|
||||
"input_schema": parameters,
|
||||
}
|
||||
responses_tool = {
|
||||
"type": "function",
|
||||
"name": "echo",
|
||||
"description": "Echo text",
|
||||
"parameters": parameters,
|
||||
}
|
||||
|
||||
|
||||
def post(path, payload):
|
||||
request = urllib.request.Request(
|
||||
base_url + path,
|
||||
data=json.dumps(payload).encode(),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=120) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
first = post(
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": "deepseek-v4-flash",
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"tools": [chat_tool],
|
||||
"reasoning_effort": "none",
|
||||
"temperature": 0,
|
||||
"max_tokens": 128,
|
||||
},
|
||||
)
|
||||
call = first["choices"][0]["message"]["tool_calls"][0]
|
||||
print("chat-first", json.dumps(first["usage"], separators=(",", ":")))
|
||||
second = post(
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": "deepseek-v4-flash",
|
||||
"messages": [
|
||||
{"role": "user", "content": prompt},
|
||||
{"role": "assistant", "content": "", "tool_calls": [call]},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": call["id"],
|
||||
"content": "hi",
|
||||
},
|
||||
],
|
||||
"tools": [chat_tool],
|
||||
"reasoning_effort": "none",
|
||||
"temperature": 0,
|
||||
"max_tokens": 128,
|
||||
},
|
||||
)
|
||||
print("chat", json.dumps(second, separators=(",", ":")))
|
||||
|
||||
first = post(
|
||||
"/v1/messages",
|
||||
{
|
||||
"model": "deepseek-v4-flash",
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"tools": [anthropic_tool],
|
||||
"thinking": {"type": "disabled"},
|
||||
"temperature": 0,
|
||||
"max_tokens": 128,
|
||||
},
|
||||
)
|
||||
call = next(block for block in first["content"] if block["type"] == "tool_use")
|
||||
print("anthropic-first", json.dumps(first["usage"], separators=(",", ":")))
|
||||
second = post(
|
||||
"/v1/messages",
|
||||
{
|
||||
"model": "deepseek-v4-flash",
|
||||
"messages": [
|
||||
{"role": "user", "content": prompt},
|
||||
{"role": "assistant", "content": [call]},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": call["id"],
|
||||
"content": "hi",
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
"tools": [anthropic_tool],
|
||||
"thinking": {"type": "disabled"},
|
||||
"temperature": 0,
|
||||
"max_tokens": 128,
|
||||
},
|
||||
)
|
||||
print("anthropic", json.dumps(second, separators=(",", ":")))
|
||||
|
||||
first = post(
|
||||
"/v1/responses",
|
||||
{
|
||||
"model": "deepseek-v4-flash",
|
||||
"input": prompt,
|
||||
"tools": [responses_tool],
|
||||
"reasoning": {"effort": "none"},
|
||||
"temperature": 0,
|
||||
"max_output_tokens": 128,
|
||||
},
|
||||
)
|
||||
call = next(item for item in first["output"] if item["type"] == "function_call")
|
||||
print("responses-first", json.dumps(first["usage"], separators=(",", ":")))
|
||||
second = post(
|
||||
"/v1/responses",
|
||||
{
|
||||
"model": "deepseek-v4-flash",
|
||||
"input": [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": prompt}],
|
||||
},
|
||||
call,
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": call["call_id"],
|
||||
"output": "hi",
|
||||
},
|
||||
],
|
||||
"tools": [responses_tool],
|
||||
"reasoning": {"effort": "none"},
|
||||
"temperature": 0,
|
||||
"max_output_tokens": 128,
|
||||
},
|
||||
)
|
||||
print("responses", json.dumps(second, separators=(",", ":")))
|
||||
101
scripts/endpoint_parity.py
Normal file
101
scripts/endpoint_parity.py
Normal file
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
|
||||
base_url = sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:9000"
|
||||
streaming = "--stream" in sys.argv
|
||||
parameters = {
|
||||
"type": "object",
|
||||
"properties": {"text": {"type": "string"}},
|
||||
"required": ["text"],
|
||||
}
|
||||
cases = [
|
||||
(
|
||||
"chat",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": "deepseek-v4-flash",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Call echo with text hi. Do not answer normally.",
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "echo",
|
||||
"description": "Echo text",
|
||||
"parameters": parameters,
|
||||
},
|
||||
}
|
||||
],
|
||||
"reasoning_effort": "none",
|
||||
"temperature": 0,
|
||||
"max_tokens": 128,
|
||||
},
|
||||
),
|
||||
(
|
||||
"anthropic",
|
||||
"/v1/messages",
|
||||
{
|
||||
"model": "deepseek-v4-flash",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Call echo with text hi. Do not answer normally.",
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"name": "echo",
|
||||
"description": "Echo text",
|
||||
"input_schema": parameters,
|
||||
}
|
||||
],
|
||||
"thinking": {"type": "disabled"},
|
||||
"temperature": 0,
|
||||
"max_tokens": 128,
|
||||
},
|
||||
),
|
||||
(
|
||||
"responses",
|
||||
"/v1/responses",
|
||||
{
|
||||
"model": "deepseek-v4-flash",
|
||||
"input": "Call echo with text hi. Do not answer normally.",
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "echo",
|
||||
"description": "Echo text",
|
||||
"parameters": parameters,
|
||||
}
|
||||
],
|
||||
"reasoning": {"effort": "none"},
|
||||
"temperature": 0,
|
||||
"max_output_tokens": 128,
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
for name, path, payload in cases:
|
||||
if streaming:
|
||||
payload["stream"] = True
|
||||
if name == "chat":
|
||||
payload["stream_options"] = {"include_usage": True}
|
||||
request = urllib.request.Request(
|
||||
base_url + path,
|
||||
data=json.dumps(payload).encode(),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=120) as response:
|
||||
if streaming:
|
||||
for line in response:
|
||||
if line.strip():
|
||||
print(name, line.decode().rstrip())
|
||||
else:
|
||||
print(name, json.dumps(json.load(response), separators=(",", ":")))
|
||||
53
scripts/endpoint_reasoning.py
Normal file
53
scripts/endpoint_reasoning.py
Normal file
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
|
||||
base_url = sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:9000"
|
||||
prompt = "Think briefly, then reply with only the number: 17 * 19."
|
||||
cases = [
|
||||
(
|
||||
"chat",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": "deepseek-v4-flash",
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"reasoning_effort": "low",
|
||||
"temperature": 0,
|
||||
"max_tokens": 96,
|
||||
},
|
||||
),
|
||||
(
|
||||
"anthropic",
|
||||
"/v1/messages",
|
||||
{
|
||||
"model": "deepseek-v4-flash",
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"thinking": {"type": "enabled", "budget_tokens": 64},
|
||||
"output_config": {"effort": "low"},
|
||||
"temperature": 0,
|
||||
"max_tokens": 96,
|
||||
},
|
||||
),
|
||||
(
|
||||
"responses",
|
||||
"/v1/responses",
|
||||
{
|
||||
"model": "deepseek-v4-flash",
|
||||
"input": prompt,
|
||||
"reasoning": {"effort": "low", "summary": "auto"},
|
||||
"temperature": 0,
|
||||
"max_output_tokens": 96,
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
for name, path, payload in cases:
|
||||
request = urllib.request.Request(
|
||||
base_url + path,
|
||||
data=json.dumps(payload).encode(),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=120) as response:
|
||||
print(name, json.dumps(json.load(response), separators=(",", ":")))
|
||||
Reference in New Issue
Block a user