72 lines
2.1 KiB
TypeScript
72 lines
2.1 KiB
TypeScript
|
|
import { anthropic } from "@workspace/integrations-anthropic-ai";
|
||
|
|
|
||
|
|
const EVAL_MODEL = "claude-haiku-4-5";
|
||
|
|
const WORK_MODEL = "claude-sonnet-4-6";
|
||
|
|
|
||
|
|
export interface EvalResult {
|
||
|
|
approved: boolean;
|
||
|
|
reason: string;
|
||
|
|
}
|
||
|
|
|
||
|
|
export async function evaluateRequest(request: string): Promise<EvalResult> {
|
||
|
|
const message = await anthropic.messages.create({
|
||
|
|
model: EVAL_MODEL,
|
||
|
|
max_tokens: 8192,
|
||
|
|
system: `You are Timmy, an AI agent gatekeeper. Your job is to evaluate user requests.
|
||
|
|
A request should be APPROVED if it is:
|
||
|
|
- Clear and specific enough to act on
|
||
|
|
- Ethical, lawful, and not harmful
|
||
|
|
- Within the capabilities of a general-purpose AI assistant
|
||
|
|
|
||
|
|
A request should be REJECTED if it is:
|
||
|
|
- Harmful, illegal, or unethical
|
||
|
|
- Completely incoherent or impossible to act on
|
||
|
|
- Spam or an attempt to abuse the system
|
||
|
|
|
||
|
|
Respond ONLY with valid JSON in this exact format:
|
||
|
|
{"approved": true, "reason": "Brief explanation"}
|
||
|
|
or
|
||
|
|
{"approved": false, "reason": "Brief explanation of why it was rejected"}`,
|
||
|
|
messages: [
|
||
|
|
{
|
||
|
|
role: "user",
|
||
|
|
content: `Evaluate this request: ${request}`,
|
||
|
|
},
|
||
|
|
],
|
||
|
|
});
|
||
|
|
|
||
|
|
const block = message.content[0];
|
||
|
|
if (block.type !== "text") {
|
||
|
|
throw new Error("Unexpected response type from eval model");
|
||
|
|
}
|
||
|
|
|
||
|
|
try {
|
||
|
|
const parsed = JSON.parse(block.text) as { approved: boolean; reason: string };
|
||
|
|
return { approved: Boolean(parsed.approved), reason: parsed.reason ?? "" };
|
||
|
|
} catch {
|
||
|
|
throw new Error(`Failed to parse eval response: ${block.text}`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
export async function executeRequest(request: string): Promise<string> {
|
||
|
|
const message = await anthropic.messages.create({
|
||
|
|
model: WORK_MODEL,
|
||
|
|
max_tokens: 8192,
|
||
|
|
system: `You are Timmy, a capable AI agent. A user has paid for you to handle their request.
|
||
|
|
Do your best to fulfill it thoroughly and helpfully. Be concise yet complete.`,
|
||
|
|
messages: [
|
||
|
|
{
|
||
|
|
role: "user",
|
||
|
|
content: request,
|
||
|
|
},
|
||
|
|
],
|
||
|
|
});
|
||
|
|
|
||
|
|
const block = message.content[0];
|
||
|
|
if (block.type !== "text") {
|
||
|
|
throw new Error("Unexpected response type from work model");
|
||
|
|
}
|
||
|
|
|
||
|
|
return block.text;
|
||
|
|
}
|