51878c3894
Harness at scripts/ab-corpus/ wires the pen-ai-skills corpus evaluator to
real model endpoints and pen-mcp handlers:
run.ts — CLI entry (--dry-run / --live / --models A,B,C / --only ID)
apply.ts — ApplyFn impl dispatching tool_call → element handler
and batch_design DSL → handleBatchDesign, against a
fresh tmp .op per run (isolated, auto-cleanup)
build-prompt.ts — B variant strips elements.md + appends batch_design
<op_tool> format instruction; T keeps elements + adds
element-tool PRIMARY / batch_design FALLBACK
instruction. Uniform <op_tool> wrapper in both arms
isolates "tool set width" as the only A/B variable.
stub-model.ts — fixture-based offline model for --dry-run
real-model.ts — router by model id (minimax* / gpt-*/o* / glm-5.1 /
glm-* / kimi-*)
clients/
openai-compat.ts — generic chat/completions POST
minimax.ts — api.minimax.io/v1, MINIMAX_API_KEY
codex-cli.ts — spawns `codex exec` (GPT-5.4 via Codex Pro sub)
bailian.ts — coding.dashscope.aliyuncs.com/v1 CP,
DASHSCOPE_BAILIAN_CODING_KEY (hosts glm-4.7, kimi-k2.5)
glm.ts — open.bigmodel.cn/api/coding/paas/v4 official CP,
GLM_OFFICIAL_CODING_KEY
write-report.ts — Report → report.md + report.json in out dir;
4-way routing breakdown table per model
Kept entirely outside packages/ — scripts are a local dev tool, not part
of the published SDK. API keys never hit disk or git.
v1 run results logged separately in openpencil-docs
superpowers/notes/2026-04-20-ab-v1-results.md (5 models × 24 prompts).
75 lines
2.5 KiB
TypeScript
75 lines
2.5 KiB
TypeScript
/**
|
|
* Generic OpenAI-compatible chat client. Most Chinese providers (MiniMax,
|
|
* Bailian/DashScope, Zhipu/GLM, Moonshot/Kimi) expose an OpenAI-compatible
|
|
* `/chat/completions` endpoint — we route every one of them through this
|
|
* single helper so the per-provider wrappers stay 3-5 lines each and the
|
|
* harness doesn't hardcode a different fetch shape for every vendor.
|
|
*
|
|
* Treats API errors as fatal for the single call (throws); the harness's
|
|
* loop catches and records them as apply-phase failures so one bad call
|
|
* never aborts a corpus sweep.
|
|
*/
|
|
|
|
interface ChatMessage {
|
|
role: 'system' | 'user';
|
|
content: string;
|
|
}
|
|
|
|
interface OpenAICompatResponse {
|
|
choices?: Array<{
|
|
message?: { content?: string };
|
|
finish_reason?: string;
|
|
}>;
|
|
error?: { message?: string; code?: string };
|
|
}
|
|
|
|
export interface CallOpenAICompatArgs {
|
|
/** Full endpoint base URL, e.g. `https://api.minimaxi.com/v1`. No trailing `/chat/completions`. */
|
|
baseURL: string;
|
|
/** Bearer token. Passed verbatim as `Authorization: Bearer <key>`. */
|
|
apiKey: string;
|
|
/** Provider-specific model id, e.g. `MiniMax-M2.7`, `glm-5`, `kimi-k2.5`. */
|
|
model: string;
|
|
system: string;
|
|
user: string;
|
|
temperature?: number;
|
|
maxTokens?: number;
|
|
/** Label for error messages so "<label> HTTP 401" points at the right provider. */
|
|
label?: string;
|
|
}
|
|
|
|
export async function callOpenAICompat(args: CallOpenAICompatArgs): Promise<string> {
|
|
const label = args.label ?? 'openai-compat';
|
|
const messages: ChatMessage[] = [
|
|
{ role: 'system', content: args.system },
|
|
{ role: 'user', content: args.user },
|
|
];
|
|
const res = await fetch(`${args.baseURL}/chat/completions`, {
|
|
method: 'POST',
|
|
headers: {
|
|
Authorization: `Bearer ${args.apiKey}`,
|
|
'Content-Type': 'application/json',
|
|
},
|
|
body: JSON.stringify({
|
|
model: args.model,
|
|
messages,
|
|
temperature: args.temperature ?? 0.2,
|
|
max_tokens: args.maxTokens ?? 4096,
|
|
stream: false,
|
|
}),
|
|
});
|
|
if (!res.ok) {
|
|
const body = await res.text();
|
|
throw new Error(`${label} HTTP ${res.status}: ${body.slice(0, 300)}`);
|
|
}
|
|
const data: OpenAICompatResponse = await res.json();
|
|
if (data.error) {
|
|
throw new Error(`${label} API error: ${data.error.message ?? data.error.code ?? 'unknown'}`);
|
|
}
|
|
const content = data.choices?.[0]?.message?.content;
|
|
if (typeof content !== 'string' || content.length === 0) {
|
|
throw new Error(`${label} returned no content in choices[0].message.content`);
|
|
}
|
|
return content;
|
|
}
|