51878c3894
Harness at scripts/ab-corpus/ wires the pen-ai-skills corpus evaluator to
real model endpoints and pen-mcp handlers:
run.ts — CLI entry (--dry-run / --live / --models A,B,C / --only ID)
apply.ts — ApplyFn impl dispatching tool_call → element handler
and batch_design DSL → handleBatchDesign, against a
fresh tmp .op per run (isolated, auto-cleanup)
build-prompt.ts — B variant strips elements.md + appends batch_design
<op_tool> format instruction; T keeps elements + adds
element-tool PRIMARY / batch_design FALLBACK
instruction. Uniform <op_tool> wrapper in both arms
isolates "tool set width" as the only A/B variable.
stub-model.ts — fixture-based offline model for --dry-run
real-model.ts — router by model id (minimax* / gpt-*/o* / glm-5.1 /
glm-* / kimi-*)
clients/
openai-compat.ts — generic chat/completions POST
minimax.ts — api.minimax.io/v1, MINIMAX_API_KEY
codex-cli.ts — spawns `codex exec` (GPT-5.4 via Codex Pro sub)
bailian.ts — coding.dashscope.aliyuncs.com/v1 CP,
DASHSCOPE_BAILIAN_CODING_KEY (hosts glm-4.7, kimi-k2.5)
glm.ts — open.bigmodel.cn/api/coding/paas/v4 official CP,
GLM_OFFICIAL_CODING_KEY
write-report.ts — Report → report.md + report.json in out dir;
4-way routing breakdown table per model
Kept entirely outside packages/ — scripts are a local dev tool, not part
of the published SDK. API keys never hit disk or git.
v1 run results logged separately in openpencil-docs
superpowers/notes/2026-04-20-ab-v1-results.md (5 models × 24 prompts).
47 lines
1.6 KiB
TypeScript
47 lines
1.6 KiB
TypeScript
/**
|
|
* 百炼 (Alibaba DashScope Bailian) Coding Plan wrapper. Exposes
|
|
* third-party models — including GLM-5 and Kimi K2.5 — through a
|
|
* single DashScope endpoint, so the same API key (`DASHSCOPE_BAILIAN_CODING_KEY`)
|
|
* covers multiple target models. Cribbed from builtin-provider-presets.ts
|
|
* `bailian-coding` preset.
|
|
*
|
|
* Default baseURL: https://coding.dashscope.aliyuncs.com/v1 (CN).
|
|
* Override via `BAILIAN_BASE_URL` if you need the -intl variant.
|
|
*/
|
|
|
|
import { callOpenAICompat } from './openai-compat';
|
|
|
|
// User supplied the 百炼 CP (Coding Plan) key, which targets the
|
|
// coding-plan endpoint (not the regular compatible-mode endpoint).
|
|
// See builtin-provider-presets.ts `bailian-coding` preset for the
|
|
// canonical URL — this is separate from plain `bailian` tier.
|
|
const BAILIAN_BASE_URL = process.env.BAILIAN_BASE_URL ?? 'https://coding.dashscope.aliyuncs.com/v1';
|
|
const BAILIAN_API_KEY_ENV = 'DASHSCOPE_BAILIAN_CODING_KEY';
|
|
|
|
export interface CallBailianArgs {
|
|
model: string;
|
|
system: string;
|
|
user: string;
|
|
temperature?: number;
|
|
maxTokens?: number;
|
|
}
|
|
|
|
export async function callBailian(args: CallBailianArgs): Promise<string> {
|
|
const apiKey = process.env[BAILIAN_API_KEY_ENV];
|
|
if (!apiKey) {
|
|
throw new Error(
|
|
`${BAILIAN_API_KEY_ENV} not set — export it (the 百炼 CP sk-sp-... key) before running with --live`,
|
|
);
|
|
}
|
|
return callOpenAICompat({
|
|
baseURL: BAILIAN_BASE_URL,
|
|
apiKey,
|
|
model: args.model,
|
|
system: args.system,
|
|
user: args.user,
|
|
temperature: args.temperature,
|
|
maxTokens: args.maxTokens,
|
|
label: 'bailian',
|
|
});
|
|
}
|