{"slug": "modeltrace-manual-test-for-ai-agents", "title": "ModelTrace manual test for AI agents", "summary": "A developer published a JavaScript implementation for a manual ModelTrace test aimed at AI agents, featuring a number-parsing routine and a Hellinger-distance-based feature extractor. The code parses numeric runs from text, builds count histograms over a 1–355 range, and combines standardized Hellinger features with ordered-block and last-digit features to score inputs against a reference bank.", "body_md": "|  | export const VALUE_MIN = 1; | \n|  | export const VALUE_MAX = 355; | \n|  | export const DIMENSION = VALUE_MAX - VALUE_MIN + 1; | \n|  | export const ALPHA = 0.5; | \n|  | export const ORDERED_BLOCK_WEIGHT = 0.25; | \n|  |  | \n|  | export function parseNumbers(text) { | \n|  | const runs = []; | \n|  | let current = []; | \n|  | let previousEnd = 0; | \n|  | for (const match of String(text).matchAll(/\\d+/g)) { | \n|  | const separator = String(text).slice(previousEnd, match.index); | \n|  | const value = Number(match[0]); | \n|  | if (current.length && /\\p{L}/u.test(separator)) { | \n|  | runs.push(current); | \n|  | current = []; | \n|  | } | \n|  | if (value >= VALUE_MIN && value <= VALUE_MAX) current.push(value); | \n|  | previousEnd = match.index + match[0].length; | \n|  | } | \n|  | if (current.length) runs.push(current); | \n|  | return runs.reduce((best, run) => run.length > best.length ? run : best, []); | \n|  | } | \n|  |  | \n|  | export function countNumbers(numbers) { | \n|  | const counts = Array(DIMENSION).fill(0); | \n|  | numbers.forEach((number) => { counts[number - VALUE_MIN] += 1; }); | \n|  | return counts; | \n|  | } | \n|  |  | \n|  | function mean(values) { | \n|  | return values.reduce((total, value) => total + value, 0) / values.length; | \n|  | } | \n|  |  | \n|  | function standardize(values) { | \n|  | const center = mean(values); | \n|  | const variance = mean(values.map((value) => (value - center) ** 2)); | \n|  | const scale = Math.max(Math.sqrt(variance), 1e-12); | \n|  | return values.map((value) => (value - center) / scale); | \n|  | } | \n|  |  | \n|  | function dot(left, right) { | \n|  | let value = 0; | \n|  | for (let index = 0; index < left.length; index += 1) value += left[index] * right[index]; | \n|  | return value; | \n|  | } | \n|  |  | \n|  | function norm(values) { | \n|  | return Math.sqrt(dot(values, values)); | \n|  | } | \n|  |  | \n|  | function normalized(values) { | \n|  | const scale = Math.max(norm(values), 1e-12); | \n|  | return values.map((value) => value / scale); | \n|  | } | \n|  |  | \n|  | function subtractBasis(values, basis) { | \n|  | const output = values.slice(); | \n|  | for (const vector of basis \\|\\| []) { | \n|  | const projection = dot(output, vector); | \n|  | for (let index = 0; index < output.length; index += 1) output[index] -= projection * vector[index]; | \n|  | } | \n|  | return output; | \n|  | } | \n|  |  | \n|  | function hellingerFeature(counts) { | \n|  | const total = counts.reduce((sum, value) => sum + value, 0) + ALPHA * DIMENSION; | \n|  | return counts.map((value) => Math.sqrt((value + ALPHA) / total)); | \n|  | } | \n|  |  | \n|  | function splitIntoFour(values) { | \n|  | const base = Math.floor(values.length / 4); | \n|  | const remainder = values.length % 4; | \n|  | const chunks = []; | \n|  | let start = 0; | \n|  | for (let index = 0; index < 4; index += 1) { | \n|  | const size = base + (index < remainder ? 1 : 0); | \n|  | chunks.push(values.slice(start, start + size)); | \n|  | start += size; | \n|  | } | \n|  | return chunks; | \n|  | } | \n|  |  | \n|  | function orderedBlockFeature(numbers) { | \n|  | const pieces = []; | \n|  | for (const chunk of splitIntoFour(numbers)) { | \n|  | const bins = Array(16).fill(0.5); | \n|  | for (const value of chunk) { | \n|  | const index = Math.min(15, Math.floor(((value - 1) / 355) * 16)); | \n|  | bins[index] += 1; | \n|  | } | \n|  | const total = bins.reduce((sum, value) => sum + value, 0); | \n|  | pieces.push(...bins.map((value) => Math.sqrt(value / total))); | \n|  | } | \n|  | const lastDigits = Array(10).fill(0.5); | \n|  | numbers.forEach((value) => { lastDigits[value % 10] += 1; }); | \n|  | const lastTotal = lastDigits.reduce((sum, value) => sum + value, 0); | \n|  | pieces.push(...lastDigits.map((value) => Math.sqrt(value / lastTotal))); | \n|  | return pieces; | \n|  | } | \n|  |  | \n|  | function robustScoreCounts(counts, bank) { | \n|  | const artifact = bank.robust.hellinger; | \n|  | const feature = hellingerFeature(counts); | \n|  | let projected = feature.map((value, index) => (value - artifact.feature_mean[index]) / artifact.feature_scale[index]); | \n|  | projected = subtractBasis(projected, artifact.nuisance_basis); | \n|  | projected = normalized(projected); | \n|  | const scores = artifact.centroids.map((centroid) => dot(projected, centroid)); | \n|  | return standardize(scores); | \n|  | } | \n|  |  | \n|  | function orderedBlockScores(numbers, bank) { | \n|  | const artifact = bank.robust.ordered_blocks; | \n|  | const feature = orderedBlockFeature(numbers); | \n|  | const standardizedFeature = feature.map((value, index) => (value - artifact.feature_mean[index]) / artifact.feature_scale[index]); | \n|  | const unit = normalized(standardizedFeature); | \n|  | const environmentScores = artifact.environment_centroids.map( | \n|  | (centroids) => centroids.map((centroid) => dot(unit, centroid)), | \n|  | ); | \n|  | const template = standardize(artifact.centroids.map((_, modelIndex) => Math.max( | \n|  | ...environmentScores.map((scores) => scores[modelIndex]), | \n|  | ))); | \n|  | const projected = normalized(subtractBasis(standardizedFeature, artifact.nuisance_basis)); | \n|  | const nuisance = standardize(artifact.centroids.map((centroid) => dot(projected, centroid))); | \n|  | return standardize(template.map((value, index) => 0.5 * value + 0.5 * nuisance[index])); | \n|  | } | \n|  |  | \n|  | function robustScoreNumbers(numbers, bank) { | \n|  | const marginal = robustScoreCounts(countNumbers(numbers), bank); | \n|  | const artifact = bank.robust.ordered_blocks; | \n|  | const weight = artifact ? Number(artifact.weight \\|\\| 0) : 0; | \n|  | if (!artifact \\|\\| weight === 0) return marginal; | \n|  | const ordered = orderedBlockScores(numbers, bank); | \n|  | return marginal.map((value, index) => (1 - weight) * value + weight * ordered[index]); | \n|  | } | \n|  |  | \n|  | function softmax(values) { | \n|  | const maximum = Math.max(...values); | \n|  | const weights = values.map((value) => Math.exp(value - maximum)); | \n|  | const total = weights.reduce((sum, value) => sum + value, 0); | \n|  | return weights.map((value) => value / total); | \n|  | } | \n|  |  | \n|  | function jsSimilarity(left, right) { | \n|  | const leftTotal = left.reduce((sum, value) => sum + value, 0); | \n|  | const rightTotal = right.reduce((sum, value) => sum + value, 0) + ALPHA * DIMENSION; | \n|  | const p = left.map((value) => value / leftTotal); | \n|  | const q = right.map((value) => (value + ALPHA) / rightTotal); | \n|  | const midpoint = p.map((value, index) => (value + q[index]) / 2); | \n|  | const divergence = (values) => values.reduce( | \n|  | (total, value, index) => total + (value ? value * Math.log(value / midpoint[index]) : 0), | \n|  | 0, | \n|  | ); | \n|  | const js = (divergence(p) + divergence(q)) / 2; | \n|  | return 1 - Math.sqrt(js / Math.log(2)); | \n|  | } | \n|  |  | \n|  | export function analyzeGlobalOutputs(outputs, bank) { | \n|  | const modelIds = bank.models.map((model) => model.id); | \n|  | const valid = []; | \n|  | const diagnostics = []; | \n|  | outputs.forEach((item, index) => { | \n|  | const expected = Number(item.expected_count \\|\\| 0); | \n|  | const numbers = parseNumbers(item.text \\|\\| \"\"); | \n|  | const minimum = expected ? Math.max(80, Math.ceil(expected * 0.55)) : 80; | \n|  | const accepted = numbers.length >= minimum; | \n|  | diagnostics.push({ index, parsed_numbers: numbers.length, minimum_numbers: minimum, accepted }); | \n|  | if (accepted) valid.push({ numbers, counts: countNumbers(numbers), scores: robustScoreNumbers(numbers, bank) }); | \n|  | }); | \n|  | if (!valid.length) throw new Error(\"没有可用回答：请粘贴完整数字序列；拒答或严重截断的回答不会计入。\"); | \n|  |  | \n|  | const combinedScores = modelIds.map((_, modelIndex) => mean(valid.map((item) => item.scores[modelIndex]))); | \n|  | const calibrationKey = String(Math.min(valid.length, 3)); | \n|  | const beta = Number(bank.calibration[calibrationKey].beta); | \n|  | const probabilities = softmax(combinedScores.map((value) => beta * value)); | \n|  | const pooledCounts = Array.from({ length: DIMENSION }, (_, index) => valid.reduce((sum, item) => sum + item.counts[index], 0)); | \n|  | const modelEntries = Object.fromEntries(bank.models.map((model) => [model.id, model])); | \n|  | const familyOrder = [...new Set(bank.models.map((model) => model.family \\|\\| \"models\"))]; | \n|  | const familyNames = Object.fromEntries(familyOrder.map((family) => [ | \n|  | family, | \n|  | bank.models.find((model) => (model.family \\|\\| \"models\") === family)?.family_name \\|\\| family, | \n|  | ])); | \n|  | const results = modelIds.map((model, index) => ({ | \n|  | model, | \n|  | display_name: modelEntries[model].display_name, | \n|  | probability: probabilities[index], | \n|  | profile_similarity: jsSimilarity(pooledCounts, modelEntries[model].counts), | \n|  | score: combinedScores[index], | \n|  | family: modelEntries[model].family \\|\\| \"models\", | \n|  | family_name: familyNames[modelEntries[model].family \\|\\| \"models\"], | \n|  | })).sort((left, right) => right.probability - left.probability); | \n|  | const familyProbabilities = Object.fromEntries(familyOrder.map((family) => [ | \n|  | family, | \n|  | results.filter((item) => item.family === family).reduce((sum, item) => sum + item.probability, 0), | \n|  | ])); | \n|  | results.forEach((item) => { item.conditional_probability = item.probability / familyProbabilities[item.family]; }); | \n|  | const winningFamily = familyOrder.reduce((best, family) => familyProbabilities[family] > familyProbabilities[best] ? family : best); | \n|  | return { | \n|  | prediction: results[0].model, | \n|  | prediction_name: results[0].display_name, | \n|  | probability: results[0].probability, | \n|  | used_outputs: valid.length, | \n|  | results, | \n|  | diagnostics, | \n|  | calibration: { queries: calibrationKey, beta, cv_accuracy: bank.calibration[calibrationKey].cv_accuracy }, | \n|  | family_prediction: winningFamily, | \n|  | family_prediction_name: familyNames[winningFamily], | \n|  | family_probability: familyProbabilities[winningFamily], | \n|  | family_probabilities: familyOrder.map((family) => ({ family, display_name: familyNames[family], probability: familyProbabilities[family] })), | \n|  | method: \"统一全局稳健数字指纹\", | \n|  | }; | \n|  | } |", "url": "https://wpnews.pro/news/modeltrace-manual-test-for-ai-agents", "canonical_source": "https://gist.github.com/fankaidev/0e08b96ad4d8e17c4ecad39770f492cf", "published_at": "2026-10-01 10:39:02+00:00", "updated_at": "2026-10-28 10:48:09.194304+00:00", "lang": "en", "topics": ["ai-agents", "ai-tools", "developer-tools"], "entities": ["ModelTrace"], "also_reported_by": [], "alternates": {"html": "https://wpnews.pro/news/modeltrace-manual-test-for-ai-agents", "markdown": "https://wpnews.pro/news/modeltrace-manual-test-for-ai-agents.md", "text": "https://wpnews.pro/news/modeltrace-manual-test-for-ai-agents.txt", "jsonld": "https://wpnews.pro/news/modeltrace-manual-test-for-ai-agents.jsonld"}}