Jaspers Screener MCP: Screens US SEC filers by fundamentals, holders and insider data, then checks criteria in words against filings via the Jaspers screener server.
| 1 | // The screener's pure half: what the panel's state means, what goes to the server and what its |
| 2 | // answers hold, how a value reads and how a typed one is read back, and the text a model is given. |
| 3 | // The server (screener-mcp) owns the fields: their names, kinds, units and ops come from its guide, |
| 4 | // so there is no list of them here. No React and no SDK: it is imported by the view and read by the |
| 5 | // tests, which run under Node on their own. |
| 6 | |
| 7 | export type Kind = 'number' | 'text' | 'date' | 'list' |
| 8 | /** How a number field reads: raw USD, a fraction shown as a percentage, a year, or a plain number. */ |
| 9 | export type Unit = 'usd' | 'fraction' | 'year' | 'number' |
| 10 | |
| 11 | export const OPS = ['gt', 'gte', 'lt', 'lte', 'eq', 'ne', 'between', 'in', 'contains', 'is_null', 'not_null'] as const |
| 12 | export type Op = (typeof OPS)[number] |
| 13 | |
| 14 | export const VERDICTS = ['pass', 'fail', 'unclear', 'unverified'] as const |
| 15 | export type Verdict = (typeof VERDICTS)[number] |
| 16 | |
| 17 | /** One field a filter, a sort or a column may name, as the server's guide describes it. */ |
| 18 | export interface Field { |
| 19 | field: string |
| 20 | group: string |
| 21 | kind: Kind |
| 22 | unit: Unit | null |
| 23 | ops: Op[] |
| 24 | sortable: boolean |
| 25 | description: string |
| 26 | } |
| 27 | |
| 28 | /** The server's filter language: raw units, joined by AND. */ |
| 29 | export interface Filter { |
| 30 | field: string |
| 31 | op: Op |
| 32 | value?: number | string | (number | string)[] |
| 33 | } |
| 34 | |
| 35 | export interface Sort { |
| 36 | field: string |
| 37 | dir: 'asc' | 'desc' |
| 38 | } |
| 39 | |
| 40 | export interface Criterion { |
| 41 | id: string |
| 42 | question: string |
| 43 | } |
| 44 | |
| 45 | /** The qualitative run of this panel. Present with no id, it is a request to start one. */ |
| 46 | export interface Run { |
| 47 | id?: string |
| 48 | status?: 'running' | 'done' | 'error' |
| 49 | error?: string |
| 50 | } |
| 51 | |
| 52 | /** What the panel holds, and so what the orchestrator sets. */ |
| 53 | export interface State { |
| 54 | filters: Filter[] |
| 55 | /** The user's own list; empty for every company. */ |
| 56 | tickers: string[] |
| 57 | includeUnlisted: boolean |
| 58 | sort: Sort |
| 59 | page: number |
| 60 | columns: string[] |
| 61 | /** The server's session, written by the view from its first answer. */ |
| 62 | sessionId: string | null |
| 63 | criteria: Criterion[] |
| 64 | run: Run | null |
| 65 | /** Which verdicts the table shows once a run has results; empty for all. */ |
| 66 | verdicts: Verdict[] |
| 67 | /** The ticker whose evidence is open; empty for none. */ |
| 68 | open: string |
| 69 | } |
| 70 | |
| 71 | export const DEFAULT_SORT: Sort = { field: 'revenue', dir: 'desc' } |
| 72 | |
| 73 | /** The columns a new screener shows. The fields filtered and sorted on are shown whatever this says. */ |
| 74 | export const DEFAULT_COLUMNS = [ |
| 75 | 'revenue', |
| 76 | 'revenue_growth_yoy', |
| 77 | 'gross_margin', |
| 78 | 'operating_margin', |
| 79 | 'net_margin', |
| 80 | 'net_income', |
| 81 | 'free_cash_flow', |
| 82 | 'roe', |
| 83 | 'debt_to_equity', |
| 84 | 'insider_net_buy_value_6m', |
| 85 | 'inst_holders', |
| 86 | 'fiscal_year', |
| 87 | ] |
| 88 | |
| 89 | /** What the server allows in one run, for the editor to stop at. */ |
| 90 | export const MAX_CRITERIA = 8 |
| 91 | /** The most columns the panel's state takes. */ |
| 92 | export const MAX_COLUMNS = 40 |
| 93 | |
| 94 | // The guide's field rows. |
| 95 | |
| 96 | const KINDS: Kind[] = ['number', 'text', 'date', 'list'] |
| 97 | const UNITS: Unit[] = ['usd', 'fraction', 'year', 'number'] |
| 98 | |
| 99 | /** The rows of get_guide("fields"), read leniently: a row that names no field is dropped, a number field with no unit is a plain number. */ |
| 100 | export function readFields(rows: Record<string, unknown>[] | undefined): Field[] { |
| 101 | const fields: Field[] = [] |
| 102 | for (const row of rows ?? []) { |
| 103 | if (typeof row.field !== 'string' || !row.field) continue |
| 104 | const kind = KINDS.includes(row.kind as Kind) ? (row.kind as Kind) : 'number' |
| 105 | const unit = UNITS.includes(row.unit as Unit) ? (row.unit as Unit) : kind === 'number' ? 'number' : null |
| 106 | fields.push({ |
| 107 | field: row.field, |
| 108 | group: typeof row.group === 'string' ? row.group : '', |
| 109 | kind, |
| 110 | unit, |
| 111 | ops: Array.isArray(row.ops) ? row.ops.filter((op): op is Op => OPS.includes(op as Op)) : [], |
| 112 | sortable: typeof row.sortable === 'boolean' ? row.sortable : kind !== 'list', |
| 113 | description: typeof row.description === 'string' ? row.description : '', |
| 114 | }) |
| 115 | } |
| 116 | return fields |
| 117 | } |
| 118 | |
| 119 | /** The fields by group, in the guide's order. */ |
| 120 | export function groupFields(fields: Field[]): [string, Field[]][] { |
| 121 | const groups = new Map<string, Field[]>() |
| 122 | for (const field of fields) groups.set(field.group, [...(groups.get(field.group) ?? []), field]) |
| 123 | return [...groups.entries()] |
| 124 | } |
| 125 | |
| 126 | /** Parts of a field's name that do not read as words. */ |
| 127 | const WORDS: Record<string, string> = { |
| 128 | yoy: 'YoY', |
| 129 | cagr: 'CAGR', |
| 130 | roe: 'ROE', |
| 131 | roa: 'ROA', |
| 132 | eps: 'EPS', |
| 133 | fcf: 'FCF', |
| 134 | ocf: 'OCF', |
| 135 | sga: 'SG&A', |
| 136 | rnd: 'R&D', |
| 137 | ebitda: 'EBITDA', |
| 138 | sic: 'SIC', |
| 139 | cik: 'CIK', |
| 140 | hq: 'HQ', |
| 141 | lt: 'LT', |
| 142 | st: 'ST', |
| 143 | wa: 'avg', |
| 144 | inst: 'Inst.', |
| 145 | pct: '%', |
| 146 | '3y': '3Y', |
| 147 | } |
| 148 | |
| 149 | /** A field's name as a column header and a chip say it: revenue_growth_yoy is "Revenue growth YoY". */ |
| 150 | export function fieldLabel(field: string): string { |
| 151 | const words = field.split('_').map((word) => WORDS[word] ?? word) |
| 152 | const first = words[0] ?? '' |
| 153 | // A word that was replaced is already written the way it reads; any other opens with a capital. |
| 154 | if (first === field.split('_')[0]) words[0] = first.charAt(0).toUpperCase() + first.slice(1) |
| 155 | return words.join(' ') |
| 156 | } |
| 157 | |
| 158 | /** What every row carries whatever is asked for. Naming one as a column says nothing, so none is drawn twice and none is sent. */ |
| 159 | const IDENTITY = new Set(['cik', 'name', 'ticker', 'tickers', 'sic_description']) |
| 160 | |
| 161 | /** The columns the table draws after the company: the fields filtered on, the sort, then the ones asked for — the server's own rule. */ |
| 162 | export function shownColumns(filters: Filter[], sort: Sort, columns: string[]): string[] { |
| 163 | return [...new Set([...filters.map((f) => f.field), sort.field, ...columns])].filter((field) => !IDENTITY.has(field)) |
| 164 | } |
| 165 | |
| 166 | // Formatting. A cell says a number the way a screener does (B, M, K); a chip and the dialog say a |
| 167 | // threshold in the unit the user types it in, so what is typed and what is shown agree. |
| 168 | |
| 169 | /** A number with K, M, B or T behind it when that says it exactly, so what is shown reads back as the same value. */ |
| 170 | function scaled(value: number, thousands: boolean): string { |
| 171 | const a = Math.abs(value) |
| 172 | const sign = value < 0 ? '-' : '' |
| 173 | for (const [suffix, scale] of SCALES) { |
| 174 | if (a < scale || (suffix === 'K' && !thousands)) continue |
| 175 | const short = String(clean(a / scale)) |
| 176 | if (short.length <= 7) return `${sign}${short}${suffix}` |
| 177 | break |
| 178 | } |
| 179 | return `${sign}${clean(a)}` |
| 180 | } |
| 181 | |
| 182 | const SCALES: [string, number][] = [ |
| 183 | ['T', 1e12], |
| 184 | ['B', 1e9], |
| 185 | ['M', 1e6], |
| 186 | ['K', 1e3], |
| 187 | ] |
| 188 | |
| 189 | function money(value: number): string { |
| 190 | const a = Math.abs(value) |
| 191 | const sign = value < 0 ? '-' : '' |
| 192 | if (a >= 1e12) return `${sign}$${(a / 1e12).toFixed(2)}T` |
| 193 | if (a >= 1e9) return `${sign}$${(a / 1e9).toFixed(2)}B` |
| 194 | if (a >= 1e6) return `${sign}$${(a / 1e6).toFixed(1)}M` |
| 195 | if (a >= 1e3) return `${sign}$${(a / 1e3).toFixed(0)}K` |
| 196 | return `${sign}$${a.toFixed(2)}` |
| 197 | } |
| 198 | |
| 199 | function plain(value: number): string { |
| 200 | const a = Math.abs(value) |
| 201 | if (a >= 1e9) return `${(value / 1e9).toFixed(2)}B` |
| 202 | if (a >= 1e6) return `${(value / 1e6).toFixed(1)}M` |
| 203 | if (Number.isInteger(value)) return value.toLocaleString('en-US') |
| 204 | return value.toFixed(2) |
| 205 | } |
| 206 | |
| 207 | /** What one cell says. A value the company never reported is a dash, not a zero. */ |
| 208 | export function cellText(value: unknown, field?: Pick<Field, 'kind' | 'unit'>): string { |
| 209 | if (value === null || value === undefined) return '—' |
| 210 | if (Array.isArray(value)) return value.length > 0 ? value.join(' ') : '—' |
| 211 | if (typeof value === 'number') { |
| 212 | if (!Number.isFinite(value)) return '—' |
| 213 | if (field?.unit === 'usd') return money(value) |
| 214 | if (field?.unit === 'fraction') return `${(value * 100).toFixed(1)}%` |
| 215 | if (field?.unit === 'year') return String(value) |
| 216 | return plain(value) |
| 217 | } |
| 218 | return String(value) |
| 219 | } |
| 220 | |
| 221 | /** Growth and returns read faster with a sign on them; nothing else is coloured. */ |
| 222 | export function signOf(field: string, value: unknown): 'up' | 'down' | null { |
| 223 | if (!/growth|cagr|change|^ro[ae]$/.test(field)) return null |
| 224 | if (typeof value !== 'number' || !Number.isFinite(value) || value === 0) return null |
| 225 | return value > 0 ? 'up' : 'down' |
| 226 | } |
| 227 | |
| 228 | /** What the dialog writes beside a value, so the user knows what the number they type means. */ |
| 229 | export function unitHint(field: Pick<Field, 'kind' | 'unit'>): string { |
| 230 | if (field.kind === 'date') return 'YYYY-MM-DD' |
| 231 | if (field.kind !== 'number') return '' |
| 232 | if (field.unit === 'usd') return '$, with K M B' |
| 233 | if (field.unit === 'fraction') return '%' |
| 234 | if (field.unit === 'year') return 'year' |
| 235 | return 'K M B allowed' |
| 236 | } |
| 237 | |
| 238 | const SCALE: Record<string, number> = { k: 1e3, m: 1e6, b: 1e9, t: 1e12 } |
| 239 | |
| 240 | /** |
| 241 | * A value as the user types it, in the server's raw units, or null when it cannot be read: "500M" and |
| 242 | * "$2B" are dollars, "20" on a fraction is 0.2, a date is ISO. Text is taken as it is. |
| 243 | */ |
| 244 | export function parseValue(text: string, field: Pick<Field, 'kind' | 'unit'>): number | string | null { |
| 245 | const typed = text.trim() |
| 246 | if (!typed) return null |
| 247 | if (field.kind === 'date') return /^\d{4}-\d{2}-\d{2}$/.test(typed) ? typed : null |
| 248 | if (field.kind !== 'number') return typed |
| 249 | if (field.unit === 'year') return /^\d{4}$/.test(typed) ? Number(typed) : null |
| 250 | const match = /^(-)?\s*\$?\s*(-)?\s*(\d[\d,]*\.?\d*|\.\d+)\s*([kmbt])?\s*(%)?$/i.exec(typed) |
| 251 | if (!match) return null |
| 252 | const number = Number(match[3]!.replace(/,/g, '')) |
| 253 | if (!Number.isFinite(number)) return null |
| 254 | const fraction = field.unit === 'fraction' |
| 255 | // A percent sign belongs on a fraction and a scale does not; anywhere else it is the other way round. |
| 256 | if (fraction ? match[4] : match[5]) return null |
| 257 | const signed = match[1] || match[2] ? -number : number |
| 258 | return clean(fraction ? signed / 100 : signed * (SCALE[match[4]?.toLowerCase() ?? ''] ?? 1)) |
| 259 | } |
| 260 | |
| 261 | /** A raw value as the user would type it, so a filter reads in the dialog the way it was entered. */ |
| 262 | export function valueText(raw: number | string, field: Pick<Field, 'kind' | 'unit'> | undefined): string { |
| 263 | if (typeof raw !== 'number') return raw |
| 264 | if (field?.unit === 'fraction') return `${clean(raw * 100)}%` |
| 265 | if (field?.unit === 'year') return String(raw) |
| 266 | if (field?.unit === 'usd') return raw < 0 ? `-$${scaled(-raw, true)}` : `$${scaled(raw, true)}` |
| 267 | return scaled(raw, false) |
| 268 | } |
| 269 | |
| 270 | const OP_TEXT: Record<Op, string> = { |
| 271 | gt: '>', |
| 272 | gte: '≥', |
| 273 | lt: '<', |
| 274 | lte: '≤', |
| 275 | eq: '=', |
| 276 | ne: '≠', |
| 277 | between: 'between', |
| 278 | in: 'in', |
| 279 | contains: 'contains', |
| 280 | is_null: 'is empty', |
| 281 | not_null: 'has a value', |
| 282 | } |
| 283 | |
| 284 | /** An op as the dialog's list and a chip say it. */ |
| 285 | export function opText(op: Op): string { |
| 286 | return OP_TEXT[op] |
| 287 | } |
| 288 | |
| 289 | /** One filter in words: "Revenue ≥ $500M", "Gross margin 20% to 40%", "HQ state in TX, FL". */ |
| 290 | export function filterText(filter: Filter, field: Pick<Field, 'kind' | 'unit'> | undefined): string { |
| 291 | const label = fieldLabel(filter.field) |
| 292 | const say = (value: number | string): string => |
| 293 | typeof value === 'string' && field?.kind !== 'date' ? `"${value}"` : valueText(value, field) |
| 294 | const { op, value } = filter |
| 295 | if (op === 'is_null' || op === 'not_null') return `${label} ${OP_TEXT[op]}` |
| 296 | if (op === 'between' && Array.isArray(value) && value.length === 2) return `${label} ${say(value[0]!)} to ${say(value[1]!)}` |
| 297 | if (Array.isArray(value)) return `${label} ${OP_TEXT[op]} ${value.map((v) => valueText(v, field)).join(', ')}` |
| 298 | return `${label} ${OP_TEXT[op]} ${value === undefined ? '?' : say(value)}` |
| 299 | } |
| 300 | |
| 301 | /** A filter as the dialog edits it: what was typed, not yet read. */ |
| 302 | export interface Draft { |
| 303 | field: string |
| 304 | op: Op |
| 305 | a: string |
| 306 | b: string |
| 307 | } |
| 308 | |
| 309 | export function toDraft(filter: Filter, field: Pick<Field, 'kind' | 'unit'> | undefined): Draft { |
| 310 | const { value } = filter |
| 311 | const text = (v: number | string | undefined): string => (v === undefined ? '' : valueText(v, field)) |
| 312 | if (filter.op === 'between' && Array.isArray(value)) return { field: filter.field, op: filter.op, a: text(value[0]), b: text(value[1]) } |
| 313 | if (Array.isArray(value)) return { field: filter.field, op: filter.op, a: value.map(text).join(', '), b: '' } |
| 314 | return { field: filter.field, op: filter.op, a: text(value), b: '' } |
| 315 | } |
| 316 | |
| 317 | /** The filter a draft says, or a sentence about what is wrong with it. */ |
| 318 | export function fromDraft(draft: Draft, field: Field | undefined): { filter: Filter } | { error: string } { |
| 319 | if (!field) return { error: 'Pick a field.' } |
| 320 | if (!field.ops.includes(draft.op)) return { error: `${fieldLabel(field.field)} takes ${field.ops.map(opText).join(', ')}.` } |
| 321 | const base = { field: field.field, op: draft.op } |
| 322 | if (draft.op === 'is_null' || draft.op === 'not_null') return { filter: base } |
| 323 | const hint = unitHint(field) |
| 324 | const wrong = (text: string): { error: string } => ({ error: `Cannot read "${text}"${hint ? ` (${hint})` : ''}.` }) |
| 325 | if (draft.op === 'between') { |
| 326 | const low = parseValue(draft.a, field) |
| 327 | const high = parseValue(draft.b, field) |
| 328 | if (low === null) return wrong(draft.a) |
| 329 | if (high === null) return wrong(draft.b) |
| 330 | return { filter: { ...base, value: [low, high] } } |
| 331 | } |
| 332 | // A list field is asked "any of", whatever the op's name: the server takes a list there. |
| 333 | if (draft.op === 'in') { |
| 334 | const parts = draft.a.split(',').map((part) => part.trim()).filter(Boolean) |
| 335 | if (parts.length === 0) return { error: 'Give at least one value, separated by commas.' } |
| 336 | const values: (number | string)[] = [] |
| 337 | for (const part of parts) { |
| 338 | const value = parseValue(part, field) |
| 339 | if (value === null) return wrong(part) |
| 340 | values.push(value) |
| 341 | } |
| 342 | return { filter: { ...base, value: values } } |
| 343 | } |
| 344 | if (draft.op === 'contains') return draft.a.trim() ? { filter: { ...base, value: draft.a.trim() } } : { error: 'Give the text to look for.' } |
| 345 | const value = parseValue(draft.a, field) |
| 346 | return value === null ? wrong(draft.a) : { filter: { ...base, value } } |
| 347 | } |
| 348 | |
| 349 | /** Tickers as typed, as the SEC writes them: upper case, once each, split on commas, spaces and lines. */ |
| 350 | export function readTickers(text: string): string[] { |
| 351 | return [...new Set(text.split(/[\s,;]+/).map((t) => t.trim().toUpperCase()).filter(Boolean))] |
| 352 | } |
| 353 | |
| 354 | // The screen: what one call sends, and what its answer holds. |
| 355 | |
| 356 | /** What the server's session holds, as its last answer said it. */ |
| 357 | export interface Echo { |
| 358 | sessionId: string |
| 359 | filters: Filter[] |
| 360 | tickers: string[] |
| 361 | includeUnlisted: boolean |
| 362 | } |
| 363 | |
| 364 | /** A screen_companies answer without its rows: the dataset's meta, read leniently. */ |
| 365 | export interface Screen { |
| 366 | sessionId: string | null |
| 367 | universe: number |
| 368 | count: number |
| 369 | countBefore: number | null |
| 370 | applied: (Filter & { matches: number })[] |
| 371 | tickers: string[] |
| 372 | unknownTickers: string[] |
| 373 | includeUnlisted: boolean |
| 374 | ready: boolean |
| 375 | max: number |
| 376 | } |
| 377 | |
| 378 | export function readScreen(meta: Record<string, unknown>): Screen | null { |
| 379 | if (typeof meta.count !== 'number') return null |
| 380 | const applied = Array.isArray(meta.applied_filters) ? meta.applied_filters.filter(isRecord) : [] |
| 381 | return { |
| 382 | sessionId: typeof meta.session_id === 'string' ? meta.session_id : null, |
| 383 | universe: number(meta.universe), |
| 384 | count: meta.count, |
| 385 | countBefore: typeof meta.count_before === 'number' ? meta.count_before : null, |
| 386 | applied: applied |
| 387 | .filter((f) => typeof f.field === 'string' && OPS.includes(f.op as Op)) |
| 388 | .map((f) => ({ |
| 389 | field: f.field as string, |
| 390 | op: f.op as Op, |
| 391 | ...(f.value === undefined || f.value === null ? {} : { value: f.value as Filter['value'] }), |
| 392 | matches: number(f.matches), |
| 393 | })), |
| 394 | tickers: strings(meta.tickers), |
| 395 | unknownTickers: strings(meta.unknown_tickers), |
| 396 | includeUnlisted: meta.include_unlisted === true, |
| 397 | ready: meta.qualitative_ready === true, |
| 398 | max: number(meta.qualitative_max), |
| 399 | } |
| 400 | } |
| 401 | |
| 402 | export function echoOf(screen: Screen): Echo | null { |
| 403 | if (!screen.sessionId) return null |
| 404 | return { |
| 405 | sessionId: screen.sessionId, |
| 406 | filters: screen.applied.map(({ matches: _matches, ...filter }) => filter), |
| 407 | tickers: screen.tickers, |
| 408 | includeUnlisted: screen.includeUnlisted, |
| 409 | } |
| 410 | } |
| 411 | |
| 412 | type Listed = Pick<State, 'filters' | 'tickers' | 'includeUnlisted'> |
| 413 | |
| 414 | /** Whether the server's session holds the list this state asks for. A run freezes the session's list, so it waits for this. */ |
| 415 | export function inSync(state: Listed & { sessionId: string | null }, echo: Echo | null): boolean { |
| 416 | return ( |
| 417 | echo !== null && |
| 418 | echo.sessionId === state.sessionId && |
| 419 | same(echo.filters.map(parts), state.filters.map(parts)) && |
| 420 | same(echo.tickers, readTickers(state.tickers.join(' '))) && |
| 421 | echo.includeUnlisted === state.includeUnlisted |
| 422 | ) |
| 423 | } |
| 424 | |
| 425 | /** |
| 426 | * What one screen_companies call sends. The list (filters, include_unlisted) goes whole when the |
| 427 | * session does not hold it yet, and is left out when only the sort, the page or the columns moved: |
| 428 | * to the server a call that names the list is a change of it, and one that does not is a read. |
| 429 | * No tickers leave the view unless the user has a list of their own: they go as the server's |
| 430 | * tickers parameter, an empty one only to clear a list the session still holds, and never as a column. |
| 431 | */ |
| 432 | export function screenArgs( |
| 433 | state: Listed & Pick<State, 'sort' | 'page' | 'columns' | 'sessionId'>, |
| 434 | echo: Echo | null, |
| 435 | pageSize: number, |
| 436 | ): Record<string, unknown> { |
| 437 | const columns = state.columns.filter((field) => !IDENTITY.has(field)) |
| 438 | const read = { sort: state.sort, columns, limit: pageSize, offset: state.page * pageSize } |
| 439 | if (inSync(state, echo)) return { session_id: state.sessionId, ...read } |
| 440 | // Left out, the session's list stays, and a new session has none: [] is only how a list is taken back. |
| 441 | const held = (echo?.tickers.length ?? 0) > 0 |
| 442 | return { |
| 443 | ...(state.sessionId ? { session_id: state.sessionId } : {}), |
| 444 | filters: state.filters, |
| 445 | ...(state.tickers.length > 0 ? { tickers: state.tickers } : held ? { tickers: [] } : {}), |
| 446 | include_unlisted: state.includeUnlisted, |
| 447 | ...read, |
| 448 | } |
| 449 | } |
| 450 | |
| 451 | /** How far over the qualitative stage's limit the list is, in the words the view and the orchestrator are told. */ |
| 452 | export function overLimit(screen: Pick<Screen, 'count' | 'max'>): string { |
| 453 | if (screen.count === 0) return 'The list is empty: there is nothing to check.' |
| 454 | const over = screen.count - screen.max |
| 455 | const n = (value: number): string => value.toLocaleString('en-US') |
| 456 | return `The list has ${n(screen.count)} companies, ${n(over)} over the ${screen.max} the qualitative stage takes. Narrow it first.` |
| 457 | } |
| 458 | |
| 459 | // The qualitative stage. |
| 460 | |
| 461 | /** A criterion's id from its question: the words that say something, as the server's pattern wants them (^[a-z0-9_]{1,40}$), unlike any taken. */ |
| 462 | export function slug(question: string, taken: string[] = []): string { |
| 463 | const words = question.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim().split(' ').filter(Boolean) |
| 464 | // A letter on its own is what an apostrophe left behind. |
| 465 | const telling = words.filter((word) => !SMALL_WORDS.has(word) && (word.length > 1 || /\d/.test(word))) |
| 466 | // Whole words, up to five, while they fit with room left for a number behind them. |
| 467 | let base = '' |
| 468 | for (const word of (telling.length > 0 ? telling : words).slice(0, 5)) { |
| 469 | if (base && base.length + 1 + word.length > 36) break |
| 470 | base = base ? `${base}_${word}` : word.slice(0, 36) |
| 471 | } |
| 472 | base ||= 'criterion' |
| 473 | let id = base |
| 474 | for (let n = 2; taken.includes(id); n++) id = `${base}_${n}` |
| 475 | return id |
| 476 | } |
| 477 | |
| 478 | const SMALL_WORDS = new Set( |
| 479 | 'a an and any are as at be been by do does for from has have in is it its of on or that the their this to was were which with company'.split(' '), |
| 480 | ) |
| 481 | |
| 482 | /** |
| 483 | * Whether a run still has to be started. A `run` with no id is the request, from the Run button or |
| 484 | * from the orchestrator; what a model fills in beside it (a status, an empty id) does not count as |
| 485 | * started. Only what the view itself leaves does: the server's run id, or a failed start with its reason. |
| 486 | */ |
| 487 | export function needsStart(criteria: Criterion[] | undefined, run: Run | null | undefined): boolean { |
| 488 | return Boolean(criteria?.length) && run !== null && run !== undefined && !run.id && !(run.status === 'error' && run.error) |
| 489 | } |
| 490 | |
| 491 | /** |
| 492 | * Where the qualitative stage stands, in one word, for the bar and the orchestrator alike. A run asked |
| 493 | * for with no criteria never starts, and must not read as if it had: that is how a reply came to promise |
| 494 | * results of a run that did not exist. What a model fills in beside the request counts no more here than in needsStart. |
| 495 | */ |
| 496 | export type Stage = 'none' | 'no_criteria' | 'starting' | 'running' | 'done' | 'error' |
| 497 | |
| 498 | export function stageOf(criteria: Criterion[] | undefined, run: Run | null | undefined): Stage { |
| 499 | if (!run) return 'none' |
| 500 | if (run.status === 'error' && run.error) return 'error' |
| 501 | if (!run.id) return criteria?.length ? 'starting' : 'no_criteria' |
| 502 | return run.status === 'done' ? 'done' : 'running' |
| 503 | } |
| 504 | |
| 505 | /** What a started run answers with. */ |
| 506 | export function readStart(text: string): { runId: string; total: number } { |
| 507 | const value = parseJson(text) |
| 508 | if (typeof value.run_id !== 'string' || !value.run_id) throw new Error('the run did not start: no run id came back') |
| 509 | return { runId: value.run_id, total: number(value.total) } |
| 510 | } |
| 511 | |
| 512 | export interface Progress { |
| 513 | total: number |
| 514 | queued: number |
| 515 | running: number |
| 516 | error: number |
| 517 | pass: number |
| 518 | fail: number |
| 519 | unclear: number |
| 520 | unverified: number |
| 521 | } |
| 522 | |
| 523 | /** One quote of a filing that an answer rests on, already checked by the server. */ |
| 524 | export interface Citation { |
| 525 | id: string |
| 526 | title: string |
| 527 | url: string | null |
| 528 | quote: string |
| 529 | } |
| 530 | |
| 531 | export interface CriterionResult { |
| 532 | id: string |
| 533 | verdict: string |
| 534 | rationale: string |
| 535 | verification: string |
| 536 | confidence: number | null |
| 537 | note: string |
| 538 | /** Citation ids. */ |
| 539 | evidence: string[] |
| 540 | gathered: { searched: number; judged: number; paragraphs: number; relevant: number } | null |
| 541 | } |
| 542 | |
| 543 | export interface CompanyResult { |
| 544 | cik: number |
| 545 | ticker: string | null |
| 546 | name: string | null |
| 547 | status: string |
| 548 | verdict: Verdict | null |
| 549 | error: string | null |
| 550 | criteria: CriterionResult[] |
| 551 | } |
| 552 | |
| 553 | export interface RunSnapshot { |
| 554 | status: 'queued' | 'running' | 'done' |
| 555 | progress: Progress |
| 556 | criteria: Criterion[] |
| 557 | results: CompanyResult[] |
| 558 | citations: Record<string, Citation> |
| 559 | } |
| 560 | |
| 561 | /** |
| 562 | * One get_qualitative_screen answer: its results are the dataset's rows, the rest its meta. Read |
| 563 | * leniently. A criterion's evidence is a list of citation ids; the citations themselves sit once at the top. |
| 564 | */ |
| 565 | export function readRun(meta: Record<string, unknown>, rows: Record<string, unknown>[]): RunSnapshot { |
| 566 | const citations: Record<string, Citation> = {} |
| 567 | for (const entry of Array.isArray(meta.citations) ? meta.citations : []) { |
| 568 | const citation = readCitation(entry) |
| 569 | if (citation) citations[citation.id] = citation |
| 570 | } |
| 571 | const results = rows.map((row): CompanyResult => ({ |
| 572 | cik: number(row.cik), |
| 573 | ticker: typeof row.ticker === 'string' ? row.ticker : null, |
| 574 | name: typeof row.name === 'string' ? row.name : null, |
| 575 | status: typeof row.status === 'string' ? row.status : 'queued', |
| 576 | verdict: VERDICTS.includes(row.verdict as Verdict) ? (row.verdict as Verdict) : null, |
| 577 | error: typeof row.error === 'string' ? row.error : null, |
| 578 | criteria: (Array.isArray(row.criteria) ? row.criteria.filter(isRecord) : []).map((c) => readCriterion(c, citations)), |
| 579 | })) |
| 580 | const progress = isRecord(meta.progress) ? meta.progress : {} |
| 581 | return { |
| 582 | status: meta.status === 'done' ? 'done' : meta.status === 'queued' ? 'queued' : 'running', |
| 583 | progress: { |
| 584 | total: number(progress.total), |
| 585 | queued: number(progress.queued), |
| 586 | running: number(progress.running), |
| 587 | error: number(progress.error), |
| 588 | pass: number(progress.pass), |
| 589 | fail: number(progress.fail), |
| 590 | unclear: number(progress.unclear), |
| 591 | unverified: number(progress.unverified), |
| 592 | }, |
| 593 | criteria: (Array.isArray(meta.criteria) ? meta.criteria.filter(isRecord) : []) |
| 594 | .filter((c) => typeof c.id === 'string' && typeof c.question === 'string') |
| 595 | .map((c) => ({ id: c.id as string, question: c.question as string })), |
| 596 | results, |
| 597 | citations, |
| 598 | } |
| 599 | } |
| 600 | |
| 601 | /** The same answer when it came back as text rather than rows. */ |
| 602 | export function readRunText(text: string): RunSnapshot { |
| 603 | const { results, ...meta } = parseJson(text) |
| 604 | return readRun(meta, Array.isArray(results) ? results.filter(isRecord) : []) |
| 605 | } |
| 606 | |
| 607 | function readCitation(entry: unknown): Citation | null { |
| 608 | if (!isRecord(entry) || typeof entry.id !== 'string' || !entry.id) return null |
| 609 | const url = typeof entry.url === 'string' && entry.url.startsWith('https://') ? entry.url : null |
| 610 | return { |
| 611 | id: entry.id, |
| 612 | title: typeof entry.title === 'string' ? entry.title : '', |
| 613 | url, |
| 614 | quote: typeof entry.quote === 'string' ? entry.quote : '', |
| 615 | } |
| 616 | } |
| 617 | |
| 618 | function readCriterion(c: Record<string, unknown>, citations: Record<string, Citation>): CriterionResult { |
| 619 | // An id with no citation behind it has nothing to show, so it is not kept. |
| 620 | const evidence = (Array.isArray(c.evidence) ? c.evidence : []).filter((id): id is string => typeof id === 'string' && id in citations) |
| 621 | const gathered = isRecord(c.gathered) ? c.gathered : null |
| 622 | return { |
| 623 | id: typeof c.id === 'string' ? c.id : '', |
| 624 | verdict: typeof c.verdict === 'string' ? c.verdict : '', |
| 625 | rationale: typeof c.rationale === 'string' ? c.rationale : '', |
| 626 | verification: typeof c.verification === 'string' ? c.verification : '', |
| 627 | confidence: typeof c.confidence === 'number' ? c.confidence : null, |
| 628 | note: typeof c.note === 'string' ? c.note : '', |
| 629 | evidence, |
| 630 | gathered: gathered && { |
| 631 | searched: Array.isArray(gathered.searched) ? gathered.searched.length : 0, |
| 632 | judged: number(gathered.judged), |
| 633 | paragraphs: number(gathered.paragraphs), |
| 634 | relevant: Array.isArray(gathered.good) ? gathered.good.length : 0, |
| 635 | }, |
| 636 | } |
| 637 | } |
| 638 | |
| 639 | /** The companies that passed, by ticker. The table's rows are the list that was read; only these are what the user asked for. */ |
| 640 | export function passedTickers(run: Pick<RunSnapshot, 'results'> | null): string[] { |
| 641 | return (run?.results ?? []).flatMap((result) => (result.verdict === 'pass' && result.ticker ? [result.ticker] : [])) |
| 642 | } |
| 643 | |
| 644 | /** Where a run is, in one line: how many companies are read, then the counts by verdict. */ |
| 645 | export function progressText(progress: Progress, status: RunSnapshot['status']): string { |
| 646 | const read = progress.total - progress.queued - progress.running |
| 647 | const head = status === 'done' ? `${progress.total} read` : `reading: ${read} of ${progress.total}` |
| 648 | const counts = VERDICTS.map((verdict) => `${progress[verdict]} ${verdict}`) |
| 649 | if (progress.error > 0) counts.push(`${progress.error} error`) |
| 650 | return `${head} · ${counts.join(' · ')}` |
| 651 | } |
| 652 | |
| 653 | /** A verification that did not hold, in words; nothing for one that did, or for a claim of absence, which cites nothing. */ |
| 654 | export function verificationMark(verification: string): string | null { |
| 655 | if (!verification || verification === 'supports' || verification === 'not_checked') return null |
| 656 | if (verification === 'contradicts') return 'The cited evidence contradicts this verdict.' |
| 657 | if (verification === 'says_nothing') return 'The cited evidence does not show this.' |
| 658 | if (verification === 'uncertain') return 'The verifier was not confident the cited evidence carries this verdict.' |
| 659 | return `Verification: ${verification}.` |
| 660 | } |
| 661 | |
| 662 | /** What was searched and read for one criterion, in one line. */ |
| 663 | export function gatheredText(gathered: NonNullable<CriterionResult['gathered']>): string { |
| 664 | const n = (count: number, one: string, many: string): string => `${count.toLocaleString('en-US')} ${count === 1 ? one : many}` |
| 665 | return [ |
| 666 | `${n(gathered.searched, 'search', 'searches')} made`, |
| 667 | `${n(gathered.judged, 'passage', 'passages')} judged`, |
| 668 | `${n(gathered.paragraphs, 'paragraph', 'paragraphs')} read`, |
| 669 | `${gathered.relevant.toLocaleString('en-US')} relevant`, |
| 670 | ].join(', ') |
| 671 | } |
| 672 | |
| 673 | /** |
| 674 | * The open company's evidence as text a model can quote. The first line names what it is, since a |
| 675 | * citation of the panel takes that line as its title; each quote is followed by its marker, [^id], |
| 676 | * which the chat resolves against the citations of the same tool result. |
| 677 | */ |
| 678 | export function evidenceText(company: CompanyResult, questions: Criterion[], citations: Record<string, Citation>, runId: string): string { |
| 679 | const name = company.name ? ` (${company.name})` : '' |
| 680 | const lines = [`Screener evidence for ${company.ticker ?? `CIK ${company.cik}`}${name}, qualitative run ${runId}: ${company.verdict ?? company.status}`] |
| 681 | if (company.error) lines.push('', `Error: ${company.error}`) |
| 682 | for (const c of company.criteria) { |
| 683 | const question = questions.find((q) => q.id === c.id)?.question |
| 684 | const confidence = c.confidence === null ? '' : `, confidence ${c.confidence.toFixed(2)}` |
| 685 | lines.push('', `## ${c.id}${question ? `: ${question}` : ''}`, `Verdict: ${c.verdict} (verification: ${c.verification || 'none'}${confidence})`) |
| 686 | if (c.rationale) lines.push(`Rationale: ${c.rationale}`) |
| 687 | if (c.note) lines.push(`Note: ${c.note}`) |
| 688 | for (const id of c.evidence) { |
| 689 | const citation = citations[id] |
| 690 | if (citation) lines.push(`- "${citation.quote}" [^${citation.id}]${citation.title ? ` (${citation.title})` : ''}`) |
| 691 | } |
| 692 | if (c.gathered) lines.push(`Retrieval: ${gatheredText(c.gathered)}.`) |
| 693 | } |
| 694 | return lines.join('\n') |
| 695 | } |
| 696 | |
| 697 | // What the orchestrator reads. |
| 698 | |
| 699 | /** What the view publishes: the screen behind the rows on screen, and where the run is. */ |
| 700 | export interface Output { |
| 701 | session_id: string | null |
| 702 | universe: number |
| 703 | count: number |
| 704 | applied_filters: (Filter & { matches: number })[] |
| 705 | /** The tickers on this page, in order. */ |
| 706 | tickers: string[] |
| 707 | unknown_tickers: string[] |
| 708 | sort: Sort |
| 709 | qualitative_ready: boolean |
| 710 | qualitative_max: number |
| 711 | /** status is a Stage. passed: the tickers with a pass, once there are results; the rows above are only the list that was read. */ |
| 712 | qualitative?: { run_id: string | null; status: string; progress: Progress | null; passed?: string[]; error?: string } |
| 713 | open: string |
| 714 | } |
| 715 | |
| 716 | /** The app takes 4,096 bytes of output: the tickers at the end of the page give way first. */ |
| 717 | export const OUTPUT_BYTES = 3800 |
| 718 | |
| 719 | export function fitOutput(output: Output): Output { |
| 720 | let fitted = output |
| 721 | while (fitted.tickers.length > 0 && byteLength(JSON.stringify(fitted)) > OUTPUT_BYTES) { |
| 722 | fitted = { ...fitted, tickers: fitted.tickers.slice(0, Math.floor(fitted.tickers.length * 0.8)) } |
| 723 | } |
| 724 | return fitted |
| 725 | } |
| 726 | |
| 727 | /** The one line the model's map carries: how many pass, of how many, behind which screen, and where the run is. */ |
| 728 | export function summarize(state: Partial<State>, output: Partial<Output>): string { |
| 729 | const n = (value: number | undefined): string => (value ?? 0).toLocaleString('en-US') |
| 730 | const filters = state.filters ?? [] |
| 731 | const parts = [`Screener: ${n(output.count)} of ${n(output.universe)} pass`] |
| 732 | if (filters.length > 0) parts[0] += ` ${filters.length} filter${filters.length === 1 ? '' : 's'}` |
| 733 | if (state.tickers?.length) parts.push(`own list of ${state.tickers.length}`) |
| 734 | const sort = state.sort ?? DEFAULT_SORT |
| 735 | parts.push(`sort ${sort.field} ${sort.dir}`) |
| 736 | const q = output.qualitative |
| 737 | if (q) { |
| 738 | const progress = q.progress |
| 739 | parts.push( |
| 740 | q.status === 'error' |
| 741 | ? `qualitative run failed: ${q.error ?? 'unknown error'}` |
| 742 | : progress |
| 743 | ? `qualitative run ${q.status}: ${progressText(progress, q.status === 'done' ? 'done' : 'running')}` |
| 744 | : q.status === 'no_criteria' |
| 745 | ? 'a qualitative run was asked for but NOT started: state.criteria is empty' |
| 746 | : `qualitative run ${q.status}`, |
| 747 | ) |
| 748 | } else if (output.qualitative_ready) parts.push('ready for the qualitative stage') |
| 749 | if (output.open) parts.push(`evidence open for ${output.open}`) |
| 750 | return parts.join(', ') |
| 751 | } |
| 752 | |
| 753 | // Small things. |
| 754 | |
| 755 | function parseJson(text: string): Record<string, unknown> { |
| 756 | try { |
| 757 | const value: unknown = JSON.parse(text) |
| 758 | return isRecord(value) ? value : {} |
| 759 | } catch { |
| 760 | return {} |
| 761 | } |
| 762 | } |
| 763 | |
| 764 | function strings(value: unknown): string[] { |
| 765 | return Array.isArray(value) ? value.filter((v): v is string => typeof v === 'string') : [] |
| 766 | } |
| 767 | |
| 768 | function number(value: unknown): number { |
| 769 | return typeof value === 'number' && Number.isFinite(value) ? value : 0 |
| 770 | } |
| 771 | |
| 772 | function isRecord(value: unknown): value is Record<string, unknown> { |
| 773 | return typeof value === 'object' && value !== null && !Array.isArray(value) |
| 774 | } |
| 775 | |
| 776 | /** A filter by what it says, whatever order its keys were written in. */ |
| 777 | function parts(filter: Filter): unknown[] { |
| 778 | return [filter.field, filter.op, filter.value ?? null] |
| 779 | } |
| 780 | |
| 781 | function same(a: unknown, b: unknown): boolean { |
| 782 | return JSON.stringify(a) === JSON.stringify(b) |
| 783 | } |
| 784 | |
| 785 | /** A product or a quotient without the digits floating point adds at the end: 15.3 / 100 is 0.153. */ |
| 786 | function clean(value: number): number { |
| 787 | return Number(value.toPrecision(12)) |
| 788 | } |
| 789 | |
| 790 | function byteLength(text: string): number { |
| 791 | return new TextEncoder().encode(text).length |
| 792 | } |