Jaspers Jaspers Research: Research rooms where configurable analysts research companies in parallel over SEC filings data, answering with checked citations.
| 1 | import { citationLink, numberCitations, numberKey, sourceTitle, toMarkers, type Audit, type Citation } from './citations.ts' |
| 2 | |
| 3 | // A research room and its thread, as data: the shapes on disk and in live values, and the pure work |
| 4 | // on them the runner and the view share. How a room is named and numbered, what an analyst is shown |
| 5 | // of the thread, where a tool chip sits, which old tool results make way, and what the orchestrator |
| 6 | // reads back. No files and no model here, so all of it can be read in a test. |
| 7 | |
| 8 | export interface Room { |
| 9 | id: string |
| 10 | name: string |
| 11 | tickers: string[] |
| 12 | /** Analyst ids, in roster order. */ |
| 13 | analysts: string[] |
| 14 | createdAt: string |
| 15 | updatedAt: string |
| 16 | /** |
| 17 | * The workspace the room belongs to: the one it was started on. It is listed, opened, and posted to |
| 18 | * from there only. A room made before rooms belonged to one is given one when the plugin starts. |
| 19 | */ |
| 20 | workspace?: string |
| 21 | } |
| 22 | |
| 23 | export interface Step { |
| 24 | /** The length of the message's text when the call was made: where its chip sits. */ |
| 25 | at: number |
| 26 | tool: string |
| 27 | /** The arguments as JSON, cut at 200 characters. */ |
| 28 | args: string |
| 29 | summary: string | null |
| 30 | /** The saved result's ref, like r7, once it came back. */ |
| 31 | ref: string | null |
| 32 | isError: boolean |
| 33 | } |
| 34 | |
| 35 | export type MessageStatus = 'queued' | 'working' | 'done' | 'stopped' | 'error' |
| 36 | |
| 37 | export interface UserMessage { |
| 38 | seq: number |
| 39 | role: 'user' |
| 40 | text: string |
| 41 | to: string[] |
| 42 | at: string |
| 43 | /** The workspace on screen as the user posted it, whose views the analysts read. */ |
| 44 | workspace?: string |
| 45 | } |
| 46 | |
| 47 | export interface AnalystMessage { |
| 48 | seq: number |
| 49 | role: 'analyst' |
| 50 | analyst: string |
| 51 | name: string |
| 52 | color: string | null |
| 53 | inReplyTo: number |
| 54 | status: MessageStatus |
| 55 | text: string |
| 56 | steps: Step[] |
| 57 | files: { path: string; size: number }[] |
| 58 | /** The sources the answer cites, numbered as in its text, with the server's verdict on each. */ |
| 59 | citations: Citation[] |
| 60 | /** What the citation audit found, once the answer was done; null before. */ |
| 61 | audit: Audit | null |
| 62 | turns: number |
| 63 | error: string | null |
| 64 | usage: { input: number; output: number } |
| 65 | startedAt: string | null |
| 66 | endedAt: string | null |
| 67 | } |
| 68 | |
| 69 | export type Message = UserMessage | AnalystMessage |
| 70 | |
| 71 | /** What the room view publishes, for the orchestrator. */ |
| 72 | export interface RoomOutput { |
| 73 | room: string |
| 74 | name: string |
| 75 | tickers: string[] |
| 76 | analysts: { id: string; name: string; status: 'idle' | 'queued' | 'working' }[] |
| 77 | last: { seq: number; from: string; status: string; excerpt: string; files: string[] }[] |
| 78 | messages: number |
| 79 | files: number |
| 80 | } |
| 81 | |
| 82 | /** Whether a call may reach a room: one of its own workspace's, or any when the call names no workspace. */ |
| 83 | export function onWorkspace(room: Room, workspace: string | null): boolean { |
| 84 | return workspace === null || room.workspace === workspace |
| 85 | } |
| 86 | |
| 87 | /** |
| 88 | * The workspace for a room made before rooms belonged to one: the workspace its earliest message was |
| 89 | * posted from, among those still there, else `fallback`. |
| 90 | */ |
| 91 | export function claimWorkspace(messages: Message[], known: ReadonlySet<string>, fallback: string | null): string | null { |
| 92 | const first = [...messages] |
| 93 | .sort((a, b) => a.seq - b.seq) |
| 94 | .find((m): m is UserMessage => m.role === 'user' && typeof m.workspace === 'string' && known.has(m.workspace)) |
| 95 | return first?.workspace ?? fallback |
| 96 | } |
| 97 | |
| 98 | /** A tool result as the runner keeps it in the model's turns. */ |
| 99 | interface ToolTurn { |
| 100 | role: 'tool' |
| 101 | results: { callId: string; output: string; isError: boolean }[] |
| 102 | } |
| 103 | |
| 104 | const SLUG_MAX = 40 |
| 105 | const ARGS_MAX = 200 |
| 106 | const EXCERPT = 300 |
| 107 | const OUTPUT_MAX = 4096 |
| 108 | const STUB_PREVIEW = 1200 |
| 109 | |
| 110 | export function roomIdFor(name: string, random: string): string { |
| 111 | const slug = name |
| 112 | .toLowerCase() |
| 113 | .replace(/[^a-z0-9]+/g, '-') |
| 114 | .replace(/^-+|-+$/g, '') |
| 115 | .slice(0, SLUG_MAX) |
| 116 | .replace(/-+$/, '') |
| 117 | return `${slug || 'room'}-${random}` |
| 118 | } |
| 119 | |
| 120 | export function seqFile(seq: number): string { |
| 121 | return `${String(seq).padStart(6, '0')}.json` |
| 122 | } |
| 123 | |
| 124 | export function nextSeq(messages: Message[]): number { |
| 125 | return messages.reduce((max, m) => Math.max(max, m.seq), 0) + 1 |
| 126 | } |
| 127 | |
| 128 | /** |
| 129 | * What an analyst is shown of the thread, as the first user turn: the whole room, every message |
| 130 | * numbered and labeled with who said it, then the message for this analyst. Only the answer it is |
| 131 | * writing is left out. One labeled transcript keeps the analysts apart, which unlabeled history does not. |
| 132 | */ |
| 133 | export function transcript(input: { |
| 134 | messages: Message[] |
| 135 | names: Record<string, string> |
| 136 | analyst: string |
| 137 | message: UserMessage |
| 138 | /** The seq of the answer this analyst is writing. */ |
| 139 | answer: number |
| 140 | }): string { |
| 141 | const { messages, names, analyst, message, answer } = input |
| 142 | const blocks = messages.filter((m) => m.seq !== message.seq && m.seq !== answer).map((m) => messageLine(m, names)) |
| 143 | const lines: string[] = [] |
| 144 | if (blocks.length > 0) { |
| 145 | lines.push('The room so far:', '') |
| 146 | for (const block of blocks) lines.push(block, '') |
| 147 | } |
| 148 | lines.push(`New message for you, ${names[analyst] ?? analyst}:`, messageLine(message, names)) |
| 149 | return lines.join('\n') |
| 150 | } |
| 151 | |
| 152 | /** What an analyst has seen of each message, by seq, as `stateOf` puts it. */ |
| 153 | export function shownOf(messages: Message[]): Map<number, string> { |
| 154 | return new Map(messages.map((m) => [m.seq, stateOf(m)])) |
| 155 | } |
| 156 | |
| 157 | /** |
| 158 | * What came into the room since the analyst last saw it, as a turn for its model: a message it has |
| 159 | * not seen, or an answer that has finished since it saw it being written. An answer still being |
| 160 | * written waits until it is done. What comes back is noted in `shown`, so each arrives once. |
| 161 | */ |
| 162 | export function roomNews(input: { |
| 163 | messages: Message[] |
| 164 | names: Record<string, string> |
| 165 | message: UserMessage |
| 166 | answer: number |
| 167 | shown: Map<number, string> |
| 168 | }): string | null { |
| 169 | const { messages, names, message, answer, shown } = input |
| 170 | const fresh = messages.filter((m) => m.seq !== answer && stateOf(m) !== 'open' && shown.get(m.seq) !== stateOf(m)) |
| 171 | if (fresh.length === 0) return null |
| 172 | for (const m of fresh) shown.set(m.seq, stateOf(m)) |
| 173 | return [`Room update: these came in while you were working. You are still answering #${message.seq}.`, ...fresh.map((m) => messageLine(m, names))].join('\n\n') |
| 174 | } |
| 175 | |
| 176 | /** A step for a call made when the message's text had this much in it. */ |
| 177 | export function stepFor(text: string, tool: string, input: Record<string, unknown>): Step { |
| 178 | return { at: text.length, tool, args: JSON.stringify(input).slice(0, ARGS_MAX), summary: null, ref: null, isError: false } |
| 179 | } |
| 180 | |
| 181 | /** One line saying what a tool gave back, for its chip. */ |
| 182 | export function resultSummary(text: string, isError: boolean): string { |
| 183 | if (isError) return `Error: ${(text.split('\n')[0] ?? '').slice(0, 120)}` |
| 184 | const count = rowCount(text) |
| 185 | return count === null ? `${text.length.toLocaleString('en-US')} characters` : `${count} result${count === 1 ? '' : 's'}` |
| 186 | } |
| 187 | |
| 188 | /** |
| 189 | * The turns with old, large tool results swapped for stubs until what is kept fits the budget. The |
| 190 | * newest `keepNewest` results and anything under `minSize` stay whole; a stub names its ref, so the |
| 191 | * model reads the rest with fetch_result instead of running the search again. |
| 192 | */ |
| 193 | export function evictResults<T extends { role: string }>( |
| 194 | turns: T[], |
| 195 | refs: Map<string, { ref: string; tool: string; args: string }>, |
| 196 | options: { budget: number; keepNewest: number; minSize: number }, |
| 197 | ): T[] { |
| 198 | const results = turns.flatMap((turn) => (turn.role === 'tool' ? (turn as unknown as ToolTurn).results : [])) |
| 199 | let kept = results.reduce((sum, r) => sum + (isStub(r.output) ? 0 : r.output.length), 0) |
| 200 | const protectedIds = new Set(results.slice(-options.keepNewest).map((r) => r.callId)) |
| 201 | const evicted = new Map<string, string>() |
| 202 | for (const result of results) { |
| 203 | if (kept <= options.budget) break |
| 204 | if (protectedIds.has(result.callId) || isStub(result.output) || result.output.length < options.minSize) continue |
| 205 | const about = refs.get(result.callId) |
| 206 | if (!about) continue |
| 207 | evicted.set( |
| 208 | result.callId, |
| 209 | `[${about.ref} evicted: ${about.tool} ${about.args}. First ${STUB_PREVIEW} characters: ${result.output.slice(0, STUB_PREVIEW)} … Read the rest with fetch_result("${about.ref}", offset).]`, |
| 210 | ) |
| 211 | kept -= result.output.length |
| 212 | } |
| 213 | if (evicted.size === 0) return turns |
| 214 | return turns.map((turn) => { |
| 215 | if (turn.role !== 'tool') return turn |
| 216 | const tool = turn as unknown as ToolTurn |
| 217 | if (!tool.results.some((r) => evicted.has(r.callId))) return turn |
| 218 | return { ...turn, results: tool.results.map((r) => (evicted.has(r.callId) ? { ...r, output: evicted.get(r.callId)! } : r)) } |
| 219 | }) |
| 220 | } |
| 221 | |
| 222 | /** What the orchestrator reads: who is in the room and busy, and the last three messages. Within 4 KB. */ |
| 223 | export function outputOf(room: Room, messages: Message[], roster: { id: string; name: string }[]): RoomOutput { |
| 224 | const analysts = roster.map(({ id, name }) => { |
| 225 | const theirs = messages.filter((m): m is AnalystMessage => m.role === 'analyst' && m.analyst === id) |
| 226 | const status = theirs.some((m) => m.status === 'working') ? 'working' : theirs.some((m) => m.status === 'queued') ? 'queued' : 'idle' |
| 227 | return { id, name, status: status as RoomOutput['analysts'][number]['status'] } |
| 228 | }) |
| 229 | const files = messages.reduce((sum, m) => sum + (m.role === 'analyst' ? m.files.length : 0), 0) |
| 230 | for (const excerpt of [EXCERPT, 150, 60]) { |
| 231 | const output: RoomOutput = { |
| 232 | room: room.id, |
| 233 | name: room.name, |
| 234 | tickers: room.tickers, |
| 235 | analysts, |
| 236 | last: messages.slice(-3).map((m) => ({ |
| 237 | seq: m.seq, |
| 238 | from: m.role === 'user' ? 'you' : m.name, |
| 239 | status: m.role === 'user' ? 'sent' : m.status, |
| 240 | excerpt: toMarkers(m.text).slice(0, excerpt), |
| 241 | files: m.role === 'user' ? [] : m.files.map((f) => f.path.slice(f.path.lastIndexOf('/') + 1)), |
| 242 | })), |
| 243 | messages: messages.length, |
| 244 | files, |
| 245 | } |
| 246 | if (JSON.stringify(output).length <= OUTPUT_MAX) return output |
| 247 | } |
| 248 | return { room: room.id, name: room.name, tickers: room.tickers, analysts, last: [], messages: messages.length, files } |
| 249 | } |
| 250 | |
| 251 | /** The one line the model's map carries for a room. */ |
| 252 | export function roomSummary(output: RoomOutput): string { |
| 253 | const busy = output.analysts.filter((a) => a.status !== 'idle').map((a) => `${a.name} ${a.status}`) |
| 254 | const plural = (n: number, word: string): string => `${n} ${word}${n === 1 ? '' : 's'}` |
| 255 | return [output.name, busy.length > 0 ? busy.join(', ') : 'idle', plural(output.messages, 'msg'), plural(output.files, 'file')].join(' · ') |
| 256 | } |
| 257 | |
| 258 | /** |
| 259 | * The room's messages as text, for "what did they find?": every one, or the last few or one analyst's |
| 260 | * when asked, each whole, its citation tags shown as [n] markers and every source listed under it |
| 261 | * with the sec.gov link. |
| 262 | */ |
| 263 | export function threadText(messages: Message[], names: Record<string, string>, options: { last?: number; analyst?: string }): string { |
| 264 | const chosen = options.analyst ? messages.filter((m) => m.role === 'analyst' && m.analyst === options.analyst) : messages |
| 265 | return (options.last === undefined ? chosen : chosen.slice(-options.last)) |
| 266 | .map((m) => { |
| 267 | if (m.role === 'user') return `#${m.seq} ${userLine(m, names)}` |
| 268 | const text = toMarkers(m.text).trim() || (m.error ? `(error: ${m.error})` : '(nothing yet)') |
| 269 | const sources = (m.citations ?? []).map((c) => `[${c.n}] ${sourceLine(c)}`) |
| 270 | return [`#${m.seq} [${m.name}]${statusMark(m)}: ${text}`, ...(sources.length > 0 ? [`Sources: ${sources.join(' ')}`] : [])].join('\n') |
| 271 | }) |
| 272 | .join('\n\n') |
| 273 | } |
| 274 | |
| 275 | /** |
| 276 | * An answer as the user copies it: what the analyst wrote after its last tool call, since the text |
| 277 | * before that narrates the research, with its tags as the [n] markers the view shows and the sources |
| 278 | * those markers point to listed under it. Empty until the answer is in: a draft still being checked |
| 279 | * may be repaired. |
| 280 | */ |
| 281 | export function answerText(message: AnalystMessage): string { |
| 282 | if (message.status === 'queued' || message.status === 'working') return '' |
| 283 | const at = message.steps.reduce((last, step) => Math.max(last, step.at), 0) |
| 284 | const answer = message.text.slice(at) |
| 285 | const numbered = numberCitations(message.text) |
| 286 | const text = toMarkers(answer, numbered).trim() |
| 287 | if (!text) return '' |
| 288 | const cited = new Set(numberCitations(answer).map((c) => numberKey(c.attrs))) |
| 289 | const sources = numbered |
| 290 | .filter((c) => cited.has(numberKey(c.attrs))) |
| 291 | .map((c) => { |
| 292 | const citation = (message.citations ?? []).find((r) => r.n === c.n) |
| 293 | const link = citation ? citationLink(citation) : null |
| 294 | return `- [${c.n}] ${sourceTitle(c, citation)}${citation?.verified === false ? ' (quote not found)' : ''}${link ? `: ${link}` : ''}` |
| 295 | }) |
| 296 | return sources.length > 0 ? `${text}\n\n## Sources\n\n${sources.join('\n')}` : text |
| 297 | } |
| 298 | |
| 299 | /** One source in a line: what it is, whether its quote was found, and where it opens. */ |
| 300 | function sourceLine(c: Citation): string { |
| 301 | const state = c.verified === true ? 'verified' : c.verified === false ? 'unverified' : 'unchecked' |
| 302 | return [c.title ?? `document ${c.documentId}`, state, citationLink(c) ?? ''].filter(Boolean).join(' · ') |
| 303 | } |
| 304 | |
| 305 | function userLine(message: UserMessage, names: Record<string, string>): string { |
| 306 | return `[You → ${message.to.map((id) => names[id] ?? id).join(', ')}]: ${message.text}` |
| 307 | } |
| 308 | |
| 309 | /** One message as an analyst reads it: its number, who, and what it says, or where it stands when it says nothing. */ |
| 310 | function messageLine(message: Message, names: Record<string, string>): string { |
| 311 | if (message.role === 'user') return `#${message.seq} ${userLine(message, names)}` |
| 312 | const text = message.text.trim() |
| 313 | if (!text) return `#${message.seq} [${message.name}] (${message.status}${message.error ? `: ${message.error}` : ''})` |
| 314 | return `#${message.seq} [${message.name}]${statusMark(message)}: ${text}` |
| 315 | } |
| 316 | |
| 317 | /** Sent for the user's words; for an answer, open while it is queued or being written, final after. */ |
| 318 | function stateOf(message: Message): 'sent' | 'open' | 'final' { |
| 319 | if (message.role === 'user') return 'sent' |
| 320 | return message.status === 'queued' || message.status === 'working' ? 'open' : 'final' |
| 321 | } |
| 322 | |
| 323 | function statusMark(message: AnalystMessage): string { |
| 324 | return message.status === 'done' ? '' : ` (${message.status})` |
| 325 | } |
| 326 | |
| 327 | function isStub(output: string): boolean { |
| 328 | return /^\[r\d+ evicted: /.test(output) |
| 329 | } |
| 330 | |
| 331 | function rowCount(text: string): number | null { |
| 332 | try { |
| 333 | const value: unknown = JSON.parse(text) |
| 334 | if (Array.isArray(value)) return value.length |
| 335 | if (typeof value === 'object' && value !== null) { |
| 336 | for (const key of ['rows', 'results', 'items', 'data', 'companies', 'filings']) { |
| 337 | const rows = (value as Record<string, unknown>)[key] |
| 338 | if (Array.isArray(rows)) return rows.length |
| 339 | } |
| 340 | } |
| 341 | } catch { |
| 342 | // Prose, not JSON. |
| 343 | } |
| 344 | return null |
| 345 | } |